Skip to content

Commit 6d26f6f

Browse files
easwarseaswars
committed
e2e style tests for client side
1 parent a1143b6 commit 6d26f6f

File tree

2 files changed

+387
-1
lines changed

2 files changed

+387
-1
lines changed

‎internal/testutils/xds/e2e/setup_certs.go‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,11 +87,35 @@ func CreateClientTLSCredentials(t *testing.T) credentials.TransportCredentials {
8787
}
8888
roots := x509.NewCertPool()
8989
if !roots.AppendCertsFromPEM(b) {
90-
t.Fatal("failed to append certificates")
90+
t.Fatal("Failed to append certificates")
9191
}
9292
return credentials.NewTLS(&tls.Config{
9393
Certificates: []tls.Certificate{cert},
9494
RootCAs: roots,
9595
ServerName: "x.test.example.com",
9696
})
9797
}
98+
99+
// CreateServerTLSCredentials creates server-side TLS transport credentials
100+
// using certificate and key files from testdata/x509 directory.
101+
func CreateServerTLSCredentials(t *testing.T) credentials.TransportCredentials {
102+
t.Helper()
103+
104+
cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
105+
if err != nil {
106+
t.Fatalf("tls.LoadX509KeyPair(x509/server1_cert.pem, x509/server1_key.pem) failed: %v", err)
107+
}
108+
b, err := os.ReadFile(testdata.Path("x509/client_ca_cert.pem"))
109+
if err != nil {
110+
t.Fatalf("os.ReadFile(x509/client_ca_cert.pem) failed: %v", err)
111+
}
112+
ca := x509.NewCertPool()
113+
if !ca.AppendCertsFromPEM(b) {
114+
t.Fatal("Failed to append certificates")
115+
}
116+
return credentials.NewTLS(&tls.Config{
117+
ClientAuth: tls.RequireAndVerifyClientCert,
118+
Certificates: []tls.Certificate{cert},
119+
ClientCAs: ca,
120+
})
121+
}

‎test/xds/xds_client_certificate_providers_test.go‎

Lines changed: 362 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,362 @@
1+
/*
2+
*
3+
* Copyright 2023 gRPC authors.
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*
17+
*/
18+
19+
package xds_test
20+
21+
import (
22+
"context"
23+
"fmt"
24+
"strings"
25+
"testing"
26+
27+
"github.com/google/uuid"
28+
"google.golang.org/grpc"
29+
"google.golang.org/grpc/codes"
30+
"google.golang.org/grpc/connectivity"
31+
"google.golang.org/grpc/credentials/insecure"
32+
xdscreds "google.golang.org/grpc/credentials/xds"
33+
"google.golang.org/grpc/internal"
34+
"google.golang.org/grpc/internal/stubserver"
35+
"google.golang.org/grpc/internal/testutils"
36+
"google.golang.org/grpc/internal/testutils/xds/bootstrap"
37+
"google.golang.org/grpc/internal/testutils/xds/e2e"
38+
"google.golang.org/grpc/peer"
39+
"google.golang.org/grpc/resolver"
40+
"google.golang.org/grpc/status"
41+
42+
v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
43+
v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
44+
v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
45+
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
46+
v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
47+
v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
48+
testgrpc "google.golang.org/grpc/interop/grpc_testing"
49+
testpb "google.golang.org/grpc/interop/grpc_testing"
50+
)
51+
52+
// Tests the case where the bootstrap configuration contains no certificate
53+
// providers, and xDS credentials with an insecure fallback is specified at dial
54+
// time. The management server is configured to return client side xDS resources
55+
// with no security configuration. The test verifies that the gRPC client is
56+
// able to make RPCs to the backend which is configured to accept plaintext
57+
// connections. This ensures that the insecure fallback credentials are getting
58+
// used on the client.
59+
func (s) TestClientSideXDS_WithNoCertificateProvidersInBootstrap_Success(t *testing.T) {
60+
// Spin up an xDS management server.
61+
mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{})
62+
if err != nil {
63+
t.Fatalf("Failed to start management server: %v", err)
64+
}
65+
defer mgmtServer.Stop()
66+
67+
// Create bootstrap configuration with no certificate providers.
68+
nodeID := uuid.New().String()
69+
bs, err := bootstrap.Contents(bootstrap.Options{
70+
NodeID: nodeID,
71+
ServerURI: mgmtServer.Address,
72+
})
73+
if err != nil {
74+
t.Fatalf("Failed to create bootstrap configuration: %v", err)
75+
}
76+
77+
// Create an xDS resolver with the above bootstrap configuration.
78+
newResolver := internal.NewXDSResolverWithConfigForTesting
79+
if newResolver == nil {
80+
t.Fatal("internal.NewXDSResolverWithConfigForTesting is unset")
81+
}
82+
resolverBuilder, err := newResolver.(func([]byte) (resolver.Builder, error))(bs)
83+
if err != nil {
84+
t.Fatalf("Failed to create xDS resolver for testing: %v", err)
85+
}
86+
87+
// Spin up a test backend.
88+
server := stubserver.StartTestService(t, nil)
89+
defer server.Stop()
90+
91+
// Configure client side xDS resources on the management server, with no
92+
// security configuration in the Cluster resource.
93+
const serviceName = "my-service-client-side-xds"
94+
resources := e2e.DefaultClientResources(e2e.ResourceParams{
95+
DialTarget: serviceName,
96+
NodeID: nodeID,
97+
Host: "localhost",
98+
Port: testutils.ParsePort(t, server.Address),
99+
SecLevel: e2e.SecurityLevelNone,
100+
})
101+
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
102+
defer cancel()
103+
if err := mgmtServer.Update(ctx, resources); err != nil {
104+
t.Fatal(err)
105+
}
106+
107+
// Create client-side xDS credentials with an insecure fallback.
108+
creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
109+
if err != nil {
110+
t.Fatal(err)
111+
}
112+
113+
// Create a ClientConn and make a successful RPC.
114+
cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolverBuilder))
115+
if err != nil {
116+
t.Fatalf("failed to dial local test server: %v", err)
117+
}
118+
defer cc.Close()
119+
120+
client := testgrpc.NewTestServiceClient(cc)
121+
if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
122+
t.Fatalf("EmptyCall() failed: %v", err)
123+
}
124+
}
125+
126+
// Tests the case where the bootstrap configuration contains no certificate
127+
// providers, and xDS credentials with an insecure fallback is specified at dial
128+
// time. The management server is configured to return client side xDS resources
129+
// with an mTLS security configuration. The test verifies that the gRPC client
130+
// moves to TRANSIENT_FAILURE and rpcs fail with the expected error code and
131+
// string. This ensures that when the certificate provider instance name
132+
// specified in the security configuration is not present in the bootstrap,
133+
// channel creation does not fail, but it moves to TRANSIENT_FAILURE and
134+
// subsequent rpcs fail.
135+
func (s) TestClientSideXDS_WithNoCertificateProvidersInBootstrap_Failure(t *testing.T) {
136+
// Spin up an xDS management server.
137+
mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{})
138+
if err != nil {
139+
t.Fatalf("Failed to start management server: %v", err)
140+
}
141+
defer mgmtServer.Stop()
142+
143+
// Create bootstrap configuration with no certificate providers.
144+
nodeID := uuid.New().String()
145+
bs, err := bootstrap.Contents(bootstrap.Options{
146+
NodeID: nodeID,
147+
ServerURI: mgmtServer.Address,
148+
})
149+
if err != nil {
150+
t.Fatalf("Failed to create bootstrap configuration: %v", err)
151+
}
152+
153+
// Create an xDS resolver with the above bootstrap configuration.
154+
newResolver := internal.NewXDSResolverWithConfigForTesting
155+
if newResolver == nil {
156+
t.Fatal("internal.NewXDSResolverWithConfigForTesting is unset")
157+
}
158+
resolverBuilder, err := newResolver.(func([]byte) (resolver.Builder, error))(bs)
159+
if err != nil {
160+
t.Fatalf("Failed to create xDS resolver for testing: %v", err)
161+
}
162+
163+
// Spin up a test backend.
164+
server := stubserver.StartTestService(t, nil)
165+
defer server.Stop()
166+
167+
// Configure client side xDS resources on the management server, with mTLS
168+
// security configuration in the Cluster resource.
169+
const serviceName = "my-service-client-side-xds"
170+
const clusterName = "cluster-" + serviceName
171+
const endpointsName = "endpoints-" + serviceName
172+
resources := e2e.DefaultClientResources(e2e.ResourceParams{
173+
DialTarget: serviceName,
174+
NodeID: nodeID,
175+
Host: "localhost",
176+
Port: testutils.ParsePort(t, server.Address),
177+
SecLevel: e2e.SecurityLevelNone,
178+
})
179+
resources.Clusters = []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, endpointsName, e2e.SecurityLevelMTLS)}
180+
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
181+
defer cancel()
182+
if err := mgmtServer.Update(ctx, resources); err != nil {
183+
t.Fatal(err)
184+
}
185+
186+
// Create client-side xDS credentials with an insecure fallback.
187+
creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
188+
if err != nil {
189+
t.Fatal(err)
190+
}
191+
192+
// Create a ClientConn and ensure that it moves to TRANSIENT_FAILURE.
193+
cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolverBuilder))
194+
if err != nil {
195+
t.Fatalf("failed to dial local test server: %v", err)
196+
}
197+
defer cc.Close()
198+
testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
199+
200+
// Make an RPC and ensure that expected error is returned.
201+
wantErr := fmt.Sprintf("identitiy certificate provider instance name %q missing in bootstrap configuration", e2e.ClientSideCertProviderInstance)
202+
client := testgrpc.NewTestServiceClient(cc)
203+
if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) {
204+
t.Fatalf("EmptyCall() failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr)
205+
}
206+
}
207+
208+
// Tests the case where the bootstrap configuration contains one certificate
209+
// provider, and xDS credentials with an insecure fallback is specified at dial
210+
// time. The management server responds with three clusters:
211+
// 1. contains valid security configuration pointing to the certificate provider
212+
// instance specified in the bootstrap
213+
// 2. contains no security configuration, hence should use insecure fallback
214+
// 3. contains invalid security configuration pointing to a non-existent
215+
// certificate provider instance
216+
//
217+
// The test verifies that RPCs to the first two clusters succeed, while RPCs to
218+
// the third cluster fails with an appropriate code and error message.
219+
func (s) TestClientSideXDS_WithValidAndInvalidSecurityConfiguration(t *testing.T) {
220+
// Spin up an xDS management server. This uses a bootstrap config with a
221+
// certificate provider instance name e2e.ClientSideCertProviderInstance.
222+
mgmtServer, nodeID, _, resolver, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true})
223+
defer cleanup()
224+
225+
// Create test backends for all three clusters
226+
// backend1 configured with TLS creds, represents cluster1
227+
// backend2 configured with insecure creds, represents cluster2
228+
// backend3 configured with insecure creds, represents cluster3
229+
creds := e2e.CreateServerTLSCredentials(t)
230+
server1 := stubserver.StartTestService(t, nil, grpc.Creds(creds))
231+
defer server1.Stop()
232+
server2 := stubserver.StartTestService(t, nil)
233+
defer server2.Stop()
234+
server3 := stubserver.StartTestService(t, nil)
235+
defer server3.Stop()
236+
237+
// Configure client side xDS resources on the management server.
238+
const serviceName = "my-service-client-side-xds"
239+
const routeConfigName = "route-" + serviceName
240+
const clusterName1 = "cluster1-" + serviceName
241+
const clusterName2 = "cluster2-" + serviceName
242+
const clusterName3 = "cluster3-" + serviceName
243+
const endpointsName1 = "endpoints1-" + serviceName
244+
const endpointsName2 = "endpoints2-" + serviceName
245+
const endpointsName3 = "endpoints3-" + serviceName
246+
listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
247+
// Route configuration:
248+
// - "/grpc.testing.TestService/EmptyCall" --> cluster1
249+
// - "/grpc.testing.TestService/UnaryCall" --> cluster2
250+
// - "/grpc.testing.TestService/FullDuplexCall" --> cluster3
251+
routes := []*v3routepb.RouteConfiguration{{
252+
Name: routeConfigName,
253+
VirtualHosts: []*v3routepb.VirtualHost{{
254+
Domains: []string{serviceName},
255+
Routes: []*v3routepb.Route{
256+
{
257+
Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}},
258+
Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
259+
ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1},
260+
}},
261+
},
262+
{
263+
Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}},
264+
Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
265+
ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2},
266+
}},
267+
},
268+
{
269+
Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/FullDuplexCall"}},
270+
Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
271+
ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName3},
272+
}},
273+
},
274+
},
275+
}},
276+
}}
277+
// Clusters:
278+
// - cluster1 with cert provider name e2e.ClientSideCertProviderInstance.
279+
// - cluster2 with no security configuration.
280+
// - cluster3 with non-existent cert provider name.
281+
clusters := []*v3clusterpb.Cluster{
282+
e2e.DefaultCluster(clusterName1, endpointsName1, e2e.SecurityLevelMTLS),
283+
e2e.DefaultCluster(clusterName2, endpointsName2, e2e.SecurityLevelNone),
284+
func() *v3clusterpb.Cluster {
285+
cluster3 := e2e.DefaultCluster(clusterName3, endpointsName3, e2e.SecurityLevelMTLS)
286+
cluster3.TransportSocket = &v3corepb.TransportSocket{
287+
Name: "envoy.transport_sockets.tls",
288+
ConfigType: &v3corepb.TransportSocket_TypedConfig{
289+
TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
290+
CommonTlsContext: &v3tlspb.CommonTlsContext{
291+
ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
292+
ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
293+
InstanceName: "non-existent-certificate-provider-instance-name",
294+
},
295+
},
296+
TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
297+
InstanceName: "non-existent-certificate-provider-instance-name",
298+
},
299+
},
300+
}),
301+
},
302+
}
303+
return cluster3
304+
}(),
305+
}
306+
// Endpoints for each of the above clusters with backends created earlier.
307+
endpoints := []*v3endpointpb.ClusterLoadAssignment{
308+
e2e.DefaultEndpoint(endpointsName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}),
309+
e2e.DefaultEndpoint(endpointsName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}),
310+
}
311+
resources := e2e.UpdateOptions{
312+
NodeID: nodeID,
313+
Listeners: listeners,
314+
Routes: routes,
315+
Clusters: clusters,
316+
Endpoints: endpoints,
317+
SkipValidation: true,
318+
}
319+
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
320+
defer cancel()
321+
if err := mgmtServer.Update(ctx, resources); err != nil {
322+
t.Fatal(err)
323+
}
324+
325+
// Create client-side xDS credentials with an insecure fallback.
326+
creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
327+
if err != nil {
328+
t.Fatal(err)
329+
}
330+
331+
// Create a ClientConn.
332+
cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolver))
333+
if err != nil {
334+
t.Fatalf("failed to dial local test server: %v", err)
335+
}
336+
defer cc.Close()
337+
338+
// Make an RPC to be routed to cluster1 and verify that it succeeds.
339+
client := testgrpc.NewTestServiceClient(cc)
340+
peer := &peer.Peer{}
341+
if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
342+
t.Fatalf("EmptyCall() failed: %v", err)
343+
}
344+
if got, want := peer.Addr.String(), server1.Address; got != want {
345+
t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
346+
347+
}
348+
349+
// Make an RPC to be routed to cluster2 and verify that it succeeds.
350+
if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.Peer(peer)); err != nil {
351+
t.Fatalf("UnaryCall() failed: %v", err)
352+
}
353+
if got, want := peer.Addr.String(), server2.Address; got != want {
354+
t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
355+
}
356+
357+
// Make an RPC to be routed to cluster3 and verify that it fails.
358+
const wantErr = `identitiy certificate provider instance name "non-existent-certificate-provider-instance-name" missing in bootstrap configuration`
359+
if _, err := client.FullDuplexCall(ctx); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) {
360+
t.Fatalf("FullDuplexCall failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr)
361+
}
362+
}

0 commit comments

Comments
 (0)