Skip to content

Commit 6bc1906

Browse files
authored
xds: add support for mTLS Credentials in xDS bootstrap (#6757)
1 parent 71cc0f1 commit 6bc1906

File tree

11 files changed

+571
-31
lines changed

11 files changed

+571
-31
lines changed

credentials/tls/certprovider/pemfile/builder.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ import (
2929
)
3030

3131
const (
32-
pluginName = "file_watcher"
32+
PluginName = "file_watcher"
3333
defaultRefreshInterval = 10 * time.Minute
3434
)
3535

@@ -48,13 +48,13 @@ func (p *pluginBuilder) ParseConfig(c any) (*certprovider.BuildableConfig, error
4848
if err != nil {
4949
return nil, err
5050
}
51-
return certprovider.NewBuildableConfig(pluginName, opts.canonical(), func(certprovider.BuildOptions) certprovider.Provider {
51+
return certprovider.NewBuildableConfig(PluginName, opts.canonical(), func(certprovider.BuildOptions) certprovider.Provider {
5252
return newProvider(opts)
5353
}), nil
5454
}
5555

5656
func (p *pluginBuilder) Name() string {
57-
return pluginName
57+
return PluginName
5858
}
5959

6060
func pluginConfigFromJSON(jd json.RawMessage) (Options, error) {

internal/testutils/xds/e2e/setup_certs.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ func CreateClientTLSCredentials(t *testing.T) credentials.TransportCredentials {
9898

9999
// CreateServerTLSCredentials creates server-side TLS transport credentials
100100
// using certificate and key files from testdata/x509 directory.
101-
func CreateServerTLSCredentials(t *testing.T) credentials.TransportCredentials {
101+
func CreateServerTLSCredentials(t *testing.T, clientAuth tls.ClientAuthType) credentials.TransportCredentials {
102102
t.Helper()
103103

104104
cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
@@ -114,7 +114,7 @@ func CreateServerTLSCredentials(t *testing.T) credentials.TransportCredentials {
114114
t.Fatal("Failed to append certificates")
115115
}
116116
return credentials.NewTLS(&tls.Config{
117-
ClientAuth: tls.RequireAndVerifyClientCert,
117+
ClientAuth: clientAuth,
118118
Certificates: []tls.Certificate{cert},
119119
ClientCAs: ca,
120120
})

test/xds/xds_client_certificate_providers_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ package xds_test
2020

2121
import (
2222
"context"
23+
"crypto/tls"
2324
"fmt"
2425
"strings"
2526
"testing"
@@ -226,7 +227,7 @@ func (s) TestClientSideXDS_WithValidAndInvalidSecurityConfiguration(t *testing.T
226227
// backend1 configured with TLS creds, represents cluster1
227228
// backend2 configured with insecure creds, represents cluster2
228229
// backend3 configured with insecure creds, represents cluster3
229-
creds := e2e.CreateServerTLSCredentials(t)
230+
creds := e2e.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)
230231
server1 := stubserver.StartTestService(t, nil, grpc.Creds(creds))
231232
defer server1.Stop()
232233
server2 := stubserver.StartTestService(t, nil)

xds/bootstrap/bootstrap.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,10 @@ var registry = make(map[string]Credentials)
3737
// Credentials interface encapsulates a credentials.Bundle builder
3838
// that can be used for communicating with the xDS Management server.
3939
type Credentials interface {
40-
// Build returns a credential bundle associated with this credential.
41-
Build(config json.RawMessage) (credentials.Bundle, error)
40+
// Build returns a credential bundle associated with this credential, and
41+
// a function to cleans up additional resources associated with this bundle
42+
// when it is no longer needed.
43+
Build(config json.RawMessage) (credentials.Bundle, func(), error)
4244
// Name returns the credential name associated with this credential.
4345
Name() string
4446
}

xds/bootstrap/bootstrap_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,9 @@ type testCredsBuilder struct {
3636
config json.RawMessage
3737
}
3838

39-
func (t *testCredsBuilder) Build(config json.RawMessage) (credentials.Bundle, error) {
39+
func (t *testCredsBuilder) Build(config json.RawMessage) (credentials.Bundle, func(), error) {
4040
t.config = config
41-
return nil, nil
41+
return nil, nil, nil
4242
}
4343

4444
func (t *testCredsBuilder) Name() string {
@@ -53,7 +53,7 @@ func TestRegisterNew(t *testing.T) {
5353

5454
const sampleConfig = "sample_config"
5555
rawMessage := json.RawMessage(sampleConfig)
56-
if _, err := c.Build(rawMessage); err != nil {
56+
if _, _, err := c.Build(rawMessage); err != nil {
5757
t.Errorf("Build(%v) error = %v, want nil", rawMessage, err)
5858
}
5959

xds/internal/xdsclient/bootstrap/bootstrap.go

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ import (
3939
"google.golang.org/grpc/internal/envconfig"
4040
"google.golang.org/grpc/internal/pretty"
4141
"google.golang.org/grpc/xds/bootstrap"
42+
"google.golang.org/grpc/xds/internal/xdsclient/tlscreds"
4243
)
4344

4445
const (
@@ -60,6 +61,7 @@ const (
6061
func init() {
6162
bootstrap.RegisterCredentials(&insecureCredsBuilder{})
6263
bootstrap.RegisterCredentials(&googleDefaultCredsBuilder{})
64+
bootstrap.RegisterCredentials(&tlsCredsBuilder{})
6365
}
6466

6567
// For overriding in unit tests.
@@ -69,20 +71,32 @@ var bootstrapFileReadFunc = os.ReadFile
6971
// package `xds/bootstrap` and encapsulates an insecure credential.
7072
type insecureCredsBuilder struct{}
7173

72-
func (i *insecureCredsBuilder) Build(json.RawMessage) (credentials.Bundle, error) {
73-
return insecure.NewBundle(), nil
74+
func (i *insecureCredsBuilder) Build(json.RawMessage) (credentials.Bundle, func(), error) {
75+
return insecure.NewBundle(), func() {}, nil
7476
}
7577

7678
func (i *insecureCredsBuilder) Name() string {
7779
return "insecure"
7880
}
7981

82+
// tlsCredsBuilder implements the `Credentials` interface defined in
83+
// package `xds/bootstrap` and encapsulates a TLS credential.
84+
type tlsCredsBuilder struct{}
85+
86+
func (t *tlsCredsBuilder) Build(config json.RawMessage) (credentials.Bundle, func(), error) {
87+
return tlscreds.NewBundle(config)
88+
}
89+
90+
func (t *tlsCredsBuilder) Name() string {
91+
return "tls"
92+
}
93+
8094
// googleDefaultCredsBuilder implements the `Credentials` interface defined in
8195
// package `xds/boostrap` and encapsulates a Google Default credential.
8296
type googleDefaultCredsBuilder struct{}
8397

84-
func (d *googleDefaultCredsBuilder) Build(json.RawMessage) (credentials.Bundle, error) {
85-
return google.NewDefaultCredentials(), nil
98+
func (d *googleDefaultCredsBuilder) Build(json.RawMessage) (credentials.Bundle, func(), error) {
99+
return google.NewDefaultCredentials(), func() {}, nil
86100
}
87101

88102
func (d *googleDefaultCredsBuilder) Name() string {
@@ -151,6 +165,10 @@ type ServerConfig struct {
151165
// when a resource is deleted, nor will it remove the existing resource value
152166
// from its cache.
153167
IgnoreResourceDeletion bool
168+
169+
// Cleanups are called when the xDS client for this server is closed. Allows
170+
// cleaning up resources created specifically for this ServerConfig.
171+
Cleanups []func()
154172
}
155173

156174
// CredsDialOption returns the configured credentials as a grpc dial option.
@@ -206,12 +224,13 @@ func (sc *ServerConfig) UnmarshalJSON(data []byte) error {
206224
if c == nil {
207225
continue
208226
}
209-
bundle, err := c.Build(cc.Config)
227+
bundle, cancel, err := c.Build(cc.Config)
210228
if err != nil {
211229
return fmt.Errorf("failed to build credentials bundle from bootstrap for %q: %v", cc.Type, err)
212230
}
213231
sc.Creds = ChannelCreds(cc)
214232
sc.credsDialOption = grpc.WithCredentialsBundle(bundle)
233+
sc.Cleanups = append(sc.Cleanups, cancel)
215234
break
216235
}
217236
return nil

xds/internal/xdsclient/bootstrap/bootstrap_test.go

Lines changed: 38 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1008,30 +1008,53 @@ func TestServerConfigMarshalAndUnmarshal(t *testing.T) {
10081008
}
10091009

10101010
func TestDefaultBundles(t *testing.T) {
1011-
if c := bootstrap.GetCredentials("google_default"); c == nil {
1012-
t.Errorf(`bootstrap.GetCredentials("google_default") credential is nil, want non-nil`)
1013-
}
1011+
tests := []string{"google_default", "insecure", "tls"}
10141012

1015-
if c := bootstrap.GetCredentials("insecure"); c == nil {
1016-
t.Errorf(`bootstrap.GetCredentials("insecure") credential is nil, want non-nil`)
1013+
for _, typename := range tests {
1014+
t.Run(typename, func(t *testing.T) {
1015+
if c := bootstrap.GetCredentials(typename); c == nil {
1016+
t.Errorf(`bootstrap.GetCredentials(%s) credential is nil, want non-nil`, typename)
1017+
}
1018+
})
10171019
}
10181020
}
10191021

10201022
func TestCredsBuilders(t *testing.T) {
1021-
b := &googleDefaultCredsBuilder{}
1022-
if _, err := b.Build(nil); err != nil {
1023-
t.Errorf("googleDefaultCredsBuilder.Build failed: %v", err)
1023+
tests := []struct {
1024+
typename string
1025+
builder bootstrap.Credentials
1026+
}{
1027+
{"google_default", &googleDefaultCredsBuilder{}},
1028+
{"insecure", &insecureCredsBuilder{}},
1029+
{"tls", &tlsCredsBuilder{}},
10241030
}
1025-
if got, want := b.Name(), "google_default"; got != want {
1026-
t.Errorf("googleDefaultCredsBuilder.Name = %v, want %v", got, want)
1031+
1032+
for _, test := range tests {
1033+
t.Run(test.typename, func(t *testing.T) {
1034+
if got, want := test.builder.Name(), test.typename; got != want {
1035+
t.Errorf("%T.Name = %v, want %v", test.builder, got, want)
1036+
}
1037+
1038+
_, stop, err := test.builder.Build(nil)
1039+
if err != nil {
1040+
t.Fatalf("%T.Build failed: %v", test.builder, err)
1041+
}
1042+
stop()
1043+
})
10271044
}
1045+
}
10281046

1029-
i := &insecureCredsBuilder{}
1030-
if _, err := i.Build(nil); err != nil {
1031-
t.Errorf("insecureCredsBuilder.Build failed: %v", err)
1047+
func TestTlsCredsBuilder(t *testing.T) {
1048+
tls := &tlsCredsBuilder{}
1049+
_, stop, err := tls.Build(json.RawMessage(`{}`))
1050+
if err != nil {
1051+
t.Fatalf("tls.Build() failed with error %s when expected to succeed", err)
10321052
}
1053+
stop()
10331054

1034-
if got, want := i.Name(), "insecure"; got != want {
1035-
t.Errorf("insecureCredsBuilder.Name = %v, want %v", got, want)
1055+
if _, stop, err := tls.Build(json.RawMessage(`{"ca_certificate_file":"/ca_certificates.pem","refresh_interval": "asdf"}`)); err == nil {
1056+
t.Errorf("tls.Build() succeeded with an invalid refresh interval, when expected to fail")
1057+
stop()
10361058
}
1059+
// package internal/xdsclient/tlscreds has tests for config validity.
10371060
}

xds/internal/xdsclient/clientimpl.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,5 +85,17 @@ func (c *clientImpl) close() {
8585
c.authorityMu.Unlock()
8686
c.serializerClose()
8787

88+
for _, f := range c.config.XDSServer.Cleanups {
89+
f()
90+
}
91+
for _, a := range c.config.Authorities {
92+
if a.XDSServer == nil {
93+
// The server for this authority is the top-level one, cleaned up above.
94+
continue
95+
}
96+
for _, f := range a.XDSServer.Cleanups {
97+
f()
98+
}
99+
}
88100
c.logger.Infof("Shutdown")
89101
}
Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
/*
2+
*
3+
* Copyright 2023 gRPC authors.
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*
17+
*/
18+
19+
// Package tlscreds implements mTLS Credentials in xDS Bootstrap File.
20+
// See gRFC A65: github.com/grpc/proposal/blob/master/A65-xds-mtls-creds-in-bootstrap.md.
21+
package tlscreds
22+
23+
import (
24+
"context"
25+
"crypto/tls"
26+
"encoding/json"
27+
"errors"
28+
"fmt"
29+
"net"
30+
31+
"google.golang.org/grpc/credentials"
32+
"google.golang.org/grpc/credentials/tls/certprovider"
33+
"google.golang.org/grpc/credentials/tls/certprovider/pemfile"
34+
"google.golang.org/grpc/internal/grpcsync"
35+
)
36+
37+
// bundle is an implementation of credentials.Bundle which implements mTLS
38+
// Credentials in xDS Bootstrap File.
39+
type bundle struct {
40+
transportCredentials credentials.TransportCredentials
41+
}
42+
43+
// NewBundle returns a credentials.Bundle which implements mTLS Credentials in xDS
44+
// Bootstrap File. It delegates certificate loading to a file_watcher provider
45+
// if either client certificates or server root CA is specified. The second
46+
// return value is a close func that should be called when the caller no longer
47+
// needs this bundle.
48+
// See gRFC A65: github.com/grpc/proposal/blob/master/A65-xds-mtls-creds-in-bootstrap.md
49+
func NewBundle(jd json.RawMessage) (credentials.Bundle, func(), error) {
50+
cfg := &struct {
51+
CertificateFile string `json:"certificate_file"`
52+
CACertificateFile string `json:"ca_certificate_file"`
53+
PrivateKeyFile string `json:"private_key_file"`
54+
}{}
55+
56+
if jd != nil {
57+
if err := json.Unmarshal(jd, cfg); err != nil {
58+
return nil, nil, fmt.Errorf("failed to unmarshal config: %v", err)
59+
}
60+
} // Else the config field is absent. Treat it as an empty config.
61+
62+
if cfg.CACertificateFile == "" && cfg.CertificateFile == "" && cfg.PrivateKeyFile == "" {
63+
// We cannot use (and do not need) a file_watcher provider in this case,
64+
// and can simply directly use the TLS transport credentials.
65+
// Quoting A65:
66+
//
67+
// > The only difference between the file-watcher certificate provider
68+
// > config and this one is that in the file-watcher certificate
69+
// > provider, at least one of the "certificate_file" or
70+
// > "ca_certificate_file" fields must be specified, whereas in this
71+
// > configuration, it is acceptable to specify neither one.
72+
return &bundle{transportCredentials: credentials.NewTLS(&tls.Config{})}, func() {}, nil
73+
}
74+
// Otherwise we need to use a file_watcher provider to watch the CA,
75+
// private and public keys.
76+
77+
// The pemfile plugin (file_watcher) currently ignores BuildOptions.
78+
provider, err := certprovider.GetProvider(pemfile.PluginName, jd, certprovider.BuildOptions{})
79+
if err != nil {
80+
return nil, nil, err
81+
}
82+
return &bundle{
83+
transportCredentials: &reloadingCreds{provider: provider},
84+
}, grpcsync.OnceFunc(func() { provider.Close() }), nil
85+
}
86+
87+
func (t *bundle) TransportCredentials() credentials.TransportCredentials {
88+
return t.transportCredentials
89+
}
90+
91+
func (t *bundle) PerRPCCredentials() credentials.PerRPCCredentials {
92+
// mTLS provides transport credentials only. There are no per-RPC
93+
// credentials.
94+
return nil
95+
}
96+
97+
func (t *bundle) NewWithMode(string) (credentials.Bundle, error) {
98+
// This bundle has a single mode which only uses TLS transport credentials,
99+
// so there is no legitimate case where callers would call NewWithMode.
100+
return nil, fmt.Errorf("xDS TLS credentials only support one mode")
101+
}
102+
103+
// reloadingCreds is a credentials.TransportCredentials for client
104+
// side mTLS that reloads the server root CA certificate and the client
105+
// certificates from the provider on every client handshake. This is necessary
106+
// because the standard TLS credentials do not support reloading CA
107+
// certificates.
108+
type reloadingCreds struct {
109+
provider certprovider.Provider
110+
}
111+
112+
func (c *reloadingCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
113+
km, err := c.provider.KeyMaterial(ctx)
114+
if err != nil {
115+
return nil, nil, err
116+
}
117+
config := &tls.Config{
118+
RootCAs: km.Roots,
119+
Certificates: km.Certs,
120+
}
121+
return credentials.NewTLS(config).ClientHandshake(ctx, authority, rawConn)
122+
}
123+
124+
func (c *reloadingCreds) Info() credentials.ProtocolInfo {
125+
return credentials.ProtocolInfo{SecurityProtocol: "tls"}
126+
}
127+
128+
func (c *reloadingCreds) Clone() credentials.TransportCredentials {
129+
return &reloadingCreds{provider: c.provider}
130+
}
131+
132+
func (c *reloadingCreds) OverrideServerName(string) error {
133+
return errors.New("overriding server name is not supported by xDS client TLS credentials")
134+
}
135+
136+
func (c *reloadingCreds) ServerHandshake(net.Conn) (net.Conn, credentials.AuthInfo, error) {
137+
return nil, nil, errors.New("server handshake is not supported by xDS client TLS credentials")
138+
}

0 commit comments

Comments
 (0)