From a1f262c29f5cd4ab5dc5153c87050b08e9c3b371 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Tue, 9 Jul 2024 02:43:20 +0530 Subject: [PATCH 01/62] Use one subconn per address in pickfirst --- balancer/pickfirst/pickfirst.go | 446 ++++++++++++++++++----- clientconn_test.go | 38 +- test/clientconn_state_transition_test.go | 21 +- 3 files changed, 401 insertions(+), 104 deletions(-) diff --git a/balancer/pickfirst/pickfirst.go b/balancer/pickfirst/pickfirst.go index 07527603f1d4..64072cb65de2 100644 --- a/balancer/pickfirst/pickfirst.go +++ b/balancer/pickfirst/pickfirst.go @@ -24,6 +24,8 @@ import ( "errors" "fmt" "math/rand" + "slices" + "sync" "google.golang.org/grpc/balancer" "google.golang.org/grpc/connectivity" @@ -35,6 +37,12 @@ import ( "google.golang.org/grpc/serviceconfig" ) +const ( + subConnListPending = iota + subConnListActive + subConnListClosed +) + func init() { balancer.Register(pickfirstBuilder{}) internal.ShuffleAddressListForTesting = func(n int, swap func(i, j int)) { rand.Shuffle(n, swap) } @@ -69,6 +77,238 @@ type pfConfig struct { ShuffleAddressList bool `json:"shuffleAddressList"` } +// subConnList stores provides functions to connect to a list of addresses using +// the pick-first algorithm. +type subConnList struct { + subConns []*scWrapper + // The index within the subConns list for the subConn being tried. + attemptingIndex int + b *pickfirstBalancer + // The most recent failure during the initial connection attempt over the + // entire sunConns list. + lastFailure error + // Whether all the subConns have reported a transient failure once. + inTransientFailure bool + // Use a mutex to guard the list status as it can be changed concurrently by + // the idlePicker. Note that only calls to change the state to active are + // triggered from the picker and can happen concurrently. + mu sync.RWMutex + status int +} + +// scWrapper keeps track of the current state of the subConn. +type scWrapper struct { + subConn balancer.SubConn + conState connectivity.State + addr resolver.Address +} + +func newScWrapper(b *pickfirstBalancer, addr resolver.Address, listener func(state balancer.SubConnState)) (*scWrapper, error) { + scw := &scWrapper{ + conState: connectivity.Idle, + addr: addr, + } + sc, err := b.cc.NewSubConn([]resolver.Address{addr}, balancer.NewSubConnOptions{ + StateListener: func(scs balancer.SubConnState) { + // Store the state and delegate. + scw.conState = scs.ConnectivityState + listener(scs) + }, + }) + if err != nil { + return nil, err + } + scw.subConn = sc + return scw, nil +} + +func newSubConnList(addrs []resolver.Address, b *pickfirstBalancer) *subConnList { + sl := &subConnList{ + b: b, + status: subConnListPending, + } + + for _, addr := range addrs { + var scw *scWrapper + scw, err := newScWrapper(b, addr, func(state balancer.SubConnState) { + sl.stateListener(scw, state) + }) + if err != nil { + if b.logger.V(2) { + b.logger.Infof("Ignoring failure, could not create a subConn for address %q due to error: %v", addr, err) + } + continue + } + if b.logger.V(2) { + b.logger.Infof("Created a subConn for address %q", addr) + } + sl.subConns = append(sl.subConns, scw) + } + return sl +} + +func (sl *subConnList) startConnectingIfNeeded() { + if sl == nil { + return + } + sl.mu.Lock() + defer sl.mu.Unlock() + if sl.status != subConnListPending { + return + } + sl.status = subConnListActive + sl.startConnectingNextSubConn() +} + +func (sl *subConnList) stateListener(scw *scWrapper, state balancer.SubConnState) { + if sl.b.logger.V(2) { + sl.b.logger.Infof("Received SubConn state update: %p, %+v", scw, state) + } + if scw == sl.b.selectedSubConn { + // As we set the selected subConn only once it's ready, the only + // possible transitions are to IDLE and SHUTDOWN. + switch state.ConnectivityState { + case connectivity.Shutdown: + case connectivity.Idle: + sl.b.goIdle() + default: + sl.b.logger.Warningf("Ignoring unexpected transition of selected subConn %p to %v", &scw.subConn, state.ConnectivityState) + } + return + } + // If this list is already closed, ignore the update. + if sl.status != subConnListActive { + if sl.b.logger.V(2) { + sl.b.logger.Infof("Ignoring state update for non active subConn %p to %v", &scw.subConn, state.ConnectivityState) + } + return + } + if !sl.inTransientFailure { + // We are still trying to connect to each subConn once. + switch state.ConnectivityState { + case connectivity.TransientFailure: + if sl.b.logger.V(2) { + sl.b.logger.Infof("SubConn %p failed to connect due to error: %v", &scw.subConn, state.ConnectionError) + } + sl.attemptingIndex++ + sl.lastFailure = state.ConnectionError + sl.startConnectingNextSubConn() + case connectivity.Ready: + // Cleanup and update the picker to use the subconn. + sl.selectSubConn(scw) + case connectivity.Connecting: + // Move the channel to connecting if this is the first subConn to + // start connecting. + if sl.b.state == connectivity.Idle { + sl.b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &picker{err: balancer.ErrNoSubConnAvailable}, + }) + } + default: + if sl.b.logger.V(2) { + sl.b.logger.Infof("Ignoring update for the subConn %p to state %v", &scw.subConn, state.ConnectivityState) + } + } + return + } + + // We have attempted to connect to all the subConns once and failed. + switch state.ConnectivityState { + case connectivity.TransientFailure: + if sl.b.logger.V(2) { + sl.b.logger.Infof("SubConn %p failed to connect due to error: %v", &scw.subConn, state.ConnectionError) + } + // If its the initial connection attempt, try to connect to the next subConn. + if sl.attemptingIndex < len(sl.subConns) && sl.subConns[sl.attemptingIndex] == scw { + // Going over all the subConns and try connecting to ones that are idle. + sl.attemptingIndex++ + sl.startConnectingNextSubConn() + } + case connectivity.Ready: + // Cleanup and update the picker to use the subconn. + sl.selectSubConn(scw) + case connectivity.Idle: + // Trigger re-connection. + scw.subConn.Connect() + default: + if sl.b.logger.V(2) { + sl.b.logger.Infof("Ignoring update for the subConn %p to state %v", &scw.subConn, state.ConnectivityState) + } + } +} + +func (sl *subConnList) selectSubConn(scw *scWrapper) { + if sl.b.logger.V(2) { + sl.b.logger.Infof("Selected subConn %p", &scw.subConn) + } + sl.b.unsetSelectedSubConn() + sl.b.selectedSubConn = scw + sl.b.state = connectivity.Ready + sl.inTransientFailure = false + sl.close() + sl.b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &picker{result: balancer.PickResult{SubConn: scw.subConn}}, + }) +} + +func (sl *subConnList) close() { + if sl == nil { + return + } + sl.mu.Lock() + defer sl.mu.Unlock() + if sl.status == subConnListClosed { + return + } + sl.status = subConnListClosed + // Close all the subConns except the selected one. The selected subConn + // will be closed by the balancer. + for _, sc := range sl.subConns { + if sc == sl.b.selectedSubConn { + continue + } + sc.subConn.Shutdown() + } + sl.subConns = nil +} + +func (sl *subConnList) startConnectingNextSubConn() { + if sl.status != subConnListActive { + return + } + if !sl.inTransientFailure && sl.attemptingIndex < len(sl.subConns) { + // Try to connect to the next subConn. + sl.subConns[sl.attemptingIndex].subConn.Connect() + return + } + if !sl.inTransientFailure { + // Failed to connect to each subConn once, enter transient failure. + sl.b.state = connectivity.TransientFailure + sl.inTransientFailure = true + sl.attemptingIndex = 0 + sl.b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: sl.lastFailure}, + }) + // Drop the existing (working) connection, if any. This may be + // sub-optimal, but we can't ignore what the control plane told us. + sl.b.unsetSelectedSubConn() + } + + // Attempt to connect to addresses that have already completed the back-off. + for ; sl.attemptingIndex < len(sl.subConns); sl.attemptingIndex++ { + sc := sl.subConns[sl.attemptingIndex] + if sc.conState == connectivity.TransientFailure { + continue + } + sl.subConns[sl.attemptingIndex].subConn.Connect() + return + } + // Wait for the next subConn to enter idle state, then re-connect. +} + func (pickfirstBuilder) ParseConfig(js json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { var cfg pfConfig if err := json.Unmarshal(js, &cfg); err != nil { @@ -78,17 +318,24 @@ func (pickfirstBuilder) ParseConfig(js json.RawMessage) (serviceconfig.LoadBalan } type pickfirstBalancer struct { - logger *internalgrpclog.PrefixLogger - state connectivity.State - cc balancer.ClientConn - subConn balancer.SubConn + logger *internalgrpclog.PrefixLogger + state connectivity.State + cc balancer.ClientConn + // Pointer to the subConn list currently connecting. Always close the + // current list before replacing it to ensure resources are freed. + subConnList *subConnList + selectedSubConn *scWrapper + latestAddressList []resolver.Address + shuttingDown bool } func (b *pickfirstBalancer) ResolverError(err error) { if b.logger.V(2) { b.logger.Infof("Received error from the name resolver: %v", err) } - if b.subConn == nil { + if len(b.latestAddressList) == 0 { + // The picker will not change since the balancer does not currently + // report an error. b.state = connectivity.TransientFailure } @@ -103,22 +350,99 @@ func (b *pickfirstBalancer) ResolverError(err error) { }) } +func (b *pickfirstBalancer) unsetSelectedSubConn() { + if b.selectedSubConn != nil { + b.selectedSubConn.subConn.Shutdown() + b.selectedSubConn = nil + } +} + +func (b *pickfirstBalancer) goIdle() { + b.unsetSelectedSubConn() + b.refreshSunConnList() + if len(b.subConnList.subConns) == 0 { + b.state = connectivity.TransientFailure + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: fmt.Errorf("empty address list")}, + }) + b.unsetSelectedSubConn() + b.subConnList.close() + b.cc.ResolveNow(resolver.ResolveNowOptions{}) + return + } + + callback := func() { + b.subConnList.startConnectingIfNeeded() + } + + nextState := connectivity.Idle + if b.state == connectivity.TransientFailure { + // We stay in TransientFailure until we are Ready. See A62. + nextState = connectivity.TransientFailure + } else { + b.state = connectivity.Idle + } + b.cc.UpdateState(balancer.State{ + ConnectivityState: nextState, + Picker: &idlePicker{ + callback: callback, + }, + }) +} + type Shuffler interface { ShuffleAddressListForTesting(n int, swap func(i, j int)) } func ShuffleAddressListForTesting(n int, swap func(i, j int)) { rand.Shuffle(n, swap) } +func (b *pickfirstBalancer) refreshSunConnList() { + subConnList := newSubConnList(b.latestAddressList, b) + + // Reset the previous subConnList to release resources. + if b.subConnList != nil { + if b.logger.V(2) { + b.logger.Infof("Closing older subConnList") + } + b.subConnList.close() + } + b.subConnList = subConnList +} + +func (b *pickfirstBalancer) connectUsingLatestAddrs() error { + b.refreshSunConnList() + if len(b.subConnList.subConns) == 0 { + b.state = connectivity.TransientFailure + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: fmt.Errorf("empty address list")}, + }) + b.unsetSelectedSubConn() + b.subConnList.close() + return balancer.ErrBadResolverState + } + + if b.state != connectivity.TransientFailure { + // We stay in TransientFailure until we are Ready. See A62. + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &picker{err: balancer.ErrNoSubConnAvailable}, + }) + } + b.subConnList.startConnectingIfNeeded() + return nil +} + func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState) error { if len(state.ResolverState.Addresses) == 0 && len(state.ResolverState.Endpoints) == 0 { // The resolver reported an empty address list. Treat it like an error by // calling b.ResolverError. - if b.subConn != nil { - // Shut down the old subConn. All addresses were removed, so it is - // no longer valid. - b.subConn.Shutdown() - b.subConn = nil - } + b.unsetSelectedSubConn() + // Shut down the old subConnList. All addresses were removed, so it is + // no longer valid. + b.subConnList.close() + b.latestAddressList = nil b.ResolverError(errors.New("produced zero addresses")) return balancer.ErrBadResolverState } @@ -155,7 +479,7 @@ func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState // Endpoints not set, process addresses until we migrate resolver // emissions fully to Endpoints. The top channel does wrap emitted // addresses with endpoints, however some balancers such as weighted - // target do not forwarrd the corresponding correct endpoints down/split + // target do not forward the corresponding correct endpoints down/split // endpoints properly. Once all balancers correctly forward endpoints // down, can delete this else conditional. addrs = state.ResolverState.Addresses @@ -165,36 +489,18 @@ func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState } } - if b.subConn != nil { - b.cc.UpdateAddresses(b.subConn, addrs) - return nil - } - - var subConn balancer.SubConn - subConn, err := b.cc.NewSubConn(addrs, balancer.NewSubConnOptions{ - StateListener: func(state balancer.SubConnState) { - b.updateSubConnState(subConn, state) - }, - }) - if err != nil { + b.latestAddressList = addrs + // If the selected subConn's address is present in the list, don't attempt + // to re-connect. + if b.selectedSubConn != nil && slices.ContainsFunc(addrs, func(addr resolver.Address) bool { + return addr.Equal(b.selectedSubConn.addr) + }) { if b.logger.V(2) { - b.logger.Infof("Failed to create new SubConn: %v", err) + b.logger.Infof("Not attempting to re-connect since selected address %q is present in new address list", b.selectedSubConn.addr.String()) } - b.state = connectivity.TransientFailure - b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.TransientFailure, - Picker: &picker{err: fmt.Errorf("error creating connection: %v", err)}, - }) - return balancer.ErrBadResolverState + return nil } - b.subConn = subConn - b.state = connectivity.Idle - b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.Connecting, - Picker: &picker{err: balancer.ErrNoSubConnAvailable}, - }) - b.subConn.Connect() - return nil + return b.connectUsingLatestAddrs() } // UpdateSubConnState is unused as a StateListener is always registered when @@ -203,63 +509,17 @@ func (b *pickfirstBalancer) UpdateSubConnState(subConn balancer.SubConn, state b b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", subConn, state) } -func (b *pickfirstBalancer) updateSubConnState(subConn balancer.SubConn, state balancer.SubConnState) { - if b.logger.V(2) { - b.logger.Infof("Received SubConn state update: %p, %+v", subConn, state) - } - if b.subConn != subConn { - if b.logger.V(2) { - b.logger.Infof("Ignored state change because subConn is not recognized") - } - return - } - if state.ConnectivityState == connectivity.Shutdown { - b.subConn = nil - return - } - - switch state.ConnectivityState { - case connectivity.Ready: - b.cc.UpdateState(balancer.State{ - ConnectivityState: state.ConnectivityState, - Picker: &picker{result: balancer.PickResult{SubConn: subConn}}, - }) - case connectivity.Connecting: - if b.state == connectivity.TransientFailure { - // We stay in TransientFailure until we are Ready. See A62. - return - } - b.cc.UpdateState(balancer.State{ - ConnectivityState: state.ConnectivityState, - Picker: &picker{err: balancer.ErrNoSubConnAvailable}, - }) - case connectivity.Idle: - if b.state == connectivity.TransientFailure { - // We stay in TransientFailure until we are Ready. Also kick the - // subConn out of Idle into Connecting. See A62. - b.subConn.Connect() - return - } - b.cc.UpdateState(balancer.State{ - ConnectivityState: state.ConnectivityState, - Picker: &idlePicker{subConn: subConn}, - }) - case connectivity.TransientFailure: - b.cc.UpdateState(balancer.State{ - ConnectivityState: state.ConnectivityState, - Picker: &picker{err: state.ConnectionError}, - }) - } - b.state = state.ConnectivityState -} - func (b *pickfirstBalancer) Close() { + b.shuttingDown = true + b.unsetSelectedSubConn() + b.subConnList.close() } func (b *pickfirstBalancer) ExitIdle() { - if b.subConn != nil && b.state == connectivity.Idle { - b.subConn.Connect() + if b.shuttingDown { + return } + b.subConnList.startConnectingIfNeeded() } type picker struct { @@ -274,10 +534,10 @@ func (p *picker) Pick(balancer.PickInfo) (balancer.PickResult, error) { // idlePicker is used when the SubConn is IDLE and kicks the SubConn into // CONNECTING when Pick is called. type idlePicker struct { - subConn balancer.SubConn + callback func() } func (i *idlePicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { - i.subConn.Connect() + i.callback() return balancer.PickResult{}, balancer.ErrNoSubConnAvailable } diff --git a/clientconn_test.go b/clientconn_test.go index 34d22c684eee..c87f505fc735 100644 --- a/clientconn_test.go +++ b/clientconn_test.go @@ -417,18 +417,16 @@ func (s) TestWithTransportCredentialsTLS(t *testing.T) { } // When creating a transport configured with n addresses, only calculate the -// backoff once per "round" of attempts instead of once per address (n times -// per "round" of attempts). -func (s) TestDial_OneBackoffPerRetryGroup(t *testing.T) { +// backoff n times per "round" of attempts instead of once per. +func (s) TestDial_NBackoffPerRetryGroup(t *testing.T) { var attempts uint32 getMinConnectTimeout := func() time.Duration { - if atomic.AddUint32(&attempts, 1) == 1 { + if atomic.AddUint32(&attempts, 1) > 2 { // Once all addresses are exhausted, hang around and wait for the // client.Close to happen rather than re-starting a new round of // attempts. return time.Hour } - t.Error("only one attempt backoff calculation, but got more") return 0 } @@ -499,6 +497,10 @@ func (s) TestDial_OneBackoffPerRetryGroup(t *testing.T) { t.Fatal("timed out waiting for test to finish") case <-server2Done: } + + if atomic.LoadUint32(&attempts) != 2 { + t.Errorf("Back-off attempts=%d, want=2", attempts) + } } func (s) TestDialContextCancel(t *testing.T) { @@ -1061,18 +1063,28 @@ func (s) TestUpdateAddresses_NoopIfCalledWithSameAddresses(t *testing.T) { t.Fatal("timed out waiting for server2 to be contacted") } - // Grab the addrConn and call tryUpdateAddrs. + // Grab the addrConn for server 2 and call tryUpdateAddrs. var ac *addrConn client.mu.Lock() for clientAC := range client.conns { - ac = clientAC - break + if got := len(clientAC.addrs); got != 1 { + t.Errorf("len(AddrConn.addrs)=%d, want=1", got) + continue + } + if clientAC.addrs[0].Addr == lis2.Addr().String() { + ac = clientAC + break + } } client.mu.Unlock() + if ac == nil { + t.Fatal("Coudn't find the subConn for server 2") + } + // Call UpdateAddresses with the same list of addresses, it should be a noop // (even when the SubConn is Connecting, and doesn't have a curAddr). - ac.acbw.UpdateAddresses(addrsList) + ac.acbw.UpdateAddresses(addrsList[1:2]) // We've called tryUpdateAddrs - now let's make server2 close the // connection and check that it continues to server3. @@ -1215,6 +1227,12 @@ func (b *stateRecordingBalancer) Close() { b.Balancer.Close() } +func (b *stateRecordingBalancer) ExitIdle() { + if ib, ok := b.Balancer.(balancer.ExitIdler); ok { + ib.ExitIdle() + } +} + type stateRecordingBalancerBuilder struct { mu sync.Mutex notifier chan connectivity.State // The notifier used in the last Balancer. @@ -1229,7 +1247,7 @@ func (b *stateRecordingBalancerBuilder) Name() string { } func (b *stateRecordingBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { - stateNotifications := make(chan connectivity.State, 10) + stateNotifications := make(chan connectivity.State, 20) b.mu.Lock() b.notifier = stateNotifications b.mu.Unlock() diff --git a/test/clientconn_state_transition_test.go b/test/clientconn_state_transition_test.go index 6e9bfb37289d..4e4af17e5cbe 100644 --- a/test/clientconn_state_transition_test.go +++ b/test/clientconn_state_transition_test.go @@ -250,6 +250,7 @@ func (s) TestStateTransitions_ReadyToConnecting(t *testing.T) { connectivity.Connecting, connectivity.Ready, connectivity.Idle, + connectivity.Shutdown, connectivity.Connecting, } for i := 0; i < len(want); i++ { @@ -323,7 +324,15 @@ func (s) TestStateTransitions_TriesAllAddrsBeforeTransientFailure(t *testing.T) client, err := grpc.Dial("whatever:///this-gets-overwritten", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateRecordingBalancerName)), - grpc.WithResolvers(rb)) + grpc.WithResolvers(rb), + grpc.WithConnectParams(grpc.ConnectParams{ + // Set a really long back-off delay to ensure the first subConn does + // not enter ready before the second subConn connects. + Backoff: backoff.Config{ + BaseDelay: 1 * time.Hour, + }, + }), + ) if err != nil { t.Fatal(err) } @@ -331,6 +340,8 @@ func (s) TestStateTransitions_TriesAllAddrsBeforeTransientFailure(t *testing.T) stateNotifications := testBalancerBuilder.nextStateNotifier() want := []connectivity.State{ + connectivity.Connecting, + connectivity.TransientFailure, connectivity.Connecting, connectivity.Ready, } @@ -423,7 +434,9 @@ func (s) TestStateTransitions_MultipleAddrsEntersReady(t *testing.T) { want := []connectivity.State{ connectivity.Connecting, connectivity.Ready, + connectivity.Shutdown, // The second subConn is closed once the first one connects. connectivity.Idle, + connectivity.Shutdown, // The subConn will be closed and pickfirst will run on the latest address list. connectivity.Connecting, } for i := 0; i < len(want); i++ { @@ -454,6 +467,12 @@ func (b *stateRecordingBalancer) Close() { b.Balancer.Close() } +func (b *stateRecordingBalancer) ExitIdle() { + if ib, ok := b.Balancer.(balancer.ExitIdler); ok { + ib.ExitIdle() + } +} + type stateRecordingBalancerBuilder struct { mu sync.Mutex notifier chan connectivity.State // The notifier used in the last Balancer. From 90430796063861ff778954df62eb52699ab7b494 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Fri, 12 Jul 2024 01:08:59 +0530 Subject: [PATCH 02/62] address review comments --- balancer/pickfirst/pickfirst.go | 44 ++++++++++++++------------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/balancer/pickfirst/pickfirst.go b/balancer/pickfirst/pickfirst.go index 64072cb65de2..3a64ca814769 100644 --- a/balancer/pickfirst/pickfirst.go +++ b/balancer/pickfirst/pickfirst.go @@ -37,8 +37,10 @@ import ( "google.golang.org/grpc/serviceconfig" ) +type subConnListState int + const ( - subConnListPending = iota + subConnListPending subConnListState = iota subConnListActive subConnListClosed ) @@ -93,25 +95,25 @@ type subConnList struct { // the idlePicker. Note that only calls to change the state to active are // triggered from the picker and can happen concurrently. mu sync.RWMutex - status int + status subConnListState } // scWrapper keeps track of the current state of the subConn. type scWrapper struct { - subConn balancer.SubConn - conState connectivity.State - addr resolver.Address + subConn balancer.SubConn + state connectivity.State + addr resolver.Address } func newScWrapper(b *pickfirstBalancer, addr resolver.Address, listener func(state balancer.SubConnState)) (*scWrapper, error) { scw := &scWrapper{ - conState: connectivity.Idle, - addr: addr, + state: connectivity.Idle, + addr: addr, } sc, err := b.cc.NewSubConn([]resolver.Address{addr}, balancer.NewSubConnOptions{ StateListener: func(scs balancer.SubConnState) { // Store the state and delegate. - scw.conState = scs.ConnectivityState + scw.state = scs.ConnectivityState listener(scs) }, }) @@ -300,7 +302,7 @@ func (sl *subConnList) startConnectingNextSubConn() { // Attempt to connect to addresses that have already completed the back-off. for ; sl.attemptingIndex < len(sl.subConns); sl.attemptingIndex++ { sc := sl.subConns[sl.attemptingIndex] - if sc.conState == connectivity.TransientFailure { + if sc.state == connectivity.TransientFailure { continue } sl.subConns[sl.attemptingIndex].subConn.Connect() @@ -359,7 +361,7 @@ func (b *pickfirstBalancer) unsetSelectedSubConn() { func (b *pickfirstBalancer) goIdle() { b.unsetSelectedSubConn() - b.refreshSunConnList() + b.refreshSubConnList() if len(b.subConnList.subConns) == 0 { b.state = connectivity.TransientFailure b.cc.UpdateState(balancer.State{ @@ -386,18 +388,12 @@ func (b *pickfirstBalancer) goIdle() { b.cc.UpdateState(balancer.State{ ConnectivityState: nextState, Picker: &idlePicker{ - callback: callback, + exitIdle: callback, }, }) } -type Shuffler interface { - ShuffleAddressListForTesting(n int, swap func(i, j int)) -} - -func ShuffleAddressListForTesting(n int, swap func(i, j int)) { rand.Shuffle(n, swap) } - -func (b *pickfirstBalancer) refreshSunConnList() { +func (b *pickfirstBalancer) refreshSubConnList() { subConnList := newSubConnList(b.latestAddressList, b) // Reset the previous subConnList to release resources. @@ -411,7 +407,7 @@ func (b *pickfirstBalancer) refreshSunConnList() { } func (b *pickfirstBalancer) connectUsingLatestAddrs() error { - b.refreshSunConnList() + b.refreshSubConnList() if len(b.subConnList.subConns) == 0 { b.state = connectivity.TransientFailure b.cc.UpdateState(balancer.State{ @@ -436,13 +432,11 @@ func (b *pickfirstBalancer) connectUsingLatestAddrs() error { func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState) error { if len(state.ResolverState.Addresses) == 0 && len(state.ResolverState.Endpoints) == 0 { - // The resolver reported an empty address list. Treat it like an error by - // calling b.ResolverError. + // Cleanup state pertaining to the previous resolver state. b.unsetSelectedSubConn() - // Shut down the old subConnList. All addresses were removed, so it is - // no longer valid. b.subConnList.close() b.latestAddressList = nil + // Treat an empty address list like an error by calling b.ResolverError. b.ResolverError(errors.New("produced zero addresses")) return balancer.ErrBadResolverState } @@ -534,10 +528,10 @@ func (p *picker) Pick(balancer.PickInfo) (balancer.PickResult, error) { // idlePicker is used when the SubConn is IDLE and kicks the SubConn into // CONNECTING when Pick is called. type idlePicker struct { - callback func() + exitIdle func() } func (i *idlePicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { - i.callback() + i.exitIdle() return balancer.PickResult{}, balancer.ErrNoSubConnAvailable } From 2541d51e52b8eab1aaf10920d4b14d4603d590db Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Fri, 12 Jul 2024 01:33:38 +0530 Subject: [PATCH 03/62] ensure subConnList is never nil --- balancer/pickfirst/pickfirst.go | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/balancer/pickfirst/pickfirst.go b/balancer/pickfirst/pickfirst.go index 3a64ca814769..44b80f183004 100644 --- a/balancer/pickfirst/pickfirst.go +++ b/balancer/pickfirst/pickfirst.go @@ -62,6 +62,7 @@ type pickfirstBuilder struct{} func (pickfirstBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer { b := &pickfirstBalancer{cc: cc} + b.subConnList = newSubConnList([]resolver.Address{}, b) b.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(logPrefix, b)) return b } @@ -150,9 +151,6 @@ func newSubConnList(addrs []resolver.Address, b *pickfirstBalancer) *subConnList } func (sl *subConnList) startConnectingIfNeeded() { - if sl == nil { - return - } sl.mu.Lock() defer sl.mu.Unlock() if sl.status != subConnListPending { @@ -256,9 +254,6 @@ func (sl *subConnList) selectSubConn(scw *scWrapper) { } func (sl *subConnList) close() { - if sl == nil { - return - } sl.mu.Lock() defer sl.mu.Unlock() if sl.status == subConnListClosed { @@ -374,7 +369,7 @@ func (b *pickfirstBalancer) goIdle() { return } - callback := func() { + exitIdle := func() { b.subConnList.startConnectingIfNeeded() } @@ -388,7 +383,7 @@ func (b *pickfirstBalancer) goIdle() { b.cc.UpdateState(balancer.State{ ConnectivityState: nextState, Picker: &idlePicker{ - exitIdle: callback, + exitIdle: exitIdle, }, }) } @@ -397,12 +392,10 @@ func (b *pickfirstBalancer) refreshSubConnList() { subConnList := newSubConnList(b.latestAddressList, b) // Reset the previous subConnList to release resources. - if b.subConnList != nil { - if b.logger.V(2) { - b.logger.Infof("Closing older subConnList") - } - b.subConnList.close() + if b.logger.V(2) { + b.logger.Infof("Closing older subConnList") } + b.subConnList.close() b.subConnList = subConnList } From f961973f491b06bb8143dcb270945f5b5474ee8d Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Fri, 12 Jul 2024 14:13:51 +0530 Subject: [PATCH 04/62] Defer subConnList creation till conection start --- balancer/pickfirst/pickfirst.go | 93 +++++++++++++++++---------------- 1 file changed, 47 insertions(+), 46 deletions(-) diff --git a/balancer/pickfirst/pickfirst.go b/balancer/pickfirst/pickfirst.go index 44b80f183004..d4cbbb7e5048 100644 --- a/balancer/pickfirst/pickfirst.go +++ b/balancer/pickfirst/pickfirst.go @@ -40,8 +40,7 @@ import ( type subConnListState int const ( - subConnListPending subConnListState = iota - subConnListActive + subConnListActive subConnListState = iota subConnListClosed ) @@ -92,11 +91,11 @@ type subConnList struct { lastFailure error // Whether all the subConns have reported a transient failure once. inTransientFailure bool - // Use a mutex to guard the list status as it can be changed concurrently by - // the idlePicker. Note that only calls to change the state to active are - // triggered from the picker and can happen concurrently. - mu sync.RWMutex - status subConnListState + // A mutex to guard the list during close calls as they can be be + // triggered concurrently by the idlePicker and subchannel/resolver + // updates. + closeMu sync.RWMutex + status subConnListState } // scWrapper keeps track of the current state of the subConn. @@ -125,10 +124,12 @@ func newScWrapper(b *pickfirstBalancer, addr resolver.Address, listener func(sta return scw, nil } +// newSubConnList creates a new list and starts connecting using it. A new list +// list should be created only when we want to start connecting. func newSubConnList(addrs []resolver.Address, b *pickfirstBalancer) *subConnList { sl := &subConnList{ b: b, - status: subConnListPending, + status: subConnListActive, } for _, addr := range addrs { @@ -147,17 +148,12 @@ func newSubConnList(addrs []resolver.Address, b *pickfirstBalancer) *subConnList } sl.subConns = append(sl.subConns, scw) } - return sl -} - -func (sl *subConnList) startConnectingIfNeeded() { - sl.mu.Lock() - defer sl.mu.Unlock() - if sl.status != subConnListPending { - return + if len(sl.subConns) > 0 { + sl.startConnectingNextSubConn() + } else { + sl.close() } - sl.status = subConnListActive - sl.startConnectingNextSubConn() + return sl } func (sl *subConnList) stateListener(scw *scWrapper, state balancer.SubConnState) { @@ -239,9 +235,7 @@ func (sl *subConnList) stateListener(scw *scWrapper, state balancer.SubConnState } func (sl *subConnList) selectSubConn(scw *scWrapper) { - if sl.b.logger.V(2) { - sl.b.logger.Infof("Selected subConn %p", &scw.subConn) - } + sl.b.logger.Infof("Selected subConn %p", &scw.subConn) sl.b.unsetSelectedSubConn() sl.b.selectedSubConn = scw sl.b.state = connectivity.Ready @@ -254,8 +248,8 @@ func (sl *subConnList) selectSubConn(scw *scWrapper) { } func (sl *subConnList) close() { - sl.mu.Lock() - defer sl.mu.Unlock() + sl.closeMu.Lock() + defer sl.closeMu.Unlock() if sl.status == subConnListClosed { return } @@ -320,7 +314,10 @@ type pickfirstBalancer struct { cc balancer.ClientConn // Pointer to the subConn list currently connecting. Always close the // current list before replacing it to ensure resources are freed. - subConnList *subConnList + subConnList *subConnList + // A mutex to guard the swapping of subConnLists and it can be triggered + // concurrently by the idlePicker and resolver updates. + subConnListMu sync.Mutex selectedSubConn *scWrapper latestAddressList []resolver.Address shuttingDown bool @@ -356,22 +353,6 @@ func (b *pickfirstBalancer) unsetSelectedSubConn() { func (b *pickfirstBalancer) goIdle() { b.unsetSelectedSubConn() - b.refreshSubConnList() - if len(b.subConnList.subConns) == 0 { - b.state = connectivity.TransientFailure - b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.TransientFailure, - Picker: &picker{err: fmt.Errorf("empty address list")}, - }) - b.unsetSelectedSubConn() - b.subConnList.close() - b.cc.ResolveNow(resolver.ResolveNowOptions{}) - return - } - - exitIdle := func() { - b.subConnList.startConnectingIfNeeded() - } nextState := connectivity.Idle if b.state == connectivity.TransientFailure { @@ -383,12 +364,12 @@ func (b *pickfirstBalancer) goIdle() { b.cc.UpdateState(balancer.State{ ConnectivityState: nextState, Picker: &idlePicker{ - exitIdle: exitIdle, + exitIdle: b.ExitIdle, }, }) } -func (b *pickfirstBalancer) refreshSubConnList() { +func (b *pickfirstBalancer) refreshSubConnListLocked() { subConnList := newSubConnList(b.latestAddressList, b) // Reset the previous subConnList to release resources. @@ -400,7 +381,9 @@ func (b *pickfirstBalancer) refreshSubConnList() { } func (b *pickfirstBalancer) connectUsingLatestAddrs() error { - b.refreshSubConnList() + b.subConnListMu.Lock() + b.refreshSubConnListLocked() + b.subConnListMu.Unlock() if len(b.subConnList.subConns) == 0 { b.state = connectivity.TransientFailure b.cc.UpdateState(balancer.State{ @@ -419,7 +402,6 @@ func (b *pickfirstBalancer) connectUsingLatestAddrs() error { Picker: &picker{err: balancer.ErrNoSubConnAvailable}, }) } - b.subConnList.startConnectingIfNeeded() return nil } @@ -503,10 +485,29 @@ func (b *pickfirstBalancer) Close() { } func (b *pickfirstBalancer) ExitIdle() { - if b.shuttingDown { + b.subConnListMu.Lock() + defer b.subConnListMu.Unlock() + b.subConnList.closeMu.RLock() + if b.subConnList.status == subConnListActive || b.selectedSubConn != nil { + // Already exited idle, nothing to do. + b.subConnList.closeMu.RUnlock() + return + } + + b.subConnList.closeMu.RUnlock() + // The current subConnList is closed, create a new subConnList and + // start connecting. + b.refreshSubConnListLocked() + if len(b.subConnList.subConns) == 0 { + b.state = connectivity.TransientFailure + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: fmt.Errorf("empty address list")}, + }) + b.subConnList.close() + b.cc.ResolveNow(resolver.ResolveNowOptions{}) return } - b.subConnList.startConnectingIfNeeded() } type picker struct { From cb5acc6fee6a5ce2f65ae07a385af6b7e32c2cfa Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Fri, 12 Jul 2024 15:27:55 +0530 Subject: [PATCH 05/62] Move empty subConnList handling into refreshList function --- balancer/pickfirst/pickfirst.go | 40 ++++++++++++++++----------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/balancer/pickfirst/pickfirst.go b/balancer/pickfirst/pickfirst.go index d4cbbb7e5048..958a3338fa90 100644 --- a/balancer/pickfirst/pickfirst.go +++ b/balancer/pickfirst/pickfirst.go @@ -369,31 +369,34 @@ func (b *pickfirstBalancer) goIdle() { }) } -func (b *pickfirstBalancer) refreshSubConnListLocked() { +func (b *pickfirstBalancer) refreshSubConnListLocked() error { + b.subConnList.close() subConnList := newSubConnList(b.latestAddressList, b) + if len(subConnList.subConns) == 0 { + b.state = connectivity.TransientFailure + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: fmt.Errorf("empty address list")}, + }) + b.unsetSelectedSubConn() + return balancer.ErrBadResolverState + } // Reset the previous subConnList to release resources. if b.logger.V(2) { b.logger.Infof("Closing older subConnList") } - b.subConnList.close() b.subConnList = subConnList + return nil } func (b *pickfirstBalancer) connectUsingLatestAddrs() error { b.subConnListMu.Lock() - b.refreshSubConnListLocked() - b.subConnListMu.Unlock() - if len(b.subConnList.subConns) == 0 { - b.state = connectivity.TransientFailure - b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.TransientFailure, - Picker: &picker{err: fmt.Errorf("empty address list")}, - }) - b.unsetSelectedSubConn() - b.subConnList.close() - return balancer.ErrBadResolverState + if err := b.refreshSubConnListLocked(); err != nil { + b.subConnListMu.Unlock() + return err } + b.subConnListMu.Unlock() if b.state != connectivity.TransientFailure { // We stay in TransientFailure until we are Ready. See A62. @@ -497,14 +500,9 @@ func (b *pickfirstBalancer) ExitIdle() { b.subConnList.closeMu.RUnlock() // The current subConnList is closed, create a new subConnList and // start connecting. - b.refreshSubConnListLocked() - if len(b.subConnList.subConns) == 0 { - b.state = connectivity.TransientFailure - b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.TransientFailure, - Picker: &picker{err: fmt.Errorf("empty address list")}, - }) - b.subConnList.close() + if err := b.refreshSubConnListLocked(); errors.Is(err, balancer.ErrBadResolverState) { + // If creation of the subConnList fails, request for re-resolution to + // get a new list of addresses. b.cc.ResolveNow(resolver.ResolveNowOptions{}) return } From 22e0bd41720f9df706c70cd5fd2f4d6db08cfaf4 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Fri, 12 Jul 2024 17:52:43 +0530 Subject: [PATCH 06/62] Fix race conditions --- balancer/pickfirst/pickfirst.go | 84 ++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 33 deletions(-) diff --git a/balancer/pickfirst/pickfirst.go b/balancer/pickfirst/pickfirst.go index 958a3338fa90..188b508255a0 100644 --- a/balancer/pickfirst/pickfirst.go +++ b/balancer/pickfirst/pickfirst.go @@ -40,7 +40,8 @@ import ( type subConnListState int const ( - subConnListActive subConnListState = iota + subConnListConnecting subConnListState = iota + subConnListConnected subConnListClosed ) @@ -61,7 +62,8 @@ type pickfirstBuilder struct{} func (pickfirstBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer { b := &pickfirstBalancer{cc: cc} - b.subConnList = newSubConnList([]resolver.Address{}, b) + b.subConnList = newSubConnList(b) + b.subConnList.close() b.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(logPrefix, b)) return b } @@ -91,11 +93,11 @@ type subConnList struct { lastFailure error // Whether all the subConns have reported a transient failure once. inTransientFailure bool - // A mutex to guard the list during close calls as they can be be - // triggered concurrently by the idlePicker and subchannel/resolver - // updates. - closeMu sync.RWMutex - status subConnListState + // A mutex to guard the list state during state updates. State updates + // are synchronized by the clientConn, but the picker may attempt to read + // the state in parallel. + stateMu sync.RWMutex + state subConnListState } // scWrapper keeps track of the current state of the subConn. @@ -124,15 +126,15 @@ func newScWrapper(b *pickfirstBalancer, addr resolver.Address, listener func(sta return scw, nil } -// newSubConnList creates a new list and starts connecting using it. A new list -// list should be created only when we want to start connecting. -func newSubConnList(addrs []resolver.Address, b *pickfirstBalancer) *subConnList { +// newSubConnList creates a new list. A new list should be created only when we +// want to start connecting to minimize creation of subConns. +func newSubConnList(b *pickfirstBalancer) *subConnList { sl := &subConnList{ - b: b, - status: subConnListActive, + b: b, + state: subConnListConnecting, } - for _, addr := range addrs { + for _, addr := range b.latestAddressList { var scw *scWrapper scw, err := newScWrapper(b, addr, func(state balancer.SubConnState) { sl.stateListener(scw, state) @@ -148,11 +150,6 @@ func newSubConnList(addrs []resolver.Address, b *pickfirstBalancer) *subConnList } sl.subConns = append(sl.subConns, scw) } - if len(sl.subConns) > 0 { - sl.startConnectingNextSubConn() - } else { - sl.close() - } return sl } @@ -173,7 +170,7 @@ func (sl *subConnList) stateListener(scw *scWrapper, state balancer.SubConnState return } // If this list is already closed, ignore the update. - if sl.status != subConnListActive { + if sl.state != subConnListConnecting { if sl.b.logger.V(2) { sl.b.logger.Infof("Ignoring state update for non active subConn %p to %v", &scw.subConn, state.ConnectivityState) } @@ -235,12 +232,24 @@ func (sl *subConnList) stateListener(scw *scWrapper, state balancer.SubConnState } func (sl *subConnList) selectSubConn(scw *scWrapper) { + sl.stateMu.Lock() + defer sl.stateMu.Unlock() + if sl.state == subConnListClosed { + return + } + sl.state = subConnListConnected sl.b.logger.Infof("Selected subConn %p", &scw.subConn) sl.b.unsetSelectedSubConn() sl.b.selectedSubConn = scw sl.b.state = connectivity.Ready sl.inTransientFailure = false - sl.close() + for _, sc := range sl.subConns { + if sc == sl.b.selectedSubConn { + continue + } + sc.subConn.Shutdown() + } + sl.subConns = nil sl.b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.Ready, Picker: &picker{result: balancer.PickResult{SubConn: scw.subConn}}, @@ -248,12 +257,12 @@ func (sl *subConnList) selectSubConn(scw *scWrapper) { } func (sl *subConnList) close() { - sl.closeMu.Lock() - defer sl.closeMu.Unlock() - if sl.status == subConnListClosed { + sl.stateMu.Lock() + defer sl.stateMu.Unlock() + if sl.state == subConnListClosed { return } - sl.status = subConnListClosed + sl.state = subConnListClosed // Close all the subConns except the selected one. The selected subConn // will be closed by the balancer. for _, sc := range sl.subConns { @@ -266,7 +275,7 @@ func (sl *subConnList) close() { } func (sl *subConnList) startConnectingNextSubConn() { - if sl.status != subConnListActive { + if sl.state != subConnListConnecting { return } if !sl.inTransientFailure && sl.attemptingIndex < len(sl.subConns) { @@ -353,6 +362,7 @@ func (b *pickfirstBalancer) unsetSelectedSubConn() { func (b *pickfirstBalancer) goIdle() { b.unsetSelectedSubConn() + b.subConnList.close() nextState := connectivity.Idle if b.state == connectivity.TransientFailure { @@ -369,10 +379,13 @@ func (b *pickfirstBalancer) goIdle() { }) } +// refreshSubConnListLocked replaces the current subConnList with a newly created +// one. The caller is responsible to close the existing list and ensure its +// closed synchronously during a ClientConn update or a subConn state update. func (b *pickfirstBalancer) refreshSubConnListLocked() error { - b.subConnList.close() - subConnList := newSubConnList(b.latestAddressList, b) + subConnList := newSubConnList(b) if len(subConnList.subConns) == 0 { + subConnList.close() b.state = connectivity.TransientFailure b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.TransientFailure, @@ -387,10 +400,14 @@ func (b *pickfirstBalancer) refreshSubConnListLocked() error { b.logger.Infof("Closing older subConnList") } b.subConnList = subConnList + subConnList.startConnectingNextSubConn() + // Don't read any fields on subConnList after we start connecting as it will + // cause races with subConn state updates being handled by the stateListener. return nil } func (b *pickfirstBalancer) connectUsingLatestAddrs() error { + b.subConnList.close() b.subConnListMu.Lock() if err := b.refreshSubConnListLocked(); err != nil { b.subConnListMu.Unlock() @@ -487,18 +504,19 @@ func (b *pickfirstBalancer) Close() { b.subConnList.close() } +// ExitIdle moves the balancer out of idle state. It can be called concurrently +// by the idlePicker and clientConn so access to variables should be synchronized. func (b *pickfirstBalancer) ExitIdle() { b.subConnListMu.Lock() defer b.subConnListMu.Unlock() - b.subConnList.closeMu.RLock() - if b.subConnList.status == subConnListActive || b.selectedSubConn != nil { + b.subConnList.stateMu.RLock() + if b.subConnList.state != subConnListClosed { // Already exited idle, nothing to do. - b.subConnList.closeMu.RUnlock() + b.subConnList.stateMu.RUnlock() return } - - b.subConnList.closeMu.RUnlock() - // The current subConnList is closed, create a new subConnList and + b.subConnList.stateMu.RUnlock() + // The current subConnList is still closed, create a new subConnList and // start connecting. if err := b.refreshSubConnListLocked(); errors.Is(err, balancer.ErrBadResolverState) { // If creation of the subConnList fails, request for re-resolution to From 258145ac46f589ec884df22e93a0e4ea5fa18eaf Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Sat, 13 Jul 2024 03:03:36 +0530 Subject: [PATCH 07/62] Replace mutex with atomic for subConnList state --- balancer/pickfirst/pickfirst.go | 36 +++++++++++---------------------- 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/balancer/pickfirst/pickfirst.go b/balancer/pickfirst/pickfirst.go index 188b508255a0..a211f3efa32e 100644 --- a/balancer/pickfirst/pickfirst.go +++ b/balancer/pickfirst/pickfirst.go @@ -26,6 +26,7 @@ import ( "math/rand" "slices" "sync" + "sync/atomic" "google.golang.org/grpc/balancer" "google.golang.org/grpc/connectivity" @@ -37,10 +38,8 @@ import ( "google.golang.org/grpc/serviceconfig" ) -type subConnListState int - const ( - subConnListConnecting subConnListState = iota + subConnListConnecting uint32 = iota subConnListConnected subConnListClosed ) @@ -93,11 +92,9 @@ type subConnList struct { lastFailure error // Whether all the subConns have reported a transient failure once. inTransientFailure bool - // A mutex to guard the list state during state updates. State updates - // are synchronized by the clientConn, but the picker may attempt to read - // the state in parallel. - stateMu sync.RWMutex - state subConnListState + // State updates are serialized by the clientConn, but the picker may attempt + // to read the state in parallel, so we use an atomic. + state atomic.Uint32 } // scWrapper keeps track of the current state of the subConn. @@ -130,9 +127,9 @@ func newScWrapper(b *pickfirstBalancer, addr resolver.Address, listener func(sta // want to start connecting to minimize creation of subConns. func newSubConnList(b *pickfirstBalancer) *subConnList { sl := &subConnList{ - b: b, - state: subConnListConnecting, + b: b, } + sl.state.Store(subConnListConnecting) for _, addr := range b.latestAddressList { var scw *scWrapper @@ -170,7 +167,7 @@ func (sl *subConnList) stateListener(scw *scWrapper, state balancer.SubConnState return } // If this list is already closed, ignore the update. - if sl.state != subConnListConnecting { + if sl.state.Load() != subConnListConnecting { if sl.b.logger.V(2) { sl.b.logger.Infof("Ignoring state update for non active subConn %p to %v", &scw.subConn, state.ConnectivityState) } @@ -232,12 +229,9 @@ func (sl *subConnList) stateListener(scw *scWrapper, state balancer.SubConnState } func (sl *subConnList) selectSubConn(scw *scWrapper) { - sl.stateMu.Lock() - defer sl.stateMu.Unlock() - if sl.state == subConnListClosed { + if !sl.state.CompareAndSwap(subConnListConnecting, subConnListConnected) { return } - sl.state = subConnListConnected sl.b.logger.Infof("Selected subConn %p", &scw.subConn) sl.b.unsetSelectedSubConn() sl.b.selectedSubConn = scw @@ -257,12 +251,9 @@ func (sl *subConnList) selectSubConn(scw *scWrapper) { } func (sl *subConnList) close() { - sl.stateMu.Lock() - defer sl.stateMu.Unlock() - if sl.state == subConnListClosed { + if prevState := sl.state.Swap(subConnListClosed); prevState == subConnListClosed { return } - sl.state = subConnListClosed // Close all the subConns except the selected one. The selected subConn // will be closed by the balancer. for _, sc := range sl.subConns { @@ -275,7 +266,7 @@ func (sl *subConnList) close() { } func (sl *subConnList) startConnectingNextSubConn() { - if sl.state != subConnListConnecting { + if sl.state.Load() != subConnListConnecting { return } if !sl.inTransientFailure && sl.attemptingIndex < len(sl.subConns) { @@ -509,13 +500,10 @@ func (b *pickfirstBalancer) Close() { func (b *pickfirstBalancer) ExitIdle() { b.subConnListMu.Lock() defer b.subConnListMu.Unlock() - b.subConnList.stateMu.RLock() - if b.subConnList.state != subConnListClosed { + if b.subConnList.state.Load() != subConnListClosed { // Already exited idle, nothing to do. - b.subConnList.stateMu.RUnlock() return } - b.subConnList.stateMu.RUnlock() // The current subConnList is still closed, create a new subConnList and // start connecting. if err := b.refreshSubConnListLocked(); errors.Is(err, balancer.ErrBadResolverState) { From eeacec3c123c0f3884d8c2d3cb957b5bb268846b Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Mon, 15 Jul 2024 13:00:27 +0530 Subject: [PATCH 08/62] Use a go routine to manage phase 1 connection attempts --- balancer/pickfirst/pickfirst.go | 124 +++++++++++++++++--------------- 1 file changed, 68 insertions(+), 56 deletions(-) diff --git a/balancer/pickfirst/pickfirst.go b/balancer/pickfirst/pickfirst.go index a211f3efa32e..1375aa163577 100644 --- a/balancer/pickfirst/pickfirst.go +++ b/balancer/pickfirst/pickfirst.go @@ -84,17 +84,18 @@ type pfConfig struct { // the pick-first algorithm. type subConnList struct { subConns []*scWrapper - // The index within the subConns list for the subConn being tried. - attemptingIndex int - b *pickfirstBalancer + b *pickfirstBalancer // The most recent failure during the initial connection attempt over the // entire sunConns list. lastFailure error - // Whether all the subConns have reported a transient failure once. - inTransientFailure bool + // The number of transient failures seen while connecting. + transientFailuresCount int // State updates are serialized by the clientConn, but the picker may attempt // to read the state in parallel, so we use an atomic. state atomic.Uint32 + // connectingCh is used to signal the transition of the subConnList out of + // the connecting state. + connectingCh chan struct{} } // scWrapper keeps track of the current state of the subConn. @@ -102,12 +103,16 @@ type scWrapper struct { subConn balancer.SubConn state connectivity.State addr resolver.Address + // failureChan is used to communicate the failure when connection fails. + failureChan chan error + hasFailedPreviously bool } func newScWrapper(b *pickfirstBalancer, addr resolver.Address, listener func(state balancer.SubConnState)) (*scWrapper, error) { scw := &scWrapper{ - state: connectivity.Idle, - addr: addr, + state: connectivity.Idle, + addr: addr, + failureChan: make(chan error, 1), } sc, err := b.cc.NewSubConn([]resolver.Address{addr}, balancer.NewSubConnOptions{ StateListener: func(scs balancer.SubConnState) { @@ -127,7 +132,8 @@ func newScWrapper(b *pickfirstBalancer, addr resolver.Address, listener func(sta // want to start connecting to minimize creation of subConns. func newSubConnList(b *pickfirstBalancer) *subConnList { sl := &subConnList{ - b: b, + b: b, + connectingCh: make(chan struct{}), } sl.state.Store(subConnListConnecting) @@ -173,16 +179,39 @@ func (sl *subConnList) stateListener(scw *scWrapper, state balancer.SubConnState } return } - if !sl.inTransientFailure { - // We are still trying to connect to each subConn once. + if !scw.hasFailedPreviously { + // This subConn is attempting to connect for the first time, we're in the + // initial pass. switch state.ConnectivityState { case connectivity.TransientFailure: if sl.b.logger.V(2) { sl.b.logger.Infof("SubConn %p failed to connect due to error: %v", &scw.subConn, state.ConnectionError) } - sl.attemptingIndex++ + scw.hasFailedPreviously = true sl.lastFailure = state.ConnectionError - sl.startConnectingNextSubConn() + sl.transientFailuresCount++ + // If we've seen one failure from each subConn, the first pass ends + // and we can set the channel state to transient failure. + if sl.transientFailuresCount == len(sl.subConns) { + sl.b.logger.Infof("Received one failure from each subConn in list %p, waiting for any subConn to connect.", sl) + // Phase 1 is over, start phase 2. + sl.b.state = connectivity.TransientFailure + sl.b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: sl.lastFailure}, + }) + + // In phase 2, we attempt to connect to all the subConns in parallel. + // Connect to addresses that have already completed the back-off. + for idx := 0; idx < len(sl.subConns); idx++ { + sc := sl.subConns[idx] + if sc.state == connectivity.TransientFailure { + continue + } + sl.subConns[idx].subConn.Connect() + } + } + scw.failureChan <- state.ConnectionError case connectivity.Ready: // Cleanup and update the picker to use the subconn. sl.selectSubConn(scw) @@ -203,20 +232,19 @@ func (sl *subConnList) stateListener(scw *scWrapper, state balancer.SubConnState return } - // We have attempted to connect to all the subConns once and failed. + if sl.transientFailuresCount < len(sl.subConns) { + // This subConn has failed once, but other subConns are still connecting. + // Wait for other subConns to complete. + return + } + + // We have completed the first phase. switch state.ConnectivityState { case connectivity.TransientFailure: if sl.b.logger.V(2) { sl.b.logger.Infof("SubConn %p failed to connect due to error: %v", &scw.subConn, state.ConnectionError) } - // If its the initial connection attempt, try to connect to the next subConn. - if sl.attemptingIndex < len(sl.subConns) && sl.subConns[sl.attemptingIndex] == scw { - // Going over all the subConns and try connecting to ones that are idle. - sl.attemptingIndex++ - sl.startConnectingNextSubConn() - } case connectivity.Ready: - // Cleanup and update the picker to use the subconn. sl.selectSubConn(scw) case connectivity.Idle: // Trigger re-connection. @@ -232,18 +260,17 @@ func (sl *subConnList) selectSubConn(scw *scWrapper) { if !sl.state.CompareAndSwap(subConnListConnecting, subConnListConnected) { return } + close(sl.connectingCh) sl.b.logger.Infof("Selected subConn %p", &scw.subConn) sl.b.unsetSelectedSubConn() sl.b.selectedSubConn = scw sl.b.state = connectivity.Ready - sl.inTransientFailure = false for _, sc := range sl.subConns { if sc == sl.b.selectedSubConn { continue } sc.subConn.Shutdown() } - sl.subConns = nil sl.b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.Ready, Picker: &picker{result: balancer.PickResult{SubConn: scw.subConn}}, @@ -251,9 +278,13 @@ func (sl *subConnList) selectSubConn(scw *scWrapper) { } func (sl *subConnList) close() { - if prevState := sl.state.Swap(subConnListClosed); prevState == subConnListClosed { + prevState := sl.state.Swap(subConnListClosed) + if prevState == subConnListClosed { return } + if prevState == subConnListConnecting { + close(sl.connectingCh) + } // Close all the subConns except the selected one. The selected subConn // will be closed by the balancer. for _, sc := range sl.subConns { @@ -262,42 +293,23 @@ func (sl *subConnList) close() { } sc.subConn.Shutdown() } - sl.subConns = nil } -func (sl *subConnList) startConnectingNextSubConn() { - if sl.state.Load() != subConnListConnecting { - return - } - if !sl.inTransientFailure && sl.attemptingIndex < len(sl.subConns) { - // Try to connect to the next subConn. - sl.subConns[sl.attemptingIndex].subConn.Connect() - return - } - if !sl.inTransientFailure { - // Failed to connect to each subConn once, enter transient failure. - sl.b.state = connectivity.TransientFailure - sl.inTransientFailure = true - sl.attemptingIndex = 0 - sl.b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.TransientFailure, - Picker: &picker{err: sl.lastFailure}, - }) - // Drop the existing (working) connection, if any. This may be - // sub-optimal, but we can't ignore what the control plane told us. - sl.b.unsetSelectedSubConn() - } - - // Attempt to connect to addresses that have already completed the back-off. - for ; sl.attemptingIndex < len(sl.subConns); sl.attemptingIndex++ { - sc := sl.subConns[sl.attemptingIndex] - if sc.state == connectivity.TransientFailure { - continue +// connect attempts to connect to subConns till it finds a healthy one. +func (sl *subConnList) connect() { + // Attempt to connect to each subConn once. + for _, scw := range sl.subConns { + scw.subConn.Connect() + select { + case <-sl.connectingCh: + return + case err := <-scw.failureChan: + if err == nil { + // Connected successfully. + return + } } - sl.subConns[sl.attemptingIndex].subConn.Connect() - return } - // Wait for the next subConn to enter idle state, then re-connect. } func (pickfirstBuilder) ParseConfig(js json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { @@ -391,7 +403,7 @@ func (b *pickfirstBalancer) refreshSubConnListLocked() error { b.logger.Infof("Closing older subConnList") } b.subConnList = subConnList - subConnList.startConnectingNextSubConn() + go subConnList.connect() // Don't read any fields on subConnList after we start connecting as it will // cause races with subConn state updates being handled by the stateListener. return nil From 632f4dfaefbc60133fa335bb4a3015725b531bb1 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Wed, 7 Aug 2024 18:52:26 +0530 Subject: [PATCH 09/62] support both pickfirst together --- balancer/pickfirst/pickfirst.go | 460 ++----- balancer/pickfirst_leaf/pickfirst_leaf.go | 547 ++++++++ clientconn_pickfirst_leaf_test.go | 1179 +++++++++++++++++ clientconn_test.go | 47 +- ...n_state_transition_pick_first_leaf_test.go | 490 +++++++ test/clientconn_state_transition_test.go | 42 +- test/pickfirst_leaf_test.go | 910 +++++++++++++ 7 files changed, 3261 insertions(+), 414 deletions(-) create mode 100644 balancer/pickfirst_leaf/pickfirst_leaf.go create mode 100644 clientconn_pickfirst_leaf_test.go create mode 100644 test/clientconn_state_transition_pick_first_leaf_test.go create mode 100644 test/pickfirst_leaf_test.go diff --git a/balancer/pickfirst/pickfirst.go b/balancer/pickfirst/pickfirst.go index 1375aa163577..07527603f1d4 100644 --- a/balancer/pickfirst/pickfirst.go +++ b/balancer/pickfirst/pickfirst.go @@ -24,9 +24,6 @@ import ( "errors" "fmt" "math/rand" - "slices" - "sync" - "sync/atomic" "google.golang.org/grpc/balancer" "google.golang.org/grpc/connectivity" @@ -38,12 +35,6 @@ import ( "google.golang.org/grpc/serviceconfig" ) -const ( - subConnListConnecting uint32 = iota - subConnListConnected - subConnListClosed -) - func init() { balancer.Register(pickfirstBuilder{}) internal.ShuffleAddressListForTesting = func(n int, swap func(i, j int)) { rand.Shuffle(n, swap) } @@ -61,8 +52,6 @@ type pickfirstBuilder struct{} func (pickfirstBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer { b := &pickfirstBalancer{cc: cc} - b.subConnList = newSubConnList(b) - b.subConnList.close() b.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(logPrefix, b)) return b } @@ -80,238 +69,6 @@ type pfConfig struct { ShuffleAddressList bool `json:"shuffleAddressList"` } -// subConnList stores provides functions to connect to a list of addresses using -// the pick-first algorithm. -type subConnList struct { - subConns []*scWrapper - b *pickfirstBalancer - // The most recent failure during the initial connection attempt over the - // entire sunConns list. - lastFailure error - // The number of transient failures seen while connecting. - transientFailuresCount int - // State updates are serialized by the clientConn, but the picker may attempt - // to read the state in parallel, so we use an atomic. - state atomic.Uint32 - // connectingCh is used to signal the transition of the subConnList out of - // the connecting state. - connectingCh chan struct{} -} - -// scWrapper keeps track of the current state of the subConn. -type scWrapper struct { - subConn balancer.SubConn - state connectivity.State - addr resolver.Address - // failureChan is used to communicate the failure when connection fails. - failureChan chan error - hasFailedPreviously bool -} - -func newScWrapper(b *pickfirstBalancer, addr resolver.Address, listener func(state balancer.SubConnState)) (*scWrapper, error) { - scw := &scWrapper{ - state: connectivity.Idle, - addr: addr, - failureChan: make(chan error, 1), - } - sc, err := b.cc.NewSubConn([]resolver.Address{addr}, balancer.NewSubConnOptions{ - StateListener: func(scs balancer.SubConnState) { - // Store the state and delegate. - scw.state = scs.ConnectivityState - listener(scs) - }, - }) - if err != nil { - return nil, err - } - scw.subConn = sc - return scw, nil -} - -// newSubConnList creates a new list. A new list should be created only when we -// want to start connecting to minimize creation of subConns. -func newSubConnList(b *pickfirstBalancer) *subConnList { - sl := &subConnList{ - b: b, - connectingCh: make(chan struct{}), - } - sl.state.Store(subConnListConnecting) - - for _, addr := range b.latestAddressList { - var scw *scWrapper - scw, err := newScWrapper(b, addr, func(state balancer.SubConnState) { - sl.stateListener(scw, state) - }) - if err != nil { - if b.logger.V(2) { - b.logger.Infof("Ignoring failure, could not create a subConn for address %q due to error: %v", addr, err) - } - continue - } - if b.logger.V(2) { - b.logger.Infof("Created a subConn for address %q", addr) - } - sl.subConns = append(sl.subConns, scw) - } - return sl -} - -func (sl *subConnList) stateListener(scw *scWrapper, state balancer.SubConnState) { - if sl.b.logger.V(2) { - sl.b.logger.Infof("Received SubConn state update: %p, %+v", scw, state) - } - if scw == sl.b.selectedSubConn { - // As we set the selected subConn only once it's ready, the only - // possible transitions are to IDLE and SHUTDOWN. - switch state.ConnectivityState { - case connectivity.Shutdown: - case connectivity.Idle: - sl.b.goIdle() - default: - sl.b.logger.Warningf("Ignoring unexpected transition of selected subConn %p to %v", &scw.subConn, state.ConnectivityState) - } - return - } - // If this list is already closed, ignore the update. - if sl.state.Load() != subConnListConnecting { - if sl.b.logger.V(2) { - sl.b.logger.Infof("Ignoring state update for non active subConn %p to %v", &scw.subConn, state.ConnectivityState) - } - return - } - if !scw.hasFailedPreviously { - // This subConn is attempting to connect for the first time, we're in the - // initial pass. - switch state.ConnectivityState { - case connectivity.TransientFailure: - if sl.b.logger.V(2) { - sl.b.logger.Infof("SubConn %p failed to connect due to error: %v", &scw.subConn, state.ConnectionError) - } - scw.hasFailedPreviously = true - sl.lastFailure = state.ConnectionError - sl.transientFailuresCount++ - // If we've seen one failure from each subConn, the first pass ends - // and we can set the channel state to transient failure. - if sl.transientFailuresCount == len(sl.subConns) { - sl.b.logger.Infof("Received one failure from each subConn in list %p, waiting for any subConn to connect.", sl) - // Phase 1 is over, start phase 2. - sl.b.state = connectivity.TransientFailure - sl.b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.TransientFailure, - Picker: &picker{err: sl.lastFailure}, - }) - - // In phase 2, we attempt to connect to all the subConns in parallel. - // Connect to addresses that have already completed the back-off. - for idx := 0; idx < len(sl.subConns); idx++ { - sc := sl.subConns[idx] - if sc.state == connectivity.TransientFailure { - continue - } - sl.subConns[idx].subConn.Connect() - } - } - scw.failureChan <- state.ConnectionError - case connectivity.Ready: - // Cleanup and update the picker to use the subconn. - sl.selectSubConn(scw) - case connectivity.Connecting: - // Move the channel to connecting if this is the first subConn to - // start connecting. - if sl.b.state == connectivity.Idle { - sl.b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.Connecting, - Picker: &picker{err: balancer.ErrNoSubConnAvailable}, - }) - } - default: - if sl.b.logger.V(2) { - sl.b.logger.Infof("Ignoring update for the subConn %p to state %v", &scw.subConn, state.ConnectivityState) - } - } - return - } - - if sl.transientFailuresCount < len(sl.subConns) { - // This subConn has failed once, but other subConns are still connecting. - // Wait for other subConns to complete. - return - } - - // We have completed the first phase. - switch state.ConnectivityState { - case connectivity.TransientFailure: - if sl.b.logger.V(2) { - sl.b.logger.Infof("SubConn %p failed to connect due to error: %v", &scw.subConn, state.ConnectionError) - } - case connectivity.Ready: - sl.selectSubConn(scw) - case connectivity.Idle: - // Trigger re-connection. - scw.subConn.Connect() - default: - if sl.b.logger.V(2) { - sl.b.logger.Infof("Ignoring update for the subConn %p to state %v", &scw.subConn, state.ConnectivityState) - } - } -} - -func (sl *subConnList) selectSubConn(scw *scWrapper) { - if !sl.state.CompareAndSwap(subConnListConnecting, subConnListConnected) { - return - } - close(sl.connectingCh) - sl.b.logger.Infof("Selected subConn %p", &scw.subConn) - sl.b.unsetSelectedSubConn() - sl.b.selectedSubConn = scw - sl.b.state = connectivity.Ready - for _, sc := range sl.subConns { - if sc == sl.b.selectedSubConn { - continue - } - sc.subConn.Shutdown() - } - sl.b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.Ready, - Picker: &picker{result: balancer.PickResult{SubConn: scw.subConn}}, - }) -} - -func (sl *subConnList) close() { - prevState := sl.state.Swap(subConnListClosed) - if prevState == subConnListClosed { - return - } - if prevState == subConnListConnecting { - close(sl.connectingCh) - } - // Close all the subConns except the selected one. The selected subConn - // will be closed by the balancer. - for _, sc := range sl.subConns { - if sc == sl.b.selectedSubConn { - continue - } - sc.subConn.Shutdown() - } -} - -// connect attempts to connect to subConns till it finds a healthy one. -func (sl *subConnList) connect() { - // Attempt to connect to each subConn once. - for _, scw := range sl.subConns { - scw.subConn.Connect() - select { - case <-sl.connectingCh: - return - case err := <-scw.failureChan: - if err == nil { - // Connected successfully. - return - } - } - } -} - func (pickfirstBuilder) ParseConfig(js json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { var cfg pfConfig if err := json.Unmarshal(js, &cfg); err != nil { @@ -321,27 +78,17 @@ func (pickfirstBuilder) ParseConfig(js json.RawMessage) (serviceconfig.LoadBalan } type pickfirstBalancer struct { - logger *internalgrpclog.PrefixLogger - state connectivity.State - cc balancer.ClientConn - // Pointer to the subConn list currently connecting. Always close the - // current list before replacing it to ensure resources are freed. - subConnList *subConnList - // A mutex to guard the swapping of subConnLists and it can be triggered - // concurrently by the idlePicker and resolver updates. - subConnListMu sync.Mutex - selectedSubConn *scWrapper - latestAddressList []resolver.Address - shuttingDown bool + logger *internalgrpclog.PrefixLogger + state connectivity.State + cc balancer.ClientConn + subConn balancer.SubConn } func (b *pickfirstBalancer) ResolverError(err error) { if b.logger.V(2) { b.logger.Infof("Received error from the name resolver: %v", err) } - if len(b.latestAddressList) == 0 { - // The picker will not change since the balancer does not currently - // report an error. + if b.subConn == nil { b.state = connectivity.TransientFailure } @@ -356,85 +103,22 @@ func (b *pickfirstBalancer) ResolverError(err error) { }) } -func (b *pickfirstBalancer) unsetSelectedSubConn() { - if b.selectedSubConn != nil { - b.selectedSubConn.subConn.Shutdown() - b.selectedSubConn = nil - } -} - -func (b *pickfirstBalancer) goIdle() { - b.unsetSelectedSubConn() - b.subConnList.close() - - nextState := connectivity.Idle - if b.state == connectivity.TransientFailure { - // We stay in TransientFailure until we are Ready. See A62. - nextState = connectivity.TransientFailure - } else { - b.state = connectivity.Idle - } - b.cc.UpdateState(balancer.State{ - ConnectivityState: nextState, - Picker: &idlePicker{ - exitIdle: b.ExitIdle, - }, - }) -} - -// refreshSubConnListLocked replaces the current subConnList with a newly created -// one. The caller is responsible to close the existing list and ensure its -// closed synchronously during a ClientConn update or a subConn state update. -func (b *pickfirstBalancer) refreshSubConnListLocked() error { - subConnList := newSubConnList(b) - if len(subConnList.subConns) == 0 { - subConnList.close() - b.state = connectivity.TransientFailure - b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.TransientFailure, - Picker: &picker{err: fmt.Errorf("empty address list")}, - }) - b.unsetSelectedSubConn() - return balancer.ErrBadResolverState - } - - // Reset the previous subConnList to release resources. - if b.logger.V(2) { - b.logger.Infof("Closing older subConnList") - } - b.subConnList = subConnList - go subConnList.connect() - // Don't read any fields on subConnList after we start connecting as it will - // cause races with subConn state updates being handled by the stateListener. - return nil +type Shuffler interface { + ShuffleAddressListForTesting(n int, swap func(i, j int)) } -func (b *pickfirstBalancer) connectUsingLatestAddrs() error { - b.subConnList.close() - b.subConnListMu.Lock() - if err := b.refreshSubConnListLocked(); err != nil { - b.subConnListMu.Unlock() - return err - } - b.subConnListMu.Unlock() - - if b.state != connectivity.TransientFailure { - // We stay in TransientFailure until we are Ready. See A62. - b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.Connecting, - Picker: &picker{err: balancer.ErrNoSubConnAvailable}, - }) - } - return nil -} +func ShuffleAddressListForTesting(n int, swap func(i, j int)) { rand.Shuffle(n, swap) } func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState) error { if len(state.ResolverState.Addresses) == 0 && len(state.ResolverState.Endpoints) == 0 { - // Cleanup state pertaining to the previous resolver state. - b.unsetSelectedSubConn() - b.subConnList.close() - b.latestAddressList = nil - // Treat an empty address list like an error by calling b.ResolverError. + // The resolver reported an empty address list. Treat it like an error by + // calling b.ResolverError. + if b.subConn != nil { + // Shut down the old subConn. All addresses were removed, so it is + // no longer valid. + b.subConn.Shutdown() + b.subConn = nil + } b.ResolverError(errors.New("produced zero addresses")) return balancer.ErrBadResolverState } @@ -471,7 +155,7 @@ func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState // Endpoints not set, process addresses until we migrate resolver // emissions fully to Endpoints. The top channel does wrap emitted // addresses with endpoints, however some balancers such as weighted - // target do not forward the corresponding correct endpoints down/split + // target do not forwarrd the corresponding correct endpoints down/split // endpoints properly. Once all balancers correctly forward endpoints // down, can delete this else conditional. addrs = state.ResolverState.Addresses @@ -481,18 +165,36 @@ func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState } } - b.latestAddressList = addrs - // If the selected subConn's address is present in the list, don't attempt - // to re-connect. - if b.selectedSubConn != nil && slices.ContainsFunc(addrs, func(addr resolver.Address) bool { - return addr.Equal(b.selectedSubConn.addr) - }) { + if b.subConn != nil { + b.cc.UpdateAddresses(b.subConn, addrs) + return nil + } + + var subConn balancer.SubConn + subConn, err := b.cc.NewSubConn(addrs, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { + b.updateSubConnState(subConn, state) + }, + }) + if err != nil { if b.logger.V(2) { - b.logger.Infof("Not attempting to re-connect since selected address %q is present in new address list", b.selectedSubConn.addr.String()) + b.logger.Infof("Failed to create new SubConn: %v", err) } - return nil + b.state = connectivity.TransientFailure + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: fmt.Errorf("error creating connection: %v", err)}, + }) + return balancer.ErrBadResolverState } - return b.connectUsingLatestAddrs() + b.subConn = subConn + b.state = connectivity.Idle + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &picker{err: balancer.ErrNoSubConnAvailable}, + }) + b.subConn.Connect() + return nil } // UpdateSubConnState is unused as a StateListener is always registered when @@ -501,28 +203,62 @@ func (b *pickfirstBalancer) UpdateSubConnState(subConn balancer.SubConn, state b b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", subConn, state) } +func (b *pickfirstBalancer) updateSubConnState(subConn balancer.SubConn, state balancer.SubConnState) { + if b.logger.V(2) { + b.logger.Infof("Received SubConn state update: %p, %+v", subConn, state) + } + if b.subConn != subConn { + if b.logger.V(2) { + b.logger.Infof("Ignored state change because subConn is not recognized") + } + return + } + if state.ConnectivityState == connectivity.Shutdown { + b.subConn = nil + return + } + + switch state.ConnectivityState { + case connectivity.Ready: + b.cc.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: &picker{result: balancer.PickResult{SubConn: subConn}}, + }) + case connectivity.Connecting: + if b.state == connectivity.TransientFailure { + // We stay in TransientFailure until we are Ready. See A62. + return + } + b.cc.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: &picker{err: balancer.ErrNoSubConnAvailable}, + }) + case connectivity.Idle: + if b.state == connectivity.TransientFailure { + // We stay in TransientFailure until we are Ready. Also kick the + // subConn out of Idle into Connecting. See A62. + b.subConn.Connect() + return + } + b.cc.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: &idlePicker{subConn: subConn}, + }) + case connectivity.TransientFailure: + b.cc.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: &picker{err: state.ConnectionError}, + }) + } + b.state = state.ConnectivityState +} + func (b *pickfirstBalancer) Close() { - b.shuttingDown = true - b.unsetSelectedSubConn() - b.subConnList.close() } -// ExitIdle moves the balancer out of idle state. It can be called concurrently -// by the idlePicker and clientConn so access to variables should be synchronized. func (b *pickfirstBalancer) ExitIdle() { - b.subConnListMu.Lock() - defer b.subConnListMu.Unlock() - if b.subConnList.state.Load() != subConnListClosed { - // Already exited idle, nothing to do. - return - } - // The current subConnList is still closed, create a new subConnList and - // start connecting. - if err := b.refreshSubConnListLocked(); errors.Is(err, balancer.ErrBadResolverState) { - // If creation of the subConnList fails, request for re-resolution to - // get a new list of addresses. - b.cc.ResolveNow(resolver.ResolveNowOptions{}) - return + if b.subConn != nil && b.state == connectivity.Idle { + b.subConn.Connect() } } @@ -538,10 +274,10 @@ func (p *picker) Pick(balancer.PickInfo) (balancer.PickResult, error) { // idlePicker is used when the SubConn is IDLE and kicks the SubConn into // CONNECTING when Pick is called. type idlePicker struct { - exitIdle func() + subConn balancer.SubConn } func (i *idlePicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { - i.exitIdle() + i.subConn.Connect() return balancer.PickResult{}, balancer.ErrNoSubConnAvailable } diff --git a/balancer/pickfirst_leaf/pickfirst_leaf.go b/balancer/pickfirst_leaf/pickfirst_leaf.go new file mode 100644 index 000000000000..e1558fc3509d --- /dev/null +++ b/balancer/pickfirst_leaf/pickfirst_leaf.go @@ -0,0 +1,547 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package pickfirst_leaf contains the pick_first load balancing policy which +// will be the universal leaf policy after Dual Stack changes are implemented. +package pickfirst_leaf + +import ( + "encoding/json" + "errors" + "fmt" + "math/rand" + "slices" + "sync" + "sync/atomic" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +const ( + subConnListConnecting uint32 = iota + subConnListConnected + subConnListClosed +) + +func init() { + balancer.Register(pickfirstBuilder{}) +} + +var logger = grpclog.Component("pick-first-leaf-lb") + +const ( + // Name is the name of the pick_first balancer. + Name = "pick_first_leaf" + logPrefix = "[pick-first-leaf-lb %p] " +) + +type pickfirstBuilder struct{} + +func (pickfirstBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer { + b := &pickfirstBalancer{cc: cc} + b.subConnList = newSubConnList(b) + b.subConnList.close() + b.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(logPrefix, b)) + return b +} + +func (pickfirstBuilder) Name() string { + return Name +} + +type pfConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + // If set to true, instructs the LB policy to shuffle the order of the list + // of endpoints received from the name resolver before attempting to + // connect to them. + ShuffleAddressList bool `json:"shuffleAddressList"` +} + +// subConnList stores provides functions to connect to a list of addresses using +// the pick-first algorithm. +type subConnList struct { + subConns []*scWrapper + b *pickfirstBalancer + // The most recent failure during the initial connection attempt over the + // entire sunConns list. + lastFailure error + // The number of transient failures seen while connecting. + transientFailuresCount int + // State updates are serialized by the clientConn, but the picker may attempt + // to read the state in parallel, so we use an atomic. + state atomic.Uint32 + // connectingCh is used to signal the transition of the subConnList out of + // the connecting state. + connectingCh chan struct{} +} + +// scWrapper keeps track of the current state of the subConn. +type scWrapper struct { + subConn balancer.SubConn + state connectivity.State + addr resolver.Address + // failureChan is used to communicate the failure when connection fails. + failureChan chan error + hasFailedPreviously bool +} + +func newScWrapper(b *pickfirstBalancer, addr resolver.Address, listener func(state balancer.SubConnState)) (*scWrapper, error) { + scw := &scWrapper{ + state: connectivity.Idle, + addr: addr, + failureChan: make(chan error, 1), + } + sc, err := b.cc.NewSubConn([]resolver.Address{addr}, balancer.NewSubConnOptions{ + StateListener: func(scs balancer.SubConnState) { + // Store the state and delegate. + scw.state = scs.ConnectivityState + listener(scs) + }, + }) + if err != nil { + return nil, err + } + scw.subConn = sc + return scw, nil +} + +// newSubConnList creates a new list. A new list should be created only when we +// want to start connecting to minimize creation of subConns. +func newSubConnList(b *pickfirstBalancer) *subConnList { + sl := &subConnList{ + b: b, + connectingCh: make(chan struct{}), + } + sl.state.Store(subConnListConnecting) + + for _, addr := range b.latestAddressList { + var scw *scWrapper + scw, err := newScWrapper(b, addr, func(state balancer.SubConnState) { + sl.stateListener(scw, state) + }) + if err != nil { + if b.logger.V(2) { + b.logger.Infof("Ignoring failure, could not create a subConn for address %q due to error: %v", addr, err) + } + continue + } + if b.logger.V(2) { + b.logger.Infof("Created a subConn for address %q", addr) + } + sl.subConns = append(sl.subConns, scw) + } + return sl +} + +func (sl *subConnList) stateListener(scw *scWrapper, state balancer.SubConnState) { + if sl.b.logger.V(2) { + sl.b.logger.Infof("Received SubConn state update: %p, %+v", scw, state) + } + if scw == sl.b.selectedSubConn { + // As we set the selected subConn only once it's ready, the only + // possible transitions are to IDLE and SHUTDOWN. + switch state.ConnectivityState { + case connectivity.Shutdown: + case connectivity.Idle: + sl.b.goIdle() + default: + sl.b.logger.Warningf("Ignoring unexpected transition of selected subConn %p to %v", &scw.subConn, state.ConnectivityState) + } + return + } + // If this list is already closed, ignore the update. + if sl.state.Load() != subConnListConnecting { + if sl.b.logger.V(2) { + sl.b.logger.Infof("Ignoring state update for non active subConn %p to %v", &scw.subConn, state.ConnectivityState) + } + return + } + if !scw.hasFailedPreviously { + // This subConn is attempting to connect for the first time, we're in the + // initial pass. + switch state.ConnectivityState { + case connectivity.TransientFailure: + if sl.b.logger.V(2) { + sl.b.logger.Infof("SubConn %p failed to connect due to error: %v", &scw.subConn, state.ConnectionError) + } + scw.hasFailedPreviously = true + sl.lastFailure = state.ConnectionError + sl.transientFailuresCount++ + // If we've seen one failure from each subConn, the first pass ends + // and we can set the channel state to transient failure. + if sl.transientFailuresCount == len(sl.subConns) { + sl.b.logger.Infof("Received one failure from each subConn in list %p, waiting for any subConn to connect.", sl) + // Phase 1 is over, start phase 2. + sl.b.state = connectivity.TransientFailure + sl.b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: sl.lastFailure}, + }) + + // In phase 2, we attempt to connect to all the subConns in parallel. + // Connect to addresses that have already completed the back-off. + for idx := 0; idx < len(sl.subConns); idx++ { + sc := sl.subConns[idx] + if sc.state == connectivity.TransientFailure { + continue + } + sl.subConns[idx].subConn.Connect() + } + } + scw.failureChan <- state.ConnectionError + case connectivity.Ready: + // Cleanup and update the picker to use the subconn. + sl.selectSubConn(scw) + case connectivity.Connecting: + // Move the channel to connecting if this is the first subConn to + // start connecting. + if sl.b.state == connectivity.Idle { + sl.b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &picker{err: balancer.ErrNoSubConnAvailable}, + }) + } + default: + if sl.b.logger.V(2) { + sl.b.logger.Infof("Ignoring update for the subConn %p to state %v", &scw.subConn, state.ConnectivityState) + } + } + return + } + + if sl.transientFailuresCount < len(sl.subConns) { + // This subConn has failed once, but other subConns are still connecting. + // Wait for other subConns to complete. + return + } + + // We have completed the first phase. + switch state.ConnectivityState { + case connectivity.TransientFailure: + if sl.b.logger.V(2) { + sl.b.logger.Infof("SubConn %p failed to connect due to error: %v", &scw.subConn, state.ConnectionError) + } + case connectivity.Ready: + sl.selectSubConn(scw) + case connectivity.Idle: + // Trigger re-connection. + scw.subConn.Connect() + default: + if sl.b.logger.V(2) { + sl.b.logger.Infof("Ignoring update for the subConn %p to state %v", &scw.subConn, state.ConnectivityState) + } + } +} + +func (sl *subConnList) selectSubConn(scw *scWrapper) { + if !sl.state.CompareAndSwap(subConnListConnecting, subConnListConnected) { + return + } + close(sl.connectingCh) + sl.b.logger.Infof("Selected subConn %p", &scw.subConn) + sl.b.unsetSelectedSubConn() + sl.b.selectedSubConn = scw + sl.b.state = connectivity.Ready + for _, sc := range sl.subConns { + if sc == sl.b.selectedSubConn { + continue + } + sc.subConn.Shutdown() + } + sl.b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &picker{result: balancer.PickResult{SubConn: scw.subConn}}, + }) +} + +func (sl *subConnList) close() { + prevState := sl.state.Swap(subConnListClosed) + if prevState == subConnListClosed { + return + } + if prevState == subConnListConnecting { + close(sl.connectingCh) + } + // Close all the subConns except the selected one. The selected subConn + // will be closed by the balancer. + for _, sc := range sl.subConns { + if sc == sl.b.selectedSubConn { + continue + } + sc.subConn.Shutdown() + } +} + +// connect attempts to connect to subConns till it finds a healthy one. +func (sl *subConnList) connect() { + // Attempt to connect to each subConn once. + for _, scw := range sl.subConns { + scw.subConn.Connect() + select { + case <-sl.connectingCh: + return + case err := <-scw.failureChan: + if err == nil { + // Connected successfully. + return + } + } + } +} + +func (pickfirstBuilder) ParseConfig(js json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + var cfg pfConfig + if err := json.Unmarshal(js, &cfg); err != nil { + return nil, fmt.Errorf("pickfirst: unable to unmarshal LB policy config: %s, error: %v", string(js), err) + } + return cfg, nil +} + +type pickfirstBalancer struct { + logger *internalgrpclog.PrefixLogger + state connectivity.State + cc balancer.ClientConn + // Pointer to the subConn list currently connecting. Always close the + // current list before replacing it to ensure resources are freed. + subConnList *subConnList + // A mutex to guard the swapping of subConnLists and it can be triggered + // concurrently by the idlePicker and resolver updates. + subConnListMu sync.Mutex + selectedSubConn *scWrapper + latestAddressList []resolver.Address + shuttingDown bool +} + +func (b *pickfirstBalancer) ResolverError(err error) { + if b.logger.V(2) { + b.logger.Infof("Received error from the name resolver: %v", err) + } + if len(b.latestAddressList) == 0 { + // The picker will not change since the balancer does not currently + // report an error. + b.state = connectivity.TransientFailure + } + + if b.state != connectivity.TransientFailure { + // The picker will not change since the balancer does not currently + // report an error. + return + } + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: fmt.Errorf("name resolver error: %v", err)}, + }) +} + +func (b *pickfirstBalancer) unsetSelectedSubConn() { + if b.selectedSubConn != nil { + b.selectedSubConn.subConn.Shutdown() + b.selectedSubConn = nil + } +} + +func (b *pickfirstBalancer) goIdle() { + b.unsetSelectedSubConn() + b.subConnList.close() + + nextState := connectivity.Idle + if b.state == connectivity.TransientFailure { + // We stay in TransientFailure until we are Ready. See A62. + nextState = connectivity.TransientFailure + } else { + b.state = connectivity.Idle + } + b.cc.UpdateState(balancer.State{ + ConnectivityState: nextState, + Picker: &idlePicker{ + exitIdle: b.ExitIdle, + }, + }) +} + +// refreshSubConnListLocked replaces the current subConnList with a newly created +// one. The caller is responsible to close the existing list and ensure its +// closed synchronously during a ClientConn update or a subConn state update. +func (b *pickfirstBalancer) refreshSubConnListLocked() error { + subConnList := newSubConnList(b) + if len(subConnList.subConns) == 0 { + subConnList.close() + b.state = connectivity.TransientFailure + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: fmt.Errorf("empty address list")}, + }) + b.unsetSelectedSubConn() + return balancer.ErrBadResolverState + } + + // Reset the previous subConnList to release resources. + if b.logger.V(2) { + b.logger.Infof("Closing older subConnList") + } + b.subConnList = subConnList + go subConnList.connect() + // Don't read any fields on subConnList after we start connecting as it will + // cause races with subConn state updates being handled by the stateListener. + return nil +} + +func (b *pickfirstBalancer) connectUsingLatestAddrs() error { + b.subConnList.close() + b.subConnListMu.Lock() + if err := b.refreshSubConnListLocked(); err != nil { + b.subConnListMu.Unlock() + return err + } + b.subConnListMu.Unlock() + + if b.state != connectivity.TransientFailure { + // We stay in TransientFailure until we are Ready. See A62. + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &picker{err: balancer.ErrNoSubConnAvailable}, + }) + } + return nil +} + +func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState) error { + if len(state.ResolverState.Addresses) == 0 && len(state.ResolverState.Endpoints) == 0 { + // Cleanup state pertaining to the previous resolver state. + b.unsetSelectedSubConn() + b.subConnList.close() + b.latestAddressList = nil + // Treat an empty address list like an error by calling b.ResolverError. + b.ResolverError(errors.New("produced zero addresses")) + return balancer.ErrBadResolverState + } + // We don't have to guard this block with the env var because ParseConfig + // already does so. + cfg, ok := state.BalancerConfig.(pfConfig) + if state.BalancerConfig != nil && !ok { + return fmt.Errorf("pickfirst: received illegal BalancerConfig (type %T): %v", state.BalancerConfig, state.BalancerConfig) + } + + if b.logger.V(2) { + b.logger.Infof("Received new config %s, resolver state %s", pretty.ToJSON(cfg), pretty.ToJSON(state.ResolverState)) + } + + var addrs []resolver.Address + if endpoints := state.ResolverState.Endpoints; len(endpoints) != 0 { + // Perform the optional shuffling described in gRFC A62. The shuffling will + // change the order of endpoints but not touch the order of the addresses + // within each endpoint. - A61 + if cfg.ShuffleAddressList { + endpoints = append([]resolver.Endpoint{}, endpoints...) + internal.ShuffleAddressListForTesting.(func(int, func(int, int)))(len(endpoints), func(i, j int) { endpoints[i], endpoints[j] = endpoints[j], endpoints[i] }) + } + + // "Flatten the list by concatenating the ordered list of addresses for each + // of the endpoints, in order." - A61 + for _, endpoint := range endpoints { + // "In the flattened list, interleave addresses from the two address + // families, as per RFC-8304 section 4." - A61 + // TODO: support the above language. + addrs = append(addrs, endpoint.Addresses...) + } + } else { + // Endpoints not set, process addresses until we migrate resolver + // emissions fully to Endpoints. The top channel does wrap emitted + // addresses with endpoints, however some balancers such as weighted + // target do not forward the corresponding correct endpoints down/split + // endpoints properly. Once all balancers correctly forward endpoints + // down, can delete this else conditional. + addrs = state.ResolverState.Addresses + if cfg.ShuffleAddressList { + addrs = append([]resolver.Address{}, addrs...) + rand.Shuffle(len(addrs), func(i, j int) { addrs[i], addrs[j] = addrs[j], addrs[i] }) + } + } + + b.latestAddressList = addrs + // If the selected subConn's address is present in the list, don't attempt + // to re-connect. + if b.selectedSubConn != nil && slices.ContainsFunc(addrs, func(addr resolver.Address) bool { + return addr.Equal(b.selectedSubConn.addr) + }) { + if b.logger.V(2) { + b.logger.Infof("Not attempting to re-connect since selected address %q is present in new address list", b.selectedSubConn.addr.String()) + } + return nil + } + return b.connectUsingLatestAddrs() +} + +// UpdateSubConnState is unused as a StateListener is always registered when +// creating SubConns. +func (b *pickfirstBalancer) UpdateSubConnState(subConn balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", subConn, state) +} + +func (b *pickfirstBalancer) Close() { + b.shuttingDown = true + b.unsetSelectedSubConn() + b.subConnList.close() +} + +// ExitIdle moves the balancer out of idle state. It can be called concurrently +// by the idlePicker and clientConn so access to variables should be synchronized. +func (b *pickfirstBalancer) ExitIdle() { + b.subConnListMu.Lock() + defer b.subConnListMu.Unlock() + if b.subConnList.state.Load() != subConnListClosed { + // Already exited idle, nothing to do. + return + } + // The current subConnList is still closed, create a new subConnList and + // start connecting. + if err := b.refreshSubConnListLocked(); errors.Is(err, balancer.ErrBadResolverState) { + // If creation of the subConnList fails, request for re-resolution to + // get a new list of addresses. + b.cc.ResolveNow(resolver.ResolveNowOptions{}) + return + } +} + +type picker struct { + result balancer.PickResult + err error +} + +func (p *picker) Pick(balancer.PickInfo) (balancer.PickResult, error) { + return p.result, p.err +} + +// idlePicker is used when the SubConn is IDLE and kicks the SubConn into +// CONNECTING when Pick is called. +type idlePicker struct { + exitIdle func() +} + +func (i *idlePicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { + i.exitIdle() + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable +} diff --git a/clientconn_pickfirst_leaf_test.go b/clientconn_pickfirst_leaf_test.go new file mode 100644 index 000000000000..64d6c76ff0b4 --- /dev/null +++ b/clientconn_pickfirst_leaf_test.go @@ -0,0 +1,1179 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "errors" + "fmt" + "net" + "strings" + "sync/atomic" + "testing" + "time" + + "golang.org/x/net/http2" + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/pickfirst_leaf" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + internalbackoff "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/testdata" +) + +const ( + stateRecordingBalancerWithLeafPickFirstName = "state_recording_balancer_new_pick_first" +) + +var ( + testBalancerBuilderPickFirstLeaf = newStateRecordingBalancerBuilder(stateRecordingBalancerWithLeafPickFirstName, pickfirst_leaf.Name) + pickFirstLeafServiceConfig = fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, pickfirst_leaf.Name) +) + +func init() { + balancer.Register(testBalancerBuilderPickFirstLeaf) +} + +func (s) TestPickFirstLeaf_DialWithTimeout(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis.Close() + lisAddr := resolver.Address{Addr: lis.Addr().String()} + lisDone := make(chan struct{}) + dialDone := make(chan struct{}) + // 1st listener accepts the connection and then does nothing + go func() { + defer close(lisDone) + conn, err := lis.Accept() + if err != nil { + t.Errorf("Error while accepting. Err: %v", err) + return + } + framer := http2.NewFramer(conn, conn) + if err := framer.WriteSettings(http2.Setting{}); err != nil { + t.Errorf("Error while writing settings. Err: %v", err) + return + } + <-dialDone // Close conn only after dial returns. + }() + + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{Addresses: []resolver.Address{lisAddr}}) + client, err := Dial(r.Scheme()+":///test.server", WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r), WithTimeout(5*time.Second), WithDefaultServiceConfig(pickFirstLeafServiceConfig)) + close(dialDone) + if err != nil { + t.Fatalf("Dial failed. Err: %v", err) + } + defer client.Close() + timeout := time.After(1 * time.Second) + select { + case <-timeout: + t.Fatal("timed out waiting for server to finish") + case <-lisDone: + } +} + +func (s) TestPickFirstLeaf_DialWithMultipleBackendsNotSendingServerPreface(t *testing.T) { + lis1, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis1.Close() + lis1Addr := resolver.Address{Addr: lis1.Addr().String()} + lis1Done := make(chan struct{}) + // 1st listener accepts the connection and immediately closes it. + go func() { + defer close(lis1Done) + conn, err := lis1.Accept() + if err != nil { + t.Errorf("Error while accepting. Err: %v", err) + return + } + conn.Close() + }() + + lis2, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis2.Close() + lis2Done := make(chan struct{}) + lis2Addr := resolver.Address{Addr: lis2.Addr().String()} + // 2nd listener should get a connection attempt since the first one failed. + go func() { + defer close(lis2Done) + _, err := lis2.Accept() // Closing the client will clean up this conn. + if err != nil { + t.Errorf("Error while accepting. Err: %v", err) + return + } + }() + + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{Addresses: []resolver.Address{lis1Addr, lis2Addr}}) + client, err := Dial(r.Scheme()+":///test.server", WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r), WithDefaultServiceConfig(pickFirstLeafServiceConfig)) + if err != nil { + t.Fatalf("Dial failed. Err: %v", err) + } + defer client.Close() + timeout := time.After(5 * time.Second) + select { + case <-timeout: + t.Fatal("timed out waiting for server 1 to finish") + case <-lis1Done: + } + select { + case <-timeout: + t.Fatal("timed out waiting for server 2 to finish") + case <-lis2Done: + } +} + +func (s) TestPickFirstLeaf_DialWaitsForServerSettings(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis.Close() + done := make(chan struct{}) + sent := make(chan struct{}) + dialDone := make(chan struct{}) + go func() { // Launch the server. + defer func() { + close(done) + }() + conn, err := lis.Accept() + if err != nil { + t.Errorf("Error while accepting. Err: %v", err) + return + } + defer conn.Close() + // Sleep for a little bit to make sure that Dial on client + // side blocks until settings are received. + time.Sleep(100 * time.Millisecond) + framer := http2.NewFramer(conn, conn) + close(sent) + if err := framer.WriteSettings(http2.Setting{}); err != nil { + t.Errorf("Error while writing settings. Err: %v", err) + return + } + <-dialDone // Close conn only after dial returns. + }() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + client, err := DialContext(ctx, lis.Addr().String(), + WithTransportCredentials(insecure.NewCredentials()), + WithBlock(), + WithDefaultServiceConfig(pickFirstLeafServiceConfig), + ) + close(dialDone) + if err != nil { + t.Fatalf("Error while dialing. Err: %v", err) + } + defer client.Close() + select { + case <-sent: + default: + t.Fatalf("Dial returned before server settings were sent") + } + <-done +} + +func (s) TestPickFirstLeaf_DialWaitsForServerSettingsAndFails(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + done := make(chan struct{}) + numConns := 0 + go func() { // Launch the server. + defer func() { + close(done) + }() + for { + conn, err := lis.Accept() + if err != nil { + break + } + numConns++ + defer conn.Close() + } + }() + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + client, err := DialContext(ctx, + lis.Addr().String(), + WithTransportCredentials(insecure.NewCredentials()), + WithReturnConnectionError(), + WithConnectParams(ConnectParams{ + Backoff: backoff.Config{}, + MinConnectTimeout: 250 * time.Millisecond, + }), + WithDefaultServiceConfig(pickFirstLeafServiceConfig)) + lis.Close() + if err == nil { + client.Close() + t.Fatalf("Unexpected success (err=nil) while dialing") + } + expectedMsg := "server preface" + if !strings.Contains(err.Error(), context.DeadlineExceeded.Error()) || !strings.Contains(err.Error(), expectedMsg) { + t.Fatalf("DialContext(_) = %v; want a message that includes both %q and %q", err, context.DeadlineExceeded.Error(), expectedMsg) + } + <-done + if numConns < 2 { + t.Fatalf("dial attempts: %v; want > 1", numConns) + } +} + +// 1. Client connects to a server that doesn't send preface. +// 2. After minConnectTimeout(500 ms here), client disconnects and retries. +// 3. The new server sends its preface. +// 4. Client doesn't kill the connection this time. +func (s) TestPickFirstLeaf_CloseConnectionWhenServerPrefaceNotReceived(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + var ( + conn2 net.Conn + over uint32 + ) + defer func() { + lis.Close() + // conn2 shouldn't be closed until the client has + // observed a successful test. + if conn2 != nil { + conn2.Close() + } + }() + done := make(chan struct{}) + accepted := make(chan struct{}) + go func() { // Launch the server. + defer close(done) + conn1, err := lis.Accept() + if err != nil { + t.Errorf("Error while accepting. Err: %v", err) + return + } + defer conn1.Close() + // Don't send server settings and the client should close the connection and try again. + conn2, err = lis.Accept() // Accept a reconnection request from client. + if err != nil { + t.Errorf("Error while accepting. Err: %v", err) + return + } + close(accepted) + framer := http2.NewFramer(conn2, conn2) + if err = framer.WriteSettings(http2.Setting{}); err != nil { + t.Errorf("Error while writing settings. Err: %v", err) + return + } + b := make([]byte, 8) + for { + _, err = conn2.Read(b) + if err == nil { + continue + } + if atomic.LoadUint32(&over) == 1 { + // The connection stayed alive for the timer. + // Success. + return + } + t.Errorf("Unexpected error while reading. Err: %v, want timeout error", err) + break + } + }() + client, err := Dial(lis.Addr().String(), + WithTransportCredentials(insecure.NewCredentials()), + withMinConnectDeadline(func() time.Duration { return time.Millisecond * 500 }), + WithDefaultServiceConfig(pickFirstLeafServiceConfig)) + if err != nil { + t.Fatalf("Error while dialing. Err: %v", err) + } + + go stayConnected(client) + + // wait for connection to be accepted on the server. + timer := time.NewTimer(time.Second * 10) + select { + case <-accepted: + case <-timer.C: + t.Fatalf("Client didn't make another connection request in time.") + } + // Make sure the connection stays alive for sometime. + time.Sleep(time.Second) + atomic.StoreUint32(&over, 1) + client.Close() + <-done +} + +func (s) TestPickFirstLeaf_BackoffWhenNoServerPrefaceReceived(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Unexpected error from net.Listen(%q, %q): %v", "tcp", "localhost:0", err) + } + defer lis.Close() + done := make(chan struct{}) + go func() { // Launch the server. + defer close(done) + conn, err := lis.Accept() // Accept the connection only to close it immediately. + if err != nil { + t.Errorf("Error while accepting. Err: %v", err) + return + } + prevAt := time.Now() + conn.Close() + var prevDuration time.Duration + // Make sure the retry attempts are backed off properly. + for i := 0; i < 3; i++ { + conn, err := lis.Accept() + if err != nil { + t.Errorf("Error while accepting. Err: %v", err) + return + } + meow := time.Now() + conn.Close() + dr := meow.Sub(prevAt) + if dr <= prevDuration { + t.Errorf("Client backoff did not increase with retries. Previous duration: %v, current duration: %v", prevDuration, dr) + return + } + prevDuration = dr + prevAt = meow + } + }() + bc := backoff.Config{ + BaseDelay: 200 * time.Millisecond, + Multiplier: 2.0, + Jitter: 0, + MaxDelay: 120 * time.Second, + } + cp := ConnectParams{ + Backoff: bc, + MinConnectTimeout: 1 * time.Second, + } + cc, err := Dial(lis.Addr().String(), + WithTransportCredentials(insecure.NewCredentials()), + WithConnectParams(cp), + WithDefaultServiceConfig(pickFirstLeafServiceConfig)) + if err != nil { + t.Fatalf("Unexpected error from Dial(%v) = %v", lis.Addr(), err) + } + defer cc.Close() + go stayConnected(cc) + <-done +} + +func (s) TestPickFirstLeaf_WithTimeout(t *testing.T) { + conn, err := Dial("passthrough:///Non-Existent.Server:80", + WithTimeout(time.Millisecond), + WithBlock(), + WithTransportCredentials(insecure.NewCredentials()), + WithDefaultServiceConfig(pickFirstLeafServiceConfig)) + if err == nil { + conn.Close() + } + if err != context.DeadlineExceeded { + t.Fatalf("Dial(_, _) = %v, %v, want %v", conn, err, context.DeadlineExceeded) + } +} + +func (s) TestPickFirstLeaf_WithTransportCredentialsTLS(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) + defer cancel() + creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com") + if err != nil { + t.Fatalf("Failed to create credentials %v", err) + } + conn, err := DialContext(ctx, "passthrough:///Non-Existent.Server:80", + WithTransportCredentials(creds), + WithBlock(), + WithDefaultServiceConfig(pickFirstLeafServiceConfig)) + if err == nil { + conn.Close() + } + if err != context.DeadlineExceeded { + t.Fatalf("Dial(_, _) = %v, %v, want %v", conn, err, context.DeadlineExceeded) + } +} + +// When creating a transport configured with n addresses, only calculate the +// backoff n times per "round" of attempts instead of once per. +func (s) TestPickFirstLeafDial_NBackoffPerRetryGroup(t *testing.T) { + var attempts uint32 + getMinConnectTimeout := func() time.Duration { + if atomic.AddUint32(&attempts, 1) > 2 { + // Once all addresses are exhausted, hang around and wait for the + // client.Close to happen rather than re-starting a new round of + // attempts. + return time.Hour + } + return 0 + } + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + lis1, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis1.Close() + + lis2, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis2.Close() + + server1Done := make(chan struct{}) + server2Done := make(chan struct{}) + + // Launch server 1. + go func() { + conn, err := lis1.Accept() + if err != nil { + t.Error(err) + return + } + + conn.Close() + close(server1Done) + }() + // Launch server 2. + go func() { + conn, err := lis2.Accept() + if err != nil { + t.Error(err) + return + } + conn.Close() + close(server2Done) + }() + + rb := manual.NewBuilderWithScheme("whatever") + rb.InitialState(resolver.State{Addresses: []resolver.Address{ + {Addr: lis1.Addr().String()}, + {Addr: lis2.Addr().String()}, + }}) + client, err := DialContext(ctx, "whatever:///this-gets-overwritten", + WithTransportCredentials(insecure.NewCredentials()), + WithResolvers(rb), + withMinConnectDeadline(getMinConnectTimeout), + WithDefaultServiceConfig(pickFirstLeafServiceConfig)) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + timeout := time.After(15 * time.Second) + + select { + case <-timeout: + t.Fatal("timed out waiting for test to finish") + case <-server1Done: + } + + select { + case <-timeout: + t.Fatal("timed out waiting for test to finish") + case <-server2Done: + } + + if atomic.LoadUint32(&attempts) != 2 { + t.Errorf("Back-off attempts=%d, want=2", attempts) + } +} + +func (s) TestPickFirstLeaf_DialContextCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := DialContext(ctx, "Non-Existent.Server:80", + WithBlock(), + WithTransportCredentials(insecure.NewCredentials()), + WithDefaultServiceConfig(pickFirstLeafServiceConfig)); err != context.Canceled { + t.Fatalf("DialContext(%v, _) = _, %v, want _, %v", ctx, err, context.Canceled) + } +} + +func (s) TestPickFirstLeaf_DialContextFailFast(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + failErr := failFastError{} + dialer := func(string, time.Duration) (net.Conn, error) { + return nil, failErr + } + + _, err := DialContext(ctx, "Non-Existent.Server:80", WithBlock(), + WithTransportCredentials(insecure.NewCredentials()), + WithDialer(dialer), + FailOnNonTempDialError(true), + WithDefaultServiceConfig(pickFirstLeafServiceConfig)) + if terr, ok := err.(transport.ConnectionError); !ok || terr.Origin() != failErr { + t.Fatalf("DialContext() = _, %v, want _, %v", err, failErr) + } +} + +func (s) TestPickFirstLeaf_CredentialsMisuse(t *testing.T) { + // Use of no transport creds and no creds bundle must fail. + if _, err := Dial("passthrough:///Non-Existent.Server:80", WithDefaultServiceConfig(pickFirstLeafServiceConfig)); err != errNoTransportSecurity { + t.Fatalf("Dial(_, _) = _, %v, want _, %v", err, errNoTransportSecurity) + } + + // Use of both transport creds and creds bundle must fail. + creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com") + if err != nil { + t.Fatalf("Failed to create authenticator %v", err) + } + dopts := []DialOption{ + WithTransportCredentials(creds), + WithCredentialsBundle(&fakeBundleCreds{transportCreds: creds}), + WithDefaultServiceConfig(pickFirstLeafServiceConfig), + } + if _, err := Dial("passthrough:///Non-Existent.Server:80", dopts...); err != errTransportCredsAndBundle { + t.Fatalf("Dial(_, _) = _, %v, want _, %v", err, errTransportCredsAndBundle) + } + + // Use of perRPC creds requiring transport security over an insecure + // transport must fail. + if _, err := Dial("passthrough:///Non-Existent.Server:80", + WithPerRPCCredentials(securePerRPCCredentials{}), + WithTransportCredentials(insecure.NewCredentials()), + WithDefaultServiceConfig(pickFirstLeafServiceConfig)); err != errTransportCredentialsMissing { + t.Fatalf("Dial(_, _) = _, %v, want _, %v", err, errTransportCredentialsMissing) + } + + // Use of a creds bundle with nil transport credentials must fail. + if _, err := Dial("passthrough:///Non-Existent.Server:80", + WithCredentialsBundle(&fakeBundleCreds{}), + WithDefaultServiceConfig(pickFirstLeafServiceConfig)); err != errNoTransportCredsInBundle { + t.Fatalf("Dial(_, _) = _, %v, want _, %v", err, errTransportCredsAndBundle) + } +} + +func (s) TestPickFirstLeaf_WithBackoffConfigDefault(t *testing.T) { + testBackoffConfigSet(t, internalbackoff.DefaultExponential, WithDefaultServiceConfig(pickFirstLeafServiceConfig)) +} + +func (s) TestPickFirstLeaf_WithBackoffConfig(t *testing.T) { + b := BackoffConfig{MaxDelay: DefaultBackoffConfig.MaxDelay / 2} + bc := backoff.DefaultConfig + bc.MaxDelay = b.MaxDelay + wantBackoff := internalbackoff.Exponential{Config: bc} + testBackoffConfigSet(t, wantBackoff, WithBackoffConfig(b), WithDefaultServiceConfig(pickFirstLeafServiceConfig)) +} + +func (s) TestPickFirstLeaf_WithBackoffMaxDelay(t *testing.T) { + md := DefaultBackoffConfig.MaxDelay / 2 + bc := backoff.DefaultConfig + bc.MaxDelay = md + wantBackoff := internalbackoff.Exponential{Config: bc} + testBackoffConfigSet(t, wantBackoff, WithBackoffMaxDelay(md), WithDefaultServiceConfig(pickFirstLeafServiceConfig)) +} + +func (s) TestPickFirstLeaf_WithConnectParams(t *testing.T) { + bd := 2 * time.Second + mltpr := 2.0 + jitter := 0.0 + bc := backoff.Config{BaseDelay: bd, Multiplier: mltpr, Jitter: jitter} + + crt := ConnectParams{Backoff: bc} + // MaxDelay is not set in the ConnectParams. So it should not be set on + // internalbackoff.Exponential as well. + wantBackoff := internalbackoff.Exponential{Config: bc} + testBackoffConfigSet(t, wantBackoff, WithConnectParams(crt), WithDefaultServiceConfig(pickFirstLeafServiceConfig)) +} + +func (s) TestPickFirstLeaf_ConnectParamsWithMinConnectTimeout(t *testing.T) { + // Default value specified for minConnectTimeout in the spec is 20 seconds. + mct := 1 * time.Minute + conn, err := Dial("passthrough:///foo:80", + WithTransportCredentials(insecure.NewCredentials()), + WithConnectParams(ConnectParams{MinConnectTimeout: mct}), + WithDefaultServiceConfig(pickFirstLeafServiceConfig)) + if err != nil { + t.Fatalf("unexpected error dialing connection: %v", err) + } + defer conn.Close() + + if got := conn.dopts.minConnectTimeout(); got != mct { + t.Errorf("unexpect minConnectTimeout on the connection: %v, want %v", got, mct) + } +} + +func (s) TestPickFirstLeaf_ResolverServiceConfigBeforeAddressNotPanic(t *testing.T) { + r := manual.NewBuilderWithScheme("whatever") + + cc, err := Dial(r.Scheme()+":///test.server", WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r)) + if err != nil { + t.Fatalf("failed to dial: %v", err) + } + defer cc.Close() + + // SwitchBalancer before NewAddress. There was no balancer created, this + // makes sure we don't call close on nil balancerWrapper. + r.UpdateState(resolver.State{ServiceConfig: parseCfg(r, `{"loadBalancingPolicy": "round_robin"}`)}) // This should not panic. + + time.Sleep(time.Second) // Sleep to make sure the service config is handled by ClientConn. +} + +func (s) TestPickFirstLeaf_ResolverServiceConfigWhileClosingNotPanic(t *testing.T) { + for i := 0; i < 10; i++ { // Run this multiple times to make sure it doesn't panic. + r := manual.NewBuilderWithScheme(fmt.Sprintf("whatever-%d", i)) + + cc, err := Dial(r.Scheme()+":///test.server", WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r)) + if err != nil { + t.Fatalf("failed to dial: %v", err) + } + // Send a new service config while closing the ClientConn. + go cc.Close() + go r.UpdateState(resolver.State{ServiceConfig: parseCfg(r, `{"loadBalancingPolicy": "round_robin"}`)}) // This should not panic. + } +} + +func (s) TestPickFirstLeaf_ResolverEmptyUpdateNotPanic(t *testing.T) { + r := manual.NewBuilderWithScheme("whatever") + + cc, err := Dial(r.Scheme()+":///test.server", WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r)) + if err != nil { + t.Fatalf("failed to dial: %v", err) + } + defer cc.Close() + + // This make sure we don't create addrConn with empty address list. + r.UpdateState(resolver.State{}) // This should not panic. + + time.Sleep(time.Second) // Sleep to make sure the service config is handled by ClientConn. +} + +func (s) TestPickFirstLeaf_ClientUpdatesParamsAfterGoAway(t *testing.T) { + grpctest.TLogger.ExpectError("Client received GoAway with error code ENHANCE_YOUR_CALM and debug data equal to ASCII \"too_many_pings\"") + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Failed to listen. Err: %v", err) + } + defer lis.Close() + connected := grpcsync.NewEvent() + defer connected.Fire() + go func() { + conn, err := lis.Accept() + if err != nil { + t.Errorf("error accepting connection: %v", err) + return + } + defer conn.Close() + f := http2.NewFramer(conn, conn) + // Start a goroutine to read from the conn to prevent the client from + // blocking after it writes its preface. + go func() { + for { + if _, err := f.ReadFrame(); err != nil { + return + } + } + }() + if err := f.WriteSettings(http2.Setting{}); err != nil { + t.Errorf("error writing settings: %v", err) + return + } + <-connected.Done() + if err := f.WriteGoAway(0, http2.ErrCodeEnhanceYourCalm, []byte("too_many_pings")); err != nil { + t.Errorf("error writing GOAWAY: %v", err) + return + } + }() + addr := lis.Addr().String() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + cc, err := DialContext(ctx, addr, WithBlock(), WithTransportCredentials(insecure.NewCredentials()), WithKeepaliveParams(keepalive.ClientParameters{ + Time: 10 * time.Second, + Timeout: 100 * time.Millisecond, + PermitWithoutStream: true, + }), + WithDefaultServiceConfig(pickFirstLeafServiceConfig)) + if err != nil { + t.Fatalf("Dial(%s, _) = _, %v, want _, ", addr, err) + } + defer cc.Close() + connected.Fire() + for { + time.Sleep(10 * time.Millisecond) + cc.mu.RLock() + v := cc.mkp.Time + cc.mu.RUnlock() + if v == 20*time.Second { + // Success + return + } + if ctx.Err() != nil { + // Timeout + t.Fatalf("cc.dopts.copts.Keepalive.Time = %v , want 20s", v) + } + } +} + +func (s) TestPickFirstLeaf_DisableServiceConfigOption(t *testing.T) { + r := manual.NewBuilderWithScheme("whatever") + addr := r.Scheme() + ":///non.existent" + cc, err := Dial(addr, WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r), WithDisableServiceConfig()) + if err != nil { + t.Fatalf("Dial(%s, _) = _, %v, want _, ", addr, err) + } + defer cc.Close() + r.UpdateState(resolver.State{ServiceConfig: parseCfg(r, `{ + "loadBalancingConfig": [{"pick_first_leaf":{}}], + "methodConfig": [ + { + "name": [ + { + "service": "foo", + "method": "Bar" + } + ], + "waitForReady": true + } + ] +}`)}) + time.Sleep(1 * time.Second) + m := cc.GetMethodConfig("/foo/Bar") + if m.WaitForReady != nil { + t.Fatalf("want: method (\"/foo/bar/\") config to be empty, got: %+v", m) + } +} + +func (s) TestPickFirstLeaf_MethodConfigDefaultService(t *testing.T) { + addr := "nonexist:///non.existent" + cc, err := Dial(addr, WithTransportCredentials(insecure.NewCredentials()), WithDefaultServiceConfig(`{ + "loadBalancingConfig": [{"pick_first_leaf":{}}], + "methodConfig": [{ + "name": [ + { + "service": "" + } + ], + "waitForReady": true + }] +}`)) + if err != nil { + t.Fatalf("Dial(%s, _) = _, %v, want _, ", addr, err) + } + defer cc.Close() + + m := cc.GetMethodConfig("/foo/Bar") + if m.WaitForReady == nil { + t.Fatalf("want: method (%q) config to fallback to the default service", "/foo/Bar") + } +} + +func (s) TestPickFirstLeaf_ClientConnCanonicalTarget(t *testing.T) { + tests := []struct { + name string + addr string + canonicalTargetWant string + }{ + { + name: "normal-case", + addr: "dns://a.server.com/google.com", + canonicalTargetWant: "dns://a.server.com/google.com", + }, + { + name: "canonical-target-not-specified", + addr: "no.scheme", + canonicalTargetWant: "passthrough:///no.scheme", + }, + { + name: "canonical-target-nonexistent", + addr: "nonexist:///non.existent", + canonicalTargetWant: "passthrough:///nonexist:///non.existent", + }, + { + name: "canonical-target-add-colon-slash", + addr: "dns:hostname:port", + canonicalTargetWant: "dns:///hostname:port", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cc, err := Dial(test.addr, + WithTransportCredentials(insecure.NewCredentials()), + WithDefaultServiceConfig(pickFirstLeafServiceConfig), + ) + if err != nil { + t.Fatalf("Dial(%s, _) = _, %v, want _, ", test.addr, err) + } + defer cc.Close() + if cc.Target() != test.addr { + t.Fatalf("Target() = %s, want %s", cc.Target(), test.addr) + } + if cc.CanonicalTarget() != test.canonicalTargetWant { + t.Fatalf("CanonicalTarget() = %s, want %s", cc.CanonicalTarget(), test.canonicalTargetWant) + } + }) + } +} + +func (s) TestPickFirstLeaf_ResetConnectBackoff(t *testing.T) { + dials := make(chan struct{}) + defer func() { // If we fail, let the http2client break out of dialing. + select { + case <-dials: + default: + } + }() + dialer := func(string, time.Duration) (net.Conn, error) { + dials <- struct{}{} + return nil, errors.New("failed to fake dial") + } + cc, err := Dial("any", WithTransportCredentials(insecure.NewCredentials()), + WithDialer(dialer), + withBackoff(backoffForever{}), + WithDefaultServiceConfig(pickFirstLeafServiceConfig), + ) + if err != nil { + t.Fatalf("Dial() = _, %v; want _, nil", err) + } + defer cc.Close() + go stayConnected(cc) + select { + case <-dials: + case <-time.NewTimer(10 * time.Second).C: + t.Fatal("Failed to call dial within 10s") + } + + select { + case <-dials: + t.Fatal("Dial called unexpectedly before resetting backoff") + case <-time.NewTimer(100 * time.Millisecond).C: + } + + cc.ResetConnectBackoff() + + select { + case <-dials: + case <-time.NewTimer(10 * time.Second).C: + t.Fatal("Failed to call dial within 10s after resetting backoff") + } +} + +func (s) TestPickFirstLeaf_BackoffCancel(t *testing.T) { + dialStrCh := make(chan string) + cc, err := Dial("any", WithTransportCredentials(insecure.NewCredentials()), + WithDialer(func(t string, _ time.Duration) (net.Conn, error) { + dialStrCh <- t + return nil, fmt.Errorf("test dialer, always error") + }), + WithDefaultServiceConfig(pickFirstLeafServiceConfig), + ) + if err != nil { + t.Fatalf("Failed to create ClientConn: %v", err) + } + defer cc.Close() + + select { + case <-time.After(defaultTestTimeout): + t.Fatal("Timeout when waiting for custom dialer to be invoked during Dial") + case <-dialStrCh: + } +} + +// TestUpdateAddresses_NoopIfCalledWithSameAddresses tests that UpdateAddresses +// should be noop if UpdateAddresses is called with the same list of addresses, +// even when the SubConn is in Connecting and doesn't have a current address. +func (s) TestPickFirstLeaf_UpdateAddresses_NoopIfCalledWithSameAddresses(t *testing.T) { + lis1, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis1.Close() + + lis2, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis2.Close() + + lis3, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis3.Close() + + closeServer2 := make(chan struct{}) + exitCh := make(chan struct{}) + server1ContactedFirstTime := make(chan struct{}) + server1ContactedSecondTime := make(chan struct{}) + server2ContactedFirstTime := make(chan struct{}) + server2ContactedSecondTime := make(chan struct{}) + server3Contacted := make(chan struct{}) + + defer close(exitCh) + + // Launch server 1. + go func() { + // First, let's allow the initial connection to go READY. We need to do + // this because tryUpdateAddrs only works after there's some non-nil + // address on the ac, and curAddress is only set after READY. + conn1, err := lis1.Accept() + if err != nil { + t.Error(err) + return + } + go keepReading(conn1) + + framer := http2.NewFramer(conn1, conn1) + if err := framer.WriteSettings(http2.Setting{}); err != nil { + t.Errorf("Error while writing settings frame. %v", err) + return + } + + // nextStateNotifier() is updated after balancerBuilder.Build(), which is + // called by grpc.Dial. It's safe to do it here because lis1.Accept blocks + // until balancer is built to process the addresses. + stateNotifications := testBalancerBuilderPickFirstLeaf.nextStateNotifier() + // Wait for the transport to become ready. + for { + select { + case st := <-stateNotifications: + if st == connectivity.Ready { + goto ready + } + case <-exitCh: + return + } + } + + ready: + // Once it's ready, curAddress has been set. So let's close this + // connection prompting the first reconnect cycle. + conn1.Close() + + // Accept and immediately close, causing it to go to server2. + conn2, err := lis1.Accept() + if err != nil { + t.Error(err) + return + } + close(server1ContactedFirstTime) + conn2.Close() + + // Hopefully it picks this server after tryUpdateAddrs. + lis1.Accept() + close(server1ContactedSecondTime) + }() + // Launch server 2. + go func() { + // Accept and then hang waiting for the test call tryUpdateAddrs and + // then signal to this server to close. After this server closes, it + // should start from the top instead of trying server2 or continuing + // to server3. + conn, err := lis2.Accept() + if err != nil { + t.Error(err) + return + } + + close(server2ContactedFirstTime) + <-closeServer2 + conn.Close() + + // After tryUpdateAddrs, it should NOT try server2. + lis2.Accept() + close(server2ContactedSecondTime) + }() + // Launch server 3. + go func() { + // After tryUpdateAddrs, it should NOT try server3. (or any other time) + lis3.Accept() + close(server3Contacted) + }() + + addrsList := []resolver.Address{ + {Addr: lis1.Addr().String()}, + {Addr: lis2.Addr().String()}, + {Addr: lis3.Addr().String()}, + } + rb := manual.NewBuilderWithScheme("whatever") + rb.InitialState(resolver.State{Addresses: addrsList}) + + client, err := Dial("whatever:///this-gets-overwritten", + WithTransportCredentials(insecure.NewCredentials()), + WithResolvers(rb), + WithConnectParams(ConnectParams{ + Backoff: backoff.Config{}, + MinConnectTimeout: time.Hour, + }), + WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateRecordingBalancerWithLeafPickFirstName))) + if err != nil { + t.Fatal(err) + } + defer client.Close() + go stayConnected(client) + + timeout := time.After(5 * time.Second) + + // Wait for server1 to be contacted (which will immediately fail), then + // server2 (which will hang waiting for our signal). + select { + case <-server1ContactedFirstTime: + case <-timeout: + t.Fatal("timed out waiting for server1 to be contacted") + } + select { + case <-server2ContactedFirstTime: + case <-timeout: + t.Fatal("timed out waiting for server2 to be contacted") + } + + // Grab the addrConn for server 2 and call tryUpdateAddrs. + var ac *addrConn + client.mu.Lock() + for clientAC := range client.conns { + if got := len(clientAC.addrs); got != 1 { + t.Errorf("len(AddrConn.addrs)=%d, want=1", got) + continue + } + if clientAC.addrs[0].Addr == lis2.Addr().String() { + ac = clientAC + break + } + } + client.mu.Unlock() + + if ac == nil { + t.Fatal("Coudn't find the subConn for server 2") + } + + // Call UpdateAddresses with the same list of addresses, it should be a noop + // (even when the SubConn is Connecting, and doesn't have a curAddr). + ac.acbw.UpdateAddresses(addrsList[1:2]) + + // We've called tryUpdateAddrs - now let's make server2 close the + // connection and check that it continues to server3. + close(closeServer2) + + select { + case <-server1ContactedSecondTime: + t.Fatal("server1 was contacted a second time, but it should have continued to server 3") + case <-server2ContactedSecondTime: + t.Fatal("server2 was contacted a second time, but it should have continued to server 3") + case <-server3Contacted: + case <-timeout: + t.Fatal("timed out waiting for any server to be contacted after tryUpdateAddrs") + } +} + +func (s) TestPickFirstLeaf_DefaultServiceConfig(t *testing.T) { + const defaultSC = ` +{ + "loadBalancingConfig": [{"pick_first_leaf":{}}], + "methodConfig": [ + { + "name": [ + { + "service": "foo", + "method": "bar" + } + ], + "waitForReady": true + } + ] +}` + tests := []struct { + name string + testF func(t *testing.T, r *manual.Resolver, addr, sc string) + sc string + }{ + { + name: "invalid-service-config", + testF: testInvalidDefaultServiceConfig, + sc: "", + }, + { + name: "resolver-service-config-disabled", + testF: testDefaultServiceConfigWhenResolverServiceConfigDisabled, + sc: defaultSC, + }, + { + name: "resolver-does-not-return-service-config", + testF: testDefaultServiceConfigWhenResolverDoesNotReturnServiceConfig, + sc: defaultSC, + }, + { + name: "resolver-returns-invalid-service-config", + testF: testDefaultServiceConfigWhenResolverReturnInvalidServiceConfig, + sc: defaultSC, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + r := manual.NewBuilderWithScheme(test.name) + addr := r.Scheme() + ":///non.existent" + test.testF(t, r, addr, test.sc) + }) + } +} + +func (s) TestPickFirstLeaf_URLAuthorityEscape(t *testing.T) { + tests := []struct { + name string + authority string + want string + }{ + { + name: "ipv6_authority", + authority: "[::1]", + want: "[::1]", + }, + { + name: "with_user_and_host", + authority: "userinfo@host:10001", + want: "userinfo@host:10001", + }, + { + name: "with_multiple_slashes", + authority: "projects/123/network/abc/service", + want: "projects%2F123%2Fnetwork%2Fabc%2Fservice", + }, + { + name: "all_possible_allowed_chars", + authority: "abc123-._~!$&'()*+,;=@:[]", + want: "abc123-._~!$&'()*+,;=@:[]", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got, want := encodeAuthority(test.authority), test.want; got != want { + t.Errorf("encodeAuthority(%s) = %s, want %s", test.authority, got, test.want) + } + }) + } +} diff --git a/clientconn_test.go b/clientconn_test.go index c87f505fc735..85b5eef236df 100644 --- a/clientconn_test.go +++ b/clientconn_test.go @@ -52,7 +52,7 @@ const ( stateRecordingBalancerName = "state_recording_balancer" ) -var testBalancerBuilder = newStateRecordingBalancerBuilder() +var testBalancerBuilder = newStateRecordingBalancerBuilder(stateRecordingBalancerName, "pick_first") func init() { balancer.Register(testBalancerBuilder) @@ -417,16 +417,18 @@ func (s) TestWithTransportCredentialsTLS(t *testing.T) { } // When creating a transport configured with n addresses, only calculate the -// backoff n times per "round" of attempts instead of once per. -func (s) TestDial_NBackoffPerRetryGroup(t *testing.T) { +// backoff once per "round" of attempts instead of once per address (n times +// per "round" of attempts). +func (s) TestDial_OneBackoffPerRetryGroup(t *testing.T) { var attempts uint32 getMinConnectTimeout := func() time.Duration { - if atomic.AddUint32(&attempts, 1) > 2 { + if atomic.AddUint32(&attempts, 1) == 1 { // Once all addresses are exhausted, hang around and wait for the // client.Close to happen rather than re-starting a new round of // attempts. return time.Hour } + t.Error("only one attempt backoff calculation, but got more") return 0 } @@ -497,10 +499,6 @@ func (s) TestDial_NBackoffPerRetryGroup(t *testing.T) { t.Fatal("timed out waiting for test to finish") case <-server2Done: } - - if atomic.LoadUint32(&attempts) != 2 { - t.Errorf("Back-off attempts=%d, want=2", attempts) - } } func (s) TestDialContextCancel(t *testing.T) { @@ -1067,24 +1065,14 @@ func (s) TestUpdateAddresses_NoopIfCalledWithSameAddresses(t *testing.T) { var ac *addrConn client.mu.Lock() for clientAC := range client.conns { - if got := len(clientAC.addrs); got != 1 { - t.Errorf("len(AddrConn.addrs)=%d, want=1", got) - continue - } - if clientAC.addrs[0].Addr == lis2.Addr().String() { - ac = clientAC - break - } + ac = clientAC + break } client.mu.Unlock() - if ac == nil { - t.Fatal("Coudn't find the subConn for server 2") - } - // Call UpdateAddresses with the same list of addresses, it should be a noop // (even when the SubConn is Connecting, and doesn't have a curAddr). - ac.acbw.UpdateAddresses(addrsList[1:2]) + ac.acbw.UpdateAddresses(addrsList) // We've called tryUpdateAddrs - now let's make server2 close the // connection and check that it continues to server3. @@ -1234,16 +1222,21 @@ func (b *stateRecordingBalancer) ExitIdle() { } type stateRecordingBalancerBuilder struct { - mu sync.Mutex - notifier chan connectivity.State // The notifier used in the last Balancer. + mu sync.Mutex + notifier chan connectivity.State // The notifier used in the last Balancer. + policyName string + childPolicyName string } -func newStateRecordingBalancerBuilder() *stateRecordingBalancerBuilder { - return &stateRecordingBalancerBuilder{} +func newStateRecordingBalancerBuilder(policyName, childPolicyName string) *stateRecordingBalancerBuilder { + return &stateRecordingBalancerBuilder{ + childPolicyName: childPolicyName, + policyName: policyName, + } } func (b *stateRecordingBalancerBuilder) Name() string { - return stateRecordingBalancerName + return b.policyName } func (b *stateRecordingBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { @@ -1252,7 +1245,7 @@ func (b *stateRecordingBalancerBuilder) Build(cc balancer.ClientConn, opts balan b.notifier = stateNotifications b.mu.Unlock() return &stateRecordingBalancer{ - Balancer: balancer.Get("pick_first").Build(&stateRecordingCCWrapper{cc, stateNotifications}, opts), + Balancer: balancer.Get(b.childPolicyName).Build(&stateRecordingCCWrapper{cc, stateNotifications}, opts), } } diff --git a/test/clientconn_state_transition_pick_first_leaf_test.go b/test/clientconn_state_transition_pick_first_leaf_test.go new file mode 100644 index 000000000000..156c55f13a55 --- /dev/null +++ b/test/clientconn_state_transition_pick_first_leaf_test.go @@ -0,0 +1,490 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "fmt" + "net" + "testing" + "time" + + "golang.org/x/net/http2" + "google.golang.org/grpc" + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/pickfirst_leaf" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" +) + +const stateRecordingPickFirstLeafBalancerName = "state_recording_pick_first_leaf_balancer" + +var testPickFirstLeafBalancerBuilder = newStateRecordingBalancerBuilder(stateRecordingPickFirstLeafBalancerName, pickfirst_leaf.Name) + +func init() { + balancer.Register(testPickFirstLeafBalancerBuilder) +} + +// These tests use a pipeListener. This listener is similar to net.Listener +// except that it is unbuffered, so each read and write will wait for the other +// side's corresponding write or read. +func (s) TestPickFirstLeafStateTransitions_SingleAddress(t *testing.T) { + for _, test := range []struct { + desc string + want []connectivity.State + server func(net.Listener) net.Conn + }{ + { + desc: "When the server returns server preface, the client enters READY.", + want: []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + }, + server: func(lis net.Listener) net.Conn { + conn, err := lis.Accept() + if err != nil { + t.Error(err) + return nil + } + + go keepReading(conn) + + framer := http2.NewFramer(conn, conn) + if err := framer.WriteSettings(http2.Setting{}); err != nil { + t.Errorf("Error while writing settings frame. %v", err) + return nil + } + + return conn + }, + }, + { + desc: "When the connection is closed before the preface is sent, the client enters TRANSIENT FAILURE.", + want: []connectivity.State{ + connectivity.Connecting, + connectivity.TransientFailure, + }, + server: func(lis net.Listener) net.Conn { + conn, err := lis.Accept() + if err != nil { + t.Error(err) + return nil + } + + conn.Close() + return nil + }, + }, + { + desc: `When the server sends its connection preface, but the connection dies before the client can write its +connection preface, the client enters TRANSIENT FAILURE.`, + want: []connectivity.State{ + connectivity.Connecting, + connectivity.TransientFailure, + }, + server: func(lis net.Listener) net.Conn { + conn, err := lis.Accept() + if err != nil { + t.Error(err) + return nil + } + + framer := http2.NewFramer(conn, conn) + if err := framer.WriteSettings(http2.Setting{}); err != nil { + t.Errorf("Error while writing settings frame. %v", err) + return nil + } + + conn.Close() + return nil + }, + }, + { + desc: `When the server reads the client connection preface but does not send its connection preface, the +client enters TRANSIENT FAILURE.`, + want: []connectivity.State{ + connectivity.Connecting, + connectivity.TransientFailure, + }, + server: func(lis net.Listener) net.Conn { + conn, err := lis.Accept() + if err != nil { + t.Error(err) + return nil + } + + go keepReading(conn) + + return conn + }, + }, + } { + t.Log(test.desc) + testStateTransitionSingleAddress(t, test.want, test.server, testPickFirstLeafBalancerBuilder) + } +} + +// When a READY connection is closed, the client enters IDLE then CONNECTING. +func (s) TestPickFirstLeafStateTransitions_ReadyToConnecting(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis.Close() + + sawReady := make(chan struct{}, 1) + defer close(sawReady) + + // Launch the server. + go func() { + conn, err := lis.Accept() + if err != nil { + t.Error(err) + return + } + + go keepReading(conn) + + framer := http2.NewFramer(conn, conn) + if err := framer.WriteSettings(http2.Setting{}); err != nil { + t.Errorf("Error while writing settings frame. %v", err) + return + } + + // Prevents race between onPrefaceReceipt and onClose. + <-sawReady + + conn.Close() + }() + + client, err := grpc.Dial(lis.Addr().String(), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateRecordingPickFirstLeafBalancerName))) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + go testutils.StayConnected(ctx, client) + + stateNotifications := testPickFirstLeafBalancerBuilder.nextStateNotifier() + + want := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Idle, + connectivity.Shutdown, + connectivity.Connecting, + } + for i := 0; i < len(want); i++ { + select { + case <-time.After(defaultTestTimeout): + t.Fatalf("timed out waiting for state %d (%v) in flow %v", i, want[i], want) + case seen := <-stateNotifications: + if seen == connectivity.Ready { + sawReady <- struct{}{} + } + if seen != want[i] { + t.Fatalf("expected to see %v at position %d in flow %v, got %v", want[i], i, want, seen) + } + } + } +} + +// When the first connection is closed, the client stays in CONNECTING until it +// tries the second address (which succeeds, and then it enters READY). +func (s) TestPickFirstLeafStateTransitions_TriesAllAddrsBeforeTransientFailure(t *testing.T) { + lis1, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis1.Close() + + lis2, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis2.Close() + + server1Done := make(chan struct{}) + server2Done := make(chan struct{}) + + // Launch server 1. + go func() { + conn, err := lis1.Accept() + if err != nil { + t.Error(err) + return + } + + conn.Close() + close(server1Done) + }() + // Launch server 2. + go func() { + conn, err := lis2.Accept() + if err != nil { + t.Error(err) + return + } + + go keepReading(conn) + + framer := http2.NewFramer(conn, conn) + if err := framer.WriteSettings(http2.Setting{}); err != nil { + t.Errorf("Error while writing settings frame. %v", err) + return + } + + close(server2Done) + }() + + rb := manual.NewBuilderWithScheme("whatever") + rb.InitialState(resolver.State{Addresses: []resolver.Address{ + {Addr: lis1.Addr().String()}, + {Addr: lis2.Addr().String()}, + }}) + client, err := grpc.Dial("whatever:///this-gets-overwritten", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateRecordingPickFirstLeafBalancerName)), + grpc.WithResolvers(rb), + grpc.WithConnectParams(grpc.ConnectParams{ + // Set a really long back-off delay to ensure the first subConn does + // not enter ready before the second subConn connects. + Backoff: backoff.Config{ + BaseDelay: 1 * time.Hour, + }, + }), + ) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + stateNotifications := testPickFirstLeafBalancerBuilder.nextStateNotifier() + want := []connectivity.State{ + connectivity.Connecting, + connectivity.TransientFailure, + connectivity.Connecting, + connectivity.Ready, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for i := 0; i < len(want); i++ { + select { + case <-ctx.Done(): + t.Fatalf("timed out waiting for state %d (%v) in flow %v", i, want[i], want) + case seen := <-stateNotifications: + if seen != want[i] { + t.Fatalf("expected to see %v at position %d in flow %v, got %v", want[i], i, want, seen) + } + } + } + select { + case <-ctx.Done(): + t.Fatal("saw the correct state transitions, but timed out waiting for client to finish interactions with server 1") + case <-server1Done: + } + select { + case <-ctx.Done(): + t.Fatal("saw the correct state transitions, but timed out waiting for client to finish interactions with server 2") + case <-server2Done: + } +} + +// When there are multiple addresses, and we enter READY on one of them, a +// later closure should cause the client to enter CONNECTING +func (s) TestPickFirstLeafStateTransitions_MultipleAddrsEntersReady(t *testing.T) { + lis1, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis1.Close() + + // Never actually gets used; we just want it to be alive so that the resolver has two addresses to target. + lis2, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + defer lis2.Close() + + server1Done := make(chan struct{}) + sawReady := make(chan struct{}, 1) + defer close(sawReady) + + // Launch server 1. + go func() { + conn, err := lis1.Accept() + if err != nil { + t.Error(err) + return + } + + go keepReading(conn) + + framer := http2.NewFramer(conn, conn) + if err := framer.WriteSettings(http2.Setting{}); err != nil { + t.Errorf("Error while writing settings frame. %v", err) + return + } + + <-sawReady + + conn.Close() + + close(server1Done) + }() + + rb := manual.NewBuilderWithScheme("whatever") + rb.InitialState(resolver.State{Addresses: []resolver.Address{ + {Addr: lis1.Addr().String()}, + {Addr: lis2.Addr().String()}, + }}) + client, err := grpc.Dial("whatever:///this-gets-overwritten", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateRecordingPickFirstLeafBalancerName)), + grpc.WithResolvers(rb)) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + go testutils.StayConnected(ctx, client) + + stateNotifications := testPickFirstLeafBalancerBuilder.nextStateNotifier() + want := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Shutdown, // The second subConn is closed once the first one connects. + connectivity.Idle, + connectivity.Shutdown, // The subConn will be closed and pickfirst will run on the latest address list. + connectivity.Connecting, + } + for i := 0; i < len(want); i++ { + select { + case <-ctx.Done(): + t.Fatalf("timed out waiting for state %d (%v) in flow %v", i, want[i], want) + case seen := <-stateNotifications: + if seen == connectivity.Ready { + sawReady <- struct{}{} + } + if seen != want[i] { + t.Fatalf("expected to see %v at position %d in flow %v, got %v", want[i], i, want, seen) + } + } + } + select { + case <-ctx.Done(): + t.Fatal("saw the correct state transitions, but timed out waiting for client to finish interactions with server 1") + case <-server1Done: + } +} + +// TestPickFirstLeafConnectivityStateSubscriber confirms updates sent by the balancer in +// rapid succession are not missed by the subscriber. +func (s) TestPickFirstLeafConnectivityStateSubscriber(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + sendStates := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Idle, + connectivity.Connecting, + connectivity.Idle, + connectivity.Connecting, + connectivity.Ready, + } + wantStates := append(sendStates, connectivity.Shutdown) + + const testBalName = "any" + bf := stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error { + // Send the expected states in rapid succession. + for _, s := range sendStates { + t.Logf("Sending state update %s", s) + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: s}) + } + return nil + }, + } + stub.Register(testBalName, bf) + + // Create the ClientConn. + const testResName = "any" + rb := manual.NewBuilderWithScheme(testResName) + cc, err := grpc.Dial(testResName+":///", + grpc.WithResolvers(rb), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, testBalName)), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + t.Fatalf("Unexpected error from grpc.Dial: %v", err) + } + + // Subscribe to state updates. Use a buffer size of 1 to allow the + // Shutdown state to go into the channel when Close()ing. + connCh := make(chan connectivity.State, 1) + s := &funcConnectivityStateSubscriber{ + onMsg: func(s connectivity.State) { + select { + case connCh <- s: + case <-ctx.Done(): + } + if s == connectivity.Shutdown { + close(connCh) + } + }, + } + + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, s) + + // Send an update from the resolver that will trigger the LB policy's UpdateClientConnState. + go rb.UpdateState(resolver.State{}) + + // Verify the resulting states. + for i, want := range wantStates { + if i == len(sendStates) { + // Trigger Shutdown to be sent by the channel. Use a goroutine to + // ensure the operation does not block. + cc.Close() + } + select { + case got := <-connCh: + if got != want { + t.Errorf("Update %v was %s; want %s", i, got, want) + } else { + t.Logf("Update %v was %s as expected", i, got) + } + case <-ctx.Done(): + t.Fatalf("Timed out waiting for state update %v: %s", i, want) + } + } +} diff --git a/test/clientconn_state_transition_test.go b/test/clientconn_state_transition_test.go index 4e4af17e5cbe..b6e308fb85bc 100644 --- a/test/clientconn_state_transition_test.go +++ b/test/clientconn_state_transition_test.go @@ -42,7 +42,7 @@ import ( const stateRecordingBalancerName = "state_recording_balancer" -var testBalancerBuilder = newStateRecordingBalancerBuilder() +var testBalancerBuilder = newStateRecordingBalancerBuilder(stateRecordingBalancerName, "pick_first") func init() { balancer.Register(testBalancerBuilder) @@ -143,11 +143,11 @@ client enters TRANSIENT FAILURE.`, }, } { t.Log(test.desc) - testStateTransitionSingleAddress(t, test.want, test.server) + testStateTransitionSingleAddress(t, test.want, test.server, testBalancerBuilder) } } -func testStateTransitionSingleAddress(t *testing.T, want []connectivity.State, server func(net.Listener) net.Conn) { +func testStateTransitionSingleAddress(t *testing.T, want []connectivity.State, server func(net.Listener) net.Conn, bb *stateRecordingBalancerBuilder) { pl := testutils.NewPipeListener() defer pl.Close() @@ -162,7 +162,7 @@ func testStateTransitionSingleAddress(t *testing.T, want []connectivity.State, s client, err := grpc.Dial("", grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateRecordingBalancerName)), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, bb.Name())), grpc.WithDialer(pl.Dialer()), grpc.WithConnectParams(grpc.ConnectParams{ Backoff: backoff.Config{}, @@ -177,7 +177,7 @@ func testStateTransitionSingleAddress(t *testing.T, want []connectivity.State, s defer cancel() go testutils.StayConnected(ctx, client) - stateNotifications := testBalancerBuilder.nextStateNotifier() + stateNotifications := bb.nextStateNotifier() for i := 0; i < len(want); i++ { select { case <-time.After(defaultTestTimeout): @@ -250,7 +250,6 @@ func (s) TestStateTransitions_ReadyToConnecting(t *testing.T) { connectivity.Connecting, connectivity.Ready, connectivity.Idle, - connectivity.Shutdown, connectivity.Connecting, } for i := 0; i < len(want); i++ { @@ -324,15 +323,7 @@ func (s) TestStateTransitions_TriesAllAddrsBeforeTransientFailure(t *testing.T) client, err := grpc.Dial("whatever:///this-gets-overwritten", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateRecordingBalancerName)), - grpc.WithResolvers(rb), - grpc.WithConnectParams(grpc.ConnectParams{ - // Set a really long back-off delay to ensure the first subConn does - // not enter ready before the second subConn connects. - Backoff: backoff.Config{ - BaseDelay: 1 * time.Hour, - }, - }), - ) + grpc.WithResolvers(rb)) if err != nil { t.Fatal(err) } @@ -340,8 +331,6 @@ func (s) TestStateTransitions_TriesAllAddrsBeforeTransientFailure(t *testing.T) stateNotifications := testBalancerBuilder.nextStateNotifier() want := []connectivity.State{ - connectivity.Connecting, - connectivity.TransientFailure, connectivity.Connecting, connectivity.Ready, } @@ -434,9 +423,7 @@ func (s) TestStateTransitions_MultipleAddrsEntersReady(t *testing.T) { want := []connectivity.State{ connectivity.Connecting, connectivity.Ready, - connectivity.Shutdown, // The second subConn is closed once the first one connects. connectivity.Idle, - connectivity.Shutdown, // The subConn will be closed and pickfirst will run on the latest address list. connectivity.Connecting, } for i := 0; i < len(want); i++ { @@ -474,16 +461,21 @@ func (b *stateRecordingBalancer) ExitIdle() { } type stateRecordingBalancerBuilder struct { - mu sync.Mutex - notifier chan connectivity.State // The notifier used in the last Balancer. + mu sync.Mutex + notifier chan connectivity.State // The notifier used in the last Balancer. + balancerName string + childName string } -func newStateRecordingBalancerBuilder() *stateRecordingBalancerBuilder { - return &stateRecordingBalancerBuilder{} +func newStateRecordingBalancerBuilder(balancerName, childName string) *stateRecordingBalancerBuilder { + return &stateRecordingBalancerBuilder{ + balancerName: balancerName, + childName: childName, + } } func (b *stateRecordingBalancerBuilder) Name() string { - return stateRecordingBalancerName + return b.balancerName } func (b *stateRecordingBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { @@ -492,7 +484,7 @@ func (b *stateRecordingBalancerBuilder) Build(cc balancer.ClientConn, opts balan b.notifier = stateNotifications b.mu.Unlock() return &stateRecordingBalancer{ - Balancer: balancer.Get("pick_first").Build(&stateRecordingCCWrapper{cc, stateNotifications}, opts), + Balancer: balancer.Get(b.childName).Build(&stateRecordingCCWrapper{cc, stateNotifications}, opts), } } diff --git a/test/pickfirst_leaf_test.go b/test/pickfirst_leaf_test.go new file mode 100644 index 000000000000..d5daea344bb7 --- /dev/null +++ b/test/pickfirst_leaf_test.go @@ -0,0 +1,910 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/balancer/pickfirst_leaf" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/pickfirst" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +var pickFirstLeafServiceConfig = fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, pickfirst_leaf.Name) + +// setupPickFirstLeaf performs steps required for pick_first tests. It starts a +// bunch of backends exporting the TestService, creates a ClientConn to them +// with service config specifying the use of the pick_first LB policy. +func setupPickFirstLeaf(t *testing.T, backendCount int, opts ...grpc.DialOption) (*grpc.ClientConn, *manual.Resolver, []*stubserver.StubServer) { + t.Helper() + + r := manual.NewBuilderWithScheme("whatever") + + backends := make([]*stubserver.StubServer, backendCount) + addrs := make([]resolver.Address, backendCount) + for i := 0; i < backendCount; i++ { + backend := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started TestService backend at: %q", backend.Address) + t.Cleanup(func() { backend.Stop() }) + + backends[i] = backend + addrs[i] = resolver.Address{Addr: backend.Address} + } + + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(pickFirstLeafServiceConfig), + } + dopts = append(dopts, opts...) + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + // At this point, the resolver has not returned any addresses to the channel. + // This RPC must block until the context expires. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("EmptyCall() = %s, want %s", status.Code(err), codes.DeadlineExceeded) + } + return cc, r, backends +} + +// TestPickFirstLeaf_OneBackend tests the most basic scenario for pick_first. It +// brings up a single backend and verifies that all RPCs get routed to it. +func (s) TestPickFirstLeaf_OneBackend(t *testing.T) { + cc, r, backends := setupPickFirstLeaf(t, 1) + + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } +} + +// TestPickFirstLeaf_MultipleBackends tests the scenario with multiple backends and +// verifies that all RPCs get routed to the first one. +func (s) TestPickFirstLeaf_MultipleBackends(t *testing.T) { + cc, r, backends := setupPickFirstLeaf(t, 2) + + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } +} + +// TestPickFirstLeaf_OneServerDown tests the scenario where we have multiple +// backends and pick_first is working as expected. Verifies that RPCs get routed +// to the next backend in the list when the first one goes down. +func (s) TestPickFirstLeaf_OneServerDown(t *testing.T) { + cc, r, backends := setupPickFirstLeaf(t, 2) + + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Stop the backend which is currently being used. RPCs should get routed to + // the next backend in the list. + backends[0].Stop() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } +} + +// TestPickFirstLeaf_AllServersDown tests the scenario where we have multiple +// backends and pick_first is working as expected. When all backends go down, +// the test verifies that RPCs fail with appropriate status code. +func (s) TestPickFirstLeaf_AllServersDown(t *testing.T) { + cc, r, backends := setupPickFirstLeaf(t, 2) + + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + for _, b := range backends { + b.Stop() + } + + client := testgrpc.NewTestServiceClient(cc) + for { + if ctx.Err() != nil { + t.Fatalf("channel failed to move to Unavailable after all backends were stopped: %v", ctx.Err()) + } + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) == codes.Unavailable { + return + } + time.Sleep(defaultTestShortTimeout) + } +} + +// TestPickFirstLeaf_AddressesRemoved tests the scenario where we have multiple +// backends and pick_first is working as expected. It then verifies that when +// addresses are removed by the name resolver, RPCs get routed appropriately. +func (s) TestPickFirstLeaf_AddressesRemoved(t *testing.T) { + cc, r, backends := setupPickFirstLeaf(t, 3) + + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Remove the first backend from the list of addresses originally pushed. + // RPCs should get routed to the first backend in the new list. + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[1], addrs[2]}}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + + // Append the backend that we just removed to the end of the list. + // Nothing should change. + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[1], addrs[2], addrs[0]}}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + + // Remove the first backend from the existing list of addresses. + // RPCs should get routed to the first backend in the new list. + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[2], addrs[0]}}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[2]); err != nil { + t.Fatal(err) + } + + // Remove the first backend from the existing list of addresses. + // RPCs should get routed to the first backend in the new list. + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0]}}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } +} + +// TestPickFirstLeaf_NewAddressWhileBlocking tests the case where pick_first is +// configured on a channel, things are working as expected and then a resolver +// updates removes all addresses. An RPC attempted at this point in time will be +// blocked because there are no valid backends. This test verifies that when new +// backends are added, the RPC is able to complete. +func (s) TestPickFirstLeaf_NewAddressWhileBlocking(t *testing.T) { + cc, r, backends := setupPickFirstLeaf(t, 2) + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Send a resolver update with no addresses. This should push the channel into + // TransientFailure. + r.UpdateState(resolver.State{}) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + doneCh := make(chan struct{}) + client := testgrpc.NewTestServiceClient(cc) + go func() { + // The channel is currently in TransientFailure and this RPC will block + // until the channel becomes Ready, which will only happen when we push a + // resolver update with a valid backend address. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Errorf("EmptyCall() = %v, want ", err) + } + close(doneCh) + }() + + // Make sure that there is one pending RPC on the ClientConn before attempting + // to push new addresses through the name resolver. If we don't do this, the + // resolver update can happen before the above goroutine gets to make the RPC. + for { + if err := ctx.Err(); err != nil { + t.Fatal(err) + } + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + t.Fatalf("there should only be one top channel, not %d", len(tcs)) + } + started := tcs[0].ChannelMetrics.CallsStarted.Load() + completed := tcs[0].ChannelMetrics.CallsSucceeded.Load() + tcs[0].ChannelMetrics.CallsFailed.Load() + if (started - completed) == 1 { + break + } + time.Sleep(defaultTestShortTimeout) + } + + // Send a resolver update with a valid backend to push the channel to Ready + // and unblock the above RPC. + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: backends[0].Address}}}) + + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for blocked RPC to complete") + case <-doneCh: + } +} + +// TestPickFirstLeaf_StickyTransientFailure tests the case where pick_first is +// configured on a channel, and the backend is configured to close incoming +// connections as soon as they are accepted. The test verifies that the channel +// enters TransientFailure and stays there. The test also verifies that the +// pick_first LB policy is constantly trying to reconnect to the backend. +func (s) TestPickFirstLeaf_StickyTransientFailure(t *testing.T) { + // Spin up a local server which closes the connection as soon as it receives + // one. It also sends a signal on a channel whenever it received a connection. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to create listener: %v", err) + } + t.Cleanup(func() { lis.Close() }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + connCh := make(chan struct{}, 1) + go func() { + for { + conn, err := lis.Accept() + if err != nil { + return + } + select { + case connCh <- struct{}{}: + conn.Close() + case <-ctx.Done(): + return + } + } + }() + + // Dial the above server with a ConnectParams that does a constant backoff + // of defaultTestShortTimeout duration. + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(pickFirstLeafServiceConfig), + grpc.WithConnectParams(grpc.ConnectParams{ + Backoff: backoff.Config{ + BaseDelay: defaultTestShortTimeout, + Multiplier: float64(0), + Jitter: float64(0), + MaxDelay: defaultTestShortTimeout, + }, + }), + } + cc, err := grpc.Dial(lis.Addr().String(), dopts...) + if err != nil { + t.Fatalf("Failed to dial server at %q: %v", lis.Addr(), err) + } + t.Cleanup(func() { cc.Close() }) + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Spawn a goroutine to ensure that the channel stays in TransientFailure. + // The call to cc.WaitForStateChange will return false when the main + // goroutine exits and the context is cancelled. + go func() { + if cc.WaitForStateChange(ctx, connectivity.TransientFailure) { + if state := cc.GetState(); state != connectivity.Shutdown { + t.Errorf("Unexpected state change from TransientFailure to %s", cc.GetState()) + } + } + }() + + // Ensures that the pick_first LB policy is constantly trying to reconnect. + for i := 0; i < 10; i++ { + select { + case <-connCh: + case <-time.After(2 * defaultTestShortTimeout): + t.Error("Timeout when waiting for pick_first to reconnect") + } + } +} + +// Tests the PF LB policy with shuffling enabled. +func (s) TestPickFirstLeaf_ShuffleAddressList(t *testing.T) { + const serviceConfig = `{"loadBalancingConfig": [{"pick_first_leaf":{ "shuffleAddressList": true }}]}` + + // Install a shuffler that always reverses two entries. + origShuf := internal.ShuffleAddressListForTesting + defer func() { internal.ShuffleAddressListForTesting = origShuf }() + internal.ShuffleAddressListForTesting = func(n int, f func(int, int)) { + if n != 2 { + t.Errorf("Shuffle called with n=%v; want 2", n) + return + } + f(0, 1) // reverse the two addresses + } + + // Set up our backends. + cc, r, backends := setupPickFirstLeaf(t, 2) + addrs := stubBackendsToResolverAddrs(backends) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Push an update with both addresses and shuffling disabled. We should + // connect to backend 0. + r.UpdateState(resolver.State{Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{addrs[0]}}, + {Addresses: []resolver.Address{addrs[1]}}, + }}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Send a config with shuffling enabled. This will reverse the addresses, + // but the channel should still be connected to backend 0. + shufState := resolver.State{ + ServiceConfig: parseServiceConfig(t, r, serviceConfig), + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{addrs[0]}}, + {Addresses: []resolver.Address{addrs[1]}}, + }, + } + r.UpdateState(shufState) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Send a resolver update with no addresses. This should push the channel + // into TransientFailure. + r.UpdateState(resolver.State{}) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Send the same config as last time with shuffling enabled. Since we are + // not connected to backend 0, we should connect to backend 1. + r.UpdateState(shufState) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } +} + +// Test config parsing with the env var turned on and off for various scenarios. +func (s) TestPickFirstLeaf_ParseConfig_Success(t *testing.T) { + // Install a shuffler that always reverses two entries. + origShuf := internal.ShuffleAddressListForTesting + defer func() { internal.ShuffleAddressListForTesting = origShuf }() + internal.ShuffleAddressListForTesting = func(n int, f func(int, int)) { + if n != 2 { + t.Errorf("Shuffle called with n=%v; want 2", n) + return + } + f(0, 1) // reverse the two addresses + } + + tests := []struct { + name string + serviceConfig string + wantFirstAddr bool + }{ + { + name: "empty pickfirst config", + serviceConfig: `{"loadBalancingConfig": [{"pick_first_leaf":{}}]}`, + wantFirstAddr: true, + }, + { + name: "empty good pickfirst config", + serviceConfig: `{"loadBalancingConfig": [{"pick_first_leaf":{ "shuffleAddressList": true }}]}`, + wantFirstAddr: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Set up our backends. + cc, r, backends := setupPickFirstLeaf(t, 2) + addrs := stubBackendsToResolverAddrs(backends) + + r.UpdateState(resolver.State{ + ServiceConfig: parseServiceConfig(t, r, test.serviceConfig), + Addresses: addrs, + }) + + // Some tests expect address shuffling to happen, and indicate that + // by setting wantFirstAddr to false (since our shuffling function + // defined at the top of this test, simply reverses the list of + // addresses provided to it). + wantAddr := addrs[0] + if !test.wantFirstAddr { + wantAddr = addrs[1] + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, wantAddr); err != nil { + t.Fatal(err) + } + }) + } +} + +// Test config parsing for a bad service config. +func (s) TestPickFirstLeaf_ParseConfig_Failure(t *testing.T) { + // Service config should fail with the below config. Name resolvers are + // expected to perform this parsing before they push the parsed service + // config to the channel. + const sc = `{"loadBalancingConfig": [{"pick_first_leaf":{ "shuffleAddressList": 666 }}]}` + scpr := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(sc) + if scpr.Err == nil { + t.Fatalf("ParseConfig() succeeded and returned %+v, when expected to fail", scpr) + } +} + +// setupPickFirstLeafWithListenerWrapper is very similar to setupPickFirstLeaf, but uses +// a wrapped listener that the test can use to track accepted connections. +func setupPickFirstLeafWithListenerWrapper(t *testing.T, backendCount int, opts ...grpc.DialOption) (*grpc.ClientConn, *manual.Resolver, []*stubserver.StubServer, []*testutils.ListenerWrapper) { + t.Helper() + + backends := make([]*stubserver.StubServer, backendCount) + addrs := make([]resolver.Address, backendCount) + listeners := make([]*testutils.ListenerWrapper, backendCount) + for i := 0; i < backendCount; i++ { + lis := testutils.NewListenerWrapper(t, nil) + backend := &stubserver.StubServer{ + Listener: lis, + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started TestService backend at: %q", backend.Address) + t.Cleanup(func() { backend.Stop() }) + + backends[i] = backend + addrs[i] = resolver.Address{Addr: backend.Address} + listeners[i] = lis + } + + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(pickFirstLeafServiceConfig), + } + dopts = append(dopts, opts...) + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + // At this point, the resolver has not returned any addresses to the channel. + // This RPC must block until the context expires. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("EmptyCall() = %s, want %s", status.Code(err), codes.DeadlineExceeded) + } + return cc, r, backends, listeners +} + +// TestPickFirstLeaf_AddressUpdateWithAttributes tests the case where an address +// update received by the pick_first LB policy differs in attributes. Addresses +// which differ in attributes are considered different from the perspective of +// subconn creation and connection establishment and the test verifies that new +// connections are created when attributes change. +func (s) TestPickFirstLeaf_AddressUpdateWithAttributes(t *testing.T) { + cc, r, backends, listeners := setupPickFirstLeafWithListenerWrapper(t, 2) + + // Add a set of attributes to the addresses before pushing them to the + // pick_first LB policy through the manual resolver. + addrs := stubBackendsToResolverAddrs(backends) + for i := range addrs { + addrs[i].Attributes = addrs[i].Attributes.WithValue("test-attribute-1", fmt.Sprintf("%d", i)) + } + r.UpdateState(resolver.State{Addresses: addrs}) + + // Ensure that RPCs succeed to the first backend in the list. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Grab the wrapped connection from the listener wrapper. This will be used + // to verify the connection is closed. + val, err := listeners[0].NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Failed to receive new connection from wrapped listener: %v", err) + } + conn := val.(*testutils.ConnWrapper) + + // Add another set of attributes to the addresses, and push them to the + // pick_first LB policy through the manual resolver. Leave the order of the + // addresses unchanged. + for i := range addrs { + addrs[i].Attributes = addrs[i].Attributes.WithValue("test-attribute-2", fmt.Sprintf("%d", i)) + } + r.UpdateState(resolver.State{Addresses: addrs}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // A change in the address attributes results in the new address being + // considered different to the current address. This will result in the old + // connection being closed and a new connection to the same backend (since + // address order is not modified). + if _, err := conn.CloseCh.Receive(ctx); err != nil { + t.Fatalf("Timeout when expecting existing connection to be closed: %v", err) + } + val, err = listeners[0].NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Failed to receive new connection from wrapped listener: %v", err) + } + conn = val.(*testutils.ConnWrapper) + + // Add another set of attributes to the addresses, and push them to the + // pick_first LB policy through the manual resolver. Reverse of the order + // of addresses. + for i := range addrs { + addrs[i].Attributes = addrs[i].Attributes.WithValue("test-attribute-3", fmt.Sprintf("%d", i)) + } + addrs[0], addrs[1] = addrs[1], addrs[0] + r.UpdateState(resolver.State{Addresses: addrs}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Ensure that the old connection is closed and a new connection is + // established to the first address in the new list. + if _, err := conn.CloseCh.Receive(ctx); err != nil { + t.Fatalf("Timeout when expecting existing connection to be closed: %v", err) + } + _, err = listeners[1].NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Failed to receive new connection from wrapped listener: %v", err) + } +} + +// TestPickFirstLeaf_AddressUpdateWithBalancerAttributes tests the case where an +// address update received by the pick_first LB policy differs in balancer +// attributes, which are meant only for consumption by LB policies. In this +// case, the test verifies that new connections are not created when the address +// update only changes the balancer attributes. +func (s) TestPickFirstLeaf_AddressUpdateWithBalancerAttributes(t *testing.T) { + cc, r, backends, listeners := setupPickFirstLeafWithListenerWrapper(t, 2) + + // Add a set of balancer attributes to the addresses before pushing them to + // the pick_first LB policy through the manual resolver. + addrs := stubBackendsToResolverAddrs(backends) + for i := range addrs { + addrs[i].BalancerAttributes = addrs[i].BalancerAttributes.WithValue("test-attribute-1", fmt.Sprintf("%d", i)) + } + r.UpdateState(resolver.State{Addresses: addrs}) + + // Ensure that RPCs succeed to the expected backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Grab the wrapped connection from the listener wrapper. This will be used + // to verify the connection is not closed. + val, err := listeners[0].NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Failed to receive new connection from wrapped listener: %v", err) + } + conn := val.(*testutils.ConnWrapper) + + // Add a set of balancer attributes to the addresses before pushing them to + // the pick_first LB policy through the manual resolver. Leave the order of + // the addresses unchanged. + for i := range addrs { + addrs[i].BalancerAttributes = addrs[i].BalancerAttributes.WithValue("test-attribute-2", fmt.Sprintf("%d", i)) + } + r.UpdateState(resolver.State{Addresses: addrs}) + + // Ensure that no new connection is established, and ensure that the old + // connection is not closed. + for i := range listeners { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := listeners[i].NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatalf("Unexpected error when expecting no new connection: %v", err) + } + } + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := conn.CloseCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatalf("Unexpected error when expecting existing connection to stay active: %v", err) + } + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Add a set of balancer attributes to the addresses before pushing them to + // the pick_first LB policy through the manual resolver. Reverse of the + // order of addresses. + for i := range addrs { + addrs[i].BalancerAttributes = addrs[i].BalancerAttributes.WithValue("test-attribute-3", fmt.Sprintf("%d", i)) + } + addrs[0], addrs[1] = addrs[1], addrs[0] + r.UpdateState(resolver.State{Addresses: addrs}) + + // Ensure that no new connection is established, and ensure that the old + // connection is not closed. + for i := range listeners { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := listeners[i].NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatalf("Unexpected error when expecting no new connection: %v", err) + } + } + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := conn.CloseCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatalf("Unexpected error when expecting existing connection to stay active: %v", err) + } + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } +} + +// Tests the case where the pick_first LB policy receives an error from the name +// resolver without previously receiving a good update. Verifies that the +// channel moves to TRANSIENT_FAILURE and that error received from the name +// resolver is propagated to the caller of an RPC. +func (s) TestPickFirstLeaf_ResolverError_NoPreviousUpdate(t *testing.T) { + cc, r, _ := setupPickFirstLeaf(t, 0) + + nrErr := errors.New("error from name resolver") + r.ReportError(nrErr) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + client := testgrpc.NewTestServiceClient(cc) + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if err == nil { + t.Fatalf("EmptyCall() succeeded when expected to fail with error: %v", nrErr) + } + if !strings.Contains(err.Error(), nrErr.Error()) { + t.Fatalf("EmptyCall() failed with error: %v, want error: %v", err, nrErr) + } +} + +// Tests the case where the pick_first LB policy receives an error from the name +// resolver after receiving a good update (and the channel is currently READY). +// The test verifies that the channel continues to use the previously received +// good update. +func (s) TestPickFirstLeaf_ResolverError_WithPreviousUpdate_Ready(t *testing.T) { + cc, r, backends := setupPickFirstLeaf(t, 1) + + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + nrErr := errors.New("error from name resolver") + r.ReportError(nrErr) + + // Ensure that RPCs continue to succeed for the next second. + client := testgrpc.NewTestServiceClient(cc) + for end := time.Now().Add(time.Second); time.Now().Before(end); <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + } +} + +// Tests the case where the pick_first LB policy receives an error from the name +// resolver after receiving a good update (and the channel is currently in +// CONNECTING state). The test verifies that the channel continues to use the +// previously received good update, and that RPCs don't fail with the error +// received from the name resolver. +func (s) TestPickFirstLeaf_ResolverError_WithPreviousUpdate_Connecting(t *testing.T) { + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + + // Listen on a local port and act like a server that blocks until the + // channel reaches CONNECTING and closes the connection without sending a + // server preface. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + waitForConnecting := make(chan struct{}) + go func() { + conn, err := lis.Accept() + if err != nil { + t.Errorf("Unexpected error when accepting a connection: %v", err) + } + defer conn.Close() + + select { + case <-waitForConnecting: + case <-ctx.Done(): + t.Error("Timeout when waiting for channel to move to CONNECTING state") + } + }() + + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(pickFirstLeafServiceConfig), + } + cc, err := grpc.Dial(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + addrs := []resolver.Address{{Addr: lis.Addr().String()}} + r.UpdateState(resolver.State{Addresses: addrs}) + testutils.AwaitState(ctx, t, cc, connectivity.Connecting) + + nrErr := errors.New("error from name resolver") + r.ReportError(nrErr) + + // RPCs should fail with deadline exceed error as long as they are in + // CONNECTING and not the error returned by the name resolver. + client := testgrpc.NewTestServiceClient(cc) + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); !strings.Contains(err.Error(), context.DeadlineExceeded.Error()) { + t.Fatalf("EmptyCall() failed with error: %v, want error: %v", err, context.DeadlineExceeded) + } + + // Closing this channel leads to closing of the connection by our listener. + // gRPC should see this as a connection error. + close(waitForConnecting) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + checkForConnectionError(ctx, t, cc) +} + +// Tests the case where the pick_first LB policy receives an error from the name +// resolver after receiving a good update. The previous good update though has +// seen the channel move to TRANSIENT_FAILURE. The test verifies that the +// channel fails RPCs with the new error from the resolver. +func (s) TestPickFirstLeaf_ResolverError_WithPreviousUpdate_TransientFailure(t *testing.T) { + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + + // Listen on a local port and act like a server that closes the connection + // without sending a server preface. + go func() { + conn, err := lis.Accept() + if err != nil { + t.Errorf("Unexpected error when accepting a connection: %v", err) + } + conn.Close() + }() + + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(pickFirstLeafServiceConfig), + } + cc, err := grpc.Dial(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + addrs := []resolver.Address{{Addr: lis.Addr().String()}} + r.UpdateState(resolver.State{Addresses: addrs}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + checkForConnectionError(ctx, t, cc) + + // An error from the name resolver should result in RPCs failing with that + // error instead of the old error that caused the channel to move to + // TRANSIENT_FAILURE in the first place. + nrErr := errors.New("error from name resolver") + r.ReportError(nrErr) + client := testgrpc.NewTestServiceClient(cc) + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); strings.Contains(err.Error(), nrErr.Error()) { + break + } + } + if ctx.Err() != nil { + t.Fatal("Timeout when waiting for RPCs to fail with error returned by the name resolver") + } +} + +// Tests the case where the pick_first LB policy receives an update from the +// name resolver with no addresses after receiving a good update. The test +// verifies that the channel fails RPCs with an error indicating the fact that +// the name resolver returned no addresses. +func (s) TestPickFirstLeaf_ResolverError_ZeroAddresses_WithPreviousUpdate(t *testing.T) { + cc, r, backends := setupPickFirstLeaf(t, 1) + + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + r.UpdateState(resolver.State{}) + wantErr := "produced zero addresses" + client := testgrpc.NewTestServiceClient(cc) + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); strings.Contains(err.Error(), wantErr) { + break + } + } + if ctx.Err() != nil { + t.Fatal("Timeout when waiting for RPCs to fail with error returned by the name resolver") + } +} From 69c6c7bd5aa998b413d607ba607ee57644eac058 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Thu, 8 Aug 2024 10:34:09 +0530 Subject: [PATCH 10/62] Use Java style implementation --- balancer/pickfirst_leaf/pickfirst_leaf.go | 758 +++++++++++----------- 1 file changed, 384 insertions(+), 374 deletions(-) diff --git a/balancer/pickfirst_leaf/pickfirst_leaf.go b/balancer/pickfirst_leaf/pickfirst_leaf.go index e1558fc3509d..97d12fe8594f 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf.go +++ b/balancer/pickfirst_leaf/pickfirst_leaf.go @@ -24,10 +24,6 @@ import ( "encoding/json" "errors" "fmt" - "math/rand" - "slices" - "sync" - "sync/atomic" "google.golang.org/grpc/balancer" "google.golang.org/grpc/connectivity" @@ -60,9 +56,11 @@ const ( type pickfirstBuilder struct{} func (pickfirstBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer { - b := &pickfirstBalancer{cc: cc} - b.subConnList = newSubConnList(b) - b.subConnList.close() + b := &pickfirstBalancer{ + cc: cc, + addressIndex: newIndex(nil), + subConns: map[string]*scData{}, + } b.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(logPrefix, b)) return b } @@ -71,6 +69,14 @@ func (pickfirstBuilder) Name() string { return Name } +func (pickfirstBuilder) ParseConfig(js json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + var cfg pfConfig + if err := json.Unmarshal(js, &cfg); err != nil { + return nil, fmt.Errorf("pickfirst: unable to unmarshal LB policy config: %s, error: %v", string(js), err) + } + return cfg, nil +} + type pfConfig struct { serviceconfig.LoadBalancingConfig `json:"-"` @@ -80,449 +86,381 @@ type pfConfig struct { ShuffleAddressList bool `json:"shuffleAddressList"` } -// subConnList stores provides functions to connect to a list of addresses using -// the pick-first algorithm. -type subConnList struct { - subConns []*scWrapper - b *pickfirstBalancer - // The most recent failure during the initial connection attempt over the - // entire sunConns list. - lastFailure error - // The number of transient failures seen while connecting. - transientFailuresCount int - // State updates are serialized by the clientConn, but the picker may attempt - // to read the state in parallel, so we use an atomic. - state atomic.Uint32 - // connectingCh is used to signal the transition of the subConnList out of - // the connecting state. - connectingCh chan struct{} -} - -// scWrapper keeps track of the current state of the subConn. -type scWrapper struct { +// scData keeps track of the current state of the subConn. +type scData struct { subConn balancer.SubConn state connectivity.State - addr resolver.Address - // failureChan is used to communicate the failure when connection fails. - failureChan chan error - hasFailedPreviously bool + addr *resolver.Address } -func newScWrapper(b *pickfirstBalancer, addr resolver.Address, listener func(state balancer.SubConnState)) (*scWrapper, error) { - scw := &scWrapper{ - state: connectivity.Idle, - addr: addr, - failureChan: make(chan error, 1), +func newScData(b *pickfirstBalancer, addr resolver.Address) (*scData, error) { + sd := &scData{ + state: connectivity.Idle, + addr: &addr, } sc, err := b.cc.NewSubConn([]resolver.Address{addr}, balancer.NewSubConnOptions{ - StateListener: func(scs balancer.SubConnState) { + StateListener: func(state balancer.SubConnState) { // Store the state and delegate. - scw.state = scs.ConnectivityState - listener(scs) + sd.state = state.ConnectivityState + b.updateSubConnState(sd, state) }, }) if err != nil { return nil, err } - scw.subConn = sc - return scw, nil + sd.subConn = sc + return sd, nil } -// newSubConnList creates a new list. A new list should be created only when we -// want to start connecting to minimize creation of subConns. -func newSubConnList(b *pickfirstBalancer) *subConnList { - sl := &subConnList{ - b: b, - connectingCh: make(chan struct{}), - } - sl.state.Store(subConnListConnecting) - - for _, addr := range b.latestAddressList { - var scw *scWrapper - scw, err := newScWrapper(b, addr, func(state balancer.SubConnState) { - sl.stateListener(scw, state) - }) - if err != nil { - if b.logger.V(2) { - b.logger.Infof("Ignoring failure, could not create a subConn for address %q due to error: %v", addr, err) - } - continue - } - if b.logger.V(2) { - b.logger.Infof("Created a subConn for address %q", addr) - } - sl.subConns = append(sl.subConns, scw) - } - return sl +type pickfirstBalancer struct { + logger *internalgrpclog.PrefixLogger + state connectivity.State + cc balancer.ClientConn + subConns map[string]*scData + addressIndex index + firstPass bool + firstErr error + numTf int } -func (sl *subConnList) stateListener(scw *scWrapper, state balancer.SubConnState) { - if sl.b.logger.V(2) { - sl.b.logger.Infof("Received SubConn state update: %p, %+v", scw, state) +func (b *pickfirstBalancer) ResolverError(err error) { + if b.logger.V(2) { + b.logger.Infof("Received error from the name resolver: %v", err) } - if scw == sl.b.selectedSubConn { - // As we set the selected subConn only once it's ready, the only - // possible transitions are to IDLE and SHUTDOWN. - switch state.ConnectivityState { - case connectivity.Shutdown: - case connectivity.Idle: - sl.b.goIdle() - default: - sl.b.logger.Warningf("Ignoring unexpected transition of selected subConn %p to %v", &scw.subConn, state.ConnectivityState) - } + if b.state == connectivity.Shutdown { return } - // If this list is already closed, ignore the update. - if sl.state.Load() != subConnListConnecting { - if sl.b.logger.V(2) { - sl.b.logger.Infof("Ignoring state update for non active subConn %p to %v", &scw.subConn, state.ConnectivityState) - } + // The picker will not change since the balancer does not currently + // report an error. + if b.state != connectivity.TransientFailure { return } - if !scw.hasFailedPreviously { - // This subConn is attempting to connect for the first time, we're in the - // initial pass. - switch state.ConnectivityState { - case connectivity.TransientFailure: - if sl.b.logger.V(2) { - sl.b.logger.Infof("SubConn %p failed to connect due to error: %v", &scw.subConn, state.ConnectionError) - } - scw.hasFailedPreviously = true - sl.lastFailure = state.ConnectionError - sl.transientFailuresCount++ - // If we've seen one failure from each subConn, the first pass ends - // and we can set the channel state to transient failure. - if sl.transientFailuresCount == len(sl.subConns) { - sl.b.logger.Infof("Received one failure from each subConn in list %p, waiting for any subConn to connect.", sl) - // Phase 1 is over, start phase 2. - sl.b.state = connectivity.TransientFailure - sl.b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.TransientFailure, - Picker: &picker{err: sl.lastFailure}, - }) - // In phase 2, we attempt to connect to all the subConns in parallel. - // Connect to addresses that have already completed the back-off. - for idx := 0; idx < len(sl.subConns); idx++ { - sc := sl.subConns[idx] - if sc.state == connectivity.TransientFailure { - continue - } - sl.subConns[idx].subConn.Connect() - } - } - scw.failureChan <- state.ConnectionError - case connectivity.Ready: - // Cleanup and update the picker to use the subconn. - sl.selectSubConn(scw) - case connectivity.Connecting: - // Move the channel to connecting if this is the first subConn to - // start connecting. - if sl.b.state == connectivity.Idle { - sl.b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.Connecting, - Picker: &picker{err: balancer.ErrNoSubConnAvailable}, - }) - } - default: - if sl.b.logger.V(2) { - sl.b.logger.Infof("Ignoring update for the subConn %p to state %v", &scw.subConn, state.ConnectivityState) - } - } - return + for _, sd := range b.subConns { + sd.subConn.Shutdown() } + for k := range b.subConns { + delete(b.subConns, k) + } + b.addressIndex.updateEndpointList(nil) + b.state = connectivity.TransientFailure + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: fmt.Errorf("name resolver error: %v", err)}, + }) +} - if sl.transientFailuresCount < len(sl.subConns) { - // This subConn has failed once, but other subConns are still connecting. - // Wait for other subConns to complete. - return +func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState) error { + if b.state == connectivity.Shutdown { + return fmt.Errorf("balancer is already closed") + } + if len(state.ResolverState.Addresses) == 0 && len(state.ResolverState.Endpoints) == 0 { + // Cleanup state pertaining to the previous resolver state. + // Treat an empty address list like an error by calling b.ResolverError. + b.state = connectivity.TransientFailure + b.ResolverError(errors.New("produced zero addresses")) + return balancer.ErrBadResolverState + } + // We don't have to guard this block with the env var because ParseConfig + // already does so. + cfg, ok := state.BalancerConfig.(pfConfig) + if state.BalancerConfig != nil && !ok { + return fmt.Errorf("pickfirst: received illegal BalancerConfig (type %T): %v", state.BalancerConfig, state.BalancerConfig) } - // We have completed the first phase. - switch state.ConnectivityState { - case connectivity.TransientFailure: - if sl.b.logger.V(2) { - sl.b.logger.Infof("SubConn %p failed to connect due to error: %v", &scw.subConn, state.ConnectionError) - } - case connectivity.Ready: - sl.selectSubConn(scw) - case connectivity.Idle: - // Trigger re-connection. - scw.subConn.Connect() - default: - if sl.b.logger.V(2) { - sl.b.logger.Infof("Ignoring update for the subConn %p to state %v", &scw.subConn, state.ConnectivityState) - } + if b.logger.V(2) { + b.logger.Infof("Received new config %s, resolver state %s", pretty.ToJSON(cfg), pretty.ToJSON(state.ResolverState)) } -} -func (sl *subConnList) selectSubConn(scw *scWrapper) { - if !sl.state.CompareAndSwap(subConnListConnecting, subConnListConnected) { - return + newEndpoints := state.ResolverState.Endpoints + if len(newEndpoints) == 0 { + return fmt.Errorf("addresses are no longer supported, resolvers should produce endpoints instead") } - close(sl.connectingCh) - sl.b.logger.Infof("Selected subConn %p", &scw.subConn) - sl.b.unsetSelectedSubConn() - sl.b.selectedSubConn = scw - sl.b.state = connectivity.Ready - for _, sc := range sl.subConns { - if sc == sl.b.selectedSubConn { - continue - } - sc.subConn.Shutdown() + + // Since we have a new set of addresses, we are again at first pass + b.firstPass = true + b.firstErr = nil + + newEndpoints = deDupAddresses(newEndpoints) + + // Perform the optional shuffling described in gRFC A62. The shuffling will + // change the order of endpoints but not touch the order of the addresses + // within each endpoint. - A61 + if cfg.ShuffleAddressList { + newEndpoints = append([]resolver.Endpoint{}, newEndpoints...) + internal.ShuffleAddressListForTesting.(func(int, func(int, int)))(len(newEndpoints), func(i, j int) { newEndpoints[i], newEndpoints[j] = newEndpoints[j], newEndpoints[i] }) } - sl.b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.Ready, - Picker: &picker{result: balancer.PickResult{SubConn: scw.subConn}}, - }) -} -func (sl *subConnList) close() { - prevState := sl.state.Swap(subConnListClosed) - if prevState == subConnListClosed { - return + if b.state == connectivity.Ready { + // If the previous ready subconn exists in new address list, + // keep this connection and don't create new subconns. + prevAddr, err := b.addressIndex.currentAddress() + if err != nil { + // This error should never happen when the state is READY if the + // index is managed correctly. + return fmt.Errorf("address index is in an invalid state: %v", err) + } + b.addressIndex.updateEndpointList(newEndpoints) + if b.addressIndex.seekTo(prevAddr) { + return nil + } else { + b.addressIndex.reset() + } + } else { + b.addressIndex.updateEndpointList(newEndpoints) } - if prevState == subConnListConnecting { - close(sl.connectingCh) + // Remove old subConns that were not in new address list. + oldAddrs := map[string]bool{} + for k, _ := range b.subConns { + oldAddrs[k] = true } - // Close all the subConns except the selected one. The selected subConn - // will be closed by the balancer. - for _, sc := range sl.subConns { - if sc == sl.b.selectedSubConn { - continue + + // Flatten the new endpoint addresses. + newAddrs := map[string]bool{} + for _, endpoint := range newEndpoints { + for _, addr := range endpoint.Addresses { + newAddrs[addrKey(&addr)] = true } - sc.subConn.Shutdown() } -} -// connect attempts to connect to subConns till it finds a healthy one. -func (sl *subConnList) connect() { - // Attempt to connect to each subConn once. - for _, scw := range sl.subConns { - scw.subConn.Connect() - select { - case <-sl.connectingCh: - return - case err := <-scw.failureChan: - if err == nil { - // Connected successfully. - return - } + // Shut them down and remove them. + for oldAddr, _ := range oldAddrs { + if _, ok := newAddrs[oldAddr]; ok { + continue } + b.subConns[oldAddr].subConn.Shutdown() + delete(b.subConns, oldAddr) } -} -func (pickfirstBuilder) ParseConfig(js json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { - var cfg pfConfig - if err := json.Unmarshal(js, &cfg); err != nil { - return nil, fmt.Errorf("pickfirst: unable to unmarshal LB policy config: %s, error: %v", string(js), err) + if len(oldAddrs) == 0 || b.state == connectivity.Ready || b.state == connectivity.Connecting { + // Start connection attempt at first address. + b.state = connectivity.Connecting + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &picker{err: balancer.ErrNoSubConnAvailable}, + }) + b.requestConnection() + } else if b.state == connectivity.Idle { + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.Idle, + Picker: &idlePicker{ + exitIdle: b.ExitIdle, + }, + }) + } else if b.state == connectivity.TransientFailure { + b.requestConnection() } - return cfg, nil + return nil } -type pickfirstBalancer struct { - logger *internalgrpclog.PrefixLogger - state connectivity.State - cc balancer.ClientConn - // Pointer to the subConn list currently connecting. Always close the - // current list before replacing it to ensure resources are freed. - subConnList *subConnList - // A mutex to guard the swapping of subConnLists and it can be triggered - // concurrently by the idlePicker and resolver updates. - subConnListMu sync.Mutex - selectedSubConn *scWrapper - latestAddressList []resolver.Address - shuttingDown bool +// UpdateSubConnState is unused as a StateListener is always registered when +// creating SubConns. +func (b *pickfirstBalancer) UpdateSubConnState(subConn balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", subConn, state) } -func (b *pickfirstBalancer) ResolverError(err error) { - if b.logger.V(2) { - b.logger.Infof("Received error from the name resolver: %v", err) - } - if len(b.latestAddressList) == 0 { - // The picker will not change since the balancer does not currently - // report an error. - b.state = connectivity.TransientFailure +func (b *pickfirstBalancer) Close() { + for _, sd := range b.subConns { + sd.subConn.Shutdown() } - - if b.state != connectivity.TransientFailure { - // The picker will not change since the balancer does not currently - // report an error. - return + for k := range b.subConns { + delete(b.subConns, k) } - b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.TransientFailure, - Picker: &picker{err: fmt.Errorf("name resolver error: %v", err)}, - }) + b.state = connectivity.Shutdown } -func (b *pickfirstBalancer) unsetSelectedSubConn() { - if b.selectedSubConn != nil { - b.selectedSubConn.subConn.Shutdown() - b.selectedSubConn = nil - } +// ExitIdle moves the balancer out of idle state. It can be called concurrently +// by the idlePicker and clientConn so access to variables should be synchronized. +func (b *pickfirstBalancer) ExitIdle() { + // TODO: Synchronize this. + b.requestConnection() } -func (b *pickfirstBalancer) goIdle() { - b.unsetSelectedSubConn() - b.subConnList.close() +// deDupAddresses ensures that each address belongs to only one endpoint. +func deDupAddresses(endpoints []resolver.Endpoint) []resolver.Endpoint { + seenAddrs := map[string]bool{} + newEndpoints := []resolver.Endpoint{} - nextState := connectivity.Idle - if b.state == connectivity.TransientFailure { - // We stay in TransientFailure until we are Ready. See A62. - nextState = connectivity.TransientFailure - } else { - b.state = connectivity.Idle + for _, ep := range endpoints { + addrs := []resolver.Address{} + for _, addr := range ep.Addresses { + if _, ok := seenAddrs[addrKey(&addr)]; ok { + continue + } + addrs = append(addrs, addr) + } + if len(addrs) == 0 { + continue + } + newEndpoints = append(newEndpoints, resolver.Endpoint{ + Addresses: addrs, + Attributes: ep.Attributes, + }) } - b.cc.UpdateState(balancer.State{ - ConnectivityState: nextState, - Picker: &idlePicker{ - exitIdle: b.ExitIdle, - }, - }) + return newEndpoints } -// refreshSubConnListLocked replaces the current subConnList with a newly created -// one. The caller is responsible to close the existing list and ensure its -// closed synchronously during a ClientConn update or a subConn state update. -func (b *pickfirstBalancer) refreshSubConnListLocked() error { - subConnList := newSubConnList(b) - if len(subConnList.subConns) == 0 { - subConnList.close() - b.state = connectivity.TransientFailure - b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.TransientFailure, - Picker: &picker{err: fmt.Errorf("empty address list")}, - }) - b.unsetSelectedSubConn() - return balancer.ErrBadResolverState +// shutdownRemaining shuts down remaining subConns. Called when a subConn +// becomes ready, which means that all other subConn must be shutdown. +func (b *pickfirstBalancer) shutdownRemaining(selected *scData) { + for _, sd := range b.subConns { + if sd.subConn != selected.subConn { + sd.subConn.Shutdown() + } } - - // Reset the previous subConnList to release resources. - if b.logger.V(2) { - b.logger.Infof("Closing older subConnList") + for k := range b.subConns { + delete(b.subConns, k) } - b.subConnList = subConnList - go subConnList.connect() - // Don't read any fields on subConnList after we start connecting as it will - // cause races with subConn state updates being handled by the stateListener. - return nil + b.subConns[addrKey(selected.addr)] = selected } -func (b *pickfirstBalancer) connectUsingLatestAddrs() error { - b.subConnList.close() - b.subConnListMu.Lock() - if err := b.refreshSubConnListLocked(); err != nil { - b.subConnListMu.Unlock() - return err +// requestConnection requests a connection to the next applicable address' +// subcon, creating one if necessary. Schedules a connection to next address in list as well. +// If the current channel has already attempted a connection, we attempt a connection +// to the next address/subconn in our list. We assume that NewSubConn will never +// return an error. +func (b *pickfirstBalancer) requestConnection() { + if !b.addressIndex.isValid() || b.state == connectivity.Shutdown { + return + } + curAddr, err := b.addressIndex.currentAddress() + if err != nil { + // This should not never happen because we already check for validity and + // return early above. + return + } + sd, ok := b.subConns[addrKey(curAddr)] + if !ok { + sd, err = newScData(b, *curAddr) + if err != nil { + // This should never happen. + b.logger.Errorf("Failed to create a subConn for address %v: %v", curAddr.String(), err) + b.addressIndex.increment() + b.requestConnection() + } + b.subConns[addrKey(curAddr)] = sd } - b.subConnListMu.Unlock() - if b.state != connectivity.TransientFailure { - // We stay in TransientFailure until we are Ready. See A62. - b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.Connecting, - Picker: &picker{err: balancer.ErrNoSubConnAvailable}, - }) + switch sd.state { + case connectivity.Idle: + sd.subConn.Connect() + case connectivity.TransientFailure: + if !b.addressIndex.increment() { + b.endFirstPass() + } + b.requestConnection() + case connectivity.Ready: + // Should never happen. + b.logger.Warningf("Requesting a connection even though we have a READY subconn") } - return nil } -func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState) error { - if len(state.ResolverState.Addresses) == 0 && len(state.ResolverState.Endpoints) == 0 { - // Cleanup state pertaining to the previous resolver state. - b.unsetSelectedSubConn() - b.subConnList.close() - b.latestAddressList = nil - // Treat an empty address list like an error by calling b.ResolverError. - b.ResolverError(errors.New("produced zero addresses")) - return balancer.ErrBadResolverState +func addrKey(addr *resolver.Address) string { + return fmt.Sprintf("%s-%s", addr.Addr, addr.Attributes) +} + +func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubConnState) { + // Previously relevant subconns can still callback with state updates. + // To prevent pickers from returning these obsolete subconns, this logic + // is included to check if the current list of active subconns includes this + // subconn. + if activeSd, found := b.subConns[addrKey(sd.addr)]; !found || activeSd != sd { + return } - // We don't have to guard this block with the env var because ParseConfig - // already does so. - cfg, ok := state.BalancerConfig.(pfConfig) - if state.BalancerConfig != nil && !ok { - return fmt.Errorf("pickfirst: received illegal BalancerConfig (type %T): %v", state.BalancerConfig, state.BalancerConfig) + if state.ConnectivityState == connectivity.Shutdown { + return } - if b.logger.V(2) { - b.logger.Infof("Received new config %s, resolver state %s", pretty.ToJSON(cfg), pretty.ToJSON(state.ResolverState)) + if state.ConnectivityState == connectivity.Ready { + b.shutdownRemaining(sd) + b.addressIndex.seekTo(sd.addr) + b.state = connectivity.Ready + b.firstErr = nil + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &picker{result: balancer.PickResult{SubConn: sd.subConn}}, + }) + return } - var addrs []resolver.Address - if endpoints := state.ResolverState.Endpoints; len(endpoints) != 0 { - // Perform the optional shuffling described in gRFC A62. The shuffling will - // change the order of endpoints but not touch the order of the addresses - // within each endpoint. - A61 - if cfg.ShuffleAddressList { - endpoints = append([]resolver.Endpoint{}, endpoints...) - internal.ShuffleAddressListForTesting.(func(int, func(int, int)))(len(endpoints), func(i, j int) { endpoints[i], endpoints[j] = endpoints[j], endpoints[i] }) - } + // If we are transitioning from READY to IDLE, shutdown and re-connect when + // prompted. + if state.ConnectivityState == connectivity.Idle && b.state == connectivity.Ready { + b.state = connectivity.Idle + b.addressIndex.reset() + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.Idle, + Picker: &idlePicker{ + exitIdle: b.ExitIdle, + }, + }) + return + } - // "Flatten the list by concatenating the ordered list of addresses for each - // of the endpoints, in order." - A61 - for _, endpoint := range endpoints { - // "In the flattened list, interleave addresses from the two address - // families, as per RFC-8304 section 4." - A61 - // TODO: support the above language. - addrs = append(addrs, endpoint.Addresses...) - } - } else { - // Endpoints not set, process addresses until we migrate resolver - // emissions fully to Endpoints. The top channel does wrap emitted - // addresses with endpoints, however some balancers such as weighted - // target do not forward the corresponding correct endpoints down/split - // endpoints properly. Once all balancers correctly forward endpoints - // down, can delete this else conditional. - addrs = state.ResolverState.Addresses - if cfg.ShuffleAddressList { - addrs = append([]resolver.Address{}, addrs...) - rand.Shuffle(len(addrs), func(i, j int) { addrs[i], addrs[j] = addrs[j], addrs[i] }) + if b.firstPass { + switch state.ConnectivityState { + case connectivity.Connecting: + if b.state == connectivity.Idle { + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &picker{err: balancer.ErrNoSubConnAvailable}, + }) + } + case connectivity.TransientFailure: + if b.firstErr == nil { + b.firstErr = state.ConnectionError + } + curAddr, err := b.addressIndex.currentAddress() + if err != nil { + // This is not expected since we end the first pass when we + // reach the end of the list. + b.logger.Errorf("Current index is invalid during first pass: %v", err) + return + } + if activeSd, found := b.subConns[addrKey(curAddr)]; !found || activeSd != sd { + return + } + if b.addressIndex.increment() { + b.requestConnection() + return + } + // End of the first pass. + b.endFirstPass() } + return } - b.latestAddressList = addrs - // If the selected subConn's address is present in the list, don't attempt - // to re-connect. - if b.selectedSubConn != nil && slices.ContainsFunc(addrs, func(addr resolver.Address) bool { - return addr.Equal(b.selectedSubConn.addr) - }) { - if b.logger.V(2) { - b.logger.Infof("Not attempting to re-connect since selected address %q is present in new address list", b.selectedSubConn.addr.String()) + // We have finished the first pass, keep re-connecting failing subconns. + switch state.ConnectivityState { + case connectivity.TransientFailure: + b.numTf += 1 + // We request re-resolution when we've seen the same number of TFs as + // subconns. It could be that a subconn has seen multiple TFs due to + // differences in back-off durations, but this is a decent approximation. + if b.numTf >= len(b.subConns) { + b.numTf = 0 + b.cc.ResolveNow(resolver.ResolveNowOptions{}) } - return nil + case connectivity.Idle: + sd.subConn.Connect() } - return b.connectUsingLatestAddrs() } -// UpdateSubConnState is unused as a StateListener is always registered when -// creating SubConns. -func (b *pickfirstBalancer) UpdateSubConnState(subConn balancer.SubConn, state balancer.SubConnState) { - b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", subConn, state) -} - -func (b *pickfirstBalancer) Close() { - b.shuttingDown = true - b.unsetSelectedSubConn() - b.subConnList.close() -} - -// ExitIdle moves the balancer out of idle state. It can be called concurrently -// by the idlePicker and clientConn so access to variables should be synchronized. -func (b *pickfirstBalancer) ExitIdle() { - b.subConnListMu.Lock() - defer b.subConnListMu.Unlock() - if b.subConnList.state.Load() != subConnListClosed { - // Already exited idle, nothing to do. - return - } - // The current subConnList is still closed, create a new subConnList and - // start connecting. - if err := b.refreshSubConnListLocked(); errors.Is(err, balancer.ErrBadResolverState) { - // If creation of the subConnList fails, request for re-resolution to - // get a new list of addresses. - b.cc.ResolveNow(resolver.ResolveNowOptions{}) - return +func (b *pickfirstBalancer) endFirstPass() { + b.firstPass = false + b.numTf = 0 + b.state = connectivity.TransientFailure + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: b.firstErr}, + }) + // Re-request resolution. + b.cc.ResolveNow(resolver.ResolveNowOptions{}) + // Start re-connecting all the subconns that are already in IDLE. + for _, sd := range b.subConns { + if sd.state == connectivity.Idle { + sd.subConn.Connect() + } } } @@ -545,3 +483,75 @@ func (i *idlePicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { i.exitIdle() return balancer.PickResult{}, balancer.ErrNoSubConnAvailable } + +// index is an Index as in 'i', the pointer to an entry. Not a "search index." +// All updates should be synchronized. +type index struct { + endpointList []resolver.Endpoint + endpointIdx int + addrIdx int +} + +// newIndex is the constructor for index. +func newIndex(endpointList []resolver.Endpoint) index { + return index{ + endpointList: endpointList, + } +} + +func (i *index) isValid() bool { + return i.endpointIdx < len(i.endpointList) +} + +// increment moves to the next index in the address list. If at the last address +// in the address list, moves to the next endpoint in the endpoint list. +// This method returns false if it went off the list, true otherwise. +func (i *index) increment() bool { + if !i.isValid() { + return false + } + ep := i.endpointList[i.endpointIdx] + i.addrIdx += 1 + if i.addrIdx >= len(ep.Addresses) { + i.endpointIdx += 1 + i.addrIdx = 0 + return i.endpointIdx < len(i.endpointList) + } + return false +} + +func (i *index) currentAddress() (*resolver.Address, error) { + if !i.isValid() { + return nil, fmt.Errorf("index is off the end of the address list") + } + return &i.endpointList[i.endpointIdx].Addresses[i.addrIdx], nil +} + +func (i *index) reset() { + i.endpointIdx = 0 + i.addrIdx = 0 +} + +func (i *index) updateEndpointList(endpointList []resolver.Endpoint) { + i.endpointList = endpointList + i.reset() +} + +// seekTo returns false if the needle was not found and the current index was left unchanged. +func (idx *index) seekTo(needle *resolver.Address) bool { + for i, endpoint := range idx.endpointList { + for j, addr := range endpoint.Addresses { + if !addr.Attributes.Equal(needle.Attributes) || addr.Addr != needle.Addr { + continue + } + idx.endpointIdx = i + idx.addrIdx = j + return true + } + } + return false +} + +func (i *index) size() int { + return len(i.endpointList) +} From 67b3e93fc7ece10bee6fe0a51be3022b9ad69bad Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Thu, 8 Aug 2024 11:00:19 +0530 Subject: [PATCH 11/62] Revert comment change --- clientconn_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clientconn_test.go b/clientconn_test.go index 85b5eef236df..3340521e031b 100644 --- a/clientconn_test.go +++ b/clientconn_test.go @@ -1061,7 +1061,7 @@ func (s) TestUpdateAddresses_NoopIfCalledWithSameAddresses(t *testing.T) { t.Fatal("timed out waiting for server2 to be contacted") } - // Grab the addrConn for server 2 and call tryUpdateAddrs. + // Grab the addrConn and call tryUpdateAddrs. var ac *addrConn client.mu.Lock() for clientAC := range client.conns { From cceb75d2edc9051c01099c287a6c6714e0c4c5ed Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Thu, 8 Aug 2024 14:29:47 +0530 Subject: [PATCH 12/62] Fix revive warnings --- balancer/pickfirst_leaf/pickfirst_leaf.go | 29 +++++++++---------- clientconn_pickfirst_leaf_test.go | 4 +-- ...n_state_transition_pick_first_leaf_test.go | 4 +-- test/pickfirst_leaf_test.go | 6 ++-- 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/balancer/pickfirst_leaf/pickfirst_leaf.go b/balancer/pickfirst_leaf/pickfirst_leaf.go index 97d12fe8594f..0082de67e9c9 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf.go +++ b/balancer/pickfirst_leaf/pickfirst_leaf.go @@ -16,9 +16,9 @@ * */ -// Package pickfirst_leaf contains the pick_first load balancing policy which +// Package pickfirstleaf contains the pick_first load balancing policy which // will be the universal leaf policy after Dual Stack changes are implemented. -package pickfirst_leaf +package pickfirstleaf import ( "encoding/json" @@ -55,7 +55,7 @@ const ( type pickfirstBuilder struct{} -func (pickfirstBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer { +func (pickfirstBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer { b := &pickfirstBalancer{ cc: cc, addressIndex: newIndex(nil), @@ -203,15 +203,14 @@ func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState b.addressIndex.updateEndpointList(newEndpoints) if b.addressIndex.seekTo(prevAddr) { return nil - } else { - b.addressIndex.reset() } + b.addressIndex.reset() } else { b.addressIndex.updateEndpointList(newEndpoints) } // Remove old subConns that were not in new address list. oldAddrs := map[string]bool{} - for k, _ := range b.subConns { + for k := range b.subConns { oldAddrs[k] = true } @@ -224,7 +223,7 @@ func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState } // Shut them down and remove them. - for oldAddr, _ := range oldAddrs { + for oldAddr := range oldAddrs { if _, ok := newAddrs[oldAddr]; ok { continue } @@ -433,7 +432,7 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon // We have finished the first pass, keep re-connecting failing subconns. switch state.ConnectivityState { case connectivity.TransientFailure: - b.numTf += 1 + b.numTf++ // We request re-resolution when we've seen the same number of TFs as // subconns. It could be that a subconn has seen multiple TFs due to // differences in back-off durations, but this is a decent approximation. @@ -511,9 +510,9 @@ func (i *index) increment() bool { return false } ep := i.endpointList[i.endpointIdx] - i.addrIdx += 1 + i.addrIdx++ if i.addrIdx >= len(ep.Addresses) { - i.endpointIdx += 1 + i.endpointIdx++ i.addrIdx = 0 return i.endpointIdx < len(i.endpointList) } @@ -538,14 +537,14 @@ func (i *index) updateEndpointList(endpointList []resolver.Endpoint) { } // seekTo returns false if the needle was not found and the current index was left unchanged. -func (idx *index) seekTo(needle *resolver.Address) bool { - for i, endpoint := range idx.endpointList { - for j, addr := range endpoint.Addresses { +func (i *index) seekTo(needle *resolver.Address) bool { + for ei, endpoint := range i.endpointList { + for ai, addr := range endpoint.Addresses { if !addr.Attributes.Equal(needle.Attributes) || addr.Addr != needle.Addr { continue } - idx.endpointIdx = i - idx.addrIdx = j + i.endpointIdx = ei + i.addrIdx = ai return true } } diff --git a/clientconn_pickfirst_leaf_test.go b/clientconn_pickfirst_leaf_test.go index 64d6c76ff0b4..6d475cb15fad 100644 --- a/clientconn_pickfirst_leaf_test.go +++ b/clientconn_pickfirst_leaf_test.go @@ -31,7 +31,7 @@ import ( "golang.org/x/net/http2" "google.golang.org/grpc/backoff" "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/pickfirst_leaf" + pickfirstleaf "google.golang.org/grpc/balancer/pickfirst_leaf" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" @@ -50,7 +50,7 @@ const ( ) var ( - testBalancerBuilderPickFirstLeaf = newStateRecordingBalancerBuilder(stateRecordingBalancerWithLeafPickFirstName, pickfirst_leaf.Name) + testBalancerBuilderPickFirstLeaf = newStateRecordingBalancerBuilder(stateRecordingBalancerWithLeafPickFirstName, pickfirstleaf.Name) pickFirstLeafServiceConfig = fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, pickfirst_leaf.Name) ) diff --git a/test/clientconn_state_transition_pick_first_leaf_test.go b/test/clientconn_state_transition_pick_first_leaf_test.go index 156c55f13a55..788b96bcd43b 100644 --- a/test/clientconn_state_transition_pick_first_leaf_test.go +++ b/test/clientconn_state_transition_pick_first_leaf_test.go @@ -29,7 +29,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/backoff" "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/pickfirst_leaf" + pickfirstleaf "google.golang.org/grpc/balancer/pickfirst_leaf" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/internal" @@ -42,7 +42,7 @@ import ( const stateRecordingPickFirstLeafBalancerName = "state_recording_pick_first_leaf_balancer" -var testPickFirstLeafBalancerBuilder = newStateRecordingBalancerBuilder(stateRecordingPickFirstLeafBalancerName, pickfirst_leaf.Name) +var testPickFirstLeafBalancerBuilder = newStateRecordingBalancerBuilder(stateRecordingPickFirstLeafBalancerName, pickfirstleaf.Name) func init() { balancer.Register(testPickFirstLeafBalancerBuilder) diff --git a/test/pickfirst_leaf_test.go b/test/pickfirst_leaf_test.go index d5daea344bb7..6267fc9ac6c0 100644 --- a/test/pickfirst_leaf_test.go +++ b/test/pickfirst_leaf_test.go @@ -28,7 +28,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/backoff" - "google.golang.org/grpc/balancer/pickfirst_leaf" + pickfirstleaf "google.golang.org/grpc/balancer/pickfirst_leaf" "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" @@ -46,7 +46,7 @@ import ( testpb "google.golang.org/grpc/interop/grpc_testing" ) -var pickFirstLeafServiceConfig = fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, pickfirst_leaf.Name) +var pickFirstLeafServiceConfig = fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, pickfirstleaf.Name) // setupPickFirstLeaf performs steps required for pick_first tests. It starts a // bunch of backends exporting the TestService, creates a ClientConn to them @@ -60,7 +60,7 @@ func setupPickFirstLeaf(t *testing.T, backendCount int, opts ...grpc.DialOption) addrs := make([]resolver.Address, backendCount) for i := 0; i < backendCount; i++ { backend := &stubserver.StubServer{ - EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + EmptyCallF: func(_ context.Context, _ *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, } From 0d43e4b4a7b080c1252e33962e6c950a4923247c Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Fri, 9 Aug 2024 00:35:01 +0530 Subject: [PATCH 13/62] Synchronize using callback serializer --- balancer/pickfirst_leaf/pickfirst_leaf.go | 164 +++++++++++++++------- clientconn_pickfirst_leaf_test.go | 2 +- 2 files changed, 114 insertions(+), 52 deletions(-) diff --git a/balancer/pickfirst_leaf/pickfirst_leaf.go b/balancer/pickfirst_leaf/pickfirst_leaf.go index 0082de67e9c9..175edcaab8ee 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf.go +++ b/balancer/pickfirst_leaf/pickfirst_leaf.go @@ -21,6 +21,7 @@ package pickfirstleaf import ( + "context" "encoding/json" "errors" "fmt" @@ -30,6 +31,7 @@ import ( "google.golang.org/grpc/grpclog" "google.golang.org/grpc/internal" internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" "google.golang.org/grpc/internal/pretty" "google.golang.org/grpc/resolver" "google.golang.org/grpc/serviceconfig" @@ -56,10 +58,13 @@ const ( type pickfirstBuilder struct{} func (pickfirstBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer { + ctx, cancel := context.WithCancel(context.Background()) b := &pickfirstBalancer{ - cc: cc, - addressIndex: newIndex(nil), - subConns: map[string]*scData{}, + cc: cc, + addressIndex: newIndex(nil), + subConns: resolver.NewAddressMap(), + serializer: *grpcsync.NewCallbackSerializer(ctx), + serializerCancel: cancel, } b.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(logPrefix, b)) return b @@ -101,8 +106,10 @@ func newScData(b *pickfirstBalancer, addr resolver.Address) (*scData, error) { sc, err := b.cc.NewSubConn([]resolver.Address{addr}, balancer.NewSubConnOptions{ StateListener: func(state balancer.SubConnState) { // Store the state and delegate. - sd.state = state.ConnectivityState - b.updateSubConnState(sd, state) + b.serializer.TrySchedule(func(_ context.Context) { + sd.state = state.ConnectivityState + b.updateSubConnState(sd, state) + }) }, }) if err != nil { @@ -116,14 +123,30 @@ type pickfirstBalancer struct { logger *internalgrpclog.PrefixLogger state connectivity.State cc balancer.ClientConn - subConns map[string]*scData + subConns *resolver.AddressMap addressIndex index firstPass bool firstErr error numTf int + // A serializer is used to ensure synchronization from updates triggered + // due to the idle picker in addition to the already serialized resolver, + // subconn state updates. + serializer grpcsync.CallbackSerializer + serializerCancel func() } func (b *pickfirstBalancer) ResolverError(err error) { + completion := make(chan struct{}) + b.serializer.ScheduleOr(func(ctx context.Context) { + b.resolverError(err, completion) + }, func() { + close(completion) + }) + <-completion +} + +func (b *pickfirstBalancer) resolverError(err error, completion chan struct{}) { + defer close(completion) if b.logger.V(2) { b.logger.Infof("Received error from the name resolver: %v", err) } @@ -136,11 +159,11 @@ func (b *pickfirstBalancer) ResolverError(err error) { return } - for _, sd := range b.subConns { - sd.subConn.Shutdown() + for _, sd := range b.subConns.Values() { + sd.(*scData).subConn.Shutdown() } - for k := range b.subConns { - delete(b.subConns, k) + for _, k := range b.subConns.Keys() { + b.subConns.Delete(k) } b.addressIndex.updateEndpointList(nil) b.state = connectivity.TransientFailure @@ -151,21 +174,34 @@ func (b *pickfirstBalancer) ResolverError(err error) { } func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState) error { + errCh := make(chan error, 1) + b.serializer.ScheduleOr(func(_ context.Context) { + b.updateClientConnState(state, errCh) + }, func() { + close(errCh) + }) + return <-errCh +} + +func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState, errCh chan error) { if b.state == connectivity.Shutdown { - return fmt.Errorf("balancer is already closed") + errCh <- fmt.Errorf("balancer is already closed") + return } if len(state.ResolverState.Addresses) == 0 && len(state.ResolverState.Endpoints) == 0 { // Cleanup state pertaining to the previous resolver state. // Treat an empty address list like an error by calling b.ResolverError. b.state = connectivity.TransientFailure - b.ResolverError(errors.New("produced zero addresses")) - return balancer.ErrBadResolverState + b.resolverError(errors.New("produced zero addresses"), make(chan struct{})) + errCh <- balancer.ErrBadResolverState + return } // We don't have to guard this block with the env var because ParseConfig // already does so. cfg, ok := state.BalancerConfig.(pfConfig) if state.BalancerConfig != nil && !ok { - return fmt.Errorf("pickfirst: received illegal BalancerConfig (type %T): %v", state.BalancerConfig, state.BalancerConfig) + errCh <- fmt.Errorf("pickfirst: received illegal BalancerConfig (type %T): %v", state.BalancerConfig, state.BalancerConfig) + return } if b.logger.V(2) { @@ -174,7 +210,8 @@ func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState newEndpoints := state.ResolverState.Endpoints if len(newEndpoints) == 0 { - return fmt.Errorf("addresses are no longer supported, resolvers should produce endpoints instead") + errCh <- fmt.Errorf("addresses are no longer supported, resolvers should produce endpoints instead") + return } // Since we have a new set of addresses, we are again at first pass @@ -198,40 +235,44 @@ func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState if err != nil { // This error should never happen when the state is READY if the // index is managed correctly. - return fmt.Errorf("address index is in an invalid state: %v", err) + errCh <- fmt.Errorf("address index is in an invalid state: %v", err) + return } b.addressIndex.updateEndpointList(newEndpoints) if b.addressIndex.seekTo(prevAddr) { - return nil + errCh <- nil + return } b.addressIndex.reset() } else { b.addressIndex.updateEndpointList(newEndpoints) } + // Remove old subConns that were not in new address list. - oldAddrs := map[string]bool{} - for k := range b.subConns { - oldAddrs[k] = true + oldAddrs := resolver.NewAddressMap() + for _, k := range b.subConns.Keys() { + oldAddrs.Set(k, true) } // Flatten the new endpoint addresses. - newAddrs := map[string]bool{} + newAddrs := resolver.NewAddressMap() for _, endpoint := range newEndpoints { for _, addr := range endpoint.Addresses { - newAddrs[addrKey(&addr)] = true + newAddrs.Set(addr, true) } } // Shut them down and remove them. - for oldAddr := range oldAddrs { - if _, ok := newAddrs[oldAddr]; ok { + for _, oldAddr := range oldAddrs.Keys() { + if _, ok := newAddrs.Get(oldAddr); ok { continue } - b.subConns[oldAddr].subConn.Shutdown() - delete(b.subConns, oldAddr) + val, _ := b.subConns.Get(oldAddr) + val.(*scData).subConn.Shutdown() + b.subConns.Delete(oldAddr) } - if len(oldAddrs) == 0 || b.state == connectivity.Ready || b.state == connectivity.Connecting { + if oldAddrs.Len() == 0 || b.state == connectivity.Ready || b.state == connectivity.Connecting { // Start connection attempt at first address. b.state = connectivity.Connecting b.cc.UpdateState(balancer.State{ @@ -249,7 +290,7 @@ func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState } else if b.state == connectivity.TransientFailure { b.requestConnection() } - return nil + errCh <- nil } // UpdateSubConnState is unused as a StateListener is always registered when @@ -259,31 +300,53 @@ func (b *pickfirstBalancer) UpdateSubConnState(subConn balancer.SubConn, state b } func (b *pickfirstBalancer) Close() { - for _, sd := range b.subConns { - sd.subConn.Shutdown() + completion := make(chan struct{}) + b.serializer.ScheduleOr(func(ctx context.Context) { + b.close(completion) + }, func() { + close(completion) + }) + <-completion + b.serializerCancel() +} + +func (b *pickfirstBalancer) close(completion chan struct{}) { + for _, sd := range b.subConns.Values() { + sd.(*scData).subConn.Shutdown() } - for k := range b.subConns { - delete(b.subConns, k) + for _, k := range b.subConns.Keys() { + b.subConns.Delete(k) } b.state = connectivity.Shutdown + close(completion) } // ExitIdle moves the balancer out of idle state. It can be called concurrently // by the idlePicker and clientConn so access to variables should be synchronized. func (b *pickfirstBalancer) ExitIdle() { - // TODO: Synchronize this. + completion := make(chan struct{}) + b.serializer.ScheduleOr(func(ctx context.Context) { + b.exitIdle(completion) + }, func() { + close(completion) + }) + <-completion +} + +func (b *pickfirstBalancer) exitIdle(completion chan struct{}) { b.requestConnection() + close(completion) } // deDupAddresses ensures that each address belongs to only one endpoint. func deDupAddresses(endpoints []resolver.Endpoint) []resolver.Endpoint { - seenAddrs := map[string]bool{} + seenAddrs := resolver.NewAddressMap() newEndpoints := []resolver.Endpoint{} for _, ep := range endpoints { addrs := []resolver.Address{} for _, addr := range ep.Addresses { - if _, ok := seenAddrs[addrKey(&addr)]; ok { + if _, ok := seenAddrs.Get(addr); ok { continue } addrs = append(addrs, addr) @@ -302,15 +365,16 @@ func deDupAddresses(endpoints []resolver.Endpoint) []resolver.Endpoint { // shutdownRemaining shuts down remaining subConns. Called when a subConn // becomes ready, which means that all other subConn must be shutdown. func (b *pickfirstBalancer) shutdownRemaining(selected *scData) { - for _, sd := range b.subConns { + for _, v := range b.subConns.Values() { + sd := v.(*scData) if sd.subConn != selected.subConn { sd.subConn.Shutdown() } } - for k := range b.subConns { - delete(b.subConns, k) + for _, k := range b.subConns.Keys() { + b.subConns.Delete(k) } - b.subConns[addrKey(selected.addr)] = selected + b.subConns.Set(*selected.addr, selected) } // requestConnection requests a connection to the next applicable address' @@ -328,7 +392,7 @@ func (b *pickfirstBalancer) requestConnection() { // return early above. return } - sd, ok := b.subConns[addrKey(curAddr)] + sd, ok := b.subConns.Get(*curAddr) if !ok { sd, err = newScData(b, *curAddr) if err != nil { @@ -337,12 +401,13 @@ func (b *pickfirstBalancer) requestConnection() { b.addressIndex.increment() b.requestConnection() } - b.subConns[addrKey(curAddr)] = sd + b.subConns.Set(*curAddr, sd) } - switch sd.state { + scd := sd.(*scData) + switch scd.state { case connectivity.Idle: - sd.subConn.Connect() + scd.subConn.Connect() case connectivity.TransientFailure: if !b.addressIndex.increment() { b.endFirstPass() @@ -354,16 +419,12 @@ func (b *pickfirstBalancer) requestConnection() { } } -func addrKey(addr *resolver.Address) string { - return fmt.Sprintf("%s-%s", addr.Addr, addr.Attributes) -} - func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubConnState) { // Previously relevant subconns can still callback with state updates. // To prevent pickers from returning these obsolete subconns, this logic // is included to check if the current list of active subconns includes this // subconn. - if activeSd, found := b.subConns[addrKey(sd.addr)]; !found || activeSd != sd { + if activeSd, found := b.subConns.Get(*sd.addr); !found || activeSd != sd { return } if state.ConnectivityState == connectivity.Shutdown { @@ -416,7 +477,7 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon b.logger.Errorf("Current index is invalid during first pass: %v", err) return } - if activeSd, found := b.subConns[addrKey(curAddr)]; !found || activeSd != sd { + if activeSd, found := b.subConns.Get(*curAddr); !found || activeSd != sd { return } if b.addressIndex.increment() { @@ -436,7 +497,7 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon // We request re-resolution when we've seen the same number of TFs as // subconns. It could be that a subconn has seen multiple TFs due to // differences in back-off durations, but this is a decent approximation. - if b.numTf >= len(b.subConns) { + if b.numTf >= b.subConns.Len() { b.numTf = 0 b.cc.ResolveNow(resolver.ResolveNowOptions{}) } @@ -456,7 +517,8 @@ func (b *pickfirstBalancer) endFirstPass() { // Re-request resolution. b.cc.ResolveNow(resolver.ResolveNowOptions{}) // Start re-connecting all the subconns that are already in IDLE. - for _, sd := range b.subConns { + for _, v := range b.subConns.Values() { + sd := v.(*scData) if sd.state == connectivity.Idle { sd.subConn.Connect() } diff --git a/clientconn_pickfirst_leaf_test.go b/clientconn_pickfirst_leaf_test.go index 6d475cb15fad..9119892060eb 100644 --- a/clientconn_pickfirst_leaf_test.go +++ b/clientconn_pickfirst_leaf_test.go @@ -51,7 +51,7 @@ const ( var ( testBalancerBuilderPickFirstLeaf = newStateRecordingBalancerBuilder(stateRecordingBalancerWithLeafPickFirstName, pickfirstleaf.Name) - pickFirstLeafServiceConfig = fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, pickfirst_leaf.Name) + pickFirstLeafServiceConfig = fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, pickfirstleaf.Name) ) func init() { From b2a092a83b8bea7f3958b85688a9aa3961391fc1 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Fri, 9 Aug 2024 03:30:02 +0530 Subject: [PATCH 14/62] Prepare for running all tests with new pf --- balancer/pickfirst/pickfirst.go | 5 + balancer/pickfirst_leaf/pickfirst_leaf.go | 49 +- clientconn_pickfirst_leaf_test.go | 1179 ----------------- clientconn_test.go | 56 +- internal/envconfig/envconfig.go | 5 + ...n_state_transition_pick_first_leaf_test.go | 490 ------- test/clientconn_state_transition_test.go | 70 +- test/pickfirst_leaf_test.go | 853 +----------- 8 files changed, 183 insertions(+), 2524 deletions(-) delete mode 100644 clientconn_pickfirst_leaf_test.go delete mode 100644 test/clientconn_state_transition_pick_first_leaf_test.go diff --git a/balancer/pickfirst/pickfirst.go b/balancer/pickfirst/pickfirst.go index 07527603f1d4..b229e92c2e90 100644 --- a/balancer/pickfirst/pickfirst.go +++ b/balancer/pickfirst/pickfirst.go @@ -26,9 +26,11 @@ import ( "math/rand" "google.golang.org/grpc/balancer" + _ "google.golang.org/grpc/balancer/pickfirst_leaf" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/envconfig" internalgrpclog "google.golang.org/grpc/internal/grpclog" "google.golang.org/grpc/internal/pretty" "google.golang.org/grpc/resolver" @@ -36,6 +38,9 @@ import ( ) func init() { + if envconfig.NewPickFirstEnabled { + return + } balancer.Register(pickfirstBuilder{}) internal.ShuffleAddressListForTesting = func(n int, swap func(i, j int)) { rand.Shuffle(n, swap) } } diff --git a/balancer/pickfirst_leaf/pickfirst_leaf.go b/balancer/pickfirst_leaf/pickfirst_leaf.go index 175edcaab8ee..d31635e17388 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf.go +++ b/balancer/pickfirst_leaf/pickfirst_leaf.go @@ -25,11 +25,13 @@ import ( "encoding/json" "errors" "fmt" + "math/rand" "google.golang.org/grpc/balancer" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/envconfig" internalgrpclog "google.golang.org/grpc/internal/grpclog" "google.golang.org/grpc/internal/grpcsync" "google.golang.org/grpc/internal/pretty" @@ -44,20 +46,29 @@ const ( ) func init() { - balancer.Register(pickfirstBuilder{}) + balancer.Register(pickfirstBuilder{name: PickFirstLeafName}) + if envconfig.NewPickFirstEnabled { + // Register as the default pickfirst balancer also. + internal.ShuffleAddressListForTesting = func(n int, swap func(i, j int)) { rand.Shuffle(n, swap) } + balancer.Register(pickfirstBuilder{name: PickFirstName}) + } } var logger = grpclog.Component("pick-first-leaf-lb") const ( - // Name is the name of the pick_first balancer. - Name = "pick_first_leaf" - logPrefix = "[pick-first-leaf-lb %p] " + // PickFirstLeafName is the name of the pick_first balancer. + PickFirstLeafName = "pick_first_leaf" + PickFirstName = "pick_first" + logPrefix = "[pick-first-leaf-lb %p] " ) -type pickfirstBuilder struct{} +type pickfirstBuilder struct { + name string +} func (pickfirstBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer { + fmt.Printf("Building a pf balancer\n") ctx, cancel := context.WithCancel(context.Background()) b := &pickfirstBalancer{ cc: cc, @@ -70,8 +81,8 @@ func (pickfirstBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) b return b } -func (pickfirstBuilder) Name() string { - return Name +func (b pickfirstBuilder) Name() string { + return b.name } func (pickfirstBuilder) ParseConfig(js json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { @@ -210,8 +221,10 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState newEndpoints := state.ResolverState.Endpoints if len(newEndpoints) == 0 { - errCh <- fmt.Errorf("addresses are no longer supported, resolvers should produce endpoints instead") - return + // Convert addresses to endpoints. + for _, addr := range state.ResolverState.Addresses { + newEndpoints = append(newEndpoints, resolver.Endpoint{Addresses: []resolver.Address{addr}}) + } } // Since we have a new set of addresses, we are again at first pass @@ -300,17 +313,20 @@ func (b *pickfirstBalancer) UpdateSubConnState(subConn balancer.SubConn, state b } func (b *pickfirstBalancer) Close() { + fmt.Printf("Arjan: Close called\n") completion := make(chan struct{}) b.serializer.ScheduleOr(func(ctx context.Context) { b.close(completion) }, func() { - close(completion) + b.close(completion) }) <-completion - b.serializerCancel() + <-b.serializer.Done() + fmt.Printf("Arjan: serializer Close called\n") } func (b *pickfirstBalancer) close(completion chan struct{}) { + b.serializerCancel() for _, sd := range b.subConns.Values() { sd.(*scData).subConn.Shutdown() } @@ -397,9 +413,14 @@ func (b *pickfirstBalancer) requestConnection() { sd, err = newScData(b, *curAddr) if err != nil { // This should never happen. - b.logger.Errorf("Failed to create a subConn for address %v: %v", curAddr.String(), err) - b.addressIndex.increment() - b.requestConnection() + b.logger.Warningf("Failed to create a subConn for address %v: %v", curAddr.String(), err) + b.state = connectivity.TransientFailure + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: fmt.Errorf("error creating connection: %v", err)}, + }) + b.addressIndex.reset() + return } b.subConns.Set(*curAddr, sd) } diff --git a/clientconn_pickfirst_leaf_test.go b/clientconn_pickfirst_leaf_test.go deleted file mode 100644 index 9119892060eb..000000000000 --- a/clientconn_pickfirst_leaf_test.go +++ /dev/null @@ -1,1179 +0,0 @@ -/* - * - * Copyright 2014 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package grpc - -import ( - "context" - "errors" - "fmt" - "net" - "strings" - "sync/atomic" - "testing" - "time" - - "golang.org/x/net/http2" - "google.golang.org/grpc/backoff" - "google.golang.org/grpc/balancer" - pickfirstleaf "google.golang.org/grpc/balancer/pickfirst_leaf" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/credentials/insecure" - internalbackoff "google.golang.org/grpc/internal/backoff" - "google.golang.org/grpc/internal/grpcsync" - "google.golang.org/grpc/internal/grpctest" - "google.golang.org/grpc/internal/transport" - "google.golang.org/grpc/keepalive" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/resolver/manual" - "google.golang.org/grpc/testdata" -) - -const ( - stateRecordingBalancerWithLeafPickFirstName = "state_recording_balancer_new_pick_first" -) - -var ( - testBalancerBuilderPickFirstLeaf = newStateRecordingBalancerBuilder(stateRecordingBalancerWithLeafPickFirstName, pickfirstleaf.Name) - pickFirstLeafServiceConfig = fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, pickfirstleaf.Name) -) - -func init() { - balancer.Register(testBalancerBuilderPickFirstLeaf) -} - -func (s) TestPickFirstLeaf_DialWithTimeout(t *testing.T) { - lis, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Error while listening. Err: %v", err) - } - defer lis.Close() - lisAddr := resolver.Address{Addr: lis.Addr().String()} - lisDone := make(chan struct{}) - dialDone := make(chan struct{}) - // 1st listener accepts the connection and then does nothing - go func() { - defer close(lisDone) - conn, err := lis.Accept() - if err != nil { - t.Errorf("Error while accepting. Err: %v", err) - return - } - framer := http2.NewFramer(conn, conn) - if err := framer.WriteSettings(http2.Setting{}); err != nil { - t.Errorf("Error while writing settings. Err: %v", err) - return - } - <-dialDone // Close conn only after dial returns. - }() - - r := manual.NewBuilderWithScheme("whatever") - r.InitialState(resolver.State{Addresses: []resolver.Address{lisAddr}}) - client, err := Dial(r.Scheme()+":///test.server", WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r), WithTimeout(5*time.Second), WithDefaultServiceConfig(pickFirstLeafServiceConfig)) - close(dialDone) - if err != nil { - t.Fatalf("Dial failed. Err: %v", err) - } - defer client.Close() - timeout := time.After(1 * time.Second) - select { - case <-timeout: - t.Fatal("timed out waiting for server to finish") - case <-lisDone: - } -} - -func (s) TestPickFirstLeaf_DialWithMultipleBackendsNotSendingServerPreface(t *testing.T) { - lis1, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Error while listening. Err: %v", err) - } - defer lis1.Close() - lis1Addr := resolver.Address{Addr: lis1.Addr().String()} - lis1Done := make(chan struct{}) - // 1st listener accepts the connection and immediately closes it. - go func() { - defer close(lis1Done) - conn, err := lis1.Accept() - if err != nil { - t.Errorf("Error while accepting. Err: %v", err) - return - } - conn.Close() - }() - - lis2, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Error while listening. Err: %v", err) - } - defer lis2.Close() - lis2Done := make(chan struct{}) - lis2Addr := resolver.Address{Addr: lis2.Addr().String()} - // 2nd listener should get a connection attempt since the first one failed. - go func() { - defer close(lis2Done) - _, err := lis2.Accept() // Closing the client will clean up this conn. - if err != nil { - t.Errorf("Error while accepting. Err: %v", err) - return - } - }() - - r := manual.NewBuilderWithScheme("whatever") - r.InitialState(resolver.State{Addresses: []resolver.Address{lis1Addr, lis2Addr}}) - client, err := Dial(r.Scheme()+":///test.server", WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r), WithDefaultServiceConfig(pickFirstLeafServiceConfig)) - if err != nil { - t.Fatalf("Dial failed. Err: %v", err) - } - defer client.Close() - timeout := time.After(5 * time.Second) - select { - case <-timeout: - t.Fatal("timed out waiting for server 1 to finish") - case <-lis1Done: - } - select { - case <-timeout: - t.Fatal("timed out waiting for server 2 to finish") - case <-lis2Done: - } -} - -func (s) TestPickFirstLeaf_DialWaitsForServerSettings(t *testing.T) { - lis, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Error while listening. Err: %v", err) - } - defer lis.Close() - done := make(chan struct{}) - sent := make(chan struct{}) - dialDone := make(chan struct{}) - go func() { // Launch the server. - defer func() { - close(done) - }() - conn, err := lis.Accept() - if err != nil { - t.Errorf("Error while accepting. Err: %v", err) - return - } - defer conn.Close() - // Sleep for a little bit to make sure that Dial on client - // side blocks until settings are received. - time.Sleep(100 * time.Millisecond) - framer := http2.NewFramer(conn, conn) - close(sent) - if err := framer.WriteSettings(http2.Setting{}); err != nil { - t.Errorf("Error while writing settings. Err: %v", err) - return - } - <-dialDone // Close conn only after dial returns. - }() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - client, err := DialContext(ctx, lis.Addr().String(), - WithTransportCredentials(insecure.NewCredentials()), - WithBlock(), - WithDefaultServiceConfig(pickFirstLeafServiceConfig), - ) - close(dialDone) - if err != nil { - t.Fatalf("Error while dialing. Err: %v", err) - } - defer client.Close() - select { - case <-sent: - default: - t.Fatalf("Dial returned before server settings were sent") - } - <-done -} - -func (s) TestPickFirstLeaf_DialWaitsForServerSettingsAndFails(t *testing.T) { - lis, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Error while listening. Err: %v", err) - } - done := make(chan struct{}) - numConns := 0 - go func() { // Launch the server. - defer func() { - close(done) - }() - for { - conn, err := lis.Accept() - if err != nil { - break - } - numConns++ - defer conn.Close() - } - }() - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - client, err := DialContext(ctx, - lis.Addr().String(), - WithTransportCredentials(insecure.NewCredentials()), - WithReturnConnectionError(), - WithConnectParams(ConnectParams{ - Backoff: backoff.Config{}, - MinConnectTimeout: 250 * time.Millisecond, - }), - WithDefaultServiceConfig(pickFirstLeafServiceConfig)) - lis.Close() - if err == nil { - client.Close() - t.Fatalf("Unexpected success (err=nil) while dialing") - } - expectedMsg := "server preface" - if !strings.Contains(err.Error(), context.DeadlineExceeded.Error()) || !strings.Contains(err.Error(), expectedMsg) { - t.Fatalf("DialContext(_) = %v; want a message that includes both %q and %q", err, context.DeadlineExceeded.Error(), expectedMsg) - } - <-done - if numConns < 2 { - t.Fatalf("dial attempts: %v; want > 1", numConns) - } -} - -// 1. Client connects to a server that doesn't send preface. -// 2. After minConnectTimeout(500 ms here), client disconnects and retries. -// 3. The new server sends its preface. -// 4. Client doesn't kill the connection this time. -func (s) TestPickFirstLeaf_CloseConnectionWhenServerPrefaceNotReceived(t *testing.T) { - lis, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Error while listening. Err: %v", err) - } - var ( - conn2 net.Conn - over uint32 - ) - defer func() { - lis.Close() - // conn2 shouldn't be closed until the client has - // observed a successful test. - if conn2 != nil { - conn2.Close() - } - }() - done := make(chan struct{}) - accepted := make(chan struct{}) - go func() { // Launch the server. - defer close(done) - conn1, err := lis.Accept() - if err != nil { - t.Errorf("Error while accepting. Err: %v", err) - return - } - defer conn1.Close() - // Don't send server settings and the client should close the connection and try again. - conn2, err = lis.Accept() // Accept a reconnection request from client. - if err != nil { - t.Errorf("Error while accepting. Err: %v", err) - return - } - close(accepted) - framer := http2.NewFramer(conn2, conn2) - if err = framer.WriteSettings(http2.Setting{}); err != nil { - t.Errorf("Error while writing settings. Err: %v", err) - return - } - b := make([]byte, 8) - for { - _, err = conn2.Read(b) - if err == nil { - continue - } - if atomic.LoadUint32(&over) == 1 { - // The connection stayed alive for the timer. - // Success. - return - } - t.Errorf("Unexpected error while reading. Err: %v, want timeout error", err) - break - } - }() - client, err := Dial(lis.Addr().String(), - WithTransportCredentials(insecure.NewCredentials()), - withMinConnectDeadline(func() time.Duration { return time.Millisecond * 500 }), - WithDefaultServiceConfig(pickFirstLeafServiceConfig)) - if err != nil { - t.Fatalf("Error while dialing. Err: %v", err) - } - - go stayConnected(client) - - // wait for connection to be accepted on the server. - timer := time.NewTimer(time.Second * 10) - select { - case <-accepted: - case <-timer.C: - t.Fatalf("Client didn't make another connection request in time.") - } - // Make sure the connection stays alive for sometime. - time.Sleep(time.Second) - atomic.StoreUint32(&over, 1) - client.Close() - <-done -} - -func (s) TestPickFirstLeaf_BackoffWhenNoServerPrefaceReceived(t *testing.T) { - lis, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Unexpected error from net.Listen(%q, %q): %v", "tcp", "localhost:0", err) - } - defer lis.Close() - done := make(chan struct{}) - go func() { // Launch the server. - defer close(done) - conn, err := lis.Accept() // Accept the connection only to close it immediately. - if err != nil { - t.Errorf("Error while accepting. Err: %v", err) - return - } - prevAt := time.Now() - conn.Close() - var prevDuration time.Duration - // Make sure the retry attempts are backed off properly. - for i := 0; i < 3; i++ { - conn, err := lis.Accept() - if err != nil { - t.Errorf("Error while accepting. Err: %v", err) - return - } - meow := time.Now() - conn.Close() - dr := meow.Sub(prevAt) - if dr <= prevDuration { - t.Errorf("Client backoff did not increase with retries. Previous duration: %v, current duration: %v", prevDuration, dr) - return - } - prevDuration = dr - prevAt = meow - } - }() - bc := backoff.Config{ - BaseDelay: 200 * time.Millisecond, - Multiplier: 2.0, - Jitter: 0, - MaxDelay: 120 * time.Second, - } - cp := ConnectParams{ - Backoff: bc, - MinConnectTimeout: 1 * time.Second, - } - cc, err := Dial(lis.Addr().String(), - WithTransportCredentials(insecure.NewCredentials()), - WithConnectParams(cp), - WithDefaultServiceConfig(pickFirstLeafServiceConfig)) - if err != nil { - t.Fatalf("Unexpected error from Dial(%v) = %v", lis.Addr(), err) - } - defer cc.Close() - go stayConnected(cc) - <-done -} - -func (s) TestPickFirstLeaf_WithTimeout(t *testing.T) { - conn, err := Dial("passthrough:///Non-Existent.Server:80", - WithTimeout(time.Millisecond), - WithBlock(), - WithTransportCredentials(insecure.NewCredentials()), - WithDefaultServiceConfig(pickFirstLeafServiceConfig)) - if err == nil { - conn.Close() - } - if err != context.DeadlineExceeded { - t.Fatalf("Dial(_, _) = %v, %v, want %v", conn, err, context.DeadlineExceeded) - } -} - -func (s) TestPickFirstLeaf_WithTransportCredentialsTLS(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) - defer cancel() - creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com") - if err != nil { - t.Fatalf("Failed to create credentials %v", err) - } - conn, err := DialContext(ctx, "passthrough:///Non-Existent.Server:80", - WithTransportCredentials(creds), - WithBlock(), - WithDefaultServiceConfig(pickFirstLeafServiceConfig)) - if err == nil { - conn.Close() - } - if err != context.DeadlineExceeded { - t.Fatalf("Dial(_, _) = %v, %v, want %v", conn, err, context.DeadlineExceeded) - } -} - -// When creating a transport configured with n addresses, only calculate the -// backoff n times per "round" of attempts instead of once per. -func (s) TestPickFirstLeafDial_NBackoffPerRetryGroup(t *testing.T) { - var attempts uint32 - getMinConnectTimeout := func() time.Duration { - if atomic.AddUint32(&attempts, 1) > 2 { - // Once all addresses are exhausted, hang around and wait for the - // client.Close to happen rather than re-starting a new round of - // attempts. - return time.Hour - } - return 0 - } - - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - - lis1, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Error while listening. Err: %v", err) - } - defer lis1.Close() - - lis2, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Error while listening. Err: %v", err) - } - defer lis2.Close() - - server1Done := make(chan struct{}) - server2Done := make(chan struct{}) - - // Launch server 1. - go func() { - conn, err := lis1.Accept() - if err != nil { - t.Error(err) - return - } - - conn.Close() - close(server1Done) - }() - // Launch server 2. - go func() { - conn, err := lis2.Accept() - if err != nil { - t.Error(err) - return - } - conn.Close() - close(server2Done) - }() - - rb := manual.NewBuilderWithScheme("whatever") - rb.InitialState(resolver.State{Addresses: []resolver.Address{ - {Addr: lis1.Addr().String()}, - {Addr: lis2.Addr().String()}, - }}) - client, err := DialContext(ctx, "whatever:///this-gets-overwritten", - WithTransportCredentials(insecure.NewCredentials()), - WithResolvers(rb), - withMinConnectDeadline(getMinConnectTimeout), - WithDefaultServiceConfig(pickFirstLeafServiceConfig)) - if err != nil { - t.Fatal(err) - } - defer client.Close() - - timeout := time.After(15 * time.Second) - - select { - case <-timeout: - t.Fatal("timed out waiting for test to finish") - case <-server1Done: - } - - select { - case <-timeout: - t.Fatal("timed out waiting for test to finish") - case <-server2Done: - } - - if atomic.LoadUint32(&attempts) != 2 { - t.Errorf("Back-off attempts=%d, want=2", attempts) - } -} - -func (s) TestPickFirstLeaf_DialContextCancel(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - cancel() - if _, err := DialContext(ctx, "Non-Existent.Server:80", - WithBlock(), - WithTransportCredentials(insecure.NewCredentials()), - WithDefaultServiceConfig(pickFirstLeafServiceConfig)); err != context.Canceled { - t.Fatalf("DialContext(%v, _) = _, %v, want _, %v", ctx, err, context.Canceled) - } -} - -func (s) TestPickFirstLeaf_DialContextFailFast(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - failErr := failFastError{} - dialer := func(string, time.Duration) (net.Conn, error) { - return nil, failErr - } - - _, err := DialContext(ctx, "Non-Existent.Server:80", WithBlock(), - WithTransportCredentials(insecure.NewCredentials()), - WithDialer(dialer), - FailOnNonTempDialError(true), - WithDefaultServiceConfig(pickFirstLeafServiceConfig)) - if terr, ok := err.(transport.ConnectionError); !ok || terr.Origin() != failErr { - t.Fatalf("DialContext() = _, %v, want _, %v", err, failErr) - } -} - -func (s) TestPickFirstLeaf_CredentialsMisuse(t *testing.T) { - // Use of no transport creds and no creds bundle must fail. - if _, err := Dial("passthrough:///Non-Existent.Server:80", WithDefaultServiceConfig(pickFirstLeafServiceConfig)); err != errNoTransportSecurity { - t.Fatalf("Dial(_, _) = _, %v, want _, %v", err, errNoTransportSecurity) - } - - // Use of both transport creds and creds bundle must fail. - creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com") - if err != nil { - t.Fatalf("Failed to create authenticator %v", err) - } - dopts := []DialOption{ - WithTransportCredentials(creds), - WithCredentialsBundle(&fakeBundleCreds{transportCreds: creds}), - WithDefaultServiceConfig(pickFirstLeafServiceConfig), - } - if _, err := Dial("passthrough:///Non-Existent.Server:80", dopts...); err != errTransportCredsAndBundle { - t.Fatalf("Dial(_, _) = _, %v, want _, %v", err, errTransportCredsAndBundle) - } - - // Use of perRPC creds requiring transport security over an insecure - // transport must fail. - if _, err := Dial("passthrough:///Non-Existent.Server:80", - WithPerRPCCredentials(securePerRPCCredentials{}), - WithTransportCredentials(insecure.NewCredentials()), - WithDefaultServiceConfig(pickFirstLeafServiceConfig)); err != errTransportCredentialsMissing { - t.Fatalf("Dial(_, _) = _, %v, want _, %v", err, errTransportCredentialsMissing) - } - - // Use of a creds bundle with nil transport credentials must fail. - if _, err := Dial("passthrough:///Non-Existent.Server:80", - WithCredentialsBundle(&fakeBundleCreds{}), - WithDefaultServiceConfig(pickFirstLeafServiceConfig)); err != errNoTransportCredsInBundle { - t.Fatalf("Dial(_, _) = _, %v, want _, %v", err, errTransportCredsAndBundle) - } -} - -func (s) TestPickFirstLeaf_WithBackoffConfigDefault(t *testing.T) { - testBackoffConfigSet(t, internalbackoff.DefaultExponential, WithDefaultServiceConfig(pickFirstLeafServiceConfig)) -} - -func (s) TestPickFirstLeaf_WithBackoffConfig(t *testing.T) { - b := BackoffConfig{MaxDelay: DefaultBackoffConfig.MaxDelay / 2} - bc := backoff.DefaultConfig - bc.MaxDelay = b.MaxDelay - wantBackoff := internalbackoff.Exponential{Config: bc} - testBackoffConfigSet(t, wantBackoff, WithBackoffConfig(b), WithDefaultServiceConfig(pickFirstLeafServiceConfig)) -} - -func (s) TestPickFirstLeaf_WithBackoffMaxDelay(t *testing.T) { - md := DefaultBackoffConfig.MaxDelay / 2 - bc := backoff.DefaultConfig - bc.MaxDelay = md - wantBackoff := internalbackoff.Exponential{Config: bc} - testBackoffConfigSet(t, wantBackoff, WithBackoffMaxDelay(md), WithDefaultServiceConfig(pickFirstLeafServiceConfig)) -} - -func (s) TestPickFirstLeaf_WithConnectParams(t *testing.T) { - bd := 2 * time.Second - mltpr := 2.0 - jitter := 0.0 - bc := backoff.Config{BaseDelay: bd, Multiplier: mltpr, Jitter: jitter} - - crt := ConnectParams{Backoff: bc} - // MaxDelay is not set in the ConnectParams. So it should not be set on - // internalbackoff.Exponential as well. - wantBackoff := internalbackoff.Exponential{Config: bc} - testBackoffConfigSet(t, wantBackoff, WithConnectParams(crt), WithDefaultServiceConfig(pickFirstLeafServiceConfig)) -} - -func (s) TestPickFirstLeaf_ConnectParamsWithMinConnectTimeout(t *testing.T) { - // Default value specified for minConnectTimeout in the spec is 20 seconds. - mct := 1 * time.Minute - conn, err := Dial("passthrough:///foo:80", - WithTransportCredentials(insecure.NewCredentials()), - WithConnectParams(ConnectParams{MinConnectTimeout: mct}), - WithDefaultServiceConfig(pickFirstLeafServiceConfig)) - if err != nil { - t.Fatalf("unexpected error dialing connection: %v", err) - } - defer conn.Close() - - if got := conn.dopts.minConnectTimeout(); got != mct { - t.Errorf("unexpect minConnectTimeout on the connection: %v, want %v", got, mct) - } -} - -func (s) TestPickFirstLeaf_ResolverServiceConfigBeforeAddressNotPanic(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - cc, err := Dial(r.Scheme()+":///test.server", WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r)) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - - // SwitchBalancer before NewAddress. There was no balancer created, this - // makes sure we don't call close on nil balancerWrapper. - r.UpdateState(resolver.State{ServiceConfig: parseCfg(r, `{"loadBalancingPolicy": "round_robin"}`)}) // This should not panic. - - time.Sleep(time.Second) // Sleep to make sure the service config is handled by ClientConn. -} - -func (s) TestPickFirstLeaf_ResolverServiceConfigWhileClosingNotPanic(t *testing.T) { - for i := 0; i < 10; i++ { // Run this multiple times to make sure it doesn't panic. - r := manual.NewBuilderWithScheme(fmt.Sprintf("whatever-%d", i)) - - cc, err := Dial(r.Scheme()+":///test.server", WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r)) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - // Send a new service config while closing the ClientConn. - go cc.Close() - go r.UpdateState(resolver.State{ServiceConfig: parseCfg(r, `{"loadBalancingPolicy": "round_robin"}`)}) // This should not panic. - } -} - -func (s) TestPickFirstLeaf_ResolverEmptyUpdateNotPanic(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - cc, err := Dial(r.Scheme()+":///test.server", WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r)) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - - // This make sure we don't create addrConn with empty address list. - r.UpdateState(resolver.State{}) // This should not panic. - - time.Sleep(time.Second) // Sleep to make sure the service config is handled by ClientConn. -} - -func (s) TestPickFirstLeaf_ClientUpdatesParamsAfterGoAway(t *testing.T) { - grpctest.TLogger.ExpectError("Client received GoAway with error code ENHANCE_YOUR_CALM and debug data equal to ASCII \"too_many_pings\"") - - lis, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Failed to listen. Err: %v", err) - } - defer lis.Close() - connected := grpcsync.NewEvent() - defer connected.Fire() - go func() { - conn, err := lis.Accept() - if err != nil { - t.Errorf("error accepting connection: %v", err) - return - } - defer conn.Close() - f := http2.NewFramer(conn, conn) - // Start a goroutine to read from the conn to prevent the client from - // blocking after it writes its preface. - go func() { - for { - if _, err := f.ReadFrame(); err != nil { - return - } - } - }() - if err := f.WriteSettings(http2.Setting{}); err != nil { - t.Errorf("error writing settings: %v", err) - return - } - <-connected.Done() - if err := f.WriteGoAway(0, http2.ErrCodeEnhanceYourCalm, []byte("too_many_pings")); err != nil { - t.Errorf("error writing GOAWAY: %v", err) - return - } - }() - addr := lis.Addr().String() - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - cc, err := DialContext(ctx, addr, WithBlock(), WithTransportCredentials(insecure.NewCredentials()), WithKeepaliveParams(keepalive.ClientParameters{ - Time: 10 * time.Second, - Timeout: 100 * time.Millisecond, - PermitWithoutStream: true, - }), - WithDefaultServiceConfig(pickFirstLeafServiceConfig)) - if err != nil { - t.Fatalf("Dial(%s, _) = _, %v, want _, ", addr, err) - } - defer cc.Close() - connected.Fire() - for { - time.Sleep(10 * time.Millisecond) - cc.mu.RLock() - v := cc.mkp.Time - cc.mu.RUnlock() - if v == 20*time.Second { - // Success - return - } - if ctx.Err() != nil { - // Timeout - t.Fatalf("cc.dopts.copts.Keepalive.Time = %v , want 20s", v) - } - } -} - -func (s) TestPickFirstLeaf_DisableServiceConfigOption(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - addr := r.Scheme() + ":///non.existent" - cc, err := Dial(addr, WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r), WithDisableServiceConfig()) - if err != nil { - t.Fatalf("Dial(%s, _) = _, %v, want _, ", addr, err) - } - defer cc.Close() - r.UpdateState(resolver.State{ServiceConfig: parseCfg(r, `{ - "loadBalancingConfig": [{"pick_first_leaf":{}}], - "methodConfig": [ - { - "name": [ - { - "service": "foo", - "method": "Bar" - } - ], - "waitForReady": true - } - ] -}`)}) - time.Sleep(1 * time.Second) - m := cc.GetMethodConfig("/foo/Bar") - if m.WaitForReady != nil { - t.Fatalf("want: method (\"/foo/bar/\") config to be empty, got: %+v", m) - } -} - -func (s) TestPickFirstLeaf_MethodConfigDefaultService(t *testing.T) { - addr := "nonexist:///non.existent" - cc, err := Dial(addr, WithTransportCredentials(insecure.NewCredentials()), WithDefaultServiceConfig(`{ - "loadBalancingConfig": [{"pick_first_leaf":{}}], - "methodConfig": [{ - "name": [ - { - "service": "" - } - ], - "waitForReady": true - }] -}`)) - if err != nil { - t.Fatalf("Dial(%s, _) = _, %v, want _, ", addr, err) - } - defer cc.Close() - - m := cc.GetMethodConfig("/foo/Bar") - if m.WaitForReady == nil { - t.Fatalf("want: method (%q) config to fallback to the default service", "/foo/Bar") - } -} - -func (s) TestPickFirstLeaf_ClientConnCanonicalTarget(t *testing.T) { - tests := []struct { - name string - addr string - canonicalTargetWant string - }{ - { - name: "normal-case", - addr: "dns://a.server.com/google.com", - canonicalTargetWant: "dns://a.server.com/google.com", - }, - { - name: "canonical-target-not-specified", - addr: "no.scheme", - canonicalTargetWant: "passthrough:///no.scheme", - }, - { - name: "canonical-target-nonexistent", - addr: "nonexist:///non.existent", - canonicalTargetWant: "passthrough:///nonexist:///non.existent", - }, - { - name: "canonical-target-add-colon-slash", - addr: "dns:hostname:port", - canonicalTargetWant: "dns:///hostname:port", - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - cc, err := Dial(test.addr, - WithTransportCredentials(insecure.NewCredentials()), - WithDefaultServiceConfig(pickFirstLeafServiceConfig), - ) - if err != nil { - t.Fatalf("Dial(%s, _) = _, %v, want _, ", test.addr, err) - } - defer cc.Close() - if cc.Target() != test.addr { - t.Fatalf("Target() = %s, want %s", cc.Target(), test.addr) - } - if cc.CanonicalTarget() != test.canonicalTargetWant { - t.Fatalf("CanonicalTarget() = %s, want %s", cc.CanonicalTarget(), test.canonicalTargetWant) - } - }) - } -} - -func (s) TestPickFirstLeaf_ResetConnectBackoff(t *testing.T) { - dials := make(chan struct{}) - defer func() { // If we fail, let the http2client break out of dialing. - select { - case <-dials: - default: - } - }() - dialer := func(string, time.Duration) (net.Conn, error) { - dials <- struct{}{} - return nil, errors.New("failed to fake dial") - } - cc, err := Dial("any", WithTransportCredentials(insecure.NewCredentials()), - WithDialer(dialer), - withBackoff(backoffForever{}), - WithDefaultServiceConfig(pickFirstLeafServiceConfig), - ) - if err != nil { - t.Fatalf("Dial() = _, %v; want _, nil", err) - } - defer cc.Close() - go stayConnected(cc) - select { - case <-dials: - case <-time.NewTimer(10 * time.Second).C: - t.Fatal("Failed to call dial within 10s") - } - - select { - case <-dials: - t.Fatal("Dial called unexpectedly before resetting backoff") - case <-time.NewTimer(100 * time.Millisecond).C: - } - - cc.ResetConnectBackoff() - - select { - case <-dials: - case <-time.NewTimer(10 * time.Second).C: - t.Fatal("Failed to call dial within 10s after resetting backoff") - } -} - -func (s) TestPickFirstLeaf_BackoffCancel(t *testing.T) { - dialStrCh := make(chan string) - cc, err := Dial("any", WithTransportCredentials(insecure.NewCredentials()), - WithDialer(func(t string, _ time.Duration) (net.Conn, error) { - dialStrCh <- t - return nil, fmt.Errorf("test dialer, always error") - }), - WithDefaultServiceConfig(pickFirstLeafServiceConfig), - ) - if err != nil { - t.Fatalf("Failed to create ClientConn: %v", err) - } - defer cc.Close() - - select { - case <-time.After(defaultTestTimeout): - t.Fatal("Timeout when waiting for custom dialer to be invoked during Dial") - case <-dialStrCh: - } -} - -// TestUpdateAddresses_NoopIfCalledWithSameAddresses tests that UpdateAddresses -// should be noop if UpdateAddresses is called with the same list of addresses, -// even when the SubConn is in Connecting and doesn't have a current address. -func (s) TestPickFirstLeaf_UpdateAddresses_NoopIfCalledWithSameAddresses(t *testing.T) { - lis1, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Error while listening. Err: %v", err) - } - defer lis1.Close() - - lis2, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Error while listening. Err: %v", err) - } - defer lis2.Close() - - lis3, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Error while listening. Err: %v", err) - } - defer lis3.Close() - - closeServer2 := make(chan struct{}) - exitCh := make(chan struct{}) - server1ContactedFirstTime := make(chan struct{}) - server1ContactedSecondTime := make(chan struct{}) - server2ContactedFirstTime := make(chan struct{}) - server2ContactedSecondTime := make(chan struct{}) - server3Contacted := make(chan struct{}) - - defer close(exitCh) - - // Launch server 1. - go func() { - // First, let's allow the initial connection to go READY. We need to do - // this because tryUpdateAddrs only works after there's some non-nil - // address on the ac, and curAddress is only set after READY. - conn1, err := lis1.Accept() - if err != nil { - t.Error(err) - return - } - go keepReading(conn1) - - framer := http2.NewFramer(conn1, conn1) - if err := framer.WriteSettings(http2.Setting{}); err != nil { - t.Errorf("Error while writing settings frame. %v", err) - return - } - - // nextStateNotifier() is updated after balancerBuilder.Build(), which is - // called by grpc.Dial. It's safe to do it here because lis1.Accept blocks - // until balancer is built to process the addresses. - stateNotifications := testBalancerBuilderPickFirstLeaf.nextStateNotifier() - // Wait for the transport to become ready. - for { - select { - case st := <-stateNotifications: - if st == connectivity.Ready { - goto ready - } - case <-exitCh: - return - } - } - - ready: - // Once it's ready, curAddress has been set. So let's close this - // connection prompting the first reconnect cycle. - conn1.Close() - - // Accept and immediately close, causing it to go to server2. - conn2, err := lis1.Accept() - if err != nil { - t.Error(err) - return - } - close(server1ContactedFirstTime) - conn2.Close() - - // Hopefully it picks this server after tryUpdateAddrs. - lis1.Accept() - close(server1ContactedSecondTime) - }() - // Launch server 2. - go func() { - // Accept and then hang waiting for the test call tryUpdateAddrs and - // then signal to this server to close. After this server closes, it - // should start from the top instead of trying server2 or continuing - // to server3. - conn, err := lis2.Accept() - if err != nil { - t.Error(err) - return - } - - close(server2ContactedFirstTime) - <-closeServer2 - conn.Close() - - // After tryUpdateAddrs, it should NOT try server2. - lis2.Accept() - close(server2ContactedSecondTime) - }() - // Launch server 3. - go func() { - // After tryUpdateAddrs, it should NOT try server3. (or any other time) - lis3.Accept() - close(server3Contacted) - }() - - addrsList := []resolver.Address{ - {Addr: lis1.Addr().String()}, - {Addr: lis2.Addr().String()}, - {Addr: lis3.Addr().String()}, - } - rb := manual.NewBuilderWithScheme("whatever") - rb.InitialState(resolver.State{Addresses: addrsList}) - - client, err := Dial("whatever:///this-gets-overwritten", - WithTransportCredentials(insecure.NewCredentials()), - WithResolvers(rb), - WithConnectParams(ConnectParams{ - Backoff: backoff.Config{}, - MinConnectTimeout: time.Hour, - }), - WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateRecordingBalancerWithLeafPickFirstName))) - if err != nil { - t.Fatal(err) - } - defer client.Close() - go stayConnected(client) - - timeout := time.After(5 * time.Second) - - // Wait for server1 to be contacted (which will immediately fail), then - // server2 (which will hang waiting for our signal). - select { - case <-server1ContactedFirstTime: - case <-timeout: - t.Fatal("timed out waiting for server1 to be contacted") - } - select { - case <-server2ContactedFirstTime: - case <-timeout: - t.Fatal("timed out waiting for server2 to be contacted") - } - - // Grab the addrConn for server 2 and call tryUpdateAddrs. - var ac *addrConn - client.mu.Lock() - for clientAC := range client.conns { - if got := len(clientAC.addrs); got != 1 { - t.Errorf("len(AddrConn.addrs)=%d, want=1", got) - continue - } - if clientAC.addrs[0].Addr == lis2.Addr().String() { - ac = clientAC - break - } - } - client.mu.Unlock() - - if ac == nil { - t.Fatal("Coudn't find the subConn for server 2") - } - - // Call UpdateAddresses with the same list of addresses, it should be a noop - // (even when the SubConn is Connecting, and doesn't have a curAddr). - ac.acbw.UpdateAddresses(addrsList[1:2]) - - // We've called tryUpdateAddrs - now let's make server2 close the - // connection and check that it continues to server3. - close(closeServer2) - - select { - case <-server1ContactedSecondTime: - t.Fatal("server1 was contacted a second time, but it should have continued to server 3") - case <-server2ContactedSecondTime: - t.Fatal("server2 was contacted a second time, but it should have continued to server 3") - case <-server3Contacted: - case <-timeout: - t.Fatal("timed out waiting for any server to be contacted after tryUpdateAddrs") - } -} - -func (s) TestPickFirstLeaf_DefaultServiceConfig(t *testing.T) { - const defaultSC = ` -{ - "loadBalancingConfig": [{"pick_first_leaf":{}}], - "methodConfig": [ - { - "name": [ - { - "service": "foo", - "method": "bar" - } - ], - "waitForReady": true - } - ] -}` - tests := []struct { - name string - testF func(t *testing.T, r *manual.Resolver, addr, sc string) - sc string - }{ - { - name: "invalid-service-config", - testF: testInvalidDefaultServiceConfig, - sc: "", - }, - { - name: "resolver-service-config-disabled", - testF: testDefaultServiceConfigWhenResolverServiceConfigDisabled, - sc: defaultSC, - }, - { - name: "resolver-does-not-return-service-config", - testF: testDefaultServiceConfigWhenResolverDoesNotReturnServiceConfig, - sc: defaultSC, - }, - { - name: "resolver-returns-invalid-service-config", - testF: testDefaultServiceConfigWhenResolverReturnInvalidServiceConfig, - sc: defaultSC, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - r := manual.NewBuilderWithScheme(test.name) - addr := r.Scheme() + ":///non.existent" - test.testF(t, r, addr, test.sc) - }) - } -} - -func (s) TestPickFirstLeaf_URLAuthorityEscape(t *testing.T) { - tests := []struct { - name string - authority string - want string - }{ - { - name: "ipv6_authority", - authority: "[::1]", - want: "[::1]", - }, - { - name: "with_user_and_host", - authority: "userinfo@host:10001", - want: "userinfo@host:10001", - }, - { - name: "with_multiple_slashes", - authority: "projects/123/network/abc/service", - want: "projects%2F123%2Fnetwork%2Fabc%2Fservice", - }, - { - name: "all_possible_allowed_chars", - authority: "abc123-._~!$&'()*+,;=@:[]", - want: "abc123-._~!$&'()*+,;=@:[]", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if got, want := encodeAuthority(test.authority), test.want; got != want { - t.Errorf("encodeAuthority(%s) = %s, want %s", test.authority, got, test.want) - } - }) - } -} diff --git a/clientconn_test.go b/clientconn_test.go index 3340521e031b..d67355f58867 100644 --- a/clientconn_test.go +++ b/clientconn_test.go @@ -37,6 +37,7 @@ import ( "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" internalbackoff "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/internal/envconfig" "google.golang.org/grpc/internal/grpcsync" "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/internal/transport" @@ -52,7 +53,7 @@ const ( stateRecordingBalancerName = "state_recording_balancer" ) -var testBalancerBuilder = newStateRecordingBalancerBuilder(stateRecordingBalancerName, "pick_first") +var testBalancerBuilder = newStateRecordingBalancerBuilder() func init() { balancer.Register(testBalancerBuilder) @@ -418,17 +419,21 @@ func (s) TestWithTransportCredentialsTLS(t *testing.T) { // When creating a transport configured with n addresses, only calculate the // backoff once per "round" of attempts instead of once per address (n times -// per "round" of attempts). -func (s) TestDial_OneBackoffPerRetryGroup(t *testing.T) { +// per "round" of attempts) for old pickfirst and once per address for new pickfirst. +func (s) TestDial_BackoffCountPerRetryGroup(t *testing.T) { var attempts uint32 + wantBackoffs := uint32(1) + if envconfig.NewPickFirstEnabled { + wantBackoffs = 2 + } getMinConnectTimeout := func() time.Duration { - if atomic.AddUint32(&attempts, 1) == 1 { + if atomic.AddUint32(&attempts, 1) <= wantBackoffs { // Once all addresses are exhausted, hang around and wait for the // client.Close to happen rather than re-starting a new round of // attempts. return time.Hour } - t.Error("only one attempt backoff calculation, but got more") + t.Errorf("only %d attempt backoff calculation, but got more", wantBackoffs) return 0 } @@ -499,6 +504,10 @@ func (s) TestDial_OneBackoffPerRetryGroup(t *testing.T) { t.Fatal("timed out waiting for test to finish") case <-server2Done: } + + if got, want := atomic.LoadUint32(&attempts), wantBackoffs; got != want { + t.Errorf("attempts = %d, want %d", got, want) + } } func (s) TestDialContextCancel(t *testing.T) { @@ -1061,19 +1070,15 @@ func (s) TestUpdateAddresses_NoopIfCalledWithSameAddresses(t *testing.T) { t.Fatal("timed out waiting for server2 to be contacted") } - // Grab the addrConn and call tryUpdateAddrs. - var ac *addrConn + // Grab the addrConn and call tryUpdateAddrs. client.mu.Lock() for clientAC := range client.conns { - ac = clientAC - break + // Call UpdateAddresses with the same list of addresses, it should be a noop + // (even when the SubConn is Connecting, and doesn't have a curAddr). + clientAC.acbw.UpdateAddresses(clientAC.addrs) } client.mu.Unlock() - // Call UpdateAddresses with the same list of addresses, it should be a noop - // (even when the SubConn is Connecting, and doesn't have a curAddr). - ac.acbw.UpdateAddresses(addrsList) - // We've called tryUpdateAddrs - now let's make server2 close the // connection and check that it continues to server3. close(closeServer2) @@ -1215,37 +1220,26 @@ func (b *stateRecordingBalancer) Close() { b.Balancer.Close() } -func (b *stateRecordingBalancer) ExitIdle() { - if ib, ok := b.Balancer.(balancer.ExitIdler); ok { - ib.ExitIdle() - } -} - type stateRecordingBalancerBuilder struct { - mu sync.Mutex - notifier chan connectivity.State // The notifier used in the last Balancer. - policyName string - childPolicyName string + mu sync.Mutex + notifier chan connectivity.State // The notifier used in the last Balancer. } -func newStateRecordingBalancerBuilder(policyName, childPolicyName string) *stateRecordingBalancerBuilder { - return &stateRecordingBalancerBuilder{ - childPolicyName: childPolicyName, - policyName: policyName, - } +func newStateRecordingBalancerBuilder() *stateRecordingBalancerBuilder { + return &stateRecordingBalancerBuilder{} } func (b *stateRecordingBalancerBuilder) Name() string { - return b.policyName + return stateRecordingBalancerName } func (b *stateRecordingBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { - stateNotifications := make(chan connectivity.State, 20) + stateNotifications := make(chan connectivity.State, 10) b.mu.Lock() b.notifier = stateNotifications b.mu.Unlock() return &stateRecordingBalancer{ - Balancer: balancer.Get(b.childPolicyName).Build(&stateRecordingCCWrapper{cc, stateNotifications}, opts), + Balancer: balancer.Get("pick_first").Build(&stateRecordingCCWrapper{cc, stateNotifications}, opts), } } diff --git a/internal/envconfig/envconfig.go b/internal/envconfig/envconfig.go index 00abc7c2beb0..e323b3703b33 100644 --- a/internal/envconfig/envconfig.go +++ b/internal/envconfig/envconfig.go @@ -50,6 +50,11 @@ var ( // xDS fallback is turned on. If this is unset or is false, only the first // xDS server in the list of server configs will be used. XDSFallbackSupport = boolFromEnv("GRPC_EXPERIMENTAL_XDS_FALLBACK", false) + // NewPickFirstEnabled is set if the new pickfirst leaf policy is to be used + // instead of the exiting pickfirst implementation. This can be enabled by + // setting the environment variable "GRPC_EXPERIMENTAL_ENABLE_NEW_PICK_FIRST" + // to "true". + NewPickFirstEnabled = boolFromEnv("GRPC_EXPERIMENTAL_ENABLE_NEW_PICK_FIRST", false) ) func boolFromEnv(envVar string, def bool) bool { diff --git a/test/clientconn_state_transition_pick_first_leaf_test.go b/test/clientconn_state_transition_pick_first_leaf_test.go deleted file mode 100644 index 788b96bcd43b..000000000000 --- a/test/clientconn_state_transition_pick_first_leaf_test.go +++ /dev/null @@ -1,490 +0,0 @@ -/* - * - * Copyright 2024 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package test - -import ( - "context" - "fmt" - "net" - "testing" - "time" - - "golang.org/x/net/http2" - "google.golang.org/grpc" - "google.golang.org/grpc/backoff" - "google.golang.org/grpc/balancer" - pickfirstleaf "google.golang.org/grpc/balancer/pickfirst_leaf" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/internal" - "google.golang.org/grpc/internal/balancer/stub" - "google.golang.org/grpc/internal/grpcsync" - "google.golang.org/grpc/internal/testutils" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/resolver/manual" -) - -const stateRecordingPickFirstLeafBalancerName = "state_recording_pick_first_leaf_balancer" - -var testPickFirstLeafBalancerBuilder = newStateRecordingBalancerBuilder(stateRecordingPickFirstLeafBalancerName, pickfirstleaf.Name) - -func init() { - balancer.Register(testPickFirstLeafBalancerBuilder) -} - -// These tests use a pipeListener. This listener is similar to net.Listener -// except that it is unbuffered, so each read and write will wait for the other -// side's corresponding write or read. -func (s) TestPickFirstLeafStateTransitions_SingleAddress(t *testing.T) { - for _, test := range []struct { - desc string - want []connectivity.State - server func(net.Listener) net.Conn - }{ - { - desc: "When the server returns server preface, the client enters READY.", - want: []connectivity.State{ - connectivity.Connecting, - connectivity.Ready, - }, - server: func(lis net.Listener) net.Conn { - conn, err := lis.Accept() - if err != nil { - t.Error(err) - return nil - } - - go keepReading(conn) - - framer := http2.NewFramer(conn, conn) - if err := framer.WriteSettings(http2.Setting{}); err != nil { - t.Errorf("Error while writing settings frame. %v", err) - return nil - } - - return conn - }, - }, - { - desc: "When the connection is closed before the preface is sent, the client enters TRANSIENT FAILURE.", - want: []connectivity.State{ - connectivity.Connecting, - connectivity.TransientFailure, - }, - server: func(lis net.Listener) net.Conn { - conn, err := lis.Accept() - if err != nil { - t.Error(err) - return nil - } - - conn.Close() - return nil - }, - }, - { - desc: `When the server sends its connection preface, but the connection dies before the client can write its -connection preface, the client enters TRANSIENT FAILURE.`, - want: []connectivity.State{ - connectivity.Connecting, - connectivity.TransientFailure, - }, - server: func(lis net.Listener) net.Conn { - conn, err := lis.Accept() - if err != nil { - t.Error(err) - return nil - } - - framer := http2.NewFramer(conn, conn) - if err := framer.WriteSettings(http2.Setting{}); err != nil { - t.Errorf("Error while writing settings frame. %v", err) - return nil - } - - conn.Close() - return nil - }, - }, - { - desc: `When the server reads the client connection preface but does not send its connection preface, the -client enters TRANSIENT FAILURE.`, - want: []connectivity.State{ - connectivity.Connecting, - connectivity.TransientFailure, - }, - server: func(lis net.Listener) net.Conn { - conn, err := lis.Accept() - if err != nil { - t.Error(err) - return nil - } - - go keepReading(conn) - - return conn - }, - }, - } { - t.Log(test.desc) - testStateTransitionSingleAddress(t, test.want, test.server, testPickFirstLeafBalancerBuilder) - } -} - -// When a READY connection is closed, the client enters IDLE then CONNECTING. -func (s) TestPickFirstLeafStateTransitions_ReadyToConnecting(t *testing.T) { - lis, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Error while listening. Err: %v", err) - } - defer lis.Close() - - sawReady := make(chan struct{}, 1) - defer close(sawReady) - - // Launch the server. - go func() { - conn, err := lis.Accept() - if err != nil { - t.Error(err) - return - } - - go keepReading(conn) - - framer := http2.NewFramer(conn, conn) - if err := framer.WriteSettings(http2.Setting{}); err != nil { - t.Errorf("Error while writing settings frame. %v", err) - return - } - - // Prevents race between onPrefaceReceipt and onClose. - <-sawReady - - conn.Close() - }() - - client, err := grpc.Dial(lis.Addr().String(), - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateRecordingPickFirstLeafBalancerName))) - if err != nil { - t.Fatal(err) - } - defer client.Close() - - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - go testutils.StayConnected(ctx, client) - - stateNotifications := testPickFirstLeafBalancerBuilder.nextStateNotifier() - - want := []connectivity.State{ - connectivity.Connecting, - connectivity.Ready, - connectivity.Idle, - connectivity.Shutdown, - connectivity.Connecting, - } - for i := 0; i < len(want); i++ { - select { - case <-time.After(defaultTestTimeout): - t.Fatalf("timed out waiting for state %d (%v) in flow %v", i, want[i], want) - case seen := <-stateNotifications: - if seen == connectivity.Ready { - sawReady <- struct{}{} - } - if seen != want[i] { - t.Fatalf("expected to see %v at position %d in flow %v, got %v", want[i], i, want, seen) - } - } - } -} - -// When the first connection is closed, the client stays in CONNECTING until it -// tries the second address (which succeeds, and then it enters READY). -func (s) TestPickFirstLeafStateTransitions_TriesAllAddrsBeforeTransientFailure(t *testing.T) { - lis1, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Error while listening. Err: %v", err) - } - defer lis1.Close() - - lis2, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Error while listening. Err: %v", err) - } - defer lis2.Close() - - server1Done := make(chan struct{}) - server2Done := make(chan struct{}) - - // Launch server 1. - go func() { - conn, err := lis1.Accept() - if err != nil { - t.Error(err) - return - } - - conn.Close() - close(server1Done) - }() - // Launch server 2. - go func() { - conn, err := lis2.Accept() - if err != nil { - t.Error(err) - return - } - - go keepReading(conn) - - framer := http2.NewFramer(conn, conn) - if err := framer.WriteSettings(http2.Setting{}); err != nil { - t.Errorf("Error while writing settings frame. %v", err) - return - } - - close(server2Done) - }() - - rb := manual.NewBuilderWithScheme("whatever") - rb.InitialState(resolver.State{Addresses: []resolver.Address{ - {Addr: lis1.Addr().String()}, - {Addr: lis2.Addr().String()}, - }}) - client, err := grpc.Dial("whatever:///this-gets-overwritten", - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateRecordingPickFirstLeafBalancerName)), - grpc.WithResolvers(rb), - grpc.WithConnectParams(grpc.ConnectParams{ - // Set a really long back-off delay to ensure the first subConn does - // not enter ready before the second subConn connects. - Backoff: backoff.Config{ - BaseDelay: 1 * time.Hour, - }, - }), - ) - if err != nil { - t.Fatal(err) - } - defer client.Close() - - stateNotifications := testPickFirstLeafBalancerBuilder.nextStateNotifier() - want := []connectivity.State{ - connectivity.Connecting, - connectivity.TransientFailure, - connectivity.Connecting, - connectivity.Ready, - } - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - for i := 0; i < len(want); i++ { - select { - case <-ctx.Done(): - t.Fatalf("timed out waiting for state %d (%v) in flow %v", i, want[i], want) - case seen := <-stateNotifications: - if seen != want[i] { - t.Fatalf("expected to see %v at position %d in flow %v, got %v", want[i], i, want, seen) - } - } - } - select { - case <-ctx.Done(): - t.Fatal("saw the correct state transitions, but timed out waiting for client to finish interactions with server 1") - case <-server1Done: - } - select { - case <-ctx.Done(): - t.Fatal("saw the correct state transitions, but timed out waiting for client to finish interactions with server 2") - case <-server2Done: - } -} - -// When there are multiple addresses, and we enter READY on one of them, a -// later closure should cause the client to enter CONNECTING -func (s) TestPickFirstLeafStateTransitions_MultipleAddrsEntersReady(t *testing.T) { - lis1, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Error while listening. Err: %v", err) - } - defer lis1.Close() - - // Never actually gets used; we just want it to be alive so that the resolver has two addresses to target. - lis2, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Error while listening. Err: %v", err) - } - defer lis2.Close() - - server1Done := make(chan struct{}) - sawReady := make(chan struct{}, 1) - defer close(sawReady) - - // Launch server 1. - go func() { - conn, err := lis1.Accept() - if err != nil { - t.Error(err) - return - } - - go keepReading(conn) - - framer := http2.NewFramer(conn, conn) - if err := framer.WriteSettings(http2.Setting{}); err != nil { - t.Errorf("Error while writing settings frame. %v", err) - return - } - - <-sawReady - - conn.Close() - - close(server1Done) - }() - - rb := manual.NewBuilderWithScheme("whatever") - rb.InitialState(resolver.State{Addresses: []resolver.Address{ - {Addr: lis1.Addr().String()}, - {Addr: lis2.Addr().String()}, - }}) - client, err := grpc.Dial("whatever:///this-gets-overwritten", - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateRecordingPickFirstLeafBalancerName)), - grpc.WithResolvers(rb)) - if err != nil { - t.Fatal(err) - } - defer client.Close() - - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - go testutils.StayConnected(ctx, client) - - stateNotifications := testPickFirstLeafBalancerBuilder.nextStateNotifier() - want := []connectivity.State{ - connectivity.Connecting, - connectivity.Ready, - connectivity.Shutdown, // The second subConn is closed once the first one connects. - connectivity.Idle, - connectivity.Shutdown, // The subConn will be closed and pickfirst will run on the latest address list. - connectivity.Connecting, - } - for i := 0; i < len(want); i++ { - select { - case <-ctx.Done(): - t.Fatalf("timed out waiting for state %d (%v) in flow %v", i, want[i], want) - case seen := <-stateNotifications: - if seen == connectivity.Ready { - sawReady <- struct{}{} - } - if seen != want[i] { - t.Fatalf("expected to see %v at position %d in flow %v, got %v", want[i], i, want, seen) - } - } - } - select { - case <-ctx.Done(): - t.Fatal("saw the correct state transitions, but timed out waiting for client to finish interactions with server 1") - case <-server1Done: - } -} - -// TestPickFirstLeafConnectivityStateSubscriber confirms updates sent by the balancer in -// rapid succession are not missed by the subscriber. -func (s) TestPickFirstLeafConnectivityStateSubscriber(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - - sendStates := []connectivity.State{ - connectivity.Connecting, - connectivity.Ready, - connectivity.Idle, - connectivity.Connecting, - connectivity.Idle, - connectivity.Connecting, - connectivity.Ready, - } - wantStates := append(sendStates, connectivity.Shutdown) - - const testBalName = "any" - bf := stub.BalancerFuncs{ - UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error { - // Send the expected states in rapid succession. - for _, s := range sendStates { - t.Logf("Sending state update %s", s) - bd.ClientConn.UpdateState(balancer.State{ConnectivityState: s}) - } - return nil - }, - } - stub.Register(testBalName, bf) - - // Create the ClientConn. - const testResName = "any" - rb := manual.NewBuilderWithScheme(testResName) - cc, err := grpc.Dial(testResName+":///", - grpc.WithResolvers(rb), - grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, testBalName)), - grpc.WithTransportCredentials(insecure.NewCredentials()), - ) - if err != nil { - t.Fatalf("Unexpected error from grpc.Dial: %v", err) - } - - // Subscribe to state updates. Use a buffer size of 1 to allow the - // Shutdown state to go into the channel when Close()ing. - connCh := make(chan connectivity.State, 1) - s := &funcConnectivityStateSubscriber{ - onMsg: func(s connectivity.State) { - select { - case connCh <- s: - case <-ctx.Done(): - } - if s == connectivity.Shutdown { - close(connCh) - } - }, - } - - internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, s) - - // Send an update from the resolver that will trigger the LB policy's UpdateClientConnState. - go rb.UpdateState(resolver.State{}) - - // Verify the resulting states. - for i, want := range wantStates { - if i == len(sendStates) { - // Trigger Shutdown to be sent by the channel. Use a goroutine to - // ensure the operation does not block. - cc.Close() - } - select { - case got := <-connCh: - if got != want { - t.Errorf("Update %v was %s; want %s", i, got, want) - } else { - t.Logf("Update %v was %s as expected", i, got) - } - case <-ctx.Done(): - t.Fatalf("Timed out waiting for state update %v: %s", i, want) - } - } -} diff --git a/test/clientconn_state_transition_test.go b/test/clientconn_state_transition_test.go index b6e308fb85bc..a6a59454c49a 100644 --- a/test/clientconn_state_transition_test.go +++ b/test/clientconn_state_transition_test.go @@ -34,6 +34,7 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/envconfig" "google.golang.org/grpc/internal/grpcsync" "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/resolver" @@ -42,7 +43,7 @@ import ( const stateRecordingBalancerName = "state_recording_balancer" -var testBalancerBuilder = newStateRecordingBalancerBuilder(stateRecordingBalancerName, "pick_first") +var testBalancerBuilder = newStateRecordingBalancerBuilder() func init() { balancer.Register(testBalancerBuilder) @@ -143,11 +144,11 @@ client enters TRANSIENT FAILURE.`, }, } { t.Log(test.desc) - testStateTransitionSingleAddress(t, test.want, test.server, testBalancerBuilder) + testStateTransitionSingleAddress(t, test.want, test.server) } } -func testStateTransitionSingleAddress(t *testing.T, want []connectivity.State, server func(net.Listener) net.Conn, bb *stateRecordingBalancerBuilder) { +func testStateTransitionSingleAddress(t *testing.T, want []connectivity.State, server func(net.Listener) net.Conn) { pl := testutils.NewPipeListener() defer pl.Close() @@ -162,7 +163,7 @@ func testStateTransitionSingleAddress(t *testing.T, want []connectivity.State, s client, err := grpc.Dial("", grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, bb.Name())), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateRecordingBalancerName)), grpc.WithDialer(pl.Dialer()), grpc.WithConnectParams(grpc.ConnectParams{ Backoff: backoff.Config{}, @@ -177,7 +178,7 @@ func testStateTransitionSingleAddress(t *testing.T, want []connectivity.State, s defer cancel() go testutils.StayConnected(ctx, client) - stateNotifications := bb.nextStateNotifier() + stateNotifications := testBalancerBuilder.nextStateNotifier() for i := 0; i < len(want); i++ { select { case <-time.After(defaultTestTimeout): @@ -252,6 +253,15 @@ func (s) TestStateTransitions_ReadyToConnecting(t *testing.T) { connectivity.Idle, connectivity.Connecting, } + if envconfig.NewPickFirstEnabled { + want = []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Idle, + connectivity.Shutdown, // Unselected subconn will shutdown. + connectivity.Connecting, + } + } for i := 0; i < len(want); i++ { select { case <-time.After(defaultTestTimeout): @@ -323,6 +333,13 @@ func (s) TestStateTransitions_TriesAllAddrsBeforeTransientFailure(t *testing.T) client, err := grpc.Dial("whatever:///this-gets-overwritten", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateRecordingBalancerName)), + grpc.WithConnectParams(grpc.ConnectParams{ + // Set a really long back-off delay to ensure the first subConn does + // not enter ready before the second subConn connects. + Backoff: backoff.Config{ + BaseDelay: 1 * time.Hour, + }, + }), grpc.WithResolvers(rb)) if err != nil { t.Fatal(err) @@ -334,6 +351,16 @@ func (s) TestStateTransitions_TriesAllAddrsBeforeTransientFailure(t *testing.T) connectivity.Connecting, connectivity.Ready, } + if envconfig.NewPickFirstEnabled { + want = []connectivity.State{ + // The first subconn fails. + connectivity.Connecting, + connectivity.TransientFailure, + // The second subconn connects. + connectivity.Connecting, + connectivity.Ready, + } + } ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() for i := 0; i < len(want); i++ { @@ -426,6 +453,16 @@ func (s) TestStateTransitions_MultipleAddrsEntersReady(t *testing.T) { connectivity.Idle, connectivity.Connecting, } + if envconfig.NewPickFirstEnabled { + want = []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Shutdown, // The second subConn is closed once the first one connects. + connectivity.Idle, + connectivity.Shutdown, // The subConn will be closed and pickfirst will run on the latest address list. + connectivity.Connecting, + } + } for i := 0; i < len(want); i++ { select { case <-ctx.Done(): @@ -454,28 +491,17 @@ func (b *stateRecordingBalancer) Close() { b.Balancer.Close() } -func (b *stateRecordingBalancer) ExitIdle() { - if ib, ok := b.Balancer.(balancer.ExitIdler); ok { - ib.ExitIdle() - } -} - type stateRecordingBalancerBuilder struct { - mu sync.Mutex - notifier chan connectivity.State // The notifier used in the last Balancer. - balancerName string - childName string + mu sync.Mutex + notifier chan connectivity.State // The notifier used in the last Balancer. } -func newStateRecordingBalancerBuilder(balancerName, childName string) *stateRecordingBalancerBuilder { - return &stateRecordingBalancerBuilder{ - balancerName: balancerName, - childName: childName, - } +func newStateRecordingBalancerBuilder() *stateRecordingBalancerBuilder { + return &stateRecordingBalancerBuilder{} } func (b *stateRecordingBalancerBuilder) Name() string { - return b.balancerName + return stateRecordingBalancerName } func (b *stateRecordingBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { @@ -484,7 +510,7 @@ func (b *stateRecordingBalancerBuilder) Build(cc balancer.ClientConn, opts balan b.notifier = stateNotifications b.mu.Unlock() return &stateRecordingBalancer{ - Balancer: balancer.Get(b.childName).Build(&stateRecordingCCWrapper{cc, stateNotifications}, opts), + Balancer: balancer.Get("pick_first").Build(&stateRecordingCCWrapper{cc, stateNotifications}, opts), } } diff --git a/test/pickfirst_leaf_test.go b/test/pickfirst_leaf_test.go index 6267fc9ac6c0..033ee5bb0b33 100644 --- a/test/pickfirst_leaf_test.go +++ b/test/pickfirst_leaf_test.go @@ -20,33 +20,28 @@ package test import ( "context" - "errors" "fmt" - "strings" + "sync" "testing" - "time" "google.golang.org/grpc" - "google.golang.org/grpc/backoff" + "google.golang.org/grpc/balancer" pickfirstleaf "google.golang.org/grpc/balancer/pickfirst_leaf" "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/internal" - "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/internal/stubserver" - "google.golang.org/grpc/internal/testutils" - "google.golang.org/grpc/internal/testutils/pickfirst" "google.golang.org/grpc/resolver" "google.golang.org/grpc/resolver/manual" - "google.golang.org/grpc/serviceconfig" "google.golang.org/grpc/status" testgrpc "google.golang.org/grpc/interop/grpc_testing" testpb "google.golang.org/grpc/interop/grpc_testing" ) -var pickFirstLeafServiceConfig = fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, pickfirstleaf.Name) +var pickFirstLeafServiceConfig = fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, pickfirstleaf.PickFirstLeafName) + +const stateStoringBalancerName = "state_storing" // setupPickFirstLeaf performs steps required for pick_first tests. It starts a // bunch of backends exporting the TestService, creates a ClientConn to them @@ -97,814 +92,96 @@ func setupPickFirstLeaf(t *testing.T, backendCount int, opts ...grpc.DialOption) return cc, r, backends } -// TestPickFirstLeaf_OneBackend tests the most basic scenario for pick_first. It -// brings up a single backend and verifies that all RPCs get routed to it. -func (s) TestPickFirstLeaf_OneBackend(t *testing.T) { - cc, r, backends := setupPickFirstLeaf(t, 1) - - addrs := stubBackendsToResolverAddrs(backends) - r.UpdateState(resolver.State{Addresses: addrs}) - - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { - t.Fatal(err) - } -} - -// TestPickFirstLeaf_MultipleBackends tests the scenario with multiple backends and -// verifies that all RPCs get routed to the first one. -func (s) TestPickFirstLeaf_MultipleBackends(t *testing.T) { - cc, r, backends := setupPickFirstLeaf(t, 2) - - addrs := stubBackendsToResolverAddrs(backends) - r.UpdateState(resolver.State{Addresses: addrs}) - - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { - t.Fatal(err) - } -} - -// TestPickFirstLeaf_OneServerDown tests the scenario where we have multiple -// backends and pick_first is working as expected. Verifies that RPCs get routed -// to the next backend in the list when the first one goes down. -func (s) TestPickFirstLeaf_OneServerDown(t *testing.T) { - cc, r, backends := setupPickFirstLeaf(t, 2) - - addrs := stubBackendsToResolverAddrs(backends) - r.UpdateState(resolver.State{Addresses: addrs}) - - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { - t.Fatal(err) - } - - // Stop the backend which is currently being used. RPCs should get routed to - // the next backend in the list. - backends[0].Stop() - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { - t.Fatal(err) - } -} - -// TestPickFirstLeaf_AllServersDown tests the scenario where we have multiple -// backends and pick_first is working as expected. When all backends go down, -// the test verifies that RPCs fail with appropriate status code. -func (s) TestPickFirstLeaf_AllServersDown(t *testing.T) { - cc, r, backends := setupPickFirstLeaf(t, 2) - - addrs := stubBackendsToResolverAddrs(backends) - r.UpdateState(resolver.State{Addresses: addrs}) - - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { - t.Fatal(err) - } - - for _, b := range backends { - b.Stop() - } - - client := testgrpc.NewTestServiceClient(cc) - for { - if ctx.Err() != nil { - t.Fatalf("channel failed to move to Unavailable after all backends were stopped: %v", ctx.Err()) - } - if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) == codes.Unavailable { - return - } - time.Sleep(defaultTestShortTimeout) - } +type stateStoringBalancer struct { + balancer.Balancer + mu sync.Mutex + scStates []*scState + ccState connectivity.State } -// TestPickFirstLeaf_AddressesRemoved tests the scenario where we have multiple -// backends and pick_first is working as expected. It then verifies that when -// addresses are removed by the name resolver, RPCs get routed appropriately. -func (s) TestPickFirstLeaf_AddressesRemoved(t *testing.T) { - cc, r, backends := setupPickFirstLeaf(t, 3) - - addrs := stubBackendsToResolverAddrs(backends) - r.UpdateState(resolver.State{Addresses: addrs}) - - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { - t.Fatal(err) - } - - // Remove the first backend from the list of addresses originally pushed. - // RPCs should get routed to the first backend in the new list. - r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[1], addrs[2]}}) - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { - t.Fatal(err) - } - - // Append the backend that we just removed to the end of the list. - // Nothing should change. - r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[1], addrs[2], addrs[0]}}) - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { - t.Fatal(err) - } - - // Remove the first backend from the existing list of addresses. - // RPCs should get routed to the first backend in the new list. - r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[2], addrs[0]}}) - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[2]); err != nil { - t.Fatal(err) - } - - // Remove the first backend from the existing list of addresses. - // RPCs should get routed to the first backend in the new list. - r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0]}}) - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { - t.Fatal(err) - } +func (b *stateStoringBalancer) Close() { + b.Balancer.Close() } -// TestPickFirstLeaf_NewAddressWhileBlocking tests the case where pick_first is -// configured on a channel, things are working as expected and then a resolver -// updates removes all addresses. An RPC attempted at this point in time will be -// blocked because there are no valid backends. This test verifies that when new -// backends are added, the RPC is able to complete. -func (s) TestPickFirstLeaf_NewAddressWhileBlocking(t *testing.T) { - cc, r, backends := setupPickFirstLeaf(t, 2) - addrs := stubBackendsToResolverAddrs(backends) - r.UpdateState(resolver.State{Addresses: addrs}) - - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { - t.Fatal(err) - } - - // Send a resolver update with no addresses. This should push the channel into - // TransientFailure. - r.UpdateState(resolver.State{}) - testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) - - doneCh := make(chan struct{}) - client := testgrpc.NewTestServiceClient(cc) - go func() { - // The channel is currently in TransientFailure and this RPC will block - // until the channel becomes Ready, which will only happen when we push a - // resolver update with a valid backend address. - if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { - t.Errorf("EmptyCall() = %v, want ", err) - } - close(doneCh) - }() - - // Make sure that there is one pending RPC on the ClientConn before attempting - // to push new addresses through the name resolver. If we don't do this, the - // resolver update can happen before the above goroutine gets to make the RPC. - for { - if err := ctx.Err(); err != nil { - t.Fatal(err) - } - tcs, _ := channelz.GetTopChannels(0, 0) - if len(tcs) != 1 { - t.Fatalf("there should only be one top channel, not %d", len(tcs)) - } - started := tcs[0].ChannelMetrics.CallsStarted.Load() - completed := tcs[0].ChannelMetrics.CallsSucceeded.Load() + tcs[0].ChannelMetrics.CallsFailed.Load() - if (started - completed) == 1 { - break - } - time.Sleep(defaultTestShortTimeout) - } - - // Send a resolver update with a valid backend to push the channel to Ready - // and unblock the above RPC. - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: backends[0].Address}}}) - - select { - case <-ctx.Done(): - t.Fatal("Timeout when waiting for blocked RPC to complete") - case <-doneCh: +func (b *stateStoringBalancer) ExitIdle() { + if ib, ok := b.Balancer.(balancer.ExitIdler); ok { + ib.ExitIdle() } } -// TestPickFirstLeaf_StickyTransientFailure tests the case where pick_first is -// configured on a channel, and the backend is configured to close incoming -// connections as soon as they are accepted. The test verifies that the channel -// enters TransientFailure and stays there. The test also verifies that the -// pick_first LB policy is constantly trying to reconnect to the backend. -func (s) TestPickFirstLeaf_StickyTransientFailure(t *testing.T) { - // Spin up a local server which closes the connection as soon as it receives - // one. It also sends a signal on a channel whenever it received a connection. - lis, err := testutils.LocalTCPListener() - if err != nil { - t.Fatalf("Failed to create listener: %v", err) - } - t.Cleanup(func() { lis.Close() }) - - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - connCh := make(chan struct{}, 1) - go func() { - for { - conn, err := lis.Accept() - if err != nil { - return - } - select { - case connCh <- struct{}{}: - conn.Close() - case <-ctx.Done(): - return - } - } - }() - - // Dial the above server with a ConnectParams that does a constant backoff - // of defaultTestShortTimeout duration. - dopts := []grpc.DialOption{ - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithDefaultServiceConfig(pickFirstLeafServiceConfig), - grpc.WithConnectParams(grpc.ConnectParams{ - Backoff: backoff.Config{ - BaseDelay: defaultTestShortTimeout, - Multiplier: float64(0), - Jitter: float64(0), - MaxDelay: defaultTestShortTimeout, - }, - }), - } - cc, err := grpc.Dial(lis.Addr().String(), dopts...) - if err != nil { - t.Fatalf("Failed to dial server at %q: %v", lis.Addr(), err) - } - t.Cleanup(func() { cc.Close() }) - - testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) - - // Spawn a goroutine to ensure that the channel stays in TransientFailure. - // The call to cc.WaitForStateChange will return false when the main - // goroutine exits and the context is cancelled. - go func() { - if cc.WaitForStateChange(ctx, connectivity.TransientFailure) { - if state := cc.GetState(); state != connectivity.Shutdown { - t.Errorf("Unexpected state change from TransientFailure to %s", cc.GetState()) - } - } - }() - - // Ensures that the pick_first LB policy is constantly trying to reconnect. - for i := 0; i < 10; i++ { - select { - case <-connCh: - case <-time.After(2 * defaultTestShortTimeout): - t.Error("Timeout when waiting for pick_first to reconnect") - } - } +type stateStoringBalancerBuilder struct { } -// Tests the PF LB policy with shuffling enabled. -func (s) TestPickFirstLeaf_ShuffleAddressList(t *testing.T) { - const serviceConfig = `{"loadBalancingConfig": [{"pick_first_leaf":{ "shuffleAddressList": true }}]}` - - // Install a shuffler that always reverses two entries. - origShuf := internal.ShuffleAddressListForTesting - defer func() { internal.ShuffleAddressListForTesting = origShuf }() - internal.ShuffleAddressListForTesting = func(n int, f func(int, int)) { - if n != 2 { - t.Errorf("Shuffle called with n=%v; want 2", n) - return - } - f(0, 1) // reverse the two addresses - } - - // Set up our backends. - cc, r, backends := setupPickFirstLeaf(t, 2) - addrs := stubBackendsToResolverAddrs(backends) - - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - - // Push an update with both addresses and shuffling disabled. We should - // connect to backend 0. - r.UpdateState(resolver.State{Endpoints: []resolver.Endpoint{ - {Addresses: []resolver.Address{addrs[0]}}, - {Addresses: []resolver.Address{addrs[1]}}, - }}) - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { - t.Fatal(err) - } - - // Send a config with shuffling enabled. This will reverse the addresses, - // but the channel should still be connected to backend 0. - shufState := resolver.State{ - ServiceConfig: parseServiceConfig(t, r, serviceConfig), - Endpoints: []resolver.Endpoint{ - {Addresses: []resolver.Address{addrs[0]}}, - {Addresses: []resolver.Address{addrs[1]}}, - }, - } - r.UpdateState(shufState) - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { - t.Fatal(err) - } - - // Send a resolver update with no addresses. This should push the channel - // into TransientFailure. - r.UpdateState(resolver.State{}) - testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) - - // Send the same config as last time with shuffling enabled. Since we are - // not connected to backend 0, we should connect to backend 1. - r.UpdateState(shufState) - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { - t.Fatal(err) - } +func newStateStoringBalancerBuilder() *stateStoringBalancerBuilder { + return &stateStoringBalancerBuilder{} } -// Test config parsing with the env var turned on and off for various scenarios. -func (s) TestPickFirstLeaf_ParseConfig_Success(t *testing.T) { - // Install a shuffler that always reverses two entries. - origShuf := internal.ShuffleAddressListForTesting - defer func() { internal.ShuffleAddressListForTesting = origShuf }() - internal.ShuffleAddressListForTesting = func(n int, f func(int, int)) { - if n != 2 { - t.Errorf("Shuffle called with n=%v; want 2", n) - return - } - f(0, 1) // reverse the two addresses - } - - tests := []struct { - name string - serviceConfig string - wantFirstAddr bool - }{ - { - name: "empty pickfirst config", - serviceConfig: `{"loadBalancingConfig": [{"pick_first_leaf":{}}]}`, - wantFirstAddr: true, - }, - { - name: "empty good pickfirst config", - serviceConfig: `{"loadBalancingConfig": [{"pick_first_leaf":{ "shuffleAddressList": true }}]}`, - wantFirstAddr: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - // Set up our backends. - cc, r, backends := setupPickFirstLeaf(t, 2) - addrs := stubBackendsToResolverAddrs(backends) - - r.UpdateState(resolver.State{ - ServiceConfig: parseServiceConfig(t, r, test.serviceConfig), - Addresses: addrs, - }) - - // Some tests expect address shuffling to happen, and indicate that - // by setting wantFirstAddr to false (since our shuffling function - // defined at the top of this test, simply reverses the list of - // addresses provided to it). - wantAddr := addrs[0] - if !test.wantFirstAddr { - wantAddr = addrs[1] - } - - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := pickfirst.CheckRPCsToBackend(ctx, cc, wantAddr); err != nil { - t.Fatal(err) - } - }) - } +func (b *stateStoringBalancerBuilder) Name() string { + return stateStoringBalancerName } -// Test config parsing for a bad service config. -func (s) TestPickFirstLeaf_ParseConfig_Failure(t *testing.T) { - // Service config should fail with the below config. Name resolvers are - // expected to perform this parsing before they push the parsed service - // config to the channel. - const sc = `{"loadBalancingConfig": [{"pick_first_leaf":{ "shuffleAddressList": 666 }}]}` - scpr := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(sc) - if scpr.Err == nil { - t.Fatalf("ParseConfig() succeeded and returned %+v, when expected to fail", scpr) - } +func (b *stateStoringBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + bal := &stateStoringBalancer{} + bal.Balancer = balancer.Get(pickfirstleaf.PickFirstLeafName).Build(&stateStoringCCWrapper{cc, bal}, opts) + return bal } -// setupPickFirstLeafWithListenerWrapper is very similar to setupPickFirstLeaf, but uses -// a wrapped listener that the test can use to track accepted connections. -func setupPickFirstLeafWithListenerWrapper(t *testing.T, backendCount int, opts ...grpc.DialOption) (*grpc.ClientConn, *manual.Resolver, []*stubserver.StubServer, []*testutils.ListenerWrapper) { - t.Helper() - - backends := make([]*stubserver.StubServer, backendCount) - addrs := make([]resolver.Address, backendCount) - listeners := make([]*testutils.ListenerWrapper, backendCount) - for i := 0; i < backendCount; i++ { - lis := testutils.NewListenerWrapper(t, nil) - backend := &stubserver.StubServer{ - Listener: lis, - EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { - return &testpb.Empty{}, nil - }, - } - if err := backend.StartServer(); err != nil { - t.Fatalf("Failed to start backend: %v", err) - } - t.Logf("Started TestService backend at: %q", backend.Address) - t.Cleanup(func() { backend.Stop() }) - - backends[i] = backend - addrs[i] = resolver.Address{Addr: backend.Address} - listeners[i] = lis - } - - r := manual.NewBuilderWithScheme("whatever") - dopts := []grpc.DialOption{ - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithResolvers(r), - grpc.WithDefaultServiceConfig(pickFirstLeafServiceConfig), - } - dopts = append(dopts, opts...) - cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) - if err != nil { - t.Fatalf("grpc.NewClient() failed: %v", err) - } - t.Cleanup(func() { cc.Close() }) - - // At this point, the resolver has not returned any addresses to the channel. - // This RPC must block until the context expires. - sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) - defer sCancel() - client := testgrpc.NewTestServiceClient(cc) - if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("EmptyCall() = %s, want %s", status.Code(err), codes.DeadlineExceeded) +func (b *stateStoringBalancer) subConns() []scState { + b.mu.Lock() + defer b.mu.Unlock() + ret := []scState{} + for _, s := range b.scStates { + ret = append(ret, *s) } - return cc, r, backends, listeners + return ret } -// TestPickFirstLeaf_AddressUpdateWithAttributes tests the case where an address -// update received by the pick_first LB policy differs in attributes. Addresses -// which differ in attributes are considered different from the perspective of -// subconn creation and connection establishment and the test verifies that new -// connections are created when attributes change. -func (s) TestPickFirstLeaf_AddressUpdateWithAttributes(t *testing.T) { - cc, r, backends, listeners := setupPickFirstLeafWithListenerWrapper(t, 2) - - // Add a set of attributes to the addresses before pushing them to the - // pick_first LB policy through the manual resolver. - addrs := stubBackendsToResolverAddrs(backends) - for i := range addrs { - addrs[i].Attributes = addrs[i].Attributes.WithValue("test-attribute-1", fmt.Sprintf("%d", i)) - } - r.UpdateState(resolver.State{Addresses: addrs}) - - // Ensure that RPCs succeed to the first backend in the list. - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { - t.Fatal(err) - } - - // Grab the wrapped connection from the listener wrapper. This will be used - // to verify the connection is closed. - val, err := listeners[0].NewConnCh.Receive(ctx) - if err != nil { - t.Fatalf("Failed to receive new connection from wrapped listener: %v", err) - } - conn := val.(*testutils.ConnWrapper) - - // Add another set of attributes to the addresses, and push them to the - // pick_first LB policy through the manual resolver. Leave the order of the - // addresses unchanged. - for i := range addrs { - addrs[i].Attributes = addrs[i].Attributes.WithValue("test-attribute-2", fmt.Sprintf("%d", i)) - } - r.UpdateState(resolver.State{Addresses: addrs}) - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { - t.Fatal(err) - } - - // A change in the address attributes results in the new address being - // considered different to the current address. This will result in the old - // connection being closed and a new connection to the same backend (since - // address order is not modified). - if _, err := conn.CloseCh.Receive(ctx); err != nil { - t.Fatalf("Timeout when expecting existing connection to be closed: %v", err) - } - val, err = listeners[0].NewConnCh.Receive(ctx) - if err != nil { - t.Fatalf("Failed to receive new connection from wrapped listener: %v", err) - } - conn = val.(*testutils.ConnWrapper) - - // Add another set of attributes to the addresses, and push them to the - // pick_first LB policy through the manual resolver. Reverse of the order - // of addresses. - for i := range addrs { - addrs[i].Attributes = addrs[i].Attributes.WithValue("test-attribute-3", fmt.Sprintf("%d", i)) - } - addrs[0], addrs[1] = addrs[1], addrs[0] - r.UpdateState(resolver.State{Addresses: addrs}) - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { - t.Fatal(err) - } - - // Ensure that the old connection is closed and a new connection is - // established to the first address in the new list. - if _, err := conn.CloseCh.Receive(ctx); err != nil { - t.Fatalf("Timeout when expecting existing connection to be closed: %v", err) - } - _, err = listeners[1].NewConnCh.Receive(ctx) - if err != nil { - t.Fatalf("Failed to receive new connection from wrapped listener: %v", err) - } +func (b *stateStoringBalancer) setCCState(state connectivity.State) { + b.mu.Lock() + b.ccState = state + b.mu.Unlock() } -// TestPickFirstLeaf_AddressUpdateWithBalancerAttributes tests the case where an -// address update received by the pick_first LB policy differs in balancer -// attributes, which are meant only for consumption by LB policies. In this -// case, the test verifies that new connections are not created when the address -// update only changes the balancer attributes. -func (s) TestPickFirstLeaf_AddressUpdateWithBalancerAttributes(t *testing.T) { - cc, r, backends, listeners := setupPickFirstLeafWithListenerWrapper(t, 2) - - // Add a set of balancer attributes to the addresses before pushing them to - // the pick_first LB policy through the manual resolver. - addrs := stubBackendsToResolverAddrs(backends) - for i := range addrs { - addrs[i].BalancerAttributes = addrs[i].BalancerAttributes.WithValue("test-attribute-1", fmt.Sprintf("%d", i)) - } - r.UpdateState(resolver.State{Addresses: addrs}) - - // Ensure that RPCs succeed to the expected backend. - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { - t.Fatal(err) - } - - // Grab the wrapped connection from the listener wrapper. This will be used - // to verify the connection is not closed. - val, err := listeners[0].NewConnCh.Receive(ctx) - if err != nil { - t.Fatalf("Failed to receive new connection from wrapped listener: %v", err) - } - conn := val.(*testutils.ConnWrapper) - - // Add a set of balancer attributes to the addresses before pushing them to - // the pick_first LB policy through the manual resolver. Leave the order of - // the addresses unchanged. - for i := range addrs { - addrs[i].BalancerAttributes = addrs[i].BalancerAttributes.WithValue("test-attribute-2", fmt.Sprintf("%d", i)) - } - r.UpdateState(resolver.State{Addresses: addrs}) - - // Ensure that no new connection is established, and ensure that the old - // connection is not closed. - for i := range listeners { - sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) - defer sCancel() - if _, err := listeners[i].NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { - t.Fatalf("Unexpected error when expecting no new connection: %v", err) - } - } - sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) - defer sCancel() - if _, err := conn.CloseCh.Receive(sCtx); err != context.DeadlineExceeded { - t.Fatalf("Unexpected error when expecting existing connection to stay active: %v", err) - } - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { - t.Fatal(err) - } - - // Add a set of balancer attributes to the addresses before pushing them to - // the pick_first LB policy through the manual resolver. Reverse of the - // order of addresses. - for i := range addrs { - addrs[i].BalancerAttributes = addrs[i].BalancerAttributes.WithValue("test-attribute-3", fmt.Sprintf("%d", i)) - } - addrs[0], addrs[1] = addrs[1], addrs[0] - r.UpdateState(resolver.State{Addresses: addrs}) - - // Ensure that no new connection is established, and ensure that the old - // connection is not closed. - for i := range listeners { - sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) - defer sCancel() - if _, err := listeners[i].NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { - t.Fatalf("Unexpected error when expecting no new connection: %v", err) - } - } - sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) - defer sCancel() - if _, err := conn.CloseCh.Receive(sCtx); err != context.DeadlineExceeded { - t.Fatalf("Unexpected error when expecting existing connection to stay active: %v", err) - } - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { - t.Fatal(err) - } +func (b *stateStoringBalancer) addScState(state *scState) { + b.mu.Lock() + b.scStates = append(b.scStates, state) + b.mu.Unlock() } -// Tests the case where the pick_first LB policy receives an error from the name -// resolver without previously receiving a good update. Verifies that the -// channel moves to TRANSIENT_FAILURE and that error received from the name -// resolver is propagated to the caller of an RPC. -func (s) TestPickFirstLeaf_ResolverError_NoPreviousUpdate(t *testing.T) { - cc, r, _ := setupPickFirstLeaf(t, 0) - - nrErr := errors.New("error from name resolver") - r.ReportError(nrErr) - - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) - - client := testgrpc.NewTestServiceClient(cc) - _, err := client.EmptyCall(ctx, &testpb.Empty{}) - if err == nil { - t.Fatalf("EmptyCall() succeeded when expected to fail with error: %v", nrErr) - } - if !strings.Contains(err.Error(), nrErr.Error()) { - t.Fatalf("EmptyCall() failed with error: %v, want error: %v", err, nrErr) - } +func (b *stateStoringBalancer) curCCState() connectivity.State { + b.mu.Lock() + ret := b.ccState + b.mu.Unlock() + return ret } -// Tests the case where the pick_first LB policy receives an error from the name -// resolver after receiving a good update (and the channel is currently READY). -// The test verifies that the channel continues to use the previously received -// good update. -func (s) TestPickFirstLeaf_ResolverError_WithPreviousUpdate_Ready(t *testing.T) { - cc, r, backends := setupPickFirstLeaf(t, 1) - - addrs := stubBackendsToResolverAddrs(backends) - r.UpdateState(resolver.State{Addresses: addrs}) - - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { - t.Fatal(err) - } - - nrErr := errors.New("error from name resolver") - r.ReportError(nrErr) - - // Ensure that RPCs continue to succeed for the next second. - client := testgrpc.NewTestServiceClient(cc) - for end := time.Now().Add(time.Second); time.Now().Before(end); <-time.After(defaultTestShortTimeout) { - if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { - t.Fatalf("EmptyCall() failed: %v", err) - } - } +type stateStoringCCWrapper struct { + balancer.ClientConn + b *stateStoringBalancer } -// Tests the case where the pick_first LB policy receives an error from the name -// resolver after receiving a good update (and the channel is currently in -// CONNECTING state). The test verifies that the channel continues to use the -// previously received good update, and that RPCs don't fail with the error -// received from the name resolver. -func (s) TestPickFirstLeaf_ResolverError_WithPreviousUpdate_Connecting(t *testing.T) { - lis, err := testutils.LocalTCPListener() - if err != nil { - t.Fatalf("net.Listen() failed: %v", err) - } - - // Listen on a local port and act like a server that blocks until the - // channel reaches CONNECTING and closes the connection without sending a - // server preface. - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - waitForConnecting := make(chan struct{}) - go func() { - conn, err := lis.Accept() - if err != nil { - t.Errorf("Unexpected error when accepting a connection: %v", err) - } - defer conn.Close() - - select { - case <-waitForConnecting: - case <-ctx.Done(): - t.Error("Timeout when waiting for channel to move to CONNECTING state") - } - }() - - r := manual.NewBuilderWithScheme("whatever") - dopts := []grpc.DialOption{ - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithResolvers(r), - grpc.WithDefaultServiceConfig(pickFirstLeafServiceConfig), +func (ccw *stateStoringCCWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + oldListener := opts.StateListener + scs := &scState{ + state: connectivity.Idle, + addrs: addrs, } - cc, err := grpc.Dial(r.Scheme()+":///test.server", dopts...) - if err != nil { - t.Fatalf("grpc.Dial() failed: %v", err) - } - t.Cleanup(func() { cc.Close() }) - - addrs := []resolver.Address{{Addr: lis.Addr().String()}} - r.UpdateState(resolver.State{Addresses: addrs}) - testutils.AwaitState(ctx, t, cc, connectivity.Connecting) - - nrErr := errors.New("error from name resolver") - r.ReportError(nrErr) - - // RPCs should fail with deadline exceed error as long as they are in - // CONNECTING and not the error returned by the name resolver. - client := testgrpc.NewTestServiceClient(cc) - sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) - defer sCancel() - if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); !strings.Contains(err.Error(), context.DeadlineExceeded.Error()) { - t.Fatalf("EmptyCall() failed with error: %v, want error: %v", err, context.DeadlineExceeded) + ccw.b.addScState(scs) + opts.StateListener = func(s balancer.SubConnState) { + ccw.b.mu.Lock() + scs.state = s.ConnectivityState + ccw.b.mu.Unlock() + oldListener(s) } - - // Closing this channel leads to closing of the connection by our listener. - // gRPC should see this as a connection error. - close(waitForConnecting) - testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) - checkForConnectionError(ctx, t, cc) + return ccw.ClientConn.NewSubConn(addrs, opts) } -// Tests the case where the pick_first LB policy receives an error from the name -// resolver after receiving a good update. The previous good update though has -// seen the channel move to TRANSIENT_FAILURE. The test verifies that the -// channel fails RPCs with the new error from the resolver. -func (s) TestPickFirstLeaf_ResolverError_WithPreviousUpdate_TransientFailure(t *testing.T) { - lis, err := testutils.LocalTCPListener() - if err != nil { - t.Fatalf("net.Listen() failed: %v", err) - } - - // Listen on a local port and act like a server that closes the connection - // without sending a server preface. - go func() { - conn, err := lis.Accept() - if err != nil { - t.Errorf("Unexpected error when accepting a connection: %v", err) - } - conn.Close() - }() - - r := manual.NewBuilderWithScheme("whatever") - dopts := []grpc.DialOption{ - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithResolvers(r), - grpc.WithDefaultServiceConfig(pickFirstLeafServiceConfig), - } - cc, err := grpc.Dial(r.Scheme()+":///test.server", dopts...) - if err != nil { - t.Fatalf("grpc.Dial() failed: %v", err) - } - t.Cleanup(func() { cc.Close() }) - - addrs := []resolver.Address{{Addr: lis.Addr().String()}} - r.UpdateState(resolver.State{Addresses: addrs}) - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) - checkForConnectionError(ctx, t, cc) - - // An error from the name resolver should result in RPCs failing with that - // error instead of the old error that caused the channel to move to - // TRANSIENT_FAILURE in the first place. - nrErr := errors.New("error from name resolver") - r.ReportError(nrErr) - client := testgrpc.NewTestServiceClient(cc) - for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { - if _, err := client.EmptyCall(ctx, &testpb.Empty{}); strings.Contains(err.Error(), nrErr.Error()) { - break - } - } - if ctx.Err() != nil { - t.Fatal("Timeout when waiting for RPCs to fail with error returned by the name resolver") - } +func (ccw *stateStoringCCWrapper) UpdateState(state balancer.State) { + ccw.b.setCCState(state.ConnectivityState) + ccw.ClientConn.UpdateState(state) } -// Tests the case where the pick_first LB policy receives an update from the -// name resolver with no addresses after receiving a good update. The test -// verifies that the channel fails RPCs with an error indicating the fact that -// the name resolver returned no addresses. -func (s) TestPickFirstLeaf_ResolverError_ZeroAddresses_WithPreviousUpdate(t *testing.T) { - cc, r, backends := setupPickFirstLeaf(t, 1) - - addrs := stubBackendsToResolverAddrs(backends) - r.UpdateState(resolver.State{Addresses: addrs}) - - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { - t.Fatal(err) - } - - r.UpdateState(resolver.State{}) - wantErr := "produced zero addresses" - client := testgrpc.NewTestServiceClient(cc) - for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { - if _, err := client.EmptyCall(ctx, &testpb.Empty{}); strings.Contains(err.Error(), wantErr) { - break - } - } - if ctx.Err() != nil { - t.Fatal("Timeout when waiting for RPCs to fail with error returned by the name resolver") - } +type scState struct { + state connectivity.State + addrs []resolver.Address } From 56bfb59d0ba9637d6107efe5b1b11e25f7ee13fb Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Fri, 9 Aug 2024 11:37:43 +0530 Subject: [PATCH 15/62] Fix test failures --- balancer/pickfirst_leaf/pickfirst_leaf.go | 11 ++++++----- balancer/rls/balancer_test.go | 3 +++ internal/balancergroup/balancergroup_test.go | 4 ++++ test/balancer_switching_test.go | 3 +++ test/balancer_test.go | 3 +++ test/clientconn_state_transition_test.go | 19 ------------------- test/resolver_update_test.go | 3 +++ 7 files changed, 22 insertions(+), 24 deletions(-) diff --git a/balancer/pickfirst_leaf/pickfirst_leaf.go b/balancer/pickfirst_leaf/pickfirst_leaf.go index d31635e17388..74a412d07672 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf.go +++ b/balancer/pickfirst_leaf/pickfirst_leaf.go @@ -68,7 +68,6 @@ type pickfirstBuilder struct { } func (pickfirstBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer { - fmt.Printf("Building a pf balancer\n") ctx, cancel := context.WithCancel(context.Background()) b := &pickfirstBalancer{ cc: cc, @@ -229,8 +228,6 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState // Since we have a new set of addresses, we are again at first pass b.firstPass = true - b.firstErr = nil - newEndpoints = deDupAddresses(newEndpoints) // Perform the optional shuffling described in gRFC A62. The shuffling will @@ -286,6 +283,7 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState } if oldAddrs.Len() == 0 || b.state == connectivity.Ready || b.state == connectivity.Connecting { + b.firstErr = nil // Start connection attempt at first address. b.state = connectivity.Connecting b.cc.UpdateState(balancer.State{ @@ -294,6 +292,7 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState }) b.requestConnection() } else if b.state == connectivity.Idle { + b.firstErr = nil b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.Idle, Picker: &idlePicker{ @@ -313,7 +312,6 @@ func (b *pickfirstBalancer) UpdateSubConnState(subConn balancer.SubConn, state b } func (b *pickfirstBalancer) Close() { - fmt.Printf("Arjan: Close called\n") completion := make(chan struct{}) b.serializer.ScheduleOr(func(ctx context.Context) { b.close(completion) @@ -322,7 +320,6 @@ func (b *pickfirstBalancer) Close() { }) <-completion <-b.serializer.Done() - fmt.Printf("Arjan: serializer Close called\n") } func (b *pickfirstBalancer) close(completion chan struct{}) { @@ -514,6 +511,10 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon // We have finished the first pass, keep re-connecting failing subconns. switch state.ConnectivityState { case connectivity.TransientFailure: + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: state.ConnectionError}, + }) b.numTf++ // We request re-resolution when we've seen the same number of TFs as // subconns. It could be that a subconn has seen multiple TFs due to diff --git a/balancer/rls/balancer_test.go b/balancer/rls/balancer_test.go index 6930e3da14f5..625bc8a2b109 100644 --- a/balancer/rls/balancer_test.go +++ b/balancer/rls/balancer_test.go @@ -923,6 +923,9 @@ func (s) TestUpdateStatePauses(t *testing.T) { Init: func(bd *stub.BalancerData) { bd.Data = balancer.Get(pickfirst.Name).Build(bd.ClientConn, bd.BuildOptions) }, + Close: func(bd *stub.BalancerData) { + bd.Data.(balancer.Balancer).Close() + }, ParseConfig: func(sc json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { cfg := &childPolicyConfig{} if err := json.Unmarshal(sc, cfg); err != nil { diff --git a/internal/balancergroup/balancergroup_test.go b/internal/balancergroup/balancergroup_test.go index 9de47f54504f..8874b1832359 100644 --- a/internal/balancergroup/balancergroup_test.go +++ b/internal/balancergroup/balancergroup_test.go @@ -575,6 +575,7 @@ func (s) TestBalancerGracefulSwitch(t *testing.T) { bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}}) bg.Start() + defer bg.Close() m1 := make(map[resolver.Address]balancer.SubConn) scs := make(map[balancer.SubConn]bool) @@ -604,6 +605,9 @@ func (s) TestBalancerGracefulSwitch(t *testing.T) { Init: func(bd *stub.BalancerData) { bd.Data = balancer.Get(pickfirst.Name).Build(bd.ClientConn, bd.BuildOptions) }, + Close: func(bd *stub.BalancerData) { + bd.Data.(balancer.Balancer).Close() + }, UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { ccs.ResolverState.Addresses = ccs.ResolverState.Addresses[1:] bal := bd.Data.(balancer.Balancer) diff --git a/test/balancer_switching_test.go b/test/balancer_switching_test.go index 34fd871b65c7..23e561eeb5f9 100644 --- a/test/balancer_switching_test.go +++ b/test/balancer_switching_test.go @@ -472,6 +472,9 @@ func (s) TestBalancerSwitch_Graceful(t *testing.T) { pf := balancer.Get(pickfirst.Name) bd.Data = pf.Build(bd.ClientConn, bd.BuildOptions) }, + Close: func(bd *stub.BalancerData) { + bd.Data.(balancer.Balancer).Close() + }, UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { bal := bd.Data.(balancer.Balancer) close(ccUpdateCh) diff --git a/test/balancer_test.go b/test/balancer_test.go index de7ab5557e80..6f90bc6bdbde 100644 --- a/test/balancer_test.go +++ b/test/balancer_test.go @@ -850,6 +850,9 @@ func (s) TestMetadataInPickResult(t *testing.T) { cc := &testCCWrapper{ClientConn: bd.ClientConn} bd.Data = balancer.Get(pickfirst.Name).Build(cc, bd.BuildOptions) }, + Close: func(bd *stub.BalancerData) { + bd.Data.(balancer.Balancer).Close() + }, UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { bal := bd.Data.(balancer.Balancer) return bal.UpdateClientConnState(ccs) diff --git a/test/clientconn_state_transition_test.go b/test/clientconn_state_transition_test.go index a6a59454c49a..89e40d9d124c 100644 --- a/test/clientconn_state_transition_test.go +++ b/test/clientconn_state_transition_test.go @@ -253,15 +253,6 @@ func (s) TestStateTransitions_ReadyToConnecting(t *testing.T) { connectivity.Idle, connectivity.Connecting, } - if envconfig.NewPickFirstEnabled { - want = []connectivity.State{ - connectivity.Connecting, - connectivity.Ready, - connectivity.Idle, - connectivity.Shutdown, // Unselected subconn will shutdown. - connectivity.Connecting, - } - } for i := 0; i < len(want); i++ { select { case <-time.After(defaultTestTimeout): @@ -453,16 +444,6 @@ func (s) TestStateTransitions_MultipleAddrsEntersReady(t *testing.T) { connectivity.Idle, connectivity.Connecting, } - if envconfig.NewPickFirstEnabled { - want = []connectivity.State{ - connectivity.Connecting, - connectivity.Ready, - connectivity.Shutdown, // The second subConn is closed once the first one connects. - connectivity.Idle, - connectivity.Shutdown, // The subConn will be closed and pickfirst will run on the latest address list. - connectivity.Connecting, - } - } for i := 0; i < len(want); i++ { select { case <-ctx.Done(): diff --git a/test/resolver_update_test.go b/test/resolver_update_test.go index 103c732f61af..2eec54491c28 100644 --- a/test/resolver_update_test.go +++ b/test/resolver_update_test.go @@ -162,6 +162,9 @@ func (s) TestResolverUpdate_InvalidServiceConfigAfterGoodUpdate(t *testing.T) { pf := balancer.Get(pickfirst.Name) bd.Data = pf.Build(bd.ClientConn, bd.BuildOptions) }, + Close: func(bd *stub.BalancerData) { + bd.Data.(balancer.Balancer).Close() + }, ParseConfig: func(lbCfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { cfg := &wrappingBalancerConfig{} if err := json.Unmarshal(lbCfg, cfg); err != nil { From 17c63d2310fb2502d6c6fe8331115f30737b486d Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Fri, 9 Aug 2024 13:26:32 +0530 Subject: [PATCH 16/62] Enable test using new pf --- .github/workflows/testing.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 67c35dc5dd97..1b86be37ece8 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -66,6 +66,11 @@ jobs: - type: tests goversion: '1.21' + - type: tests + goversion: '1.22' + testflags: -race + grpcenv: 'GRPC_EXPERIMENTAL_ENABLE_NEW_PICK_FIRST=true' + steps: # Setup the environment. - name: Setup GOARCH From f1daad1d1bf35f121ebebda5dc0bfeb09af771f7 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Fri, 9 Aug 2024 13:34:05 +0530 Subject: [PATCH 17/62] Fix tests --- balancer/rls/balancer_test.go | 6 +++--- xds/internal/balancer/clustermanager/clustermanager_test.go | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/balancer/rls/balancer_test.go b/balancer/rls/balancer_test.go index 625bc8a2b109..4184e68831d2 100644 --- a/balancer/rls/balancer_test.go +++ b/balancer/rls/balancer_test.go @@ -923,9 +923,9 @@ func (s) TestUpdateStatePauses(t *testing.T) { Init: func(bd *stub.BalancerData) { bd.Data = balancer.Get(pickfirst.Name).Build(bd.ClientConn, bd.BuildOptions) }, - Close: func(bd *stub.BalancerData) { - bd.Data.(balancer.Balancer).Close() - }, + Close: func(bd *stub.BalancerData) { + bd.Data.(balancer.Balancer).Close() + }, ParseConfig: func(sc json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { cfg := &childPolicyConfig{} if err := json.Unmarshal(sc, cfg); err != nil { diff --git a/xds/internal/balancer/clustermanager/clustermanager_test.go b/xds/internal/balancer/clustermanager/clustermanager_test.go index 1b3fa954b86f..c1df6dcd8aad 100644 --- a/xds/internal/balancer/clustermanager/clustermanager_test.go +++ b/xds/internal/balancer/clustermanager/clustermanager_test.go @@ -607,6 +607,7 @@ func TestClusterGracefulSwitch(t *testing.T) { builder := balancer.Get(balancerName) parser := builder.(balancer.ConfigParser) bal := builder.Build(cc, balancer.BuildOptions{}) + defer bal.Close() configJSON1 := `{ "children": { @@ -644,6 +645,9 @@ func TestClusterGracefulSwitch(t *testing.T) { Init: func(bd *stub.BalancerData) { bd.Data = balancer.Get(pickfirst.Name).Build(bd.ClientConn, bd.BuildOptions) }, + Close: func(bd *stub.BalancerData) { + bd.Data.(balancer.Balancer).Close() + }, UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { bal := bd.Data.(balancer.Balancer) return bal.UpdateClientConnState(ccs) @@ -730,6 +734,7 @@ func (s) TestUpdateStatePauses(t *testing.T) { builder := balancer.Get(balancerName) parser := builder.(balancer.ConfigParser) bal := builder.Build(cc, balancer.BuildOptions{}) + defer bal.Close() configJSON1 := `{ "children": { From 747bd8a50a41b2ee2a6bd55154226f5a32804d9c Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Fri, 9 Aug 2024 18:51:22 +0530 Subject: [PATCH 18/62] Add unit tests --- test/pickfirst_leaf_test.go | 243 ++++++++++++++++++++++++++++++++---- 1 file changed, 218 insertions(+), 25 deletions(-) diff --git a/test/pickfirst_leaf_test.go b/test/pickfirst_leaf_test.go index 033ee5bb0b33..4c2627697447 100644 --- a/test/pickfirst_leaf_test.go +++ b/test/pickfirst_leaf_test.go @@ -31,6 +31,7 @@ import ( "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils/pickfirst" "google.golang.org/grpc/resolver" "google.golang.org/grpc/resolver/manual" "google.golang.org/grpc/status" @@ -39,7 +40,7 @@ import ( testpb "google.golang.org/grpc/interop/grpc_testing" ) -var pickFirstLeafServiceConfig = fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, pickfirstleaf.PickFirstLeafName) +var stateStoringServiceConfig = fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateStoringBalancerName) const stateStoringBalancerName = "state_storing" @@ -72,7 +73,7 @@ func setupPickFirstLeaf(t *testing.T, backendCount int, opts ...grpc.DialOption) dopts := []grpc.DialOption{ grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r), - grpc.WithDefaultServiceConfig(pickFirstLeafServiceConfig), + grpc.WithDefaultServiceConfig(stateStoringServiceConfig), } dopts = append(dopts, opts...) cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) @@ -92,11 +93,223 @@ func setupPickFirstLeaf(t *testing.T, backendCount int, opts ...grpc.DialOption) return cc, r, backends } +// TestPickFirstLeaf_ResolverUpdate tests the behaviour of the new pick first +// policy when servers are brought down and resolver updates are received. +func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + balChan := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancerChan: balChan}) + tests := []struct { + name string + backendCount int + initialBackendIndexes []int + initialTargetBackendIndex int + wantScStates []connectivity.State + updatedBackendIndexes []int + updatedTargetBackendIndex int + wantScStatesPostUpdate []connectivity.State + restartConnected bool + }{ + { + name: "two_server_first_ready", + backendCount: 2, + initialBackendIndexes: []int{0, 1}, + initialTargetBackendIndex: 0, + wantScStates: []connectivity.State{connectivity.Ready}, + }, + { + name: "two_server_second_ready", + backendCount: 2, + initialBackendIndexes: []int{0, 1}, + initialTargetBackendIndex: 1, + wantScStates: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, + }, + { + name: "duplicate_address", + backendCount: 2, + initialBackendIndexes: []int{0, 0, 1}, + initialTargetBackendIndex: 1, + wantScStates: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, + }, + { + name: "disjoint_updated_addresses", + backendCount: 4, + initialBackendIndexes: []int{0, 1}, + initialTargetBackendIndex: 1, + wantScStates: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, + updatedBackendIndexes: []int{2, 3}, + updatedTargetBackendIndex: 3, + wantScStatesPostUpdate: []connectivity.State{connectivity.Shutdown, connectivity.Shutdown, connectivity.Shutdown, connectivity.Ready}, + }, + { + name: "active_backend_in_updated_list", + backendCount: 3, + initialBackendIndexes: []int{0, 1}, + initialTargetBackendIndex: 1, + wantScStates: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, + updatedBackendIndexes: []int{1, 2}, + updatedTargetBackendIndex: 1, + wantScStatesPostUpdate: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, + }, + { + name: "inactive_backend_in_updated_list", + backendCount: 3, + initialBackendIndexes: []int{0, 1}, + initialTargetBackendIndex: 1, + wantScStates: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, + updatedBackendIndexes: []int{0, 2}, + updatedTargetBackendIndex: 0, + wantScStatesPostUpdate: []connectivity.State{connectivity.Shutdown, connectivity.Shutdown, connectivity.Ready}, + }, + { + name: "identical_list", + backendCount: 2, + initialBackendIndexes: []int{0, 1}, + initialTargetBackendIndex: 1, + wantScStates: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, + updatedBackendIndexes: []int{0, 1}, + updatedTargetBackendIndex: 1, + wantScStatesPostUpdate: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, + }, + { + name: "first_connected_idle_reconnect", + backendCount: 2, + initialBackendIndexes: []int{0, 1}, + initialTargetBackendIndex: 0, + restartConnected: true, + wantScStates: []connectivity.State{connectivity.Ready}, + updatedBackendIndexes: []int{0, 1}, + updatedTargetBackendIndex: 0, + wantScStatesPostUpdate: []connectivity.State{connectivity.Ready}, + }, + { + name: "second_connected_idle_reconnect", + backendCount: 2, + initialBackendIndexes: []int{0, 1}, + initialTargetBackendIndex: 1, + restartConnected: true, + wantScStates: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, + updatedBackendIndexes: []int{0, 1}, + updatedTargetBackendIndex: 1, + wantScStatesPostUpdate: []connectivity.State{connectivity.Shutdown, connectivity.Ready, connectivity.Shutdown}, + }, + { + name: "second_connected_idle_reconnect_first", + backendCount: 2, + initialBackendIndexes: []int{0, 1}, + initialTargetBackendIndex: 1, + restartConnected: true, + wantScStates: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, + updatedBackendIndexes: []int{0, 1}, + updatedTargetBackendIndex: 0, + wantScStatesPostUpdate: []connectivity.State{connectivity.Shutdown, connectivity.Shutdown, connectivity.Ready}, + }, + { + name: "first_connected_idle_reconnect_second", + backendCount: 2, + initialBackendIndexes: []int{0, 1}, + initialTargetBackendIndex: 0, + restartConnected: true, + wantScStates: []connectivity.State{connectivity.Ready}, + updatedBackendIndexes: []int{0, 1}, + updatedTargetBackendIndex: 1, + wantScStatesPostUpdate: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cc, r, backends := setupPickFirstLeaf(t, tc.backendCount) + + activeBackends := []*stubserver.StubServer{} + for _, idx := range tc.initialBackendIndexes { + activeBackends = append(activeBackends, backends[idx]) + } + addrs := stubBackendsToResolverAddrs(activeBackends) + r.UpdateState(resolver.State{Addresses: addrs}) + + // shutdown all active backends except the target. + var targetAddr resolver.Address + for idx_i, idx := range tc.initialBackendIndexes { + if idx == tc.initialTargetBackendIndex { + targetAddr = addrs[idx_i] + continue + } + backends[idx].S.Stop() + } + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, targetAddr); err != nil { + t.Fatal(err) + } + bal := <-balChan + scs := bal.scStates + + if got, want := len(scs), len(tc.wantScStates); got != want { + t.Fatalf("len(subconns) = %d, want %d", got, want) + } + + for idx := range scs { + if got, want := scs[idx].state, tc.wantScStates[idx]; got != want { + t.Errorf("subconn[%d].state = %v, want = %v", idx, got, want) + } + } + + if len(tc.updatedBackendIndexes) == 0 { + return + } + + // Restart all the backends. + for i, s := range backends { + if !tc.restartConnected && i == tc.initialTargetBackendIndex { + continue + } + s.S.Stop() + if err := s.StartServer(); err != nil { + t.Fatalf("Failed to re-start test backend: %v", err) + } + } + + activeBackends = []*stubserver.StubServer{} + for _, idx := range tc.updatedBackendIndexes { + activeBackends = append(activeBackends, backends[idx]) + } + addrs = stubBackendsToResolverAddrs(activeBackends) + r.UpdateState(resolver.State{Addresses: addrs}) + + // shutdown all active backends except the target. + for idx_i, idx := range tc.updatedBackendIndexes { + if idx == tc.updatedTargetBackendIndex { + targetAddr = addrs[idx_i] + continue + } + backends[idx].S.Stop() + } + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, targetAddr); err != nil { + t.Fatal(err) + } + scs = bal.scStates + + if got, want := len(scs), len(tc.wantScStatesPostUpdate); got != want { + t.Fatalf("len(subconns) = %d, want %d", got, want) + } + + for idx := range scs { + if got, want := scs[idx].state, tc.wantScStatesPostUpdate[idx]; got != want { + t.Errorf("subconn[%d].state = %v, want = %v", idx, got, want) + } + } + + }) + } +} + +// stateStoringBalancer stores the state of the subconns being created. type stateStoringBalancer struct { balancer.Balancer mu sync.Mutex scStates []*scState - ccState connectivity.State } func (b *stateStoringBalancer) Close() { @@ -110,10 +323,7 @@ func (b *stateStoringBalancer) ExitIdle() { } type stateStoringBalancerBuilder struct { -} - -func newStateStoringBalancerBuilder() *stateStoringBalancerBuilder { - return &stateStoringBalancerBuilder{} + balancerChan chan *stateStoringBalancer } func (b *stateStoringBalancerBuilder) Name() string { @@ -123,6 +333,7 @@ func (b *stateStoringBalancerBuilder) Name() string { func (b *stateStoringBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { bal := &stateStoringBalancer{} bal.Balancer = balancer.Get(pickfirstleaf.PickFirstLeafName).Build(&stateStoringCCWrapper{cc, bal}, opts) + b.balancerChan <- bal return bal } @@ -136,25 +347,12 @@ func (b *stateStoringBalancer) subConns() []scState { return ret } -func (b *stateStoringBalancer) setCCState(state connectivity.State) { - b.mu.Lock() - b.ccState = state - b.mu.Unlock() -} - func (b *stateStoringBalancer) addScState(state *scState) { b.mu.Lock() b.scStates = append(b.scStates, state) b.mu.Unlock() } -func (b *stateStoringBalancer) curCCState() connectivity.State { - b.mu.Lock() - ret := b.ccState - b.mu.Unlock() - return ret -} - type stateStoringCCWrapper struct { balancer.ClientConn b *stateStoringBalancer @@ -176,11 +374,6 @@ func (ccw *stateStoringCCWrapper) NewSubConn(addrs []resolver.Address, opts bala return ccw.ClientConn.NewSubConn(addrs, opts) } -func (ccw *stateStoringCCWrapper) UpdateState(state balancer.State) { - ccw.b.setCCState(state.ConnectivityState) - ccw.ClientConn.UpdateState(state) -} - type scState struct { state connectivity.State addrs []resolver.Address From 84194db5007fdd8d074f3968851a4a4a418e2d9d Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Fri, 9 Aug 2024 19:04:26 +0530 Subject: [PATCH 19/62] Fix vet --- balancer/pickfirst_leaf/pickfirst_leaf.go | 10 ---------- test/pickfirst_leaf_test.go | 12 ++++++------ 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/balancer/pickfirst_leaf/pickfirst_leaf.go b/balancer/pickfirst_leaf/pickfirst_leaf.go index 74a412d07672..990e20a9fe66 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf.go +++ b/balancer/pickfirst_leaf/pickfirst_leaf.go @@ -39,12 +39,6 @@ import ( "google.golang.org/grpc/serviceconfig" ) -const ( - subConnListConnecting uint32 = iota - subConnListConnected - subConnListClosed -) - func init() { balancer.Register(pickfirstBuilder{name: PickFirstLeafName}) if envconfig.NewPickFirstEnabled { @@ -634,7 +628,3 @@ func (i *index) seekTo(needle *resolver.Address) bool { } return false } - -func (i *index) size() int { - return len(i.endpointList) -} diff --git a/test/pickfirst_leaf_test.go b/test/pickfirst_leaf_test.go index 4c2627697447..5c12a2216c4b 100644 --- a/test/pickfirst_leaf_test.go +++ b/test/pickfirst_leaf_test.go @@ -231,9 +231,9 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { // shutdown all active backends except the target. var targetAddr resolver.Address - for idx_i, idx := range tc.initialBackendIndexes { + for idxI, idx := range tc.initialBackendIndexes { if idx == tc.initialTargetBackendIndex { - targetAddr = addrs[idx_i] + targetAddr = addrs[idxI] continue } backends[idx].S.Stop() @@ -243,7 +243,7 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { t.Fatal(err) } bal := <-balChan - scs := bal.scStates + scs := bal.subConns() if got, want := len(scs), len(tc.wantScStates); got != want { t.Fatalf("len(subconns) = %d, want %d", got, want) @@ -278,9 +278,9 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { r.UpdateState(resolver.State{Addresses: addrs}) // shutdown all active backends except the target. - for idx_i, idx := range tc.updatedBackendIndexes { + for idxI, idx := range tc.updatedBackendIndexes { if idx == tc.updatedTargetBackendIndex { - targetAddr = addrs[idx_i] + targetAddr = addrs[idxI] continue } backends[idx].S.Stop() @@ -289,7 +289,7 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { if err := pickfirst.CheckRPCsToBackend(ctx, cc, targetAddr); err != nil { t.Fatal(err) } - scs = bal.scStates + scs = bal.subConns() if got, want := len(scs), len(tc.wantScStatesPostUpdate); got != want { t.Fatalf("len(subconns) = %d, want %d", got, want) From 586b091f10df7903ce298da910e78ff9ef1893d6 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Fri, 9 Aug 2024 22:01:54 +0530 Subject: [PATCH 20/62] Calculate coverage using new pickfirst also --- .github/workflows/coverage.yml | 3 +++ .github/workflows/testing.yml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 970a48ff2cc4..ef832ed8cbf0 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -19,6 +19,9 @@ jobs: - name: Run coverage run: go test -coverprofile=coverage.out -coverpkg=./... ./... + - name: Run coverage with new pickfirst + run: GRPC_EXPERIMENTAL_ENABLE_NEW_PICK_FIRST=true go test -coverprofile=coverage_new_pickfirst.out -coverpkg=./... ./... + - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 with: diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 1b86be37ece8..64b51470defd 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -86,7 +86,7 @@ jobs: - name: Setup GRPC environment if: matrix.grpcenv != '' run: echo "${{ matrix.grpcenv }}" >> $GITHUB_ENV - + - name: Checkout repo uses: actions/checkout@v4 From 31e8a1025bd7ae76767f9eb6ae94f202692a44e7 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Fri, 9 Aug 2024 22:40:57 +0530 Subject: [PATCH 21/62] Fix lint error --- balancer/pickfirst/pickfirst.go | 2 +- balancer/pickfirst_leaf/pickfirst_leaf.go | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/balancer/pickfirst/pickfirst.go b/balancer/pickfirst/pickfirst.go index b229e92c2e90..a3ea329ee6fc 100644 --- a/balancer/pickfirst/pickfirst.go +++ b/balancer/pickfirst/pickfirst.go @@ -26,7 +26,7 @@ import ( "math/rand" "google.golang.org/grpc/balancer" - _ "google.golang.org/grpc/balancer/pickfirst_leaf" + _ "google.golang.org/grpc/balancer/pickfirst_leaf" // For automatically registering the new pickfirst if required. "google.golang.org/grpc/connectivity" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/internal" diff --git a/balancer/pickfirst_leaf/pickfirst_leaf.go b/balancer/pickfirst_leaf/pickfirst_leaf.go index 990e20a9fe66..3ae14b3eb667 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf.go +++ b/balancer/pickfirst_leaf/pickfirst_leaf.go @@ -51,10 +51,11 @@ func init() { var logger = grpclog.Component("pick-first-leaf-lb") const ( - // PickFirstLeafName is the name of the pick_first balancer. + // PickFirstLeafName is the name of the pick_first_leaf balancer. PickFirstLeafName = "pick_first_leaf" - PickFirstName = "pick_first" - logPrefix = "[pick-first-leaf-lb %p] " + // PickFirstName is the name of the pick_first balancer. + PickFirstName = "pick_first" + logPrefix = "[pick-first-leaf-lb %p] " ) type pickfirstBuilder struct { From fcb0120c58a1eabd557f30ebc1d8ae728e352892 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Fri, 9 Aug 2024 23:29:07 +0530 Subject: [PATCH 22/62] simplify serializer usage --- balancer/pickfirst_leaf/pickfirst_leaf.go | 61 +++++++++++------------ 1 file changed, 29 insertions(+), 32 deletions(-) diff --git a/balancer/pickfirst_leaf/pickfirst_leaf.go b/balancer/pickfirst_leaf/pickfirst_leaf.go index 3ae14b3eb667..a3e54f3f5b68 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf.go +++ b/balancer/pickfirst_leaf/pickfirst_leaf.go @@ -141,17 +141,17 @@ type pickfirstBalancer struct { } func (b *pickfirstBalancer) ResolverError(err error) { - completion := make(chan struct{}) + ch := make(chan struct{}) b.serializer.ScheduleOr(func(ctx context.Context) { - b.resolverError(err, completion) + b.resolverError(err) + close(ch) }, func() { - close(completion) + close(ch) }) - <-completion + <-ch } -func (b *pickfirstBalancer) resolverError(err error, completion chan struct{}) { - defer close(completion) +func (b *pickfirstBalancer) resolverError(err error) { if b.logger.V(2) { b.logger.Infof("Received error from the name resolver: %v", err) } @@ -181,32 +181,30 @@ func (b *pickfirstBalancer) resolverError(err error, completion chan struct{}) { func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState) error { errCh := make(chan error, 1) b.serializer.ScheduleOr(func(_ context.Context) { - b.updateClientConnState(state, errCh) + err := b.updateClientConnState(state) + errCh <- err }, func() { close(errCh) }) return <-errCh } -func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState, errCh chan error) { +func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState) error { if b.state == connectivity.Shutdown { - errCh <- fmt.Errorf("balancer is already closed") - return + return fmt.Errorf("balancer is already closed") } if len(state.ResolverState.Addresses) == 0 && len(state.ResolverState.Endpoints) == 0 { // Cleanup state pertaining to the previous resolver state. // Treat an empty address list like an error by calling b.ResolverError. b.state = connectivity.TransientFailure - b.resolverError(errors.New("produced zero addresses"), make(chan struct{})) - errCh <- balancer.ErrBadResolverState - return + b.resolverError(errors.New("produced zero addresses")) + return balancer.ErrBadResolverState } // We don't have to guard this block with the env var because ParseConfig // already does so. cfg, ok := state.BalancerConfig.(pfConfig) if state.BalancerConfig != nil && !ok { - errCh <- fmt.Errorf("pickfirst: received illegal BalancerConfig (type %T): %v", state.BalancerConfig, state.BalancerConfig) - return + return fmt.Errorf("pickfirst: received illegal BalancerConfig (type %T): %v", state.BalancerConfig, state.BalancerConfig) } if b.logger.V(2) { @@ -240,13 +238,11 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState if err != nil { // This error should never happen when the state is READY if the // index is managed correctly. - errCh <- fmt.Errorf("address index is in an invalid state: %v", err) - return + return fmt.Errorf("address index is in an invalid state: %v", err) } b.addressIndex.updateEndpointList(newEndpoints) if b.addressIndex.seekTo(prevAddr) { - errCh <- nil - return + return nil } b.addressIndex.reset() } else { @@ -297,7 +293,7 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState } else if b.state == connectivity.TransientFailure { b.requestConnection() } - errCh <- nil + return nil } // UpdateSubConnState is unused as a StateListener is always registered when @@ -307,17 +303,19 @@ func (b *pickfirstBalancer) UpdateSubConnState(subConn balancer.SubConn, state b } func (b *pickfirstBalancer) Close() { - completion := make(chan struct{}) + ch := make(chan struct{}) b.serializer.ScheduleOr(func(ctx context.Context) { - b.close(completion) + b.close() + close(ch) }, func() { - b.close(completion) + b.close() + close(ch) }) - <-completion + <-ch <-b.serializer.Done() } -func (b *pickfirstBalancer) close(completion chan struct{}) { +func (b *pickfirstBalancer) close() { b.serializerCancel() for _, sd := range b.subConns.Values() { sd.(*scData).subConn.Shutdown() @@ -326,24 +324,23 @@ func (b *pickfirstBalancer) close(completion chan struct{}) { b.subConns.Delete(k) } b.state = connectivity.Shutdown - close(completion) } // ExitIdle moves the balancer out of idle state. It can be called concurrently // by the idlePicker and clientConn so access to variables should be synchronized. func (b *pickfirstBalancer) ExitIdle() { - completion := make(chan struct{}) + ch := make(chan struct{}) b.serializer.ScheduleOr(func(ctx context.Context) { - b.exitIdle(completion) + b.exitIdle() + close(ch) }, func() { - close(completion) + close(ch) }) - <-completion + <-ch } -func (b *pickfirstBalancer) exitIdle(completion chan struct{}) { +func (b *pickfirstBalancer) exitIdle() { b.requestConnection() - close(completion) } // deDupAddresses ensures that each address belongs to only one endpoint. From 91b63ced0de9be02ef5303895b95cfa98fb28ba8 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Sat, 10 Aug 2024 01:59:48 +0530 Subject: [PATCH 23/62] Remove re-resolution requsts since they are made by addrConn --- balancer/pickfirst_leaf/pickfirst_leaf.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/balancer/pickfirst_leaf/pickfirst_leaf.go b/balancer/pickfirst_leaf/pickfirst_leaf.go index a3e54f3f5b68..45ba77ecbad1 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf.go +++ b/balancer/pickfirst_leaf/pickfirst_leaf.go @@ -513,7 +513,6 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon // differences in back-off durations, but this is a decent approximation. if b.numTf >= b.subConns.Len() { b.numTf = 0 - b.cc.ResolveNow(resolver.ResolveNowOptions{}) } case connectivity.Idle: sd.subConn.Connect() @@ -528,8 +527,6 @@ func (b *pickfirstBalancer) endFirstPass() { ConnectivityState: connectivity.TransientFailure, Picker: &picker{err: b.firstErr}, }) - // Re-request resolution. - b.cc.ResolveNow(resolver.ResolveNowOptions{}) // Start re-connecting all the subconns that are already in IDLE. for _, v := range b.subConns.Values() { sd := v.(*scData) From 036fea3e4acbb9e4d9f9ae5dbd86cf2114c2bfad Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Sun, 11 Aug 2024 01:31:43 +0530 Subject: [PATCH 24/62] Verify server addr in subconn in tests --- test/pickfirst_leaf_test.go | 321 +++++++++++++++++++++--------------- 1 file changed, 191 insertions(+), 130 deletions(-) diff --git a/test/pickfirst_leaf_test.go b/test/pickfirst_leaf_test.go index 5c12a2216c4b..fde38bcfc1eb 100644 --- a/test/pickfirst_leaf_test.go +++ b/test/pickfirst_leaf_test.go @@ -24,6 +24,8 @@ import ( "sync" "testing" + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" "google.golang.org/grpc/balancer" pickfirstleaf "google.golang.org/grpc/balancer/pickfirst_leaf" @@ -93,6 +95,23 @@ func setupPickFirstLeaf(t *testing.T, backendCount int, opts ...grpc.DialOption) return cc, r, backends } +type scStateExpectation struct { + State connectivity.State + ServerIdx int +} + +func scStateExpectationToScState(in []scStateExpectation, serverAddrs []resolver.Address) []scState { + out := []scState{} + for _, exp := range in { + out = append(out, scState{ + State: exp.State, + Addrs: []resolver.Address{serverAddrs[exp.ServerIdx]}, + }) + } + return out + +} + // TestPickFirstLeaf_ResolverUpdate tests the behaviour of the new pick first // policy when servers are brought down and resolver updates are received. func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { @@ -101,139 +120,197 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { balChan := make(chan *stateStoringBalancer, 1) balancer.Register(&stateStoringBalancerBuilder{balancerChan: balChan}) tests := []struct { - name string - backendCount int - initialBackendIndexes []int - initialTargetBackendIndex int - wantScStates []connectivity.State - updatedBackendIndexes []int - updatedTargetBackendIndex int - wantScStatesPostUpdate []connectivity.State - restartConnected bool + name string + backendCount int + preUpdateBackendIndexes []int + preUpdateTargetBackendIndex int + wantPreUpdateScStates []scStateExpectation + updatedBackendIndexes []int + postUpdateTargetBackendIndex int + wantPostUpdateScStates []scStateExpectation + restartConnected bool }{ { - name: "two_server_first_ready", - backendCount: 2, - initialBackendIndexes: []int{0, 1}, - initialTargetBackendIndex: 0, - wantScStates: []connectivity.State{connectivity.Ready}, + name: "two_server_first_ready", + backendCount: 2, + preUpdateBackendIndexes: []int{0, 1}, + preUpdateTargetBackendIndex: 0, + wantPreUpdateScStates: []scStateExpectation{ + {State: connectivity.Ready, ServerIdx: 0}, + }, }, { - name: "two_server_second_ready", - backendCount: 2, - initialBackendIndexes: []int{0, 1}, - initialTargetBackendIndex: 1, - wantScStates: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, + name: "two_server_second_ready", + backendCount: 2, + preUpdateBackendIndexes: []int{0, 1}, + preUpdateTargetBackendIndex: 1, + wantPreUpdateScStates: []scStateExpectation{ + {State: connectivity.Shutdown, ServerIdx: 0}, + {State: connectivity.Ready, ServerIdx: 1}, + }, }, { - name: "duplicate_address", - backendCount: 2, - initialBackendIndexes: []int{0, 0, 1}, - initialTargetBackendIndex: 1, - wantScStates: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, + name: "duplicate_address", + backendCount: 2, + preUpdateBackendIndexes: []int{0, 0, 1}, + preUpdateTargetBackendIndex: 1, + wantPreUpdateScStates: []scStateExpectation{ + {State: connectivity.Shutdown, ServerIdx: 0}, + {State: connectivity.Ready, ServerIdx: 1}, + }, }, { - name: "disjoint_updated_addresses", - backendCount: 4, - initialBackendIndexes: []int{0, 1}, - initialTargetBackendIndex: 1, - wantScStates: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, - updatedBackendIndexes: []int{2, 3}, - updatedTargetBackendIndex: 3, - wantScStatesPostUpdate: []connectivity.State{connectivity.Shutdown, connectivity.Shutdown, connectivity.Shutdown, connectivity.Ready}, + name: "disjoint_updated_addresses", + backendCount: 4, + preUpdateBackendIndexes: []int{0, 1}, + preUpdateTargetBackendIndex: 1, + wantPreUpdateScStates: []scStateExpectation{ + {State: connectivity.Shutdown, ServerIdx: 0}, + {State: connectivity.Ready, ServerIdx: 1}, + }, + updatedBackendIndexes: []int{2, 3}, + postUpdateTargetBackendIndex: 3, + wantPostUpdateScStates: []scStateExpectation{ + {State: connectivity.Shutdown, ServerIdx: 0}, + {State: connectivity.Shutdown, ServerIdx: 1}, + {State: connectivity.Shutdown, ServerIdx: 2}, + {State: connectivity.Ready, ServerIdx: 3}, + }, }, { - name: "active_backend_in_updated_list", - backendCount: 3, - initialBackendIndexes: []int{0, 1}, - initialTargetBackendIndex: 1, - wantScStates: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, - updatedBackendIndexes: []int{1, 2}, - updatedTargetBackendIndex: 1, - wantScStatesPostUpdate: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, + name: "active_backend_in_updated_list", + backendCount: 3, + preUpdateBackendIndexes: []int{0, 1}, + preUpdateTargetBackendIndex: 1, + wantPreUpdateScStates: []scStateExpectation{ + {State: connectivity.Shutdown, ServerIdx: 0}, + {State: connectivity.Ready, ServerIdx: 1}, + }, + updatedBackendIndexes: []int{1, 2}, + postUpdateTargetBackendIndex: 1, + wantPostUpdateScStates: []scStateExpectation{ + {State: connectivity.Shutdown, ServerIdx: 0}, + {State: connectivity.Ready, ServerIdx: 1}, + }, }, { - name: "inactive_backend_in_updated_list", - backendCount: 3, - initialBackendIndexes: []int{0, 1}, - initialTargetBackendIndex: 1, - wantScStates: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, - updatedBackendIndexes: []int{0, 2}, - updatedTargetBackendIndex: 0, - wantScStatesPostUpdate: []connectivity.State{connectivity.Shutdown, connectivity.Shutdown, connectivity.Ready}, + name: "inactive_backend_in_updated_list", + backendCount: 3, + preUpdateBackendIndexes: []int{0, 1}, + preUpdateTargetBackendIndex: 1, + wantPreUpdateScStates: []scStateExpectation{ + {State: connectivity.Shutdown, ServerIdx: 0}, + {State: connectivity.Ready, ServerIdx: 1}, + }, + updatedBackendIndexes: []int{0, 2}, + postUpdateTargetBackendIndex: 0, + wantPostUpdateScStates: []scStateExpectation{ + {State: connectivity.Shutdown, ServerIdx: 0}, + {State: connectivity.Shutdown, ServerIdx: 1}, + {State: connectivity.Ready, ServerIdx: 0}, + }, }, { - name: "identical_list", - backendCount: 2, - initialBackendIndexes: []int{0, 1}, - initialTargetBackendIndex: 1, - wantScStates: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, - updatedBackendIndexes: []int{0, 1}, - updatedTargetBackendIndex: 1, - wantScStatesPostUpdate: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, + name: "identical_list", + backendCount: 2, + preUpdateBackendIndexes: []int{0, 1}, + preUpdateTargetBackendIndex: 1, + wantPreUpdateScStates: []scStateExpectation{ + {State: connectivity.Shutdown, ServerIdx: 0}, + {State: connectivity.Ready, ServerIdx: 1}, + }, + updatedBackendIndexes: []int{0, 1}, + postUpdateTargetBackendIndex: 1, + wantPostUpdateScStates: []scStateExpectation{ + {State: connectivity.Shutdown, ServerIdx: 0}, + {State: connectivity.Ready, ServerIdx: 1}, + }, }, { - name: "first_connected_idle_reconnect", - backendCount: 2, - initialBackendIndexes: []int{0, 1}, - initialTargetBackendIndex: 0, - restartConnected: true, - wantScStates: []connectivity.State{connectivity.Ready}, - updatedBackendIndexes: []int{0, 1}, - updatedTargetBackendIndex: 0, - wantScStatesPostUpdate: []connectivity.State{connectivity.Ready}, + name: "first_connected_idle_reconnect", + backendCount: 2, + preUpdateBackendIndexes: []int{0, 1}, + preUpdateTargetBackendIndex: 0, + restartConnected: true, + wantPreUpdateScStates: []scStateExpectation{ + {State: connectivity.Ready, ServerIdx: 0}, + }, + updatedBackendIndexes: []int{0, 1}, + postUpdateTargetBackendIndex: 0, + wantPostUpdateScStates: []scStateExpectation{ + {State: connectivity.Ready, ServerIdx: 0}, + }, }, { - name: "second_connected_idle_reconnect", - backendCount: 2, - initialBackendIndexes: []int{0, 1}, - initialTargetBackendIndex: 1, - restartConnected: true, - wantScStates: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, - updatedBackendIndexes: []int{0, 1}, - updatedTargetBackendIndex: 1, - wantScStatesPostUpdate: []connectivity.State{connectivity.Shutdown, connectivity.Ready, connectivity.Shutdown}, + name: "second_connected_idle_reconnect", + backendCount: 2, + preUpdateBackendIndexes: []int{0, 1}, + preUpdateTargetBackendIndex: 1, + restartConnected: true, + wantPreUpdateScStates: []scStateExpectation{ + {State: connectivity.Shutdown, ServerIdx: 0}, + {State: connectivity.Ready, ServerIdx: 1}, + }, + updatedBackendIndexes: []int{0, 1}, + postUpdateTargetBackendIndex: 1, + wantPostUpdateScStates: []scStateExpectation{ + {State: connectivity.Shutdown, ServerIdx: 0}, + {State: connectivity.Ready, ServerIdx: 1}, + {State: connectivity.Shutdown, ServerIdx: 0}, + }, }, { - name: "second_connected_idle_reconnect_first", - backendCount: 2, - initialBackendIndexes: []int{0, 1}, - initialTargetBackendIndex: 1, - restartConnected: true, - wantScStates: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, - updatedBackendIndexes: []int{0, 1}, - updatedTargetBackendIndex: 0, - wantScStatesPostUpdate: []connectivity.State{connectivity.Shutdown, connectivity.Shutdown, connectivity.Ready}, + name: "second_connected_idle_reconnect_first", + backendCount: 2, + preUpdateBackendIndexes: []int{0, 1}, + preUpdateTargetBackendIndex: 1, + restartConnected: true, + wantPreUpdateScStates: []scStateExpectation{ + {State: connectivity.Shutdown, ServerIdx: 0}, + {State: connectivity.Ready, ServerIdx: 1}, + }, + updatedBackendIndexes: []int{0, 1}, + postUpdateTargetBackendIndex: 0, + wantPostUpdateScStates: []scStateExpectation{ + {State: connectivity.Shutdown, ServerIdx: 0}, + {State: connectivity.Shutdown, ServerIdx: 1}, + {State: connectivity.Ready, ServerIdx: 0}, + }, }, { - name: "first_connected_idle_reconnect_second", - backendCount: 2, - initialBackendIndexes: []int{0, 1}, - initialTargetBackendIndex: 0, - restartConnected: true, - wantScStates: []connectivity.State{connectivity.Ready}, + name: "first_connected_idle_reconnect_second", + backendCount: 2, + preUpdateBackendIndexes: []int{0, 1}, + preUpdateTargetBackendIndex: 0, + restartConnected: true, + wantPreUpdateScStates: []scStateExpectation{ + {State: connectivity.Ready, ServerIdx: 0}, + }, updatedBackendIndexes: []int{0, 1}, - updatedTargetBackendIndex: 1, - wantScStatesPostUpdate: []connectivity.State{connectivity.Shutdown, connectivity.Ready}, + postUpdateTargetBackendIndex: 1, + wantPostUpdateScStates: []scStateExpectation{ + {State: connectivity.Shutdown, ServerIdx: 0}, + {State: connectivity.Ready, ServerIdx: 1}, + }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { cc, r, backends := setupPickFirstLeaf(t, tc.backendCount) + addrs := stubBackendsToResolverAddrs(backends) - activeBackends := []*stubserver.StubServer{} - for _, idx := range tc.initialBackendIndexes { - activeBackends = append(activeBackends, backends[idx]) + activeAddrs := []resolver.Address{} + for _, idx := range tc.preUpdateBackendIndexes { + activeAddrs = append(activeAddrs, addrs[idx]) } - addrs := stubBackendsToResolverAddrs(activeBackends) - r.UpdateState(resolver.State{Addresses: addrs}) + r.UpdateState(resolver.State{Addresses: activeAddrs}) // shutdown all active backends except the target. var targetAddr resolver.Address - for idxI, idx := range tc.initialBackendIndexes { - if idx == tc.initialTargetBackendIndex { - targetAddr = addrs[idxI] + for idxI, idx := range tc.preUpdateBackendIndexes { + if idx == tc.preUpdateTargetBackendIndex { + targetAddr = activeAddrs[idxI] continue } backends[idx].S.Stop() @@ -243,16 +320,9 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { t.Fatal(err) } bal := <-balChan - scs := bal.subConns() - if got, want := len(scs), len(tc.wantScStates); got != want { - t.Fatalf("len(subconns) = %d, want %d", got, want) - } - - for idx := range scs { - if got, want := scs[idx].state, tc.wantScStates[idx]; got != want { - t.Errorf("subconn[%d].state = %v, want = %v", idx, got, want) - } + if diff := cmp.Diff(scStateExpectationToScState(tc.wantPreUpdateScStates, addrs), bal.subConns()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) } if len(tc.updatedBackendIndexes) == 0 { @@ -261,7 +331,7 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { // Restart all the backends. for i, s := range backends { - if !tc.restartConnected && i == tc.initialTargetBackendIndex { + if !tc.restartConnected && i == tc.preUpdateTargetBackendIndex { continue } s.S.Stop() @@ -270,17 +340,16 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { } } - activeBackends = []*stubserver.StubServer{} + activeAddrs = []resolver.Address{} for _, idx := range tc.updatedBackendIndexes { - activeBackends = append(activeBackends, backends[idx]) + activeAddrs = append(activeAddrs, addrs[idx]) } - addrs = stubBackendsToResolverAddrs(activeBackends) - r.UpdateState(resolver.State{Addresses: addrs}) + r.UpdateState(resolver.State{Addresses: activeAddrs}) // shutdown all active backends except the target. for idxI, idx := range tc.updatedBackendIndexes { - if idx == tc.updatedTargetBackendIndex { - targetAddr = addrs[idxI] + if idx == tc.postUpdateTargetBackendIndex { + targetAddr = activeAddrs[idxI] continue } backends[idx].S.Stop() @@ -289,18 +358,10 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { if err := pickfirst.CheckRPCsToBackend(ctx, cc, targetAddr); err != nil { t.Fatal(err) } - scs = bal.subConns() - if got, want := len(scs), len(tc.wantScStatesPostUpdate); got != want { - t.Fatalf("len(subconns) = %d, want %d", got, want) + if diff := cmp.Diff(scStateExpectationToScState(tc.wantPostUpdateScStates, addrs), bal.subConns()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) } - - for idx := range scs { - if got, want := scs[idx].state, tc.wantScStatesPostUpdate[idx]; got != want { - t.Errorf("subconn[%d].state = %v, want = %v", idx, got, want) - } - } - }) } } @@ -361,13 +422,13 @@ type stateStoringCCWrapper struct { func (ccw *stateStoringCCWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { oldListener := opts.StateListener scs := &scState{ - state: connectivity.Idle, - addrs: addrs, + State: connectivity.Idle, + Addrs: addrs, } ccw.b.addScState(scs) opts.StateListener = func(s balancer.SubConnState) { ccw.b.mu.Lock() - scs.state = s.ConnectivityState + scs.State = s.ConnectivityState ccw.b.mu.Unlock() oldListener(s) } @@ -375,6 +436,6 @@ func (ccw *stateStoringCCWrapper) NewSubConn(addrs []resolver.Address, opts bala } type scState struct { - state connectivity.State - addrs []resolver.Address + State connectivity.State + Addrs []resolver.Address } From 9a2d6a9ef46f9c1671c240d2bf115f78598828b1 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Sun, 11 Aug 2024 02:24:36 +0530 Subject: [PATCH 25/62] test balancer state transitions --- test/pickfirst_leaf_test.go | 118 ++++++++++++++++++++++++++++++++++-- 1 file changed, 113 insertions(+), 5 deletions(-) diff --git a/test/pickfirst_leaf_test.go b/test/pickfirst_leaf_test.go index fde38bcfc1eb..6364a905127d 100644 --- a/test/pickfirst_leaf_test.go +++ b/test/pickfirst_leaf_test.go @@ -129,6 +129,7 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { postUpdateTargetBackendIndex int wantPostUpdateScStates []scStateExpectation restartConnected bool + wantConnStateTransitions []connectivity.State }{ { name: "two_server_first_ready", @@ -138,6 +139,10 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { wantPreUpdateScStates: []scStateExpectation{ {State: connectivity.Ready, ServerIdx: 0}, }, + wantConnStateTransitions: []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + }, }, { name: "two_server_second_ready", @@ -148,6 +153,10 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { {State: connectivity.Shutdown, ServerIdx: 0}, {State: connectivity.Ready, ServerIdx: 1}, }, + wantConnStateTransitions: []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + }, }, { name: "duplicate_address", @@ -158,6 +167,10 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { {State: connectivity.Shutdown, ServerIdx: 0}, {State: connectivity.Ready, ServerIdx: 1}, }, + wantConnStateTransitions: []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + }, }, { name: "disjoint_updated_addresses", @@ -176,6 +189,12 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { {State: connectivity.Shutdown, ServerIdx: 2}, {State: connectivity.Ready, ServerIdx: 3}, }, + wantConnStateTransitions: []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Connecting, + connectivity.Ready, + }, }, { name: "active_backend_in_updated_list", @@ -192,6 +211,10 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { {State: connectivity.Shutdown, ServerIdx: 0}, {State: connectivity.Ready, ServerIdx: 1}, }, + wantConnStateTransitions: []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + }, }, { name: "inactive_backend_in_updated_list", @@ -209,6 +232,12 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { {State: connectivity.Shutdown, ServerIdx: 1}, {State: connectivity.Ready, ServerIdx: 0}, }, + wantConnStateTransitions: []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Connecting, + connectivity.Ready, + }, }, { name: "identical_list", @@ -225,6 +254,10 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { {State: connectivity.Shutdown, ServerIdx: 0}, {State: connectivity.Ready, ServerIdx: 1}, }, + wantConnStateTransitions: []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + }, }, { name: "first_connected_idle_reconnect", @@ -240,6 +273,13 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { wantPostUpdateScStates: []scStateExpectation{ {State: connectivity.Ready, ServerIdx: 0}, }, + wantConnStateTransitions: []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Idle, + connectivity.Connecting, + connectivity.Ready, + }, }, { name: "second_connected_idle_reconnect", @@ -258,6 +298,13 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { {State: connectivity.Ready, ServerIdx: 1}, {State: connectivity.Shutdown, ServerIdx: 0}, }, + wantConnStateTransitions: []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Idle, + connectivity.Connecting, + connectivity.Ready, + }, }, { name: "second_connected_idle_reconnect_first", @@ -276,6 +323,13 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { {State: connectivity.Shutdown, ServerIdx: 1}, {State: connectivity.Ready, ServerIdx: 0}, }, + wantConnStateTransitions: []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Idle, + connectivity.Connecting, + connectivity.Ready, + }, }, { name: "first_connected_idle_reconnect_second", @@ -286,12 +340,19 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { wantPreUpdateScStates: []scStateExpectation{ {State: connectivity.Ready, ServerIdx: 0}, }, - updatedBackendIndexes: []int{0, 1}, + updatedBackendIndexes: []int{0, 1}, postUpdateTargetBackendIndex: 1, wantPostUpdateScStates: []scStateExpectation{ {State: connectivity.Shutdown, ServerIdx: 0}, {State: connectivity.Ready, ServerIdx: 1}, }, + wantConnStateTransitions: []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Idle, + connectivity.Connecting, + connectivity.Ready, + }, }, } @@ -305,6 +366,12 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { activeAddrs = append(activeAddrs, addrs[idx]) } r.UpdateState(resolver.State{Addresses: activeAddrs}) + bal := <-balChan + select { + case <-bal.resolverUpdateSeen: + case <-ctx.Done(): + t.Fatalf("Context timed out waiting for resolve update to be processed") + } // shutdown all active backends except the target. var targetAddr resolver.Address @@ -319,13 +386,15 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { if err := pickfirst.CheckRPCsToBackend(ctx, cc, targetAddr); err != nil { t.Fatal(err) } - bal := <-balChan if diff := cmp.Diff(scStateExpectationToScState(tc.wantPreUpdateScStates, addrs), bal.subConns()); diff != "" { t.Errorf("subconn states mismatch (-want +got):\n%s", diff) } if len(tc.updatedBackendIndexes) == 0 { + if diff := cmp.Diff(tc.wantConnStateTransitions, bal.connStateTransitions()); diff != "" { + t.Errorf("balancer states mismatch (-want +got):\n%s", diff) + } return } @@ -344,7 +413,13 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { for _, idx := range tc.updatedBackendIndexes { activeAddrs = append(activeAddrs, addrs[idx]) } + r.UpdateState(resolver.State{Addresses: activeAddrs}) + select { + case <-bal.resolverUpdateSeen: + case <-ctx.Done(): + t.Fatalf("Context timed out waiting for resolve update to be processed") + } // shutdown all active backends except the target. for idxI, idx := range tc.updatedBackendIndexes { @@ -362,6 +437,10 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { if diff := cmp.Diff(scStateExpectationToScState(tc.wantPostUpdateScStates, addrs), bal.subConns()); diff != "" { t.Errorf("subconn states mismatch (-want +got):\n%s", diff) } + + if diff := cmp.Diff(tc.wantConnStateTransitions, bal.connStateTransitions()); diff != "" { + t.Errorf("balancer states mismatch (-want +got):\n%s", diff) + } }) } } @@ -369,8 +448,10 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { // stateStoringBalancer stores the state of the subconns being created. type stateStoringBalancer struct { balancer.Balancer - mu sync.Mutex - scStates []*scState + mu sync.Mutex + scStates []*scState + stateTransitions []connectivity.State + resolverUpdateSeen chan struct{} } func (b *stateStoringBalancer) Close() { @@ -392,12 +473,20 @@ func (b *stateStoringBalancerBuilder) Name() string { } func (b *stateStoringBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { - bal := &stateStoringBalancer{} + bal := &stateStoringBalancer{ + resolverUpdateSeen: make(chan struct{}, 1), + } bal.Balancer = balancer.Get(pickfirstleaf.PickFirstLeafName).Build(&stateStoringCCWrapper{cc, bal}, opts) b.balancerChan <- bal return bal } +func (b *stateStoringBalancer) UpdateClientConnState(state balancer.ClientConnState) error { + err := b.Balancer.UpdateClientConnState(state) + b.resolverUpdateSeen <- struct{}{} + return err +} + func (b *stateStoringBalancer) subConns() []scState { b.mu.Lock() defer b.mu.Unlock() @@ -414,6 +503,20 @@ func (b *stateStoringBalancer) addScState(state *scState) { b.mu.Unlock() } +func (b *stateStoringBalancer) addConnState(state connectivity.State) { + b.mu.Lock() + defer b.mu.Unlock() + if len(b.stateTransitions) == 0 || state != b.stateTransitions[len(b.stateTransitions)-1] { + b.stateTransitions = append(b.stateTransitions, state) + } +} + +func (b *stateStoringBalancer) connStateTransitions() []connectivity.State { + b.mu.Lock() + defer b.mu.Unlock() + return append([]connectivity.State{}, b.stateTransitions...) +} + type stateStoringCCWrapper struct { balancer.ClientConn b *stateStoringBalancer @@ -435,6 +538,11 @@ func (ccw *stateStoringCCWrapper) NewSubConn(addrs []resolver.Address, opts bala return ccw.ClientConn.NewSubConn(addrs, opts) } +func (ccw *stateStoringCCWrapper) UpdateState(state balancer.State) { + ccw.b.addConnState(state.ConnectivityState) + ccw.ClientConn.UpdateState(state) +} + type scState struct { State connectivity.State Addrs []resolver.Address From 2a36f7f9e709f478cc4d3131f04ee6274c8eddb3 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Mon, 12 Aug 2024 23:42:17 +0530 Subject: [PATCH 26/62] Fix address to endpoint conversion logic --- balancer/pickfirst_leaf/pickfirst_leaf.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/balancer/pickfirst_leaf/pickfirst_leaf.go b/balancer/pickfirst_leaf/pickfirst_leaf.go index 45ba77ecbad1..d88361b8f601 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf.go +++ b/balancer/pickfirst_leaf/pickfirst_leaf.go @@ -213,9 +213,11 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState newEndpoints := state.ResolverState.Endpoints if len(newEndpoints) == 0 { + newEndpoints = make([]resolver.Endpoint, len(state.ResolverState.Addresses)) // Convert addresses to endpoints. - for _, addr := range state.ResolverState.Addresses { - newEndpoints = append(newEndpoints, resolver.Endpoint{Addresses: []resolver.Address{addr}}) + for i, a := range state.ResolverState.Addresses { + newEndpoints[i].Attributes = a.BalancerAttributes + newEndpoints[i].Addresses = []resolver.Address{a} } } From a8a243aeff021c9e24c21d15cbd771e992e4b927 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Thu, 15 Aug 2024 01:03:15 +0530 Subject: [PATCH 27/62] Address review comments --- balancer/pickfirst/pickfirst.go | 3 +- balancer/pickfirst_leaf/pickfirst_leaf.go | 294 +++++++++--------- .../pickfirst_leaf/pickfirst_leaf_test.go | 172 ++++++++++ test/pickfirst_leaf_test.go | 57 ++++ 4 files changed, 370 insertions(+), 156 deletions(-) create mode 100644 balancer/pickfirst_leaf/pickfirst_leaf_test.go diff --git a/balancer/pickfirst/pickfirst.go b/balancer/pickfirst/pickfirst.go index a3ea329ee6fc..b6676cf4b94b 100644 --- a/balancer/pickfirst/pickfirst.go +++ b/balancer/pickfirst/pickfirst.go @@ -26,7 +26,6 @@ import ( "math/rand" "google.golang.org/grpc/balancer" - _ "google.golang.org/grpc/balancer/pickfirst_leaf" // For automatically registering the new pickfirst if required. "google.golang.org/grpc/connectivity" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/internal" @@ -35,6 +34,8 @@ import ( "google.golang.org/grpc/internal/pretty" "google.golang.org/grpc/resolver" "google.golang.org/grpc/serviceconfig" + + _ "google.golang.org/grpc/balancer/pickfirst_leaf" // For automatically registering the new pickfirst if required. ) func init() { diff --git a/balancer/pickfirst_leaf/pickfirst_leaf.go b/balancer/pickfirst_leaf/pickfirst_leaf.go index d88361b8f601..33c68a749f90 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf.go +++ b/balancer/pickfirst_leaf/pickfirst_leaf.go @@ -17,7 +17,7 @@ */ // Package pickfirstleaf contains the pick_first load balancing policy which -// will be the universal leaf policy after Dual Stack changes are implemented. +// will be the universal leaf policy after dualstack changes are implemented. package pickfirstleaf import ( @@ -40,43 +40,43 @@ import ( ) func init() { - balancer.Register(pickfirstBuilder{name: PickFirstLeafName}) if envconfig.NewPickFirstEnabled { - // Register as the default pickfirst balancer also. internal.ShuffleAddressListForTesting = func(n int, swap func(i, j int)) { rand.Shuffle(n, swap) } - balancer.Register(pickfirstBuilder{name: PickFirstName}) + // Register as the default pick_first balancer. + PickFirstLeafName = "pick_first" } + balancer.Register(pickfirstBuilder{}) } -var logger = grpclog.Component("pick-first-leaf-lb") - -const ( +var ( + logger = grpclog.Component("pick-first-leaf-lb") + errBalancerClosed = fmt.Errorf("pickfirst: LB policy is closed") // PickFirstLeafName is the name of the pick_first_leaf balancer. + // Can be changed in init() if this balancer is to be registered as the default + // pickfirst. PickFirstLeafName = "pick_first_leaf" - // PickFirstName is the name of the pick_first balancer. - PickFirstName = "pick_first" - logPrefix = "[pick-first-leaf-lb %p] " ) -type pickfirstBuilder struct { - name string -} +const logPrefix = "[pick-first-leaf-lb %p] " + +type pickfirstBuilder struct{} func (pickfirstBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer { ctx, cancel := context.WithCancel(context.Background()) b := &pickfirstBalancer{ cc: cc, - addressIndex: newIndex(nil), + addressIndex: addressList{}, subConns: resolver.NewAddressMap(), - serializer: *grpcsync.NewCallbackSerializer(ctx), + serializer: grpcsync.NewCallbackSerializer(ctx), serializerCancel: cancel, + state: connectivity.Idle, } b.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(logPrefix, b)) return b } func (b pickfirstBuilder) Name() string { - return b.name + return PickFirstLeafName } func (pickfirstBuilder) ParseConfig(js json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { @@ -100,13 +100,13 @@ type pfConfig struct { type scData struct { subConn balancer.SubConn state connectivity.State - addr *resolver.Address + addr resolver.Address } -func newScData(b *pickfirstBalancer, addr resolver.Address) (*scData, error) { +func newSCData(b *pickfirstBalancer, addr resolver.Address) (*scData, error) { sd := &scData{ state: connectivity.Idle, - addr: &addr, + addr: addr, } sc, err := b.cc.NewSubConn([]resolver.Address{addr}, balancer.NewSubConnOptions{ StateListener: func(state balancer.SubConnState) { @@ -125,30 +125,31 @@ func newScData(b *pickfirstBalancer, addr resolver.Address) (*scData, error) { } type pickfirstBalancer struct { - logger *internalgrpclog.PrefixLogger - state connectivity.State - cc balancer.ClientConn - subConns *resolver.AddressMap - addressIndex index - firstPass bool - firstErr error - numTf int - // A serializer is used to ensure synchronization from updates triggered - // due to the idle picker in addition to the already serialized resolver, + // The following fields are initialized at build time and read-only after + // that and therefore do not need to be guarded by a mutex. + logger *internalgrpclog.PrefixLogger + cc balancer.ClientConn + + // The serializer and its cancel func are initialized at build time, and the + // rest of the fields here are only accessed from serializer callbacks (or + // from balancer.Balancer methods, which themselves are guaranteed to be + // mutually exclusive) and hence do not need to be guarded by a mutex. + // The serializer is used to ensure synchronization of updates triggered + // from the idle picker and the already serialized resolver, // subconn state updates. - serializer grpcsync.CallbackSerializer + serializer *grpcsync.CallbackSerializer serializerCancel func() + state connectivity.State + subConns *resolver.AddressMap // scData for active subonns mapped by address. + addressIndex addressList + firstPass bool + firstErr error } func (b *pickfirstBalancer) ResolverError(err error) { - ch := make(chan struct{}) - b.serializer.ScheduleOr(func(ctx context.Context) { + b.serializer.TrySchedule(func(_ context.Context) { b.resolverError(err) - close(ch) - }, func() { - close(ch) }) - <-ch } func (b *pickfirstBalancer) resolverError(err error) { @@ -164,14 +165,8 @@ func (b *pickfirstBalancer) resolverError(err error) { return } - for _, sd := range b.subConns.Values() { - sd.(*scData).subConn.Shutdown() - } - for _, k := range b.subConns.Keys() { - b.subConns.Delete(k) - } + b.closeSubConns() b.addressIndex.updateEndpointList(nil) - b.state = connectivity.TransientFailure b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.TransientFailure, Picker: &picker{err: fmt.Errorf("name resolver error: %v", err)}, @@ -184,14 +179,16 @@ func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState err := b.updateClientConnState(state) errCh <- err }, func() { - close(errCh) + errCh <- errBalancerClosed }) return <-errCh } +// updateClientConnState handles clientConn state changes. +// Only executed in the context of a serializer callback. func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState) error { if b.state == connectivity.Shutdown { - return fmt.Errorf("balancer is already closed") + return errBalancerClosed } if len(state.ResolverState.Addresses) == 0 && len(state.ResolverState.Endpoints) == 0 { // Cleanup state pertaining to the previous resolver state. @@ -218,10 +215,12 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState for i, a := range state.ResolverState.Addresses { newEndpoints[i].Attributes = a.BalancerAttributes newEndpoints[i].Addresses = []resolver.Address{a} + // We can't remove address attributes here since xds packages use + // them to store locality metadata. } } - // Since we have a new set of addresses, we are again at first pass + // Since we have a new set of addresses, we are again at first pass. b.firstPass = true newEndpoints = deDupAddresses(newEndpoints) @@ -236,17 +235,11 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState if b.state == connectivity.Ready { // If the previous ready subconn exists in new address list, // keep this connection and don't create new subconns. - prevAddr, err := b.addressIndex.currentAddress() - if err != nil { - // This error should never happen when the state is READY if the - // index is managed correctly. - return fmt.Errorf("address index is in an invalid state: %v", err) - } + prevAddr := b.addressIndex.currentAddress() b.addressIndex.updateEndpointList(newEndpoints) if b.addressIndex.seekTo(prevAddr) { return nil } - b.addressIndex.reset() } else { b.addressIndex.updateEndpointList(newEndpoints) } @@ -275,7 +268,9 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState b.subConns.Delete(oldAddr) } - if oldAddrs.Len() == 0 || b.state == connectivity.Ready || b.state == connectivity.Connecting { + // If its the first resolver update or the balancer was already READY or + // or CONNECTING, enter CONNECTING. + if b.state == connectivity.Ready || b.state == connectivity.Connecting || oldAddrs.Len() == 0 { b.firstErr = nil // Start connection attempt at first address. b.state = connectivity.Connecting @@ -288,9 +283,7 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState b.firstErr = nil b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.Idle, - Picker: &idlePicker{ - exitIdle: b.ExitIdle, - }, + Picker: &idlePicker{exitIdle: b.ExitIdle}, }) } else if b.state == connectivity.TransientFailure { b.requestConnection() @@ -305,46 +298,43 @@ func (b *pickfirstBalancer) UpdateSubConnState(subConn balancer.SubConn, state b } func (b *pickfirstBalancer) Close() { - ch := make(chan struct{}) - b.serializer.ScheduleOr(func(ctx context.Context) { - b.close() - close(ch) - }, func() { + b.serializer.TrySchedule(func(_ context.Context) { b.close() - close(ch) }) - <-ch <-b.serializer.Done() } +// close closes the balancer. +// Only executed in the context of a serializer callback. func (b *pickfirstBalancer) close() { b.serializerCancel() - for _, sd := range b.subConns.Values() { - sd.(*scData).subConn.Shutdown() - } - for _, k := range b.subConns.Keys() { - b.subConns.Delete(k) - } + b.closeSubConns() b.state = connectivity.Shutdown } // ExitIdle moves the balancer out of idle state. It can be called concurrently // by the idlePicker and clientConn so access to variables should be synchronized. func (b *pickfirstBalancer) ExitIdle() { - ch := make(chan struct{}) - b.serializer.ScheduleOr(func(ctx context.Context) { + b.serializer.TrySchedule(func(_ context.Context) { b.exitIdle() - close(ch) - }, func() { - close(ch) }) - <-ch } +// exitIdle starts a conection attempt if not already started. +// Only executed in the context of a serializer callback. func (b *pickfirstBalancer) exitIdle() { b.requestConnection() } +func (b *pickfirstBalancer) closeSubConns() { + for _, sd := range b.subConns.Values() { + sd.(*scData).subConn.Shutdown() + } + for _, k := range b.subConns.Keys() { + b.subConns.Delete(k) + } +} + // deDupAddresses ensures that each address belongs to only one endpoint. func deDupAddresses(endpoints []resolver.Endpoint) []resolver.Endpoint { seenAddrs := resolver.NewAddressMap() @@ -381,7 +371,7 @@ func (b *pickfirstBalancer) shutdownRemaining(selected *scData) { for _, k := range b.subConns.Keys() { b.subConns.Delete(k) } - b.subConns.Set(*selected.addr, selected) + b.subConns.Set(selected.addr, selected) } // requestConnection requests a connection to the next applicable address' @@ -393,27 +383,25 @@ func (b *pickfirstBalancer) requestConnection() { if !b.addressIndex.isValid() || b.state == connectivity.Shutdown { return } - curAddr, err := b.addressIndex.currentAddress() - if err != nil { - // This should not never happen because we already check for validity and - // return early above. - return - } - sd, ok := b.subConns.Get(*curAddr) + curAddr := b.addressIndex.currentAddress() + sd, ok := b.subConns.Get(curAddr) if !ok { - sd, err = newScData(b, *curAddr) + sd, err := newSCData(b, curAddr) if err != nil { // This should never happen. b.logger.Warningf("Failed to create a subConn for address %v: %v", curAddr.String(), err) b.state = connectivity.TransientFailure + b.addressIndex.reset() b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.TransientFailure, - Picker: &picker{err: fmt.Errorf("error creating connection: %v", err)}, + // Return an idle picker so that the clientConn doesn't remain + // stuck in TRANSIENT_FAILURE and attempts to re-connect the + // next time picker.Pick is called. + Picker: &idlePicker{exitIdle: b.ExitIdle}, }) - b.addressIndex.reset() return } - b.subConns.Set(*curAddr, sd) + b.subConns.Set(curAddr, sd) } scd := sd.(*scData) @@ -427,16 +415,18 @@ func (b *pickfirstBalancer) requestConnection() { b.requestConnection() case connectivity.Ready: // Should never happen. - b.logger.Warningf("Requesting a connection even though we have a READY subconn") + b.logger.Errorf("Requesting a connection even though we have a READY subconn") } } +// updateSubConnState handles subConn state updates. +// Only executed in the context of a serializer callback. func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubConnState) { // Previously relevant subconns can still callback with state updates. // To prevent pickers from returning these obsolete subconns, this logic // is included to check if the current list of active subconns includes this // subconn. - if activeSd, found := b.subConns.Get(*sd.addr); !found || activeSd != sd { + if activeSd, found := b.subConns.Get(sd.addr); !found || activeSd != sd { return } if state.ConnectivityState == connectivity.Shutdown { @@ -455,16 +445,16 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon return } - // If we are transitioning from READY to IDLE, shutdown and re-connect when + // If we are transitioning from READY to IDLE, reset index and re-connect when // prompted. - if state.ConnectivityState == connectivity.Idle && b.state == connectivity.Ready { + if b.state == connectivity.Ready && state.ConnectivityState == connectivity.Idle { + // Once a transport fails, we enter idle and start from the first address + // when the picker is used. b.state = connectivity.Idle b.addressIndex.reset() b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.Idle, - Picker: &idlePicker{ - exitIdle: b.ExitIdle, - }, + Picker: &idlePicker{exitIdle: b.ExitIdle}, }) return } @@ -472,6 +462,10 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon if b.firstPass { switch state.ConnectivityState { case connectivity.Connecting: + // We can be in either IDLE, CONNECTING or TRANSIENT_FAILURE. + // If we're in TRANSIENT_FAILURE, we stay in TRANSIENT_FAILURE until + // we're READY. See A62. + // If we're already in CONNECTING, no update is needed. if b.state == connectivity.Idle { b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.Connecting, @@ -482,14 +476,11 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon if b.firstErr == nil { b.firstErr = state.ConnectionError } - curAddr, err := b.addressIndex.currentAddress() - if err != nil { - // This is not expected since we end the first pass when we - // reach the end of the list. - b.logger.Errorf("Current index is invalid during first pass: %v", err) - return - } - if activeSd, found := b.subConns.Get(*curAddr); !found || activeSd != sd { + // Since we're re-using common subconns while handling resolver updates, + // we could receive an out of turn TRANSIENT_FAILURE from a pass + // over the previous address list. We ignore such updates. + curAddr := b.addressIndex.currentAddress() + if activeSd, found := b.subConns.Get(curAddr); !found || activeSd != sd { return } if b.addressIndex.increment() { @@ -509,13 +500,8 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon ConnectivityState: connectivity.TransientFailure, Picker: &picker{err: state.ConnectionError}, }) - b.numTf++ - // We request re-resolution when we've seen the same number of TFs as - // subconns. It could be that a subconn has seen multiple TFs due to - // differences in back-off durations, but this is a decent approximation. - if b.numTf >= b.subConns.Len() { - b.numTf = 0 - } + // We don't need to request re-resolution since the subconn already does + // that before reporting TRANSIENT_FAILURE. case connectivity.Idle: sd.subConn.Connect() } @@ -523,7 +509,6 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon func (b *pickfirstBalancer) endFirstPass() { b.firstPass = false - b.numTf = 0 b.state = connectivity.TransientFailure b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.TransientFailure, @@ -558,70 +543,69 @@ func (i *idlePicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { return balancer.PickResult{}, balancer.ErrNoSubConnAvailable } -// index is an Index as in 'i', the pointer to an entry. Not a "search index." -// All updates should be synchronized. -type index struct { - endpointList []resolver.Endpoint - endpointIdx int - addrIdx int -} - -// newIndex is the constructor for index. -func newIndex(endpointList []resolver.Endpoint) index { - return index{ - endpointList: endpointList, - } +// addressList manages sequentially iterating over addresses present in a list of +// endpoints. It provides a 1 dimensional view of the addresses present in the +// endpoints. +// This type is not safe for concurrent access. +type addressList struct { + addresses []resolver.Address + idx int } -func (i *index) isValid() bool { - return i.endpointIdx < len(i.endpointList) +func (al *addressList) isValid() bool { + return al.idx < len(al.addresses) } // increment moves to the next index in the address list. If at the last address // in the address list, moves to the next endpoint in the endpoint list. // This method returns false if it went off the list, true otherwise. -func (i *index) increment() bool { - if !i.isValid() { +func (al *addressList) increment() bool { + if !al.isValid() { return false } - ep := i.endpointList[i.endpointIdx] - i.addrIdx++ - if i.addrIdx >= len(ep.Addresses) { - i.endpointIdx++ - i.addrIdx = 0 - return i.endpointIdx < len(i.endpointList) - } - return false + al.idx++ + return al.idx < len(al.addresses) } -func (i *index) currentAddress() (*resolver.Address, error) { - if !i.isValid() { - return nil, fmt.Errorf("index is off the end of the address list") +func (al *addressList) currentAddress() resolver.Address { + if !al.isValid() { + panic("pickfirst: index is off the end of the address list") } - return &i.endpointList[i.endpointIdx].Addresses[i.addrIdx], nil + return al.addresses[al.idx] } -func (i *index) reset() { - i.endpointIdx = 0 - i.addrIdx = 0 +func (al *addressList) reset() { + al.idx = 0 } -func (i *index) updateEndpointList(endpointList []resolver.Endpoint) { - i.endpointList = endpointList - i.reset() +func (al *addressList) updateEndpointList(endpoints []resolver.Endpoint) { + // Flatten the addresses. + addrs := []resolver.Address{} + for _, e := range endpoints { + addrs = append(addrs, e.Addresses...) + } + al.addresses = addrs + al.reset() } // seekTo returns false if the needle was not found and the current index was left unchanged. -func (i *index) seekTo(needle *resolver.Address) bool { - for ei, endpoint := range i.endpointList { - for ai, addr := range endpoint.Addresses { - if !addr.Attributes.Equal(needle.Attributes) || addr.Addr != needle.Addr { - continue - } - i.endpointIdx = ei - i.addrIdx = ai - return true +func (al *addressList) seekTo(needle resolver.Address) bool { + for ai, addr := range al.addresses { + if !equalAddressIgnoringBalAttributes(&addr, &needle) { + continue } + al.idx = ai + return true } return false } + +// equalAddressIgnoringBalAttributes returns true is a and b are considered equal. +// This is different from the Equal method on the resolver.Address type which +// considers all fields to determine equality. Here, we only consider fields +// that are meaningful to the subconn. +func equalAddressIgnoringBalAttributes(a, b *resolver.Address) bool { + return a.Addr == b.Addr && a.ServerName == b.ServerName && + a.Attributes.Equal(b.Attributes) && + a.Metadata == b.Metadata +} diff --git a/balancer/pickfirst_leaf/pickfirst_leaf_test.go b/balancer/pickfirst_leaf/pickfirst_leaf_test.go new file mode 100644 index 000000000000..766714e80056 --- /dev/null +++ b/balancer/pickfirst_leaf/pickfirst_leaf_test.go @@ -0,0 +1,172 @@ +package pickfirstleaf + +import ( + "testing" + + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/resolver" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestAddressList_Iteration verifies the behaviour of the addressList while +// iterating through the entries. +func (s) TestAddressList_Iteration(t *testing.T) { + addrs := []resolver.Address{ + { + Addr: "192.168.1.1", + ServerName: "test-host-1", + Attributes: attributes.New("key-1", "val-1"), + BalancerAttributes: attributes.New("bal-key-1", "bal-val-1"), + }, + { + Addr: "192.168.1.2", + ServerName: "test-host-2", + Attributes: attributes.New("key-2", "val-2"), + BalancerAttributes: attributes.New("bal-key-2", "bal-val-2"), + }, + { + Addr: "192.168.1.3", + ServerName: "test-host-3", + Attributes: attributes.New("key-3", "val-3"), + BalancerAttributes: attributes.New("bal-key-3", "bal-val-3"), + }, + } + + endpoints := []resolver.Endpoint{ + { + Addresses: []resolver.Address{addrs[0], addrs[1]}, + }, + { + Addresses: []resolver.Address{addrs[2]}, + }, + } + + addressList := addressList{} + addressList.updateEndpointList(endpoints) + + for i := 0; i < len(addrs); i++ { + if got, want := addressList.isValid(), true; got != want { + t.Errorf("addressList.isValid() = %t, want %t", got, want) + } + if got, want := addressList.currentAddress(), addrs[i]; !want.Equal(got) { + t.Errorf("addressList.currentAddress() = %v, want %v", got, want) + } + if got, want := addressList.increment(), i+1 < len(addrs); got != want { + t.Errorf("addressList.increment() = %t, want %t", got, want) + } + } + + if got, want := addressList.isValid(), false; got != want { + t.Errorf("addressList.isValid() = %t, want %t", got, want) + } + + // increment an invalid address list. + if got, want := addressList.increment(), false; got != want { + t.Errorf("addressList.increment() = %t, want %t", got, want) + } + if got, want := addressList.isValid(), false; got != want { + t.Errorf("addressList.isValid() = %t, want %t", got, want) + } + + addressList.reset() + for i := 0; i < len(addrs); i++ { + if got, want := addressList.isValid(), true; got != want { + t.Errorf("addressList.isValid() = %t, want %t", got, want) + } + if got, want := addressList.currentAddress(), addrs[i]; !want.Equal(got) { + t.Errorf("addressList.currentAddress() = %v, want %v", got, want) + } + if got, want := addressList.increment(), i+1 < len(addrs); got != want { + t.Errorf("addressList.increment() = %t, want %t", got, want) + } + } +} + +// TestAddressList_SeekTo verifies the behaviour of addressList.seekTo. +func (s) TestAddressList_SeekTo(t *testing.T) { + addrs := []resolver.Address{ + { + Addr: "192.168.1.1", + ServerName: "test-host-1", + Attributes: attributes.New("key-1", "val-1"), + BalancerAttributes: attributes.New("bal-key-1", "bal-val-1"), + }, + { + Addr: "192.168.1.2", + ServerName: "test-host-2", + Attributes: attributes.New("key-2", "val-2"), + BalancerAttributes: attributes.New("bal-key-2", "bal-val-2"), + }, + { + Addr: "192.168.1.3", + ServerName: "test-host-3", + Attributes: attributes.New("key-3", "val-3"), + BalancerAttributes: attributes.New("bal-key-3", "bal-val-3"), + }, + } + + endpoints := []resolver.Endpoint{ + { + Addresses: []resolver.Address{addrs[0], addrs[1]}, + }, + { + Addresses: []resolver.Address{addrs[2]}, + }, + } + + addressList := addressList{} + addressList.updateEndpointList(endpoints) + + // Try finding an address in the list. + key := resolver.Address{ + Addr: "192.168.1.2", + ServerName: "test-host-2", + Attributes: attributes.New("key-2", "val-2"), + BalancerAttributes: attributes.New("ignored", "bal-val-2"), + } + + if got, want := addressList.seekTo(key), true; got != want { + t.Errorf("addressList.seekTo(%v) = %t, want %t", key, got, want) + } + + // It should be possible to increment once more now that the pointer has advanced. + if got, want := addressList.increment(), true; got != want { + t.Errorf("addressList.increment() = %t, want %t", got, want) + } + if got, want := addressList.increment(), false; got != want { + t.Errorf("addressList.increment() = %t, want %t", got, want) + } + + // Seek to the key again, it is behind the pointer now. + if got, want := addressList.seekTo(key), true; got != want { + t.Errorf("addressList.seekTo(%v) = %t, want %t", key, got, want) + } + + // Seek to a key not in the list. + key = resolver.Address{ + Addr: "192.168.1.2", + ServerName: "test-host-5", + Attributes: attributes.New("key-5", "val-5"), + BalancerAttributes: attributes.New("ignored", "bal-val-5"), + } + // Seek to the key again, it is behind the pointer now. + if got, want := addressList.seekTo(key), false; got != want { + t.Errorf("addressList.seekTo(%v) = %t, want %t", key, got, want) + } + + // It should be possible to increment once more since the pointer has not advanced. + if got, want := addressList.increment(), true; got != want { + t.Errorf("addressList.increment() = %t, want %t", got, want) + } + if got, want := addressList.increment(), false; got != want { + t.Errorf("addressList.increment() = %t, want %t", got, want) + } +} diff --git a/test/pickfirst_leaf_test.go b/test/pickfirst_leaf_test.go index 6364a905127d..3bf1f4dd48f7 100644 --- a/test/pickfirst_leaf_test.go +++ b/test/pickfirst_leaf_test.go @@ -445,6 +445,63 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { } } +func (s) TestPickFirstLeaf_EmptyAddressList(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + balChan := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancerChan: balChan}) + cc, r, backends := setupPickFirstLeaf(t, 1) + addrs := stubBackendsToResolverAddrs(backends) + + r.UpdateState(resolver.State{Addresses: addrs}) + bal := <-balChan + select { + case <-bal.resolverUpdateSeen: + case <-ctx.Done(): + t.Fatalf("Context timed out waiting for resolve update to be processed") + } + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + r.UpdateState(resolver.State{}) + select { + case <-bal.resolverUpdateSeen: + case <-ctx.Done(): + t.Fatalf("Context timed out waiting for resolve update to be processed") + } + + // The balancer should have entered transient failure. + // It should transition to CONNECTING from TRANSIENT_FAILURE as sticky TF + // only applies when the initial TF is reported due to connection failures + // and not bad resolver states. + r.UpdateState(resolver.State{Addresses: addrs}) + select { + case <-bal.resolverUpdateSeen: + case <-ctx.Done(): + t.Fatalf("Context timed out waiting for resolve update to be processed") + } + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + wantTransitions := []connectivity.State{ + // From first resolver update. + connectivity.Connecting, + connectivity.Ready, + // From second update. + connectivity.TransientFailure, + // From third update. + connectivity.Ready, + } + + if diff := cmp.Diff(wantTransitions, bal.connStateTransitions()); diff != "" { + t.Errorf("balancer states mismatch (-want +got):\n%s", diff) + } +} + // stateStoringBalancer stores the state of the subconns being created. type stateStoringBalancer struct { balancer.Balancer From abb263bb2bacd8545d2c394df0217590c98166d2 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Thu, 15 Aug 2024 01:09:24 +0530 Subject: [PATCH 28/62] Remove stale comment --- balancer/pickfirst_leaf/pickfirst_leaf.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/balancer/pickfirst_leaf/pickfirst_leaf.go b/balancer/pickfirst_leaf/pickfirst_leaf.go index 33c68a749f90..c4607ca266cb 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf.go +++ b/balancer/pickfirst_leaf/pickfirst_leaf.go @@ -197,8 +197,6 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState b.resolverError(errors.New("produced zero addresses")) return balancer.ErrBadResolverState } - // We don't have to guard this block with the env var because ParseConfig - // already does so. cfg, ok := state.BalancerConfig.(pfConfig) if state.BalancerConfig != nil && !ok { return fmt.Errorf("pickfirst: received illegal BalancerConfig (type %T): %v", state.BalancerConfig, state.BalancerConfig) From 67b9b6b65b04b142ef70a70b5a8abacca4794897 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Thu, 15 Aug 2024 01:22:14 +0530 Subject: [PATCH 29/62] Fix tests --- balancer/pickfirst_leaf/pickfirst_leaf.go | 5 ++++- balancer/pickfirst_leaf/pickfirst_leaf_test.go | 18 ++++++++++++++++++ test/pickfirst_leaf_test.go | 1 + 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/balancer/pickfirst_leaf/pickfirst_leaf.go b/balancer/pickfirst_leaf/pickfirst_leaf.go index c4607ca266cb..331ceb444009 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf.go +++ b/balancer/pickfirst_leaf/pickfirst_leaf.go @@ -384,7 +384,10 @@ func (b *pickfirstBalancer) requestConnection() { curAddr := b.addressIndex.currentAddress() sd, ok := b.subConns.Get(curAddr) if !ok { - sd, err := newSCData(b, curAddr) + var err error + // We want to assign the new scData to sd from the outer scope, hence + // we can't use := below. + sd, err = newSCData(b, curAddr) if err != nil { // This should never happen. b.logger.Warningf("Failed to create a subConn for address %v: %v", curAddr.String(), err) diff --git a/balancer/pickfirst_leaf/pickfirst_leaf_test.go b/balancer/pickfirst_leaf/pickfirst_leaf_test.go index 766714e80056..ab10f5805762 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf_test.go +++ b/balancer/pickfirst_leaf/pickfirst_leaf_test.go @@ -1,3 +1,21 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package pickfirstleaf import ( diff --git a/test/pickfirst_leaf_test.go b/test/pickfirst_leaf_test.go index 3bf1f4dd48f7..821d1f92921f 100644 --- a/test/pickfirst_leaf_test.go +++ b/test/pickfirst_leaf_test.go @@ -494,6 +494,7 @@ func (s) TestPickFirstLeaf_EmptyAddressList(t *testing.T) { // From second update. connectivity.TransientFailure, // From third update. + connectivity.Connecting, connectivity.Ready, } From 07920028b2cb3899c56c1e08083ad5111c03e615 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Thu, 15 Aug 2024 01:26:49 +0530 Subject: [PATCH 30/62] Fix vet --- balancer/pickfirst_leaf/pickfirst_leaf.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/balancer/pickfirst_leaf/pickfirst_leaf.go b/balancer/pickfirst_leaf/pickfirst_leaf.go index 331ceb444009..5fd07e097624 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf.go +++ b/balancer/pickfirst_leaf/pickfirst_leaf.go @@ -318,7 +318,7 @@ func (b *pickfirstBalancer) ExitIdle() { }) } -// exitIdle starts a conection attempt if not already started. +// exitIdle starts a connection attempt if not already started. // Only executed in the context of a serializer callback. func (b *pickfirstBalancer) exitIdle() { b.requestConnection() From 3561462d548d1e0bf6b1cc675f621d4892dba9f5 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Thu, 15 Aug 2024 01:40:21 +0530 Subject: [PATCH 31/62] start connecting in exitIdle only when balancer is in idle --- balancer/pickfirst_leaf/pickfirst_leaf.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/balancer/pickfirst_leaf/pickfirst_leaf.go b/balancer/pickfirst_leaf/pickfirst_leaf.go index 5fd07e097624..803b038d06ef 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf.go +++ b/balancer/pickfirst_leaf/pickfirst_leaf.go @@ -321,7 +321,9 @@ func (b *pickfirstBalancer) ExitIdle() { // exitIdle starts a connection attempt if not already started. // Only executed in the context of a serializer callback. func (b *pickfirstBalancer) exitIdle() { - b.requestConnection() + if b.state == connectivity.Idle { + b.requestConnection() + } } func (b *pickfirstBalancer) closeSubConns() { From d82da591bb14e88cca4f2cfeffe2bd4ea67096f9 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Fri, 16 Aug 2024 15:01:17 +0530 Subject: [PATCH 32/62] Address review comments --- balancer/pickfirst_leaf/pickfirst_leaf.go | 26 ++++++++++++----------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/balancer/pickfirst_leaf/pickfirst_leaf.go b/balancer/pickfirst_leaf/pickfirst_leaf.go index 803b038d06ef..7b697746a2a6 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf.go +++ b/balancer/pickfirst_leaf/pickfirst_leaf.go @@ -101,6 +101,7 @@ type scData struct { subConn balancer.SubConn state connectivity.State addr resolver.Address + lastErr error } func newSCData(b *pickfirstBalancer, addr resolver.Address) (*scData, error) { @@ -143,7 +144,6 @@ type pickfirstBalancer struct { subConns *resolver.AddressMap // scData for active subonns mapped by address. addressIndex addressList firstPass bool - firstErr error } func (b *pickfirstBalancer) ResolverError(err error) { @@ -213,13 +213,18 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState for i, a := range state.ResolverState.Addresses { newEndpoints[i].Attributes = a.BalancerAttributes newEndpoints[i].Addresses = []resolver.Address{a} - // We can't remove address attributes here since xds packages use + // We can't remove balancer attributes here since xds packages use // them to store locality metadata. } } // Since we have a new set of addresses, we are again at first pass. b.firstPass = true + // If multiple endpoints have the same address, they would use the same + // subconn because an AddressMap is used to store subconns. + // This would result in attempting to connect to the same subconn multiple times + // in the same pass. We don't want this, so we ensure each address is present + // in only one endpoint before moving further. newEndpoints = deDupAddresses(newEndpoints) // Perform the optional shuffling described in gRFC A62. The shuffling will @@ -269,7 +274,6 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState // If its the first resolver update or the balancer was already READY or // or CONNECTING, enter CONNECTING. if b.state == connectivity.Ready || b.state == connectivity.Connecting || oldAddrs.Len() == 0 { - b.firstErr = nil // Start connection attempt at first address. b.state = connectivity.Connecting b.cc.UpdateState(balancer.State{ @@ -278,7 +282,6 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState }) b.requestConnection() } else if b.state == connectivity.Idle { - b.firstErr = nil b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.Idle, Picker: &idlePicker{exitIdle: b.ExitIdle}, @@ -413,7 +416,7 @@ func (b *pickfirstBalancer) requestConnection() { scd.subConn.Connect() case connectivity.TransientFailure: if !b.addressIndex.increment() { - b.endFirstPass() + b.endFirstPass(scd.lastErr) } b.requestConnection() case connectivity.Ready: @@ -440,7 +443,6 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon b.shutdownRemaining(sd) b.addressIndex.seekTo(sd.addr) b.state = connectivity.Ready - b.firstErr = nil b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.Ready, Picker: &picker{result: balancer.PickResult{SubConn: sd.subConn}}, @@ -476,9 +478,7 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon }) } case connectivity.TransientFailure: - if b.firstErr == nil { - b.firstErr = state.ConnectionError - } + sd.lastErr = state.ConnectionError // Since we're re-using common subconns while handling resolver updates, // we could receive an out of turn TRANSIENT_FAILURE from a pass // over the previous address list. We ignore such updates. @@ -491,7 +491,7 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon return } // End of the first pass. - b.endFirstPass() + b.endFirstPass(state.ConnectionError) } return } @@ -499,6 +499,7 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon // We have finished the first pass, keep re-connecting failing subconns. switch state.ConnectivityState { case connectivity.TransientFailure: + sd.lastErr = state.ConnectionError b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.TransientFailure, Picker: &picker{err: state.ConnectionError}, @@ -510,12 +511,13 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon } } -func (b *pickfirstBalancer) endFirstPass() { +func (b *pickfirstBalancer) endFirstPass(lastErr error) { b.firstPass = false b.state = connectivity.TransientFailure + b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.TransientFailure, - Picker: &picker{err: b.firstErr}, + Picker: &picker{err: lastErr}, }) // Start re-connecting all the subconns that are already in IDLE. for _, v := range b.subConns.Values() { From 2ed4aff01f9d84db62b08c3e9b885c7fbbafb533 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Mon, 19 Aug 2024 15:08:39 +0530 Subject: [PATCH 33/62] Add comments to address review --- balancer/pickfirst_leaf/pickfirst_leaf.go | 8 ++++++++ clientconn.go | 2 ++ 2 files changed, 10 insertions(+) diff --git a/balancer/pickfirst_leaf/pickfirst_leaf.go b/balancer/pickfirst_leaf/pickfirst_leaf.go index 7b697746a2a6..49eff093ffd6 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf.go +++ b/balancer/pickfirst_leaf/pickfirst_leaf.go @@ -273,6 +273,10 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState // If its the first resolver update or the balancer was already READY or // or CONNECTING, enter CONNECTING. + // We may be in TRANSIENT_FAILURE due to a previous empty address list, + // we should still enter CONNECTING because the sticky TF behaviour mentioned + // in A62 applied only when the TRANSIENT_FAILURE is reported dur to connectivity + // failures. if b.state == connectivity.Ready || b.state == connectivity.Connecting || oldAddrs.Len() == 0 { // Start connection attempt at first address. b.state = connectivity.Connecting @@ -282,11 +286,15 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState }) b.requestConnection() } else if b.state == connectivity.Idle { + // If this is not the first resolver update and we're in IDLE, remain + // in IDLE until the picker is used. b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.Idle, Picker: &idlePicker{exitIdle: b.ExitIdle}, }) } else if b.state == connectivity.TransientFailure { + // If we're in TRANSIENT_FAILURE, we stay in TRANSIENT_FAILURE until + // we're READY. See A62. b.requestConnection() } return nil diff --git a/clientconn.go b/clientconn.go index cf1a7ec6895f..6ad13c85c677 100644 --- a/clientconn.go +++ b/clientconn.go @@ -1254,6 +1254,8 @@ func (ac *addrConn) resetTransportAndUnlock() { ac.mu.Unlock() if err := ac.tryAllAddrs(acCtx, addrs, connectDeadline); err != nil { + // TODO: #7534 - Move re-resolution requests into the pick_first LB policy + // to ensure one resolution request per pass instead of per subconn failure. ac.cc.resolveNow(resolver.ResolveNowOptions{}) ac.mu.Lock() if acCtx.Err() != nil { From fe8637f7c6a008dc79dfbb9b0421877ca54b7dd6 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Wed, 21 Aug 2024 01:45:21 +0530 Subject: [PATCH 34/62] Handle subconn transitions from CONNECTING to IDLE --- balancer/pickfirst_leaf/pickfirst_leaf.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/balancer/pickfirst_leaf/pickfirst_leaf.go b/balancer/pickfirst_leaf/pickfirst_leaf.go index 49eff093ffd6..0a49952d6386 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf.go +++ b/balancer/pickfirst_leaf/pickfirst_leaf.go @@ -425,6 +425,7 @@ func (b *pickfirstBalancer) requestConnection() { case connectivity.TransientFailure: if !b.addressIndex.increment() { b.endFirstPass(scd.lastErr) + return } b.requestConnection() case connectivity.Ready: @@ -440,7 +441,7 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon // To prevent pickers from returning these obsolete subconns, this logic // is included to check if the current list of active subconns includes this // subconn. - if activeSd, found := b.subConns.Get(sd.addr); !found || activeSd != sd { + if activeSD, found := b.subConns.Get(sd.addr); !found || activeSD != sd { return } if state.ConnectivityState == connectivity.Shutdown { @@ -491,7 +492,7 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon // we could receive an out of turn TRANSIENT_FAILURE from a pass // over the previous address list. We ignore such updates. curAddr := b.addressIndex.currentAddress() - if activeSd, found := b.subConns.Get(curAddr); !found || activeSd != sd { + if activeSD, found := b.subConns.Get(curAddr); !found || activeSD != sd { return } if b.addressIndex.increment() { @@ -500,6 +501,21 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon } // End of the first pass. b.endFirstPass(state.ConnectionError) + case connectivity.Idle: + // A subconn can transition from CONNECTING directly to IDLE when + // a transport is successfully created, but the connection fails before + // the subconn can send the notification for READY. We treat this + // as a successful connection and transition to IDLE. + curAddr := b.addressIndex.currentAddress() + if activeSD, found := b.subConns.Get(curAddr); !found || activeSD != sd { + return + } + b.state = connectivity.Idle + b.addressIndex.reset() + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.Idle, + Picker: &idlePicker{exitIdle: b.ExitIdle}, + }) } return } From 02267d3b08ac6de73dccd6c544fe7a430e173f2d Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Thu, 22 Aug 2024 16:50:19 +0530 Subject: [PATCH 35/62] address review comments --- balancer/pickfirst_leaf/pickfirst_leaf.go | 148 ++++++++++------------ test/pickfirst_leaf_test.go | 2 +- 2 files changed, 68 insertions(+), 82 deletions(-) diff --git a/balancer/pickfirst_leaf/pickfirst_leaf.go b/balancer/pickfirst_leaf/pickfirst_leaf.go index 0a49952d6386..8f5eda15861e 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf.go +++ b/balancer/pickfirst_leaf/pickfirst_leaf.go @@ -43,7 +43,7 @@ func init() { if envconfig.NewPickFirstEnabled { internal.ShuffleAddressListForTesting = func(n int, swap func(i, j int)) { rand.Shuffle(n, swap) } // Register as the default pick_first balancer. - PickFirstLeafName = "pick_first" + Name = "pick_first" } balancer.Register(pickfirstBuilder{}) } @@ -51,10 +51,10 @@ func init() { var ( logger = grpclog.Component("pick-first-leaf-lb") errBalancerClosed = fmt.Errorf("pickfirst: LB policy is closed") - // PickFirstLeafName is the name of the pick_first_leaf balancer. + // Name is the name of the pick_first_leaf balancer. // Can be changed in init() if this balancer is to be registered as the default // pickfirst. - PickFirstLeafName = "pick_first_leaf" + Name = "pick_first_leaf" ) const logPrefix = "[pick-first-leaf-lb %p] " @@ -69,14 +69,14 @@ func (pickfirstBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) b subConns: resolver.NewAddressMap(), serializer: grpcsync.NewCallbackSerializer(ctx), serializerCancel: cancel, - state: connectivity.Idle, + state: connectivity.Connecting, } b.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(logPrefix, b)) return b } func (b pickfirstBuilder) Name() string { - return PickFirstLeafName + return Name } func (pickfirstBuilder) ParseConfig(js json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { @@ -199,7 +199,7 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState } cfg, ok := state.BalancerConfig.(pfConfig) if state.BalancerConfig != nil && !ok { - return fmt.Errorf("pickfirst: received illegal BalancerConfig (type %T): %v", state.BalancerConfig, state.BalancerConfig) + return fmt.Errorf("pickfirst: received illegal BalancerConfig (type %T): %v: %w", state.BalancerConfig, state.BalancerConfig, balancer.ErrBadResolverState) } if b.logger.V(2) { @@ -222,9 +222,9 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState b.firstPass = true // If multiple endpoints have the same address, they would use the same // subconn because an AddressMap is used to store subconns. - // This would result in attempting to connect to the same subconn multiple times - // in the same pass. We don't want this, so we ensure each address is present - // in only one endpoint before moving further. + // This would result in attempting to connect to the same subconn multiple + // times in the same pass. We don't want this, so we ensure each address is + // present in only one endpoint before moving further. newEndpoints = deDupAddresses(newEndpoints) // Perform the optional shuffling described in gRFC A62. The shuffling will @@ -235,49 +235,24 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState internal.ShuffleAddressListForTesting.(func(int, func(int, int)))(len(newEndpoints), func(i, j int) { newEndpoints[i], newEndpoints[j] = newEndpoints[j], newEndpoints[i] }) } - if b.state == connectivity.Ready { - // If the previous ready subconn exists in new address list, - // keep this connection and don't create new subconns. - prevAddr := b.addressIndex.currentAddress() - b.addressIndex.updateEndpointList(newEndpoints) - if b.addressIndex.seekTo(prevAddr) { - return nil - } - } else { - b.addressIndex.updateEndpointList(newEndpoints) - } - - // Remove old subConns that were not in new address list. - oldAddrs := resolver.NewAddressMap() - for _, k := range b.subConns.Keys() { - oldAddrs.Set(k, true) - } - - // Flatten the new endpoint addresses. - newAddrs := resolver.NewAddressMap() - for _, endpoint := range newEndpoints { - for _, addr := range endpoint.Addresses { - newAddrs.Set(addr, true) - } - } - - // Shut them down and remove them. - for _, oldAddr := range oldAddrs.Keys() { - if _, ok := newAddrs.Get(oldAddr); ok { - continue - } - val, _ := b.subConns.Get(oldAddr) - val.(*scData).subConn.Shutdown() - b.subConns.Delete(oldAddr) + // If the previous ready subconn exists in new address list, + // keep this connection and don't create new subconns. + prevAddr := b.addressIndex.currentAddress() + prevAddrsCount := b.addressIndex.size() + b.addressIndex.updateEndpointList(newEndpoints) + if b.state == connectivity.Ready && b.addressIndex.seekTo(prevAddr) { + return nil } - // If its the first resolver update or the balancer was already READY or - // or CONNECTING, enter CONNECTING. + b.reconcileSubConns(newEndpoints) + // If its the first resolver update or the balancer was already READY + // (but the new address list does not contain the ready subconn) or + // CONNECTING, enter CONNECTING. // We may be in TRANSIENT_FAILURE due to a previous empty address list, // we should still enter CONNECTING because the sticky TF behaviour mentioned - // in A62 applied only when the TRANSIENT_FAILURE is reported dur to connectivity + // in A62 applies only when the TRANSIENT_FAILURE is reported due to connectivity // failures. - if b.state == connectivity.Ready || b.state == connectivity.Connecting || oldAddrs.Len() == 0 { + if b.state == connectivity.Ready || b.state == connectivity.Connecting || prevAddrsCount == 0 { // Start connection attempt at first address. b.state = connectivity.Connecting b.cc.UpdateState(balancer.State{ @@ -285,13 +260,6 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState Picker: &picker{err: balancer.ErrNoSubConnAvailable}, }) b.requestConnection() - } else if b.state == connectivity.Idle { - // If this is not the first resolver update and we're in IDLE, remain - // in IDLE until the picker is used. - b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.Idle, - Picker: &idlePicker{exitIdle: b.ExitIdle}, - }) } else if b.state == connectivity.TransientFailure { // If we're in TRANSIENT_FAILURE, we stay in TRANSIENT_FAILURE until // we're READY. See A62. @@ -308,42 +276,28 @@ func (b *pickfirstBalancer) UpdateSubConnState(subConn balancer.SubConn, state b func (b *pickfirstBalancer) Close() { b.serializer.TrySchedule(func(_ context.Context) { - b.close() + b.closeSubConns() + b.state = connectivity.Shutdown }) - <-b.serializer.Done() -} - -// close closes the balancer. -// Only executed in the context of a serializer callback. -func (b *pickfirstBalancer) close() { b.serializerCancel() - b.closeSubConns() - b.state = connectivity.Shutdown + <-b.serializer.Done() } // ExitIdle moves the balancer out of idle state. It can be called concurrently // by the idlePicker and clientConn so access to variables should be synchronized. func (b *pickfirstBalancer) ExitIdle() { b.serializer.TrySchedule(func(_ context.Context) { - b.exitIdle() + if b.state == connectivity.Idle { + b.requestConnection() + } }) } -// exitIdle starts a connection attempt if not already started. -// Only executed in the context of a serializer callback. -func (b *pickfirstBalancer) exitIdle() { - if b.state == connectivity.Idle { - b.requestConnection() - } -} - func (b *pickfirstBalancer) closeSubConns() { for _, sd := range b.subConns.Values() { sd.(*scData).subConn.Shutdown() } - for _, k := range b.subConns.Keys() { - b.subConns.Delete(k) - } + b.subConns = resolver.NewAddressMap() } // deDupAddresses ensures that each address belongs to only one endpoint. @@ -370,6 +324,32 @@ func deDupAddresses(endpoints []resolver.Endpoint) []resolver.Endpoint { return newEndpoints } +func (b *pickfirstBalancer) reconcileSubConns(newEndpoints []resolver.Endpoint) { + // Remove old subConns that were not in new address list. + oldAddrs := resolver.NewAddressMap() + for _, k := range b.subConns.Keys() { + oldAddrs.Set(k, true) + } + + // Flatten the new endpoint addresses. + newAddrs := resolver.NewAddressMap() + for _, endpoint := range newEndpoints { + for _, addr := range endpoint.Addresses { + newAddrs.Set(addr, true) + } + } + + // Shut them down and remove them. + for _, oldAddr := range oldAddrs.Keys() { + if _, ok := newAddrs.Get(oldAddr); ok { + continue + } + val, _ := b.subConns.Get(oldAddr) + val.(*scData).subConn.Shutdown() + b.subConns.Delete(oldAddr) + } +} + // shutdownRemaining shuts down remaining subConns. Called when a subConn // becomes ready, which means that all other subConn must be shutdown. func (b *pickfirstBalancer) shutdownRemaining(selected *scData) { @@ -402,16 +382,16 @@ func (b *pickfirstBalancer) requestConnection() { // we can't use := below. sd, err = newSCData(b, curAddr) if err != nil { - // This should never happen. + // This should never happen, unless the clientConn is being shut + // down. b.logger.Warningf("Failed to create a subConn for address %v: %v", curAddr.String(), err) + // The LB policy remains in TRANSIENT_FAILURE until a new resolver + // update is received. b.state = connectivity.TransientFailure b.addressIndex.reset() b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.TransientFailure, - // Return an idle picker so that the clientConn doesn't remain - // stuck in TRANSIENT_FAILURE and attempts to re-connect the - // next time picker.Pick is called. - Picker: &idlePicker{exitIdle: b.ExitIdle}, + Picker: &picker{err: fmt.Errorf("failed to create a new subConn: %v", err)}, }) return } @@ -585,6 +565,10 @@ func (al *addressList) isValid() bool { return al.idx < len(al.addresses) } +func (al *addressList) size() int { + return len(al.addresses) +} + // increment moves to the next index in the address list. If at the last address // in the address list, moves to the next endpoint in the endpoint list. // This method returns false if it went off the list, true otherwise. @@ -596,9 +580,11 @@ func (al *addressList) increment() bool { return al.idx < len(al.addresses) } +// currentAddress returns the current address pointed to in the addressList. +// If the list is in an invalid state, it returns an empty address instead. func (al *addressList) currentAddress() resolver.Address { if !al.isValid() { - panic("pickfirst: index is off the end of the address list") + return resolver.Address{} } return al.addresses[al.idx] } diff --git a/test/pickfirst_leaf_test.go b/test/pickfirst_leaf_test.go index 821d1f92921f..71063b23ad2c 100644 --- a/test/pickfirst_leaf_test.go +++ b/test/pickfirst_leaf_test.go @@ -534,7 +534,7 @@ func (b *stateStoringBalancerBuilder) Build(cc balancer.ClientConn, opts balance bal := &stateStoringBalancer{ resolverUpdateSeen: make(chan struct{}, 1), } - bal.Balancer = balancer.Get(pickfirstleaf.PickFirstLeafName).Build(&stateStoringCCWrapper{cc, bal}, opts) + bal.Balancer = balancer.Get(pickfirstleaf.Name).Build(&stateStoringCCWrapper{cc, bal}, opts) b.balancerChan <- bal return bal } From a3049fecfe945e4274e17b64a89b2f276978d3ca Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Fri, 23 Aug 2024 22:56:04 +0530 Subject: [PATCH 36/62] Address review comments --- balancer/pickfirst/pickfirst.go | 2 +- .../pickfirstleaf.go} | 67 ++++++++++--------- .../pickfirstleaf_test.go} | 19 +++--- .../pickfirstleaf/test/pickfirstleaf_test.go | 31 ++++++++- 4 files changed, 74 insertions(+), 45 deletions(-) rename balancer/{pickfirst_leaf/pickfirst_leaf.go => pickfirstleaf/pickfirstleaf.go} (91%) rename balancer/{pickfirst_leaf/pickfirst_leaf_test.go => pickfirstleaf/pickfirstleaf_test.go} (90%) rename test/pickfirst_leaf_test.go => balancer/pickfirstleaf/test/pickfirstleaf_test.go (95%) diff --git a/balancer/pickfirst/pickfirst.go b/balancer/pickfirst/pickfirst.go index b6676cf4b94b..8c7ee1a85360 100644 --- a/balancer/pickfirst/pickfirst.go +++ b/balancer/pickfirst/pickfirst.go @@ -35,7 +35,7 @@ import ( "google.golang.org/grpc/resolver" "google.golang.org/grpc/serviceconfig" - _ "google.golang.org/grpc/balancer/pickfirst_leaf" // For automatically registering the new pickfirst if required. + _ "google.golang.org/grpc/balancer/pickfirstleaf" // For automatically registering the new pickfirst if required. ) func init() { diff --git a/balancer/pickfirst_leaf/pickfirst_leaf.go b/balancer/pickfirstleaf/pickfirstleaf.go similarity index 91% rename from balancer/pickfirst_leaf/pickfirst_leaf.go rename to balancer/pickfirstleaf/pickfirstleaf.go index 8f5eda15861e..73aa67eec42b 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf.go +++ b/balancer/pickfirstleaf/pickfirstleaf.go @@ -65,7 +65,7 @@ func (pickfirstBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) b ctx, cancel := context.WithCancel(context.Background()) b := &pickfirstBalancer{ cc: cc, - addressIndex: addressList{}, + addressList: addressList{}, subConns: resolver.NewAddressMap(), serializer: grpcsync.NewCallbackSerializer(ctx), serializerCancel: cancel, @@ -142,7 +142,7 @@ type pickfirstBalancer struct { serializerCancel func() state connectivity.State subConns *resolver.AddressMap // scData for active subonns mapped by address. - addressIndex addressList + addressList addressList firstPass bool } @@ -166,7 +166,7 @@ func (b *pickfirstBalancer) resolverError(err error) { } b.closeSubConns() - b.addressIndex.updateEndpointList(nil) + b.addressList.updateEndpointList(nil) b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.TransientFailure, Picker: &picker{err: fmt.Errorf("name resolver error: %v", err)}, @@ -237,10 +237,10 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState // If the previous ready subconn exists in new address list, // keep this connection and don't create new subconns. - prevAddr := b.addressIndex.currentAddress() - prevAddrsCount := b.addressIndex.size() - b.addressIndex.updateEndpointList(newEndpoints) - if b.state == connectivity.Ready && b.addressIndex.seekTo(prevAddr) { + prevAddr := b.addressList.currentAddress() + prevAddrsCount := b.addressList.size() + b.addressList.updateEndpointList(newEndpoints) + if b.state == connectivity.Ready && b.addressList.seekTo(prevAddr) { return nil } @@ -365,16 +365,15 @@ func (b *pickfirstBalancer) shutdownRemaining(selected *scData) { b.subConns.Set(selected.addr, selected) } -// requestConnection requests a connection to the next applicable address' -// subcon, creating one if necessary. Schedules a connection to next address in list as well. -// If the current channel has already attempted a connection, we attempt a connection -// to the next address/subconn in our list. We assume that NewSubConn will never -// return an error. +// requestConnection starts connecting on the subchannel corresponding to the +// current address. If no subchannel exists, one is created. If the current +// subchannel is in TransientFailure, a connection to the next address is +// attempted. func (b *pickfirstBalancer) requestConnection() { - if !b.addressIndex.isValid() || b.state == connectivity.Shutdown { + if !b.addressList.isValid() || b.state == connectivity.Shutdown { return } - curAddr := b.addressIndex.currentAddress() + curAddr := b.addressList.currentAddress() sd, ok := b.subConns.Get(curAddr) if !ok { var err error @@ -388,7 +387,7 @@ func (b *pickfirstBalancer) requestConnection() { // The LB policy remains in TRANSIENT_FAILURE until a new resolver // update is received. b.state = connectivity.TransientFailure - b.addressIndex.reset() + b.addressList.reset() b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.TransientFailure, Picker: &picker{err: fmt.Errorf("failed to create a new subConn: %v", err)}, @@ -403,7 +402,7 @@ func (b *pickfirstBalancer) requestConnection() { case connectivity.Idle: scd.subConn.Connect() case connectivity.TransientFailure: - if !b.addressIndex.increment() { + if !b.addressList.increment() { b.endFirstPass(scd.lastErr) return } @@ -430,7 +429,12 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon if state.ConnectivityState == connectivity.Ready { b.shutdownRemaining(sd) - b.addressIndex.seekTo(sd.addr) + if !b.addressList.seekTo(sd.addr) { + // This should not fail as we should have only one subconn after + // entering READY. The subconn should be present in the addressList. + b.logger.Errorf("Address %q not found address list in %v", sd.addr, b.addressList.addresses) + return + } b.state = connectivity.Ready b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.Ready, @@ -442,10 +446,10 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon // If we are transitioning from READY to IDLE, reset index and re-connect when // prompted. if b.state == connectivity.Ready && state.ConnectivityState == connectivity.Idle { - // Once a transport fails, we enter idle and start from the first address - // when the picker is used. + // Once a transport fails, the balancer enters IDLE and starts from + // the first address when the picker is used. b.state = connectivity.Idle - b.addressIndex.reset() + b.addressList.reset() b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.Idle, Picker: &idlePicker{exitIdle: b.ExitIdle}, @@ -456,10 +460,10 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon if b.firstPass { switch state.ConnectivityState { case connectivity.Connecting: - // We can be in either IDLE, CONNECTING or TRANSIENT_FAILURE. - // If we're in TRANSIENT_FAILURE, we stay in TRANSIENT_FAILURE until - // we're READY. See A62. - // If we're already in CONNECTING, no update is needed. + // The balancer can be in either IDLE, CONNECTING or TRANSIENT_FAILURE. + // If it's in TRANSIENT_FAILURE, stay in TRANSIENT_FAILURE until + // it's READY. See A62. + // If the balancer is already in CONNECTING, no update is needed. if b.state == connectivity.Idle { b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.Connecting, @@ -471,11 +475,11 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon // Since we're re-using common subconns while handling resolver updates, // we could receive an out of turn TRANSIENT_FAILURE from a pass // over the previous address list. We ignore such updates. - curAddr := b.addressIndex.currentAddress() - if activeSD, found := b.subConns.Get(curAddr); !found || activeSD != sd { + + if curAddr := b.addressList.currentAddress(); !equalAddressIgnoringBalAttributes(&curAddr, &sd.addr) { return } - if b.addressIndex.increment() { + if b.addressList.increment() { b.requestConnection() return } @@ -486,12 +490,11 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon // a transport is successfully created, but the connection fails before // the subconn can send the notification for READY. We treat this // as a successful connection and transition to IDLE. - curAddr := b.addressIndex.currentAddress() - if activeSD, found := b.subConns.Get(curAddr); !found || activeSD != sd { + if curAddr := b.addressList.currentAddress(); !equalAddressIgnoringBalAttributes(&sd.addr, &curAddr) { return } b.state = connectivity.Idle - b.addressIndex.reset() + b.addressList.reset() b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.Idle, Picker: &idlePicker{exitIdle: b.ExitIdle}, @@ -510,6 +513,7 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon }) // We don't need to request re-resolution since the subconn already does // that before reporting TRANSIENT_FAILURE. + // TODO: #7534 - Move re-resolution requests from subconn into pick_first. case connectivity.Idle: sd.subConn.Connect() } @@ -569,8 +573,7 @@ func (al *addressList) size() int { return len(al.addresses) } -// increment moves to the next index in the address list. If at the last address -// in the address list, moves to the next endpoint in the endpoint list. +// increment moves to the next index in the address list. // This method returns false if it went off the list, true otherwise. func (al *addressList) increment() bool { if !al.isValid() { diff --git a/balancer/pickfirst_leaf/pickfirst_leaf_test.go b/balancer/pickfirstleaf/pickfirstleaf_test.go similarity index 90% rename from balancer/pickfirst_leaf/pickfirst_leaf_test.go rename to balancer/pickfirstleaf/pickfirstleaf_test.go index ab10f5805762..5b5285a6e0b7 100644 --- a/balancer/pickfirst_leaf/pickfirst_leaf_test.go +++ b/balancer/pickfirstleaf/pickfirstleaf_test.go @@ -72,24 +72,25 @@ func (s) TestAddressList_Iteration(t *testing.T) { for i := 0; i < len(addrs); i++ { if got, want := addressList.isValid(), true; got != want { - t.Errorf("addressList.isValid() = %t, want %t", got, want) + t.Fatalf("addressList.isValid() = %t, want %t", got, want) } if got, want := addressList.currentAddress(), addrs[i]; !want.Equal(got) { t.Errorf("addressList.currentAddress() = %v, want %v", got, want) } if got, want := addressList.increment(), i+1 < len(addrs); got != want { - t.Errorf("addressList.increment() = %t, want %t", got, want) + t.Fatalf("addressList.increment() = %t, want %t", got, want) } } if got, want := addressList.isValid(), false; got != want { - t.Errorf("addressList.isValid() = %t, want %t", got, want) + t.Fatalf("addressList.isValid() = %t, want %t", got, want) } // increment an invalid address list. if got, want := addressList.increment(), false; got != want { t.Errorf("addressList.increment() = %t, want %t", got, want) } + if got, want := addressList.isValid(), false; got != want { t.Errorf("addressList.isValid() = %t, want %t", got, want) } @@ -97,13 +98,13 @@ func (s) TestAddressList_Iteration(t *testing.T) { addressList.reset() for i := 0; i < len(addrs); i++ { if got, want := addressList.isValid(), true; got != want { - t.Errorf("addressList.isValid() = %t, want %t", got, want) + t.Fatalf("addressList.isValid() = %t, want %t", got, want) } if got, want := addressList.currentAddress(), addrs[i]; !want.Equal(got) { t.Errorf("addressList.currentAddress() = %v, want %v", got, want) } if got, want := addressList.increment(), i+1 < len(addrs); got != want { - t.Errorf("addressList.increment() = %t, want %t", got, want) + t.Fatalf("addressList.increment() = %t, want %t", got, want) } } } @@ -159,6 +160,7 @@ func (s) TestAddressList_SeekTo(t *testing.T) { if got, want := addressList.increment(), true; got != want { t.Errorf("addressList.increment() = %t, want %t", got, want) } + if got, want := addressList.increment(), false; got != want { t.Errorf("addressList.increment() = %t, want %t", got, want) } @@ -170,20 +172,17 @@ func (s) TestAddressList_SeekTo(t *testing.T) { // Seek to a key not in the list. key = resolver.Address{ - Addr: "192.168.1.2", + Addr: "192.168.1.5", ServerName: "test-host-5", Attributes: attributes.New("key-5", "val-5"), BalancerAttributes: attributes.New("ignored", "bal-val-5"), } - // Seek to the key again, it is behind the pointer now. - if got, want := addressList.seekTo(key), false; got != want { - t.Errorf("addressList.seekTo(%v) = %t, want %t", key, got, want) - } // It should be possible to increment once more since the pointer has not advanced. if got, want := addressList.increment(), true; got != want { t.Errorf("addressList.increment() = %t, want %t", got, want) } + if got, want := addressList.increment(), false; got != want { t.Errorf("addressList.increment() = %t, want %t", got, want) } diff --git a/test/pickfirst_leaf_test.go b/balancer/pickfirstleaf/test/pickfirstleaf_test.go similarity index 95% rename from test/pickfirst_leaf_test.go rename to balancer/pickfirstleaf/test/pickfirstleaf_test.go index 71063b23ad2c..a525d2ca793b 100644 --- a/test/pickfirst_leaf_test.go +++ b/balancer/pickfirstleaf/test/pickfirstleaf_test.go @@ -23,15 +23,17 @@ import ( "fmt" "sync" "testing" + "time" "github.com/google/go-cmp/cmp" "google.golang.org/grpc" "google.golang.org/grpc/balancer" - pickfirstleaf "google.golang.org/grpc/balancer/pickfirst_leaf" + "google.golang.org/grpc/balancer/pickfirstleaf" "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/internal/stubserver" "google.golang.org/grpc/internal/testutils/pickfirst" "google.golang.org/grpc/resolver" @@ -42,9 +44,24 @@ import ( testpb "google.golang.org/grpc/interop/grpc_testing" ) +const ( + // Default timeout for tests in this package. + defaultTestTimeout = 10 * time.Second + // Default short timeout, to be used when waiting for events which are not + // expected to happen. + defaultTestShortTimeout = 100 * time.Millisecond + stateStoringBalancerName = "state_storing" +) + var stateStoringServiceConfig = fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateStoringBalancerName) -const stateStoringBalancerName = "state_storing" +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} // setupPickFirstLeaf performs steps required for pick_first tests. It starts a // bunch of backends exporting the TestService, creates a ClientConn to them @@ -503,6 +520,16 @@ func (s) TestPickFirstLeaf_EmptyAddressList(t *testing.T) { } } +// stubBackendsToResolverAddrs converts from a set of stub server backends to +// resolver addresses. Useful when pushing addresses to the manual resolver. +func stubBackendsToResolverAddrs(backends []*stubserver.StubServer) []resolver.Address { + addrs := make([]resolver.Address, len(backends)) + for i, backend := range backends { + addrs[i] = resolver.Address{Addr: backend.Address} + } + return addrs +} + // stateStoringBalancer stores the state of the subconns being created. type stateStoringBalancer struct { balancer.Balancer From 11924a9e4aee52576265508f0c94f46c8489c65b Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Fri, 23 Aug 2024 23:08:21 +0530 Subject: [PATCH 37/62] fix vet --- balancer/pickfirstleaf/pickfirstleaf_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/balancer/pickfirstleaf/pickfirstleaf_test.go b/balancer/pickfirstleaf/pickfirstleaf_test.go index 5b5285a6e0b7..69cb3994c2c7 100644 --- a/balancer/pickfirstleaf/pickfirstleaf_test.go +++ b/balancer/pickfirstleaf/pickfirstleaf_test.go @@ -178,6 +178,10 @@ func (s) TestAddressList_SeekTo(t *testing.T) { BalancerAttributes: attributes.New("ignored", "bal-val-5"), } + if got, want := addressList.seekTo(key), false; got != want { + t.Errorf("addressList.seekTo(%v) = %t, want %t", key, got, want) + } + // It should be possible to increment once more since the pointer has not advanced. if got, want := addressList.increment(), true; got != want { t.Errorf("addressList.increment() = %t, want %t", got, want) From fd4817a53582239210095076afc111bdb66ae4d5 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Sat, 24 Aug 2024 01:16:17 +0530 Subject: [PATCH 38/62] User existing test utils --- .../pickfirstleaf/test/pickfirstleaf_test.go | 93 ++++++------------- 1 file changed, 29 insertions(+), 64 deletions(-) diff --git a/balancer/pickfirstleaf/test/pickfirstleaf_test.go b/balancer/pickfirstleaf/test/pickfirstleaf_test.go index a525d2ca793b..8d7c95901d57 100644 --- a/balancer/pickfirstleaf/test/pickfirstleaf_test.go +++ b/balancer/pickfirstleaf/test/pickfirstleaf_test.go @@ -33,8 +33,11 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpcsync" "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/internal/testutils/pickfirst" "google.golang.org/grpc/resolver" "google.golang.org/grpc/resolver/manual" @@ -377,19 +380,13 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { t.Run(tc.name, func(t *testing.T) { cc, r, backends := setupPickFirstLeaf(t, tc.backendCount) addrs := stubBackendsToResolverAddrs(backends) + stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) activeAddrs := []resolver.Address{} for _, idx := range tc.preUpdateBackendIndexes { activeAddrs = append(activeAddrs, addrs[idx]) } - r.UpdateState(resolver.State{Addresses: activeAddrs}) - bal := <-balChan - select { - case <-bal.resolverUpdateSeen: - case <-ctx.Done(): - t.Fatalf("Context timed out waiting for resolve update to be processed") - } - // shutdown all active backends except the target. var targetAddr resolver.Address for idxI, idx := range tc.preUpdateBackendIndexes { @@ -400,6 +397,10 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { backends[idx].S.Stop() } + r.UpdateState(resolver.State{Addresses: activeAddrs}) + bal := <-balChan + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, targetAddr); err != nil { t.Fatal(err) } @@ -409,7 +410,7 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { } if len(tc.updatedBackendIndexes) == 0 { - if diff := cmp.Diff(tc.wantConnStateTransitions, bal.connStateTransitions()); diff != "" { + if diff := cmp.Diff(tc.wantConnStateTransitions, stateSubscriber.transitions); diff != "" { t.Errorf("balancer states mismatch (-want +got):\n%s", diff) } return @@ -432,11 +433,6 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { } r.UpdateState(resolver.State{Addresses: activeAddrs}) - select { - case <-bal.resolverUpdateSeen: - case <-ctx.Done(): - t.Fatalf("Context timed out waiting for resolve update to be processed") - } // shutdown all active backends except the target. for idxI, idx := range tc.updatedBackendIndexes { @@ -455,13 +451,21 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { t.Errorf("subconn states mismatch (-want +got):\n%s", diff) } - if diff := cmp.Diff(tc.wantConnStateTransitions, bal.connStateTransitions()); diff != "" { + if diff := cmp.Diff(tc.wantConnStateTransitions, stateSubscriber.transitions); diff != "" { t.Errorf("balancer states mismatch (-want +got):\n%s", diff) } }) } } +type ccStateSubscriber struct { + transitions []connectivity.State +} + +func (c *ccStateSubscriber) OnMessage(msg any) { + c.transitions = append(c.transitions, msg.(connectivity.State)) +} + func (s) TestPickFirstLeaf_EmptyAddressList(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() @@ -470,35 +474,25 @@ func (s) TestPickFirstLeaf_EmptyAddressList(t *testing.T) { cc, r, backends := setupPickFirstLeaf(t, 1) addrs := stubBackendsToResolverAddrs(backends) + stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + r.UpdateState(resolver.State{Addresses: addrs}) - bal := <-balChan - select { - case <-bal.resolverUpdateSeen: - case <-ctx.Done(): - t.Fatalf("Context timed out waiting for resolve update to be processed") - } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { t.Fatal(err) } r.UpdateState(resolver.State{}) - select { - case <-bal.resolverUpdateSeen: - case <-ctx.Done(): - t.Fatalf("Context timed out waiting for resolve update to be processed") - } + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) // The balancer should have entered transient failure. // It should transition to CONNECTING from TRANSIENT_FAILURE as sticky TF // only applies when the initial TF is reported due to connection failures // and not bad resolver states. r.UpdateState(resolver.State{Addresses: addrs}) - select { - case <-bal.resolverUpdateSeen: - case <-ctx.Done(): - t.Fatalf("Context timed out waiting for resolve update to be processed") - } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { t.Fatal(err) @@ -515,7 +509,7 @@ func (s) TestPickFirstLeaf_EmptyAddressList(t *testing.T) { connectivity.Ready, } - if diff := cmp.Diff(wantTransitions, bal.connStateTransitions()); diff != "" { + if diff := cmp.Diff(wantTransitions, stateSubscriber.transitions); diff != "" { t.Errorf("balancer states mismatch (-want +got):\n%s", diff) } } @@ -533,10 +527,8 @@ func stubBackendsToResolverAddrs(backends []*stubserver.StubServer) []resolver.A // stateStoringBalancer stores the state of the subconns being created. type stateStoringBalancer struct { balancer.Balancer - mu sync.Mutex - scStates []*scState - stateTransitions []connectivity.State - resolverUpdateSeen chan struct{} + mu sync.Mutex + scStates []*scState } func (b *stateStoringBalancer) Close() { @@ -558,20 +550,12 @@ func (b *stateStoringBalancerBuilder) Name() string { } func (b *stateStoringBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { - bal := &stateStoringBalancer{ - resolverUpdateSeen: make(chan struct{}, 1), - } + bal := &stateStoringBalancer{} bal.Balancer = balancer.Get(pickfirstleaf.Name).Build(&stateStoringCCWrapper{cc, bal}, opts) b.balancerChan <- bal return bal } -func (b *stateStoringBalancer) UpdateClientConnState(state balancer.ClientConnState) error { - err := b.Balancer.UpdateClientConnState(state) - b.resolverUpdateSeen <- struct{}{} - return err -} - func (b *stateStoringBalancer) subConns() []scState { b.mu.Lock() defer b.mu.Unlock() @@ -588,20 +572,6 @@ func (b *stateStoringBalancer) addScState(state *scState) { b.mu.Unlock() } -func (b *stateStoringBalancer) addConnState(state connectivity.State) { - b.mu.Lock() - defer b.mu.Unlock() - if len(b.stateTransitions) == 0 || state != b.stateTransitions[len(b.stateTransitions)-1] { - b.stateTransitions = append(b.stateTransitions, state) - } -} - -func (b *stateStoringBalancer) connStateTransitions() []connectivity.State { - b.mu.Lock() - defer b.mu.Unlock() - return append([]connectivity.State{}, b.stateTransitions...) -} - type stateStoringCCWrapper struct { balancer.ClientConn b *stateStoringBalancer @@ -623,11 +593,6 @@ func (ccw *stateStoringCCWrapper) NewSubConn(addrs []resolver.Address, opts bala return ccw.ClientConn.NewSubConn(addrs, opts) } -func (ccw *stateStoringCCWrapper) UpdateState(state balancer.State) { - ccw.b.addConnState(state.ConnectivityState) - ccw.ClientConn.UpdateState(state) -} - type scState struct { State connectivity.State Addrs []resolver.Address From e266639bc1c7ba039559729c5a3a6a60d949de14 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Sat, 24 Aug 2024 21:37:33 +0530 Subject: [PATCH 39/62] make tests more readable --- .../pickfirstleaf/test/pickfirstleaf_test.go | 576 +++++++++++------- 1 file changed, 354 insertions(+), 222 deletions(-) diff --git a/balancer/pickfirstleaf/test/pickfirstleaf_test.go b/balancer/pickfirstleaf/test/pickfirstleaf_test.go index 8d7c95901d57..b1b4706be7e7 100644 --- a/balancer/pickfirstleaf/test/pickfirstleaf_test.go +++ b/balancer/pickfirstleaf/test/pickfirstleaf_test.go @@ -21,6 +21,7 @@ package test import ( "context" "fmt" + "slices" "sync" "testing" "time" @@ -69,13 +70,12 @@ func Test(t *testing.T) { // setupPickFirstLeaf performs steps required for pick_first tests. It starts a // bunch of backends exporting the TestService, creates a ClientConn to them // with service config specifying the use of the pick_first LB policy. -func setupPickFirstLeaf(t *testing.T, backendCount int, opts ...grpc.DialOption) (*grpc.ClientConn, *manual.Resolver, []*stubserver.StubServer) { +func setupPickFirstLeaf(t *testing.T, backendCount int, opts ...grpc.DialOption) (*grpc.ClientConn, *manual.Resolver, *backendManager) { t.Helper() - r := manual.NewBuilderWithScheme("whatever") - backends := make([]*stubserver.StubServer, backendCount) addrs := make([]resolver.Address, backendCount) + for i := 0; i < backendCount; i++ { backend := &stubserver.StubServer{ EmptyCallF: func(_ context.Context, _ *testpb.Empty) (*testpb.Empty, error) { @@ -112,52 +112,37 @@ func setupPickFirstLeaf(t *testing.T, backendCount int, opts ...grpc.DialOption) if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { t.Fatalf("EmptyCall() = %s, want %s", status.Code(err), codes.DeadlineExceeded) } - return cc, r, backends -} - -type scStateExpectation struct { - State connectivity.State - ServerIdx int -} - -func scStateExpectationToScState(in []scStateExpectation, serverAddrs []resolver.Address) []scState { - out := []scState{} - for _, exp := range in { - out = append(out, scState{ - State: exp.State, - Addrs: []resolver.Address{serverAddrs[exp.ServerIdx]}, - }) - } - return out - + return cc, r, &backendManager{backends} } -// TestPickFirstLeaf_ResolverUpdate tests the behaviour of the new pick first -// policy when servers are brought down and resolver updates are received. -func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { +// TestPickFirstLeaf_SimpleResolverUpdate tests the behaviour of the pick first +// policy when when given an list of addresses. The following steps are carried +// out in order: +// 1. A list of addresses are given through the resolver. Only one +// of the servers is running. +// 2. RPCs are sent to verify they reach the running server. +// +// The state transitions of the ClientConn and all the subconns created are +// verified. +func (s) TestPickFirstLeaf_SimpleResolverUpdate(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() balChan := make(chan *stateStoringBalancer, 1) - balancer.Register(&stateStoringBalancerBuilder{balancerChan: balChan}) + balancer.Register(&stateStoringBalancerBuilder{balancer: balChan}) tests := []struct { - name string - backendCount int - preUpdateBackendIndexes []int - preUpdateTargetBackendIndex int - wantPreUpdateScStates []scStateExpectation - updatedBackendIndexes []int - postUpdateTargetBackendIndex int - wantPostUpdateScStates []scStateExpectation - restartConnected bool - wantConnStateTransitions []connectivity.State + name string + backendCount int + backendIndexes []int + runningBackendIndex int + wantSCStates []scStateExpectation + wantConnStateTransitions []connectivity.State }{ { - name: "two_server_first_ready", - backendCount: 2, - preUpdateBackendIndexes: []int{0, 1}, - preUpdateTargetBackendIndex: 0, - wantPreUpdateScStates: []scStateExpectation{ - {State: connectivity.Ready, ServerIdx: 0}, + name: "two_server_first_ready", + backendIndexes: []int{0, 1}, + runningBackendIndex: 0, + wantSCStates: []scStateExpectation{ + {ServerIdx: 0, State: connectivity.Ready}, }, wantConnStateTransitions: []connectivity.State{ connectivity.Connecting, @@ -165,13 +150,12 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { }, }, { - name: "two_server_second_ready", - backendCount: 2, - preUpdateBackendIndexes: []int{0, 1}, - preUpdateTargetBackendIndex: 1, - wantPreUpdateScStates: []scStateExpectation{ - {State: connectivity.Shutdown, ServerIdx: 0}, - {State: connectivity.Ready, ServerIdx: 1}, + name: "two_server_second_ready", + backendIndexes: []int{0, 1}, + runningBackendIndex: 1, + wantSCStates: []scStateExpectation{ + {ServerIdx: 0, State: connectivity.Shutdown}, + {ServerIdx: 1, State: connectivity.Ready}, }, wantConnStateTransitions: []connectivity.State{ connectivity.Connecting, @@ -179,35 +163,97 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { }, }, { - name: "duplicate_address", - backendCount: 2, - preUpdateBackendIndexes: []int{0, 0, 1}, - preUpdateTargetBackendIndex: 1, - wantPreUpdateScStates: []scStateExpectation{ - {State: connectivity.Shutdown, ServerIdx: 0}, - {State: connectivity.Ready, ServerIdx: 1}, + name: "duplicate_address", + backendIndexes: []int{0, 0, 1}, + runningBackendIndex: 1, + wantSCStates: []scStateExpectation{ + {ServerIdx: 0, State: connectivity.Shutdown}, + {ServerIdx: 1, State: connectivity.Ready}, }, wantConnStateTransitions: []connectivity.State{ connectivity.Connecting, connectivity.Ready, }, }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cc, r, bm := setupPickFirstLeaf(t, 2) + addrs := stubBackendsToResolverAddrs(bm.backends) + stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + + activeAddrs := []resolver.Address{} + for _, idx := range tc.backendIndexes { + activeAddrs = append(activeAddrs, addrs[idx]) + } + + bm.stopAllExcept(tc.runningBackendIndex) + runningAddr := addrs[tc.runningBackendIndex] + + r.UpdateState(resolver.State{Addresses: activeAddrs}) + bal := <-balChan + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, runningAddr); err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(scStateExpectationToSCState(tc.wantSCStates, addrs), bal.subConns()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) + } + + if diff := cmp.Diff(tc.wantConnStateTransitions, stateSubscriber.transitions); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } + }) + } +} + +// TestPickFirstLeaf_ResolverUpdate tests the behaviour of the pick first +// policy when the following steps are carried out in order: +// 1. A list of addresses are given through the resolver. Only one +// of the servers is running. +// 2. RPCs are sent to verify they reach the running server. +// 3. A second resolver update is sent. Again, only one of the servers is +// running. This may not be the same server as before. +// 4. RPCs are sent to verify they reach the running server. +// +// The state transitions of the ClientConn and all the subconns created are +// verified. +func (s) TestPickFirstLeaf_MultipleResolverUpdates(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) + tests := []struct { + name string + backendCount int + backendIndexes1 []int + runningBackendIndex1 int + wantSCStates1 []scStateExpectation + backendIndexes2 []int + runningBackendIndex2 int + wantSCStates2 []scStateExpectation + wantConnStateTransitions []connectivity.State + }{ { - name: "disjoint_updated_addresses", - backendCount: 4, - preUpdateBackendIndexes: []int{0, 1}, - preUpdateTargetBackendIndex: 1, - wantPreUpdateScStates: []scStateExpectation{ - {State: connectivity.Shutdown, ServerIdx: 0}, - {State: connectivity.Ready, ServerIdx: 1}, + name: "disjoint_updated_addresses", + backendCount: 4, + backendIndexes1: []int{0, 1}, + runningBackendIndex1: 1, + wantSCStates1: []scStateExpectation{ + {ServerIdx: 0, State: connectivity.Shutdown}, + {ServerIdx: 1, State: connectivity.Ready}, }, - updatedBackendIndexes: []int{2, 3}, - postUpdateTargetBackendIndex: 3, - wantPostUpdateScStates: []scStateExpectation{ - {State: connectivity.Shutdown, ServerIdx: 0}, - {State: connectivity.Shutdown, ServerIdx: 1}, - {State: connectivity.Shutdown, ServerIdx: 2}, - {State: connectivity.Ready, ServerIdx: 3}, + backendIndexes2: []int{2, 3}, + runningBackendIndex2: 3, + wantSCStates2: []scStateExpectation{ + {ServerIdx: 0, State: connectivity.Shutdown}, + {ServerIdx: 1, State: connectivity.Shutdown}, + {ServerIdx: 2, State: connectivity.Shutdown}, + {ServerIdx: 3, State: connectivity.Ready}, }, wantConnStateTransitions: []connectivity.State{ connectivity.Connecting, @@ -217,19 +263,19 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { }, }, { - name: "active_backend_in_updated_list", - backendCount: 3, - preUpdateBackendIndexes: []int{0, 1}, - preUpdateTargetBackendIndex: 1, - wantPreUpdateScStates: []scStateExpectation{ - {State: connectivity.Shutdown, ServerIdx: 0}, - {State: connectivity.Ready, ServerIdx: 1}, + name: "active_backend_in_updated_list", + backendCount: 3, + backendIndexes1: []int{0, 1}, + runningBackendIndex1: 1, + wantSCStates1: []scStateExpectation{ + {ServerIdx: 0, State: connectivity.Shutdown}, + {ServerIdx: 1, State: connectivity.Ready}, }, - updatedBackendIndexes: []int{1, 2}, - postUpdateTargetBackendIndex: 1, - wantPostUpdateScStates: []scStateExpectation{ - {State: connectivity.Shutdown, ServerIdx: 0}, - {State: connectivity.Ready, ServerIdx: 1}, + backendIndexes2: []int{1, 2}, + runningBackendIndex2: 1, + wantSCStates2: []scStateExpectation{ + {ServerIdx: 0, State: connectivity.Shutdown}, + {ServerIdx: 1, State: connectivity.Ready}, }, wantConnStateTransitions: []connectivity.State{ connectivity.Connecting, @@ -237,20 +283,20 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { }, }, { - name: "inactive_backend_in_updated_list", - backendCount: 3, - preUpdateBackendIndexes: []int{0, 1}, - preUpdateTargetBackendIndex: 1, - wantPreUpdateScStates: []scStateExpectation{ - {State: connectivity.Shutdown, ServerIdx: 0}, - {State: connectivity.Ready, ServerIdx: 1}, + name: "inactive_backend_in_updated_list", + backendCount: 3, + backendIndexes1: []int{0, 1}, + runningBackendIndex1: 1, + wantSCStates1: []scStateExpectation{ + {ServerIdx: 0, State: connectivity.Shutdown}, + {ServerIdx: 1, State: connectivity.Ready}, }, - updatedBackendIndexes: []int{0, 2}, - postUpdateTargetBackendIndex: 0, - wantPostUpdateScStates: []scStateExpectation{ - {State: connectivity.Shutdown, ServerIdx: 0}, - {State: connectivity.Shutdown, ServerIdx: 1}, - {State: connectivity.Ready, ServerIdx: 0}, + backendIndexes2: []int{0, 2}, + runningBackendIndex2: 0, + wantSCStates2: []scStateExpectation{ + {ServerIdx: 0, State: connectivity.Shutdown}, + {ServerIdx: 1, State: connectivity.Shutdown}, + {ServerIdx: 0, State: connectivity.Ready}, }, wantConnStateTransitions: []connectivity.State{ connectivity.Connecting, @@ -260,38 +306,128 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { }, }, { - name: "identical_list", - backendCount: 2, - preUpdateBackendIndexes: []int{0, 1}, - preUpdateTargetBackendIndex: 1, - wantPreUpdateScStates: []scStateExpectation{ - {State: connectivity.Shutdown, ServerIdx: 0}, - {State: connectivity.Ready, ServerIdx: 1}, + name: "identical_list", + backendCount: 2, + backendIndexes1: []int{0, 1}, + runningBackendIndex1: 1, + wantSCStates1: []scStateExpectation{ + {ServerIdx: 0, State: connectivity.Shutdown}, + {ServerIdx: 1, State: connectivity.Ready}, }, - updatedBackendIndexes: []int{0, 1}, - postUpdateTargetBackendIndex: 1, - wantPostUpdateScStates: []scStateExpectation{ - {State: connectivity.Shutdown, ServerIdx: 0}, - {State: connectivity.Ready, ServerIdx: 1}, + backendIndexes2: []int{0, 1}, + runningBackendIndex2: 1, + wantSCStates2: []scStateExpectation{ + {ServerIdx: 0, State: connectivity.Shutdown}, + {ServerIdx: 1, State: connectivity.Ready}, }, wantConnStateTransitions: []connectivity.State{ connectivity.Connecting, connectivity.Ready, }, }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cc, r, bm := setupPickFirstLeaf(t, tc.backendCount) + addrs := stubBackendsToResolverAddrs(bm.backends) + stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + + resolverAddrs := []resolver.Address{} + for _, idx := range tc.backendIndexes1 { + resolverAddrs = append(resolverAddrs, addrs[idx]) + } + + bm.stopAllExcept(tc.runningBackendIndex1) + targetAddr := addrs[tc.runningBackendIndex1] + + r.UpdateState(resolver.State{Addresses: resolverAddrs}) + bal := <-balCh + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, targetAddr); err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(scStateExpectationToSCState(tc.wantSCStates1, addrs), bal.subConns()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) + } + + // Start the backend that needs to be connected to next. + if tc.runningBackendIndex1 != tc.runningBackendIndex2 { + if err := bm.backends[tc.runningBackendIndex2].StartServer(); err != nil { + t.Fatalf("Failed to re-start test backend: %v", err) + } + } + + resolverAddrs = []resolver.Address{} + for _, idx := range tc.backendIndexes2 { + resolverAddrs = append(resolverAddrs, addrs[idx]) + } + r.UpdateState(resolver.State{Addresses: resolverAddrs}) + + if slices.Contains(tc.backendIndexes2, tc.runningBackendIndex1) { + // Verify that the ClientConn stays in READY. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + testutils.AwaitNoStateChange(sCtx, t, cc, connectivity.Ready) + } + // We don't shut down the previous target server if it's different + // from the current target server as it can cause the ClientConn to + // go into IDLE if it has not yet processed the resolver update. + targetAddr = addrs[tc.runningBackendIndex2] + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, targetAddr); err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(scStateExpectationToSCState(tc.wantSCStates2, addrs), bal.subConns()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) + } + + if diff := cmp.Diff(tc.wantConnStateTransitions, stateSubscriber.transitions); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } + }) + } +} + +// TestPickFirstLeaf_StopConnectedServer tests the behaviour of the pick first +// policy when the connected server is shut down. It carries out the following +// steps in order: +// 1. A list of addresses are given through the resolver. Only one +// of the servers is running. +// 2. The running server is stopped, causing the ClientConn to enter IDLE. +// 3. A (possibly different) server is started. +// 4. RPCs are made to kick the ClientConn out of IDLE. The test verifies that +// the RPCs reach the running server. +// +// The test verifies the ClientConn state transitions. +func (s) TestPickFirstLeaf_StopConnectedServer(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + balChan := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balChan}) + tests := []struct { + name string + backendCount int + runningServerIndex int + wantSCStates1 []scStateExpectation + runningServerIndex2 int + wantSCStates2 []scStateExpectation + wantConnStateTransitions []connectivity.State + }{ { - name: "first_connected_idle_reconnect", - backendCount: 2, - preUpdateBackendIndexes: []int{0, 1}, - preUpdateTargetBackendIndex: 0, - restartConnected: true, - wantPreUpdateScStates: []scStateExpectation{ - {State: connectivity.Ready, ServerIdx: 0}, + name: "first_connected_idle_reconnect_first", + backendCount: 2, + runningServerIndex: 0, + wantSCStates1: []scStateExpectation{ + {ServerIdx: 0, State: connectivity.Ready}, }, - updatedBackendIndexes: []int{0, 1}, - postUpdateTargetBackendIndex: 0, - wantPostUpdateScStates: []scStateExpectation{ - {State: connectivity.Ready, ServerIdx: 0}, + runningServerIndex2: 0, + wantSCStates2: []scStateExpectation{ + {ServerIdx: 0, State: connectivity.Ready}, }, wantConnStateTransitions: []connectivity.State{ connectivity.Connecting, @@ -302,21 +438,18 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { }, }, { - name: "second_connected_idle_reconnect", - backendCount: 2, - preUpdateBackendIndexes: []int{0, 1}, - preUpdateTargetBackendIndex: 1, - restartConnected: true, - wantPreUpdateScStates: []scStateExpectation{ - {State: connectivity.Shutdown, ServerIdx: 0}, - {State: connectivity.Ready, ServerIdx: 1}, + name: "second_connected_idle_reconnect_second", + backendCount: 2, + runningServerIndex: 1, + wantSCStates1: []scStateExpectation{ + {ServerIdx: 0, State: connectivity.Shutdown}, + {ServerIdx: 1, State: connectivity.Ready}, }, - updatedBackendIndexes: []int{0, 1}, - postUpdateTargetBackendIndex: 1, - wantPostUpdateScStates: []scStateExpectation{ - {State: connectivity.Shutdown, ServerIdx: 0}, - {State: connectivity.Ready, ServerIdx: 1}, - {State: connectivity.Shutdown, ServerIdx: 0}, + runningServerIndex2: 1, + wantSCStates2: []scStateExpectation{ + {ServerIdx: 0, State: connectivity.Shutdown}, + {ServerIdx: 1, State: connectivity.Ready}, + {ServerIdx: 0, State: connectivity.Shutdown}, }, wantConnStateTransitions: []connectivity.State{ connectivity.Connecting, @@ -327,21 +460,18 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { }, }, { - name: "second_connected_idle_reconnect_first", - backendCount: 2, - preUpdateBackendIndexes: []int{0, 1}, - preUpdateTargetBackendIndex: 1, - restartConnected: true, - wantPreUpdateScStates: []scStateExpectation{ - {State: connectivity.Shutdown, ServerIdx: 0}, - {State: connectivity.Ready, ServerIdx: 1}, + name: "second_connected_idle_reconnect_first", + backendCount: 2, + runningServerIndex: 1, + wantSCStates1: []scStateExpectation{ + {ServerIdx: 0, State: connectivity.Shutdown}, + {ServerIdx: 1, State: connectivity.Ready}, }, - updatedBackendIndexes: []int{0, 1}, - postUpdateTargetBackendIndex: 0, - wantPostUpdateScStates: []scStateExpectation{ - {State: connectivity.Shutdown, ServerIdx: 0}, - {State: connectivity.Shutdown, ServerIdx: 1}, - {State: connectivity.Ready, ServerIdx: 0}, + runningServerIndex2: 0, + wantSCStates2: []scStateExpectation{ + {ServerIdx: 0, State: connectivity.Shutdown}, + {ServerIdx: 1, State: connectivity.Shutdown}, + {ServerIdx: 0, State: connectivity.Ready}, }, wantConnStateTransitions: []connectivity.State{ connectivity.Connecting, @@ -352,19 +482,16 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { }, }, { - name: "first_connected_idle_reconnect_second", - backendCount: 2, - preUpdateBackendIndexes: []int{0, 1}, - preUpdateTargetBackendIndex: 0, - restartConnected: true, - wantPreUpdateScStates: []scStateExpectation{ - {State: connectivity.Ready, ServerIdx: 0}, + name: "first_connected_idle_reconnect_second", + backendCount: 2, + runningServerIndex: 0, + wantSCStates1: []scStateExpectation{ + {ServerIdx: 0, State: connectivity.Ready}, }, - updatedBackendIndexes: []int{0, 1}, - postUpdateTargetBackendIndex: 1, - wantPostUpdateScStates: []scStateExpectation{ - {State: connectivity.Shutdown, ServerIdx: 0}, - {State: connectivity.Ready, ServerIdx: 1}, + runningServerIndex2: 1, + wantSCStates2: []scStateExpectation{ + {ServerIdx: 0, State: connectivity.Shutdown}, + {ServerIdx: 1, State: connectivity.Ready}, }, wantConnStateTransitions: []connectivity.State{ connectivity.Connecting, @@ -378,26 +505,16 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - cc, r, backends := setupPickFirstLeaf(t, tc.backendCount) - addrs := stubBackendsToResolverAddrs(backends) + cc, r, bm := setupPickFirstLeaf(t, tc.backendCount) + addrs := stubBackendsToResolverAddrs(bm.backends) stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) - activeAddrs := []resolver.Address{} - for _, idx := range tc.preUpdateBackendIndexes { - activeAddrs = append(activeAddrs, addrs[idx]) - } // shutdown all active backends except the target. - var targetAddr resolver.Address - for idxI, idx := range tc.preUpdateBackendIndexes { - if idx == tc.preUpdateTargetBackendIndex { - targetAddr = activeAddrs[idxI] - continue - } - backends[idx].S.Stop() - } + bm.stopAllExcept(tc.runningServerIndex) + targetAddr := addrs[tc.runningServerIndex] - r.UpdateState(resolver.State{Addresses: activeAddrs}) + r.UpdateState(resolver.State{Addresses: addrs}) bal := <-balChan testutils.AwaitState(ctx, t, cc, connectivity.Ready) @@ -405,74 +522,47 @@ func (s) TestPickFirstLeaf_ResolverUpdate(t *testing.T) { t.Fatal(err) } - if diff := cmp.Diff(scStateExpectationToScState(tc.wantPreUpdateScStates, addrs), bal.subConns()); diff != "" { + if diff := cmp.Diff(scStateExpectationToSCState(tc.wantSCStates1, addrs), bal.subConns()); diff != "" { t.Errorf("subconn states mismatch (-want +got):\n%s", diff) } - if len(tc.updatedBackendIndexes) == 0 { - if diff := cmp.Diff(tc.wantConnStateTransitions, stateSubscriber.transitions); diff != "" { - t.Errorf("balancer states mismatch (-want +got):\n%s", diff) - } - return - } + // Shut down the connected server. + bm.backends[tc.runningServerIndex].S.Stop() + testutils.AwaitState(ctx, t, cc, connectivity.Idle) - // Restart all the backends. - for i, s := range backends { - if !tc.restartConnected && i == tc.preUpdateTargetBackendIndex { - continue - } - s.S.Stop() - if err := s.StartServer(); err != nil { - t.Fatalf("Failed to re-start test backend: %v", err) - } - } - - activeAddrs = []resolver.Address{} - for _, idx := range tc.updatedBackendIndexes { - activeAddrs = append(activeAddrs, addrs[idx]) - } - - r.UpdateState(resolver.State{Addresses: activeAddrs}) - - // shutdown all active backends except the target. - for idxI, idx := range tc.updatedBackendIndexes { - if idx == tc.postUpdateTargetBackendIndex { - targetAddr = activeAddrs[idxI] - continue - } - backends[idx].S.Stop() + // Start the new target server. + if err := bm.backends[tc.runningServerIndex2].StartServer(); err != nil { + t.Fatalf("Failed to start server: %v", err) } + targetAddr = addrs[tc.runningServerIndex2] if err := pickfirst.CheckRPCsToBackend(ctx, cc, targetAddr); err != nil { t.Fatal(err) } - if diff := cmp.Diff(scStateExpectationToScState(tc.wantPostUpdateScStates, addrs), bal.subConns()); diff != "" { + if diff := cmp.Diff(scStateExpectationToSCState(tc.wantSCStates2, addrs), bal.subConns()); diff != "" { t.Errorf("subconn states mismatch (-want +got):\n%s", diff) } if diff := cmp.Diff(tc.wantConnStateTransitions, stateSubscriber.transitions); diff != "" { - t.Errorf("balancer states mismatch (-want +got):\n%s", diff) + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) } }) } } -type ccStateSubscriber struct { - transitions []connectivity.State -} - -func (c *ccStateSubscriber) OnMessage(msg any) { - c.transitions = append(c.transitions, msg.(connectivity.State)) -} - +// TestPickFirstLeaf_EmptyAddressList carries out the following steps in order: +// 1. Send a resolver update with one running backend. +// 2. Send an empty address list causing the balancer to enter TRANSIENT_FAILURE. +// 3. Send a resolver update with one running backend. +// The test verifies the ClientConn state transitions. func (s) TestPickFirstLeaf_EmptyAddressList(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() balChan := make(chan *stateStoringBalancer, 1) - balancer.Register(&stateStoringBalancerBuilder{balancerChan: balChan}) - cc, r, backends := setupPickFirstLeaf(t, 1) - addrs := stubBackendsToResolverAddrs(backends) + balancer.Register(&stateStoringBalancerBuilder{balancer: balChan}) + cc, r, bm := setupPickFirstLeaf(t, 1) + addrs := stubBackendsToResolverAddrs(bm.backends) stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) @@ -510,8 +600,30 @@ func (s) TestPickFirstLeaf_EmptyAddressList(t *testing.T) { } if diff := cmp.Diff(wantTransitions, stateSubscriber.transitions); diff != "" { - t.Errorf("balancer states mismatch (-want +got):\n%s", diff) + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} + +// scStateExpectation stores the expected state for a subconn created for the +// specified server. Since the servers use random ports, the server is identified +// using using its index in the list of backends. +type scStateExpectation struct { + State connectivity.State + ServerIdx int +} + +// scStateExpectationToSCState converts scStateExpectationToSCState to scState +// using the server addresses. This is required since the servers use random +// ports which are known only once the test runs. +func scStateExpectationToSCState(in []scStateExpectation, serverAddrs []resolver.Address) []scState { + out := []scState{} + for _, exp := range in { + out = append(out, scState{ + State: exp.State, + Addrs: []resolver.Address{serverAddrs[exp.ServerIdx]}, + }) } + return out } // stubBackendsToResolverAddrs converts from a set of stub server backends to @@ -542,7 +654,7 @@ func (b *stateStoringBalancer) ExitIdle() { } type stateStoringBalancerBuilder struct { - balancerChan chan *stateStoringBalancer + balancer chan *stateStoringBalancer } func (b *stateStoringBalancerBuilder) Name() string { @@ -552,7 +664,7 @@ func (b *stateStoringBalancerBuilder) Name() string { func (b *stateStoringBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { bal := &stateStoringBalancer{} bal.Balancer = balancer.Get(pickfirstleaf.Name).Build(&stateStoringCCWrapper{cc, bal}, opts) - b.balancerChan <- bal + b.balancer <- bal return bal } @@ -566,7 +678,7 @@ func (b *stateStoringBalancer) subConns() []scState { return ret } -func (b *stateStoringBalancer) addScState(state *scState) { +func (b *stateStoringBalancer) addSCState(state *scState) { b.mu.Lock() b.scStates = append(b.scStates, state) b.mu.Unlock() @@ -583,7 +695,7 @@ func (ccw *stateStoringCCWrapper) NewSubConn(addrs []resolver.Address, opts bala State: connectivity.Idle, Addrs: addrs, } - ccw.b.addScState(scs) + ccw.b.addSCState(scs) opts.StateListener = func(s balancer.SubConnState) { ccw.b.mu.Lock() scs.State = s.ConnectivityState @@ -597,3 +709,23 @@ type scState struct { State connectivity.State Addrs []resolver.Address } + +type backendManager struct { + backends []*stubserver.StubServer +} + +func (b *backendManager) stopAllExcept(index int) { + for idx, b := range b.backends { + if idx != index { + b.S.Stop() + } + } +} + +type ccStateSubscriber struct { + transitions []connectivity.State +} + +func (c *ccStateSubscriber) OnMessage(msg any) { + c.transitions = append(c.transitions, msg.(connectivity.State)) +} From c44c80a85d86ae532945d09be533796a0150e46d Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Fri, 30 Aug 2024 15:10:56 +0530 Subject: [PATCH 40/62] address review comments --- balancer/pickfirstleaf/pickfirstleaf.go | 27 ++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/balancer/pickfirstleaf/pickfirstleaf.go b/balancer/pickfirstleaf/pickfirstleaf.go index 73aa67eec42b..3b72c6d7e304 100644 --- a/balancer/pickfirstleaf/pickfirstleaf.go +++ b/balancer/pickfirstleaf/pickfirstleaf.go @@ -98,13 +98,18 @@ type pfConfig struct { // scData keeps track of the current state of the subConn. type scData struct { + // The following fields are initialized at build time and read-only after + // that. subConn balancer.SubConn - state connectivity.State addr resolver.Address + + // The following fields should only be accessed from a serializer callback + // to ensure synchronization. + state connectivity.State lastErr error } -func newSCData(b *pickfirstBalancer, addr resolver.Address) (*scData, error) { +func (b *pickfirstBalancer) newSCData(addr resolver.Address) (*scData, error) { sd := &scData{ state: connectivity.Idle, addr: addr, @@ -152,6 +157,7 @@ func (b *pickfirstBalancer) ResolverError(err error) { }) } +// Only executed in the context of a serializer callback. func (b *pickfirstBalancer) resolverError(err error) { if b.logger.V(2) { b.logger.Infof("Received error from the name resolver: %v", err) @@ -162,6 +168,9 @@ func (b *pickfirstBalancer) resolverError(err error) { // The picker will not change since the balancer does not currently // report an error. if b.state != connectivity.TransientFailure { + if b.logger.V(2) { + b.logger.Infof("Ignoring resolver error because balancer is using a previous good update.") + } return } @@ -293,6 +302,7 @@ func (b *pickfirstBalancer) ExitIdle() { }) } +// Only executed in the context of a serializer callback. func (b *pickfirstBalancer) closeSubConns() { for _, sd := range b.subConns.Values() { sd.(*scData).subConn.Shutdown() @@ -324,6 +334,7 @@ func deDupAddresses(endpoints []resolver.Endpoint) []resolver.Endpoint { return newEndpoints } +// Only executed in the context of a serializer callback. func (b *pickfirstBalancer) reconcileSubConns(newEndpoints []resolver.Endpoint) { // Remove old subConns that were not in new address list. oldAddrs := resolver.NewAddressMap() @@ -352,6 +363,7 @@ func (b *pickfirstBalancer) reconcileSubConns(newEndpoints []resolver.Endpoint) // shutdownRemaining shuts down remaining subConns. Called when a subConn // becomes ready, which means that all other subConn must be shutdown. +// Only executed in the context of a serializer callback. func (b *pickfirstBalancer) shutdownRemaining(selected *scData) { for _, v := range b.subConns.Values() { sd := v.(*scData) @@ -369,6 +381,7 @@ func (b *pickfirstBalancer) shutdownRemaining(selected *scData) { // current address. If no subchannel exists, one is created. If the current // subchannel is in TransientFailure, a connection to the next address is // attempted. +// Only executed in the context of a serializer callback. func (b *pickfirstBalancer) requestConnection() { if !b.addressList.isValid() || b.state == connectivity.Shutdown { return @@ -379,7 +392,7 @@ func (b *pickfirstBalancer) requestConnection() { var err error // We want to assign the new scData to sd from the outer scope, hence // we can't use := below. - sd, err = newSCData(b, curAddr) + sd, err = b.newSCData(curAddr) if err != nil { // This should never happen, unless the clientConn is being shut // down. @@ -413,7 +426,6 @@ func (b *pickfirstBalancer) requestConnection() { } } -// updateSubConnState handles subConn state updates. // Only executed in the context of a serializer callback. func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubConnState) { // Previously relevant subconns can still callback with state updates. @@ -443,9 +455,9 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon return } - // If we are transitioning from READY to IDLE, reset index and re-connect when - // prompted. - if b.state == connectivity.Ready && state.ConnectivityState == connectivity.Idle { + // If the LB policy is READY, and it receives a subchannel state change, + // it means that the READY subchannel has failed. + if b.state == connectivity.Ready && state.ConnectivityState != connectivity.Ready { // Once a transport fails, the balancer enters IDLE and starts from // the first address when the picker is used. b.state = connectivity.Idle @@ -519,6 +531,7 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon } } +// Only executed in the context of a serializer callback. func (b *pickfirstBalancer) endFirstPass(lastErr error) { b.firstPass = false b.state = connectivity.TransientFailure From 4a221d1eb722e2e0d45b1df8e794e06a7747e749 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Fri, 30 Aug 2024 17:10:51 +0530 Subject: [PATCH 41/62] no more table driven tests --- .../pickfirstleaf/test/pickfirstleaf_test.go | 1033 ++++++++++------- 1 file changed, 627 insertions(+), 406 deletions(-) diff --git a/balancer/pickfirstleaf/test/pickfirstleaf_test.go b/balancer/pickfirstleaf/test/pickfirstleaf_test.go index b1b4706be7e7..0b08049d1289 100644 --- a/balancer/pickfirstleaf/test/pickfirstleaf_test.go +++ b/balancer/pickfirstleaf/test/pickfirstleaf_test.go @@ -21,7 +21,6 @@ package test import ( "context" "fmt" - "slices" "sync" "testing" "time" @@ -86,7 +85,7 @@ func setupPickFirstLeaf(t *testing.T, backendCount int, opts ...grpc.DialOption) t.Fatalf("Failed to start backend: %v", err) } t.Logf("Started TestService backend at: %q", backend.Address) - t.Cleanup(func() { backend.Stop() }) + t.Cleanup(func() { backend.S.Stop() }) backends[i] = backend addrs[i] = resolver.Address{Addr: backend.Address} @@ -124,94 +123,134 @@ func setupPickFirstLeaf(t *testing.T, backendCount int, opts ...grpc.DialOption) // // The state transitions of the ClientConn and all the subconns created are // verified. -func (s) TestPickFirstLeaf_SimpleResolverUpdate(t *testing.T) { +func (s) TestPickFirstLeaf_SimpleResolverUpdate_FirstServerReady(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - balChan := make(chan *stateStoringBalancer, 1) - balancer.Register(&stateStoringBalancerBuilder{balancer: balChan}) - tests := []struct { - name string - backendCount int - backendIndexes []int - runningBackendIndex int - wantSCStates []scStateExpectation - wantConnStateTransitions []connectivity.State - }{ - { - name: "two_server_first_ready", - backendIndexes: []int{0, 1}, - runningBackendIndex: 0, - wantSCStates: []scStateExpectation{ - {ServerIdx: 0, State: connectivity.Ready}, - }, - wantConnStateTransitions: []connectivity.State{ - connectivity.Connecting, - connectivity.Ready, - }, - }, - { - name: "two_server_second_ready", - backendIndexes: []int{0, 1}, - runningBackendIndex: 1, - wantSCStates: []scStateExpectation{ - {ServerIdx: 0, State: connectivity.Shutdown}, - {ServerIdx: 1, State: connectivity.Ready}, - }, - wantConnStateTransitions: []connectivity.State{ - connectivity.Connecting, - connectivity.Ready, - }, - }, - { - name: "duplicate_address", - backendIndexes: []int{0, 0, 1}, - runningBackendIndex: 1, - wantSCStates: []scStateExpectation{ - {ServerIdx: 0, State: connectivity.Shutdown}, - {ServerIdx: 1, State: connectivity.Ready}, - }, - wantConnStateTransitions: []connectivity.State{ - connectivity.Connecting, - connectivity.Ready, - }, - }, + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) + + cc, r, bm := setupPickFirstLeaf(t, 2) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + + r.UpdateState(resolver.State{Addresses: addrs}) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Ready}, + } + if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - cc, r, bm := setupPickFirstLeaf(t, 2) - addrs := stubBackendsToResolverAddrs(bm.backends) - stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} - internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} - activeAddrs := []resolver.Address{} - for _, idx := range tc.backendIndexes { - activeAddrs = append(activeAddrs, addrs[idx]) - } +func (s) TestPickFirstLeaf_SimpleResolverUpdate_FirstServerUnReady(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) - bm.stopAllExcept(tc.runningBackendIndex) - runningAddr := addrs[tc.runningBackendIndex] + cc, r, bm := setupPickFirstLeaf(t, 2) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + bm.stopAllExcept(1) - r.UpdateState(resolver.State{Addresses: activeAddrs}) - bal := <-balChan - testutils.AwaitState(ctx, t, cc, connectivity.Ready) + r.UpdateState(resolver.State{Addresses: addrs}) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) - if err := pickfirst.CheckRPCsToBackend(ctx, cc, runningAddr); err != nil { - t.Fatal(err) - } + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } - if diff := cmp.Diff(scStateExpectationToSCState(tc.wantSCStates, addrs), bal.subConns()); diff != "" { - t.Errorf("subconn states mismatch (-want +got):\n%s", diff) - } + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) + } - if diff := cmp.Diff(tc.wantConnStateTransitions, stateSubscriber.transitions); diff != "" { - t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) - } - }) + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) } } -// TestPickFirstLeaf_ResolverUpdate tests the behaviour of the pick first +func (s) TestPickFirstLeaf_SimpleResolverUpdate_DuplicateAddrs(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) + + cc, r, bm := setupPickFirstLeaf(t, 2) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + bm.stopAllExcept(1) + + // Add a duplicate entry in the addresslist + r.UpdateState(resolver.State{ + Addresses: append([]resolver.Address{addrs[0]}, addrs...), + }) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) + } + + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} + +// TestPickFirstLeaf_ResolverUpdates_DisjointLists tests the behaviour of the pick first // policy when the following steps are carried out in order: // 1. A list of addresses are given through the resolver. Only one // of the servers is running. @@ -222,174 +261,250 @@ func (s) TestPickFirstLeaf_SimpleResolverUpdate(t *testing.T) { // // The state transitions of the ClientConn and all the subconns created are // verified. -func (s) TestPickFirstLeaf_MultipleResolverUpdates(t *testing.T) { +func (s) TestPickFirstLeaf_ResolverUpdates_DisjointLists(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() + balCh := make(chan *stateStoringBalancer, 1) balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) - tests := []struct { - name string - backendCount int - backendIndexes1 []int - runningBackendIndex1 int - wantSCStates1 []scStateExpectation - backendIndexes2 []int - runningBackendIndex2 int - wantSCStates2 []scStateExpectation - wantConnStateTransitions []connectivity.State - }{ - { - name: "disjoint_updated_addresses", - backendCount: 4, - backendIndexes1: []int{0, 1}, - runningBackendIndex1: 1, - wantSCStates1: []scStateExpectation{ - {ServerIdx: 0, State: connectivity.Shutdown}, - {ServerIdx: 1, State: connectivity.Ready}, - }, - backendIndexes2: []int{2, 3}, - runningBackendIndex2: 3, - wantSCStates2: []scStateExpectation{ - {ServerIdx: 0, State: connectivity.Shutdown}, - {ServerIdx: 1, State: connectivity.Shutdown}, - {ServerIdx: 2, State: connectivity.Shutdown}, - {ServerIdx: 3, State: connectivity.Ready}, - }, - wantConnStateTransitions: []connectivity.State{ - connectivity.Connecting, - connectivity.Ready, - connectivity.Connecting, - connectivity.Ready, - }, - }, - { - name: "active_backend_in_updated_list", - backendCount: 3, - backendIndexes1: []int{0, 1}, - runningBackendIndex1: 1, - wantSCStates1: []scStateExpectation{ - {ServerIdx: 0, State: connectivity.Shutdown}, - {ServerIdx: 1, State: connectivity.Ready}, - }, - backendIndexes2: []int{1, 2}, - runningBackendIndex2: 1, - wantSCStates2: []scStateExpectation{ - {ServerIdx: 0, State: connectivity.Shutdown}, - {ServerIdx: 1, State: connectivity.Ready}, - }, - wantConnStateTransitions: []connectivity.State{ - connectivity.Connecting, - connectivity.Ready, - }, - }, - { - name: "inactive_backend_in_updated_list", - backendCount: 3, - backendIndexes1: []int{0, 1}, - runningBackendIndex1: 1, - wantSCStates1: []scStateExpectation{ - {ServerIdx: 0, State: connectivity.Shutdown}, - {ServerIdx: 1, State: connectivity.Ready}, - }, - backendIndexes2: []int{0, 2}, - runningBackendIndex2: 0, - wantSCStates2: []scStateExpectation{ - {ServerIdx: 0, State: connectivity.Shutdown}, - {ServerIdx: 1, State: connectivity.Shutdown}, - {ServerIdx: 0, State: connectivity.Ready}, - }, - wantConnStateTransitions: []connectivity.State{ - connectivity.Connecting, - connectivity.Ready, - connectivity.Connecting, - connectivity.Ready, - }, - }, - { - name: "identical_list", - backendCount: 2, - backendIndexes1: []int{0, 1}, - runningBackendIndex1: 1, - wantSCStates1: []scStateExpectation{ - {ServerIdx: 0, State: connectivity.Shutdown}, - {ServerIdx: 1, State: connectivity.Ready}, - }, - backendIndexes2: []int{0, 1}, - runningBackendIndex2: 1, - wantSCStates2: []scStateExpectation{ - {ServerIdx: 0, State: connectivity.Shutdown}, - {ServerIdx: 1, State: connectivity.Ready}, - }, - wantConnStateTransitions: []connectivity.State{ - connectivity.Connecting, - connectivity.Ready, - }, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - cc, r, bm := setupPickFirstLeaf(t, tc.backendCount) - addrs := stubBackendsToResolverAddrs(bm.backends) - stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} - internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) - - resolverAddrs := []resolver.Address{} - for _, idx := range tc.backendIndexes1 { - resolverAddrs = append(resolverAddrs, addrs[idx]) - } - - bm.stopAllExcept(tc.runningBackendIndex1) - targetAddr := addrs[tc.runningBackendIndex1] - - r.UpdateState(resolver.State{Addresses: resolverAddrs}) - bal := <-balCh - testutils.AwaitState(ctx, t, cc, connectivity.Ready) - - if err := pickfirst.CheckRPCsToBackend(ctx, cc, targetAddr); err != nil { - t.Fatal(err) - } - - if diff := cmp.Diff(scStateExpectationToSCState(tc.wantSCStates1, addrs), bal.subConns()); diff != "" { - t.Errorf("subconn states mismatch (-want +got):\n%s", diff) - } - - // Start the backend that needs to be connected to next. - if tc.runningBackendIndex1 != tc.runningBackendIndex2 { - if err := bm.backends[tc.runningBackendIndex2].StartServer(); err != nil { - t.Fatalf("Failed to re-start test backend: %v", err) - } - } - - resolverAddrs = []resolver.Address{} - for _, idx := range tc.backendIndexes2 { - resolverAddrs = append(resolverAddrs, addrs[idx]) - } - r.UpdateState(resolver.State{Addresses: resolverAddrs}) - - if slices.Contains(tc.backendIndexes2, tc.runningBackendIndex1) { - // Verify that the ClientConn stays in READY. - sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) - defer sCancel() - testutils.AwaitNoStateChange(sCtx, t, cc, connectivity.Ready) - } - // We don't shut down the previous target server if it's different - // from the current target server as it can cause the ClientConn to - // go into IDLE if it has not yet processed the resolver update. - targetAddr = addrs[tc.runningBackendIndex2] - - if err := pickfirst.CheckRPCsToBackend(ctx, cc, targetAddr); err != nil { - t.Fatal(err) - } - - if diff := cmp.Diff(scStateExpectationToSCState(tc.wantSCStates2, addrs), bal.subConns()); diff != "" { - t.Errorf("subconn states mismatch (-want +got):\n%s", diff) - } - - if diff := cmp.Diff(tc.wantConnStateTransitions, stateSubscriber.transitions); diff != "" { - t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) - } - }) + cc, r, bm := setupPickFirstLeaf(t, 4) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + + bm.backends[0].S.Stop() + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[1]}}) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) + } + + bm.backends[2].S.Stop() + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[2], addrs[3]}}) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[3]); err != nil { + t.Fatal(err) + } + wantSCStates = []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[2]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[3]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) + } + + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} + +func (s) TestPickFirstLeaf_ResolverUpdates_ActiveBackendInUpdatedList(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) + cc, r, bm := setupPickFirstLeaf(t, 3) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + + bm.backends[0].S.Stop() + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[1]}}) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) + } + + bm.backends[2].S.Stop() + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[2], addrs[1]}}) + + // Verify that the ClientConn stays in READY. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + testutils.AwaitNoStateChange(sCtx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + wantSCStates = []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) + } + + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} + +func (s) TestPickFirstLeaf_ResolverUpdates_InActiveBackendInUpdatedList(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) + cc, r, bm := setupPickFirstLeaf(t, 3) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + + bm.backends[0].S.Stop() + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[1]}}) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) + } + + bm.backends[2].S.Stop() + if err := bm.backends[0].StartServer(); err != nil { + t.Fatalf("Failed to re-start test backend: %v", err) + } + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[2]}}) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + wantSCStates = []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) + } + + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} + +func (s) TestPickFirstLeaf_ResolverUpdates_IdenticalLists(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) + cc, r, bm := setupPickFirstLeaf(t, 2) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + + bm.backends[0].S.Stop() + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[1]}}) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) + } + + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[1]}}) + + // Verify that the ClientConn stays in READY. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + testutils.AwaitNoStateChange(sCtx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + wantSCStates = []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) + } + + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) } } @@ -404,150 +519,278 @@ func (s) TestPickFirstLeaf_MultipleResolverUpdates(t *testing.T) { // the RPCs reach the running server. // // The test verifies the ClientConn state transitions. -func (s) TestPickFirstLeaf_StopConnectedServer(t *testing.T) { +func (s) TestPickFirstLeaf_StopConnectedServer_FirstServerRestart(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - balChan := make(chan *stateStoringBalancer, 1) - balancer.Register(&stateStoringBalancerBuilder{balancer: balChan}) - tests := []struct { - name string - backendCount int - runningServerIndex int - wantSCStates1 []scStateExpectation - runningServerIndex2 int - wantSCStates2 []scStateExpectation - wantConnStateTransitions []connectivity.State - }{ - { - name: "first_connected_idle_reconnect_first", - backendCount: 2, - runningServerIndex: 0, - wantSCStates1: []scStateExpectation{ - {ServerIdx: 0, State: connectivity.Ready}, - }, - runningServerIndex2: 0, - wantSCStates2: []scStateExpectation{ - {ServerIdx: 0, State: connectivity.Ready}, - }, - wantConnStateTransitions: []connectivity.State{ - connectivity.Connecting, - connectivity.Ready, - connectivity.Idle, - connectivity.Connecting, - connectivity.Ready, - }, - }, - { - name: "second_connected_idle_reconnect_second", - backendCount: 2, - runningServerIndex: 1, - wantSCStates1: []scStateExpectation{ - {ServerIdx: 0, State: connectivity.Shutdown}, - {ServerIdx: 1, State: connectivity.Ready}, - }, - runningServerIndex2: 1, - wantSCStates2: []scStateExpectation{ - {ServerIdx: 0, State: connectivity.Shutdown}, - {ServerIdx: 1, State: connectivity.Ready}, - {ServerIdx: 0, State: connectivity.Shutdown}, - }, - wantConnStateTransitions: []connectivity.State{ - connectivity.Connecting, - connectivity.Ready, - connectivity.Idle, - connectivity.Connecting, - connectivity.Ready, - }, - }, - { - name: "second_connected_idle_reconnect_first", - backendCount: 2, - runningServerIndex: 1, - wantSCStates1: []scStateExpectation{ - {ServerIdx: 0, State: connectivity.Shutdown}, - {ServerIdx: 1, State: connectivity.Ready}, - }, - runningServerIndex2: 0, - wantSCStates2: []scStateExpectation{ - {ServerIdx: 0, State: connectivity.Shutdown}, - {ServerIdx: 1, State: connectivity.Shutdown}, - {ServerIdx: 0, State: connectivity.Ready}, - }, - wantConnStateTransitions: []connectivity.State{ - connectivity.Connecting, - connectivity.Ready, - connectivity.Idle, - connectivity.Connecting, - connectivity.Ready, - }, - }, - { - name: "first_connected_idle_reconnect_second", - backendCount: 2, - runningServerIndex: 0, - wantSCStates1: []scStateExpectation{ - {ServerIdx: 0, State: connectivity.Ready}, - }, - runningServerIndex2: 1, - wantSCStates2: []scStateExpectation{ - {ServerIdx: 0, State: connectivity.Shutdown}, - {ServerIdx: 1, State: connectivity.Ready}, - }, - wantConnStateTransitions: []connectivity.State{ - connectivity.Connecting, - connectivity.Ready, - connectivity.Idle, - connectivity.Connecting, - connectivity.Ready, - }, - }, + + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) + cc, r, bm := setupPickFirstLeaf(t, 2) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + + // shutdown all active backends except the target. + bm.stopAllExcept(0) + + r.UpdateState(resolver.State{Addresses: addrs}) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - cc, r, bm := setupPickFirstLeaf(t, tc.backendCount) - addrs := stubBackendsToResolverAddrs(bm.backends) - stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} - internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } - // shutdown all active backends except the target. - bm.stopAllExcept(tc.runningServerIndex) - targetAddr := addrs[tc.runningServerIndex] + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Ready}, + } - r.UpdateState(resolver.State{Addresses: addrs}) - bal := <-balChan - testutils.AwaitState(ctx, t, cc, connectivity.Ready) + if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) + } - if err := pickfirst.CheckRPCsToBackend(ctx, cc, targetAddr); err != nil { - t.Fatal(err) - } + // Shut down the connected server. + bm.backends[0].S.Stop() + testutils.AwaitState(ctx, t, cc, connectivity.Idle) - if diff := cmp.Diff(scStateExpectationToSCState(tc.wantSCStates1, addrs), bal.subConns()); diff != "" { - t.Errorf("subconn states mismatch (-want +got):\n%s", diff) - } + // Start the new target server. + if err := bm.backends[0].StartServer(); err != nil { + t.Fatalf("Failed to start server: %v", err) + } - // Shut down the connected server. - bm.backends[tc.runningServerIndex].S.Stop() - testutils.AwaitState(ctx, t, cc, connectivity.Idle) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) + } + + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Idle, + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} + +func (s) TestPickFirstLeaf_StopConnectedServer_SecondServerRestart(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() - // Start the new target server. - if err := bm.backends[tc.runningServerIndex2].StartServer(); err != nil { - t.Fatalf("Failed to start server: %v", err) - } + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) + cc, r, bm := setupPickFirstLeaf(t, 2) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) - targetAddr = addrs[tc.runningServerIndex2] - if err := pickfirst.CheckRPCsToBackend(ctx, cc, targetAddr); err != nil { - t.Fatal(err) - } + // shutdown all active backends except the target. + bm.stopAllExcept(1) - if diff := cmp.Diff(scStateExpectationToSCState(tc.wantSCStates2, addrs), bal.subConns()); diff != "" { - t.Errorf("subconn states mismatch (-want +got):\n%s", diff) - } + r.UpdateState(resolver.State{Addresses: addrs}) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) - if diff := cmp.Diff(tc.wantConnStateTransitions, stateSubscriber.transitions); diff != "" { - t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) - } - }) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) + } + + // Shut down the connected server. + bm.backends[1].S.Stop() + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + // Start the new target server. + if err := bm.backends[1].StartServer(); err != nil { + t.Fatalf("Failed to start server: %v", err) + } + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + + wantSCStates = []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) + } + + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Idle, + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} + +func (s) TestPickFirstLeaf_StopConnectedServer_SecondServerToFirst(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) + cc, r, bm := setupPickFirstLeaf(t, 2) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + + // shutdown all active backends except the target. + bm.stopAllExcept(1) + + r.UpdateState(resolver.State{Addresses: addrs}) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) + } + + // Shut down the connected server. + bm.backends[1].S.Stop() + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + // Start the new target server. + if err := bm.backends[0].StartServer(); err != nil { + t.Fatalf("Failed to start server: %v", err) + } + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + wantSCStates = []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) + } + + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Idle, + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} + +func (s) TestPickFirstLeaf_StopConnectedServer_FirstServerToSecond(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + balCh := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) + cc, r, bm := setupPickFirstLeaf(t, 2) + addrs := bm.resolverAddrs() + stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + + // shutdown all active backends except the target. + bm.stopAllExcept(0) + + r.UpdateState(resolver.State{Addresses: addrs}) + var bal *stateStoringBalancer + select { + case bal = <-balCh: + case <-ctx.Done(): + t.Fatal("Context expired while waiting for balancer to be built") + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + wantSCStates := []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) + } + + // Shut down the connected server. + bm.backends[0].S.Stop() + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + // Start the new target server. + if err := bm.backends[1].StartServer(); err != nil { + t.Fatalf("Failed to start server: %v", err) + } + + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + + wantSCStates = []scState{ + {Addrs: []resolver.Address{addrs[0]}, State: connectivity.Shutdown}, + {Addrs: []resolver.Address{addrs[1]}, State: connectivity.Ready}, + } + + if diff := cmp.Diff(wantSCStates, bal.subConnStates()); diff != "" { + t.Errorf("subconn states mismatch (-want +got):\n%s", diff) + } + + wantConnStateTransitions := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Idle, + connectivity.Connecting, + connectivity.Ready, + } + if diff := cmp.Diff(wantConnStateTransitions, stateSubscriber.transitions); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) } } @@ -562,7 +805,7 @@ func (s) TestPickFirstLeaf_EmptyAddressList(t *testing.T) { balChan := make(chan *stateStoringBalancer, 1) balancer.Register(&stateStoringBalancerBuilder{balancer: balChan}) cc, r, bm := setupPickFirstLeaf(t, 1) - addrs := stubBackendsToResolverAddrs(bm.backends) + addrs := bm.resolverAddrs() stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) @@ -604,38 +847,6 @@ func (s) TestPickFirstLeaf_EmptyAddressList(t *testing.T) { } } -// scStateExpectation stores the expected state for a subconn created for the -// specified server. Since the servers use random ports, the server is identified -// using using its index in the list of backends. -type scStateExpectation struct { - State connectivity.State - ServerIdx int -} - -// scStateExpectationToSCState converts scStateExpectationToSCState to scState -// using the server addresses. This is required since the servers use random -// ports which are known only once the test runs. -func scStateExpectationToSCState(in []scStateExpectation, serverAddrs []resolver.Address) []scState { - out := []scState{} - for _, exp := range in { - out = append(out, scState{ - State: exp.State, - Addrs: []resolver.Address{serverAddrs[exp.ServerIdx]}, - }) - } - return out -} - -// stubBackendsToResolverAddrs converts from a set of stub server backends to -// resolver addresses. Useful when pushing addresses to the manual resolver. -func stubBackendsToResolverAddrs(backends []*stubserver.StubServer) []resolver.Address { - addrs := make([]resolver.Address, len(backends)) - for i, backend := range backends { - addrs[i] = resolver.Address{Addr: backend.Address} - } - return addrs -} - // stateStoringBalancer stores the state of the subconns being created. type stateStoringBalancer struct { balancer.Balancer @@ -668,7 +879,7 @@ func (b *stateStoringBalancerBuilder) Build(cc balancer.ClientConn, opts balance return bal } -func (b *stateStoringBalancer) subConns() []scState { +func (b *stateStoringBalancer) subConnStates() []scState { b.mu.Lock() defer b.mu.Unlock() ret := []scState{} @@ -722,6 +933,16 @@ func (b *backendManager) stopAllExcept(index int) { } } +// resolverAddrs returns a list of resolver addresses for the stub server +// backends. Useful when pushing addresses to the manual resolver. +func (b *backendManager) resolverAddrs() []resolver.Address { + addrs := make([]resolver.Address, len(b.backends)) + for i, backend := range b.backends { + addrs[i] = resolver.Address{Addr: backend.Address} + } + return addrs +} + type ccStateSubscriber struct { transitions []connectivity.State } From 08e8cb9c3c5fdd70ff82cb06057e3b48ec66d62d Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Mon, 2 Sep 2024 13:51:48 +0530 Subject: [PATCH 42/62] Keep b.state and state rerported to clientconn in sync --- balancer/pickfirstleaf/pickfirstleaf.go | 1 + 1 file changed, 1 insertion(+) diff --git a/balancer/pickfirstleaf/pickfirstleaf.go b/balancer/pickfirstleaf/pickfirstleaf.go index 3b72c6d7e304..7f7b58d470f8 100644 --- a/balancer/pickfirstleaf/pickfirstleaf.go +++ b/balancer/pickfirstleaf/pickfirstleaf.go @@ -477,6 +477,7 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon // it's READY. See A62. // If the balancer is already in CONNECTING, no update is needed. if b.state == connectivity.Idle { + b.state = connectivity.Connecting b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.Connecting, Picker: &picker{err: balancer.ErrNoSubConnAvailable}, From 2b2c2a3caf1f13522e3178799f6b3d7003e30612 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Thu, 5 Sep 2024 16:24:19 +0530 Subject: [PATCH 43/62] Address review comments --- balancer/pickfirstleaf/pickfirstleaf.go | 26 ++++++----- .../pickfirstleaf/test/pickfirstleaf_test.go | 43 ++++++++----------- 2 files changed, 33 insertions(+), 36 deletions(-) diff --git a/balancer/pickfirstleaf/pickfirstleaf.go b/balancer/pickfirstleaf/pickfirstleaf.go index 7f7b58d470f8..bbfdc5a436cf 100644 --- a/balancer/pickfirstleaf/pickfirstleaf.go +++ b/balancer/pickfirstleaf/pickfirstleaf.go @@ -152,8 +152,12 @@ type pickfirstBalancer struct { } func (b *pickfirstBalancer) ResolverError(err error) { - b.serializer.TrySchedule(func(_ context.Context) { + doneCh := make(chan error, 1) + b.serializer.ScheduleOr(func(_ context.Context) { b.resolverError(err) + close(doneCh) + }, func() { + close(doneCh) }) } @@ -427,7 +431,7 @@ func (b *pickfirstBalancer) requestConnection() { } // Only executed in the context of a serializer callback. -func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubConnState) { +func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.SubConnState) { // Previously relevant subconns can still callback with state updates. // To prevent pickers from returning these obsolete subconns, this logic // is included to check if the current list of active subconns includes this @@ -435,11 +439,11 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon if activeSD, found := b.subConns.Get(sd.addr); !found || activeSD != sd { return } - if state.ConnectivityState == connectivity.Shutdown { + if newState.ConnectivityState == connectivity.Shutdown { return } - if state.ConnectivityState == connectivity.Ready { + if newState.ConnectivityState == connectivity.Ready { b.shutdownRemaining(sd) if !b.addressList.seekTo(sd.addr) { // This should not fail as we should have only one subconn after @@ -457,7 +461,7 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon // If the LB policy is READY, and it receives a subchannel state change, // it means that the READY subchannel has failed. - if b.state == connectivity.Ready && state.ConnectivityState != connectivity.Ready { + if b.state == connectivity.Ready && newState.ConnectivityState != connectivity.Ready { // Once a transport fails, the balancer enters IDLE and starts from // the first address when the picker is used. b.state = connectivity.Idle @@ -470,7 +474,7 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon } if b.firstPass { - switch state.ConnectivityState { + switch newState.ConnectivityState { case connectivity.Connecting: // The balancer can be in either IDLE, CONNECTING or TRANSIENT_FAILURE. // If it's in TRANSIENT_FAILURE, stay in TRANSIENT_FAILURE until @@ -484,7 +488,7 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon }) } case connectivity.TransientFailure: - sd.lastErr = state.ConnectionError + sd.lastErr = newState.ConnectionError // Since we're re-using common subconns while handling resolver updates, // we could receive an out of turn TRANSIENT_FAILURE from a pass // over the previous address list. We ignore such updates. @@ -497,7 +501,7 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon return } // End of the first pass. - b.endFirstPass(state.ConnectionError) + b.endFirstPass(newState.ConnectionError) case connectivity.Idle: // A subconn can transition from CONNECTING directly to IDLE when // a transport is successfully created, but the connection fails before @@ -517,12 +521,12 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, state balancer.SubCon } // We have finished the first pass, keep re-connecting failing subconns. - switch state.ConnectivityState { + switch newState.ConnectivityState { case connectivity.TransientFailure: - sd.lastErr = state.ConnectionError + sd.lastErr = newState.ConnectionError b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.TransientFailure, - Picker: &picker{err: state.ConnectionError}, + Picker: &picker{err: newState.ConnectionError}, }) // We don't need to request re-resolution since the subconn already does // that before reporting TRANSIENT_FAILURE. diff --git a/balancer/pickfirstleaf/test/pickfirstleaf_test.go b/balancer/pickfirstleaf/test/pickfirstleaf_test.go index 0b08049d1289..2c09360c36ec 100644 --- a/balancer/pickfirstleaf/test/pickfirstleaf_test.go +++ b/balancer/pickfirstleaf/test/pickfirstleaf_test.go @@ -68,7 +68,7 @@ func Test(t *testing.T) { // setupPickFirstLeaf performs steps required for pick_first tests. It starts a // bunch of backends exporting the TestService, creates a ClientConn to them -// with service config specifying the use of the pick_first LB policy. +// with service config specifying the use of the state_storing LB policy. func setupPickFirstLeaf(t *testing.T, backendCount int, opts ...grpc.DialOption) (*grpc.ClientConn, *manual.Resolver, *backendManager) { t.Helper() r := manual.NewBuilderWithScheme("whatever") @@ -76,17 +76,10 @@ func setupPickFirstLeaf(t *testing.T, backendCount int, opts ...grpc.DialOption) addrs := make([]resolver.Address, backendCount) for i := 0; i < backendCount; i++ { - backend := &stubserver.StubServer{ - EmptyCallF: func(_ context.Context, _ *testpb.Empty) (*testpb.Empty, error) { - return &testpb.Empty{}, nil - }, - } - if err := backend.StartServer(); err != nil { - t.Fatalf("Failed to start backend: %v", err) - } - t.Logf("Started TestService backend at: %q", backend.Address) - t.Cleanup(func() { backend.S.Stop() }) - + backend := stubserver.StartTestService(t, nil) + t.Cleanup(func() { + backend.Stop() + }) backends[i] = backend addrs[i] = resolver.Address{Addr: backend.Address} } @@ -115,7 +108,7 @@ func setupPickFirstLeaf(t *testing.T, backendCount int, opts ...grpc.DialOption) } // TestPickFirstLeaf_SimpleResolverUpdate tests the behaviour of the pick first -// policy when when given an list of addresses. The following steps are carried +// policy when given an list of addresses. The following steps are carried // out in order: // 1. A list of addresses are given through the resolver. Only one // of the servers is running. @@ -131,7 +124,7 @@ func (s) TestPickFirstLeaf_SimpleResolverUpdate_FirstServerReady(t *testing.T) { cc, r, bm := setupPickFirstLeaf(t, 2) addrs := bm.resolverAddrs() - stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + stateSubscriber := &ccStateSubscriber{} internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) r.UpdateState(resolver.State{Addresses: addrs}) @@ -171,7 +164,7 @@ func (s) TestPickFirstLeaf_SimpleResolverUpdate_FirstServerUnReady(t *testing.T) cc, r, bm := setupPickFirstLeaf(t, 2) addrs := bm.resolverAddrs() - stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + stateSubscriber := &ccStateSubscriber{} internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) bm.stopAllExcept(1) @@ -213,7 +206,7 @@ func (s) TestPickFirstLeaf_SimpleResolverUpdate_DuplicateAddrs(t *testing.T) { cc, r, bm := setupPickFirstLeaf(t, 2) addrs := bm.resolverAddrs() - stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + stateSubscriber := &ccStateSubscriber{} internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) bm.stopAllExcept(1) @@ -269,7 +262,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_DisjointLists(t *testing.T) { balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) cc, r, bm := setupPickFirstLeaf(t, 4) addrs := bm.resolverAddrs() - stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + stateSubscriber := &ccStateSubscriber{} internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) bm.backends[0].S.Stop() @@ -330,7 +323,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_ActiveBackendInUpdatedList(t *testing balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) cc, r, bm := setupPickFirstLeaf(t, 3) addrs := bm.resolverAddrs() - stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + stateSubscriber := &ccStateSubscriber{} internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) bm.backends[0].S.Stop() @@ -392,7 +385,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_InActiveBackendInUpdatedList(t *testi balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) cc, r, bm := setupPickFirstLeaf(t, 3) addrs := bm.resolverAddrs() - stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + stateSubscriber := &ccStateSubscriber{} internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) bm.backends[0].S.Stop() @@ -455,7 +448,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_IdenticalLists(t *testing.T) { balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) cc, r, bm := setupPickFirstLeaf(t, 2) addrs := bm.resolverAddrs() - stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + stateSubscriber := &ccStateSubscriber{} internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) bm.backends[0].S.Stop() @@ -527,7 +520,7 @@ func (s) TestPickFirstLeaf_StopConnectedServer_FirstServerRestart(t *testing.T) balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) cc, r, bm := setupPickFirstLeaf(t, 2) addrs := bm.resolverAddrs() - stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + stateSubscriber := &ccStateSubscriber{} internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) // shutdown all active backends except the target. @@ -591,7 +584,7 @@ func (s) TestPickFirstLeaf_StopConnectedServer_SecondServerRestart(t *testing.T) balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) cc, r, bm := setupPickFirstLeaf(t, 2) addrs := bm.resolverAddrs() - stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + stateSubscriber := &ccStateSubscriber{} internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) // shutdown all active backends except the target. @@ -662,7 +655,7 @@ func (s) TestPickFirstLeaf_StopConnectedServer_SecondServerToFirst(t *testing.T) balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) cc, r, bm := setupPickFirstLeaf(t, 2) addrs := bm.resolverAddrs() - stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + stateSubscriber := &ccStateSubscriber{} internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) // shutdown all active backends except the target. @@ -733,7 +726,7 @@ func (s) TestPickFirstLeaf_StopConnectedServer_FirstServerToSecond(t *testing.T) balancer.Register(&stateStoringBalancerBuilder{balancer: balCh}) cc, r, bm := setupPickFirstLeaf(t, 2) addrs := bm.resolverAddrs() - stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + stateSubscriber := &ccStateSubscriber{} internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) // shutdown all active backends except the target. @@ -807,7 +800,7 @@ func (s) TestPickFirstLeaf_EmptyAddressList(t *testing.T) { cc, r, bm := setupPickFirstLeaf(t, 1) addrs := bm.resolverAddrs() - stateSubscriber := &ccStateSubscriber{transitions: []connectivity.State{}} + stateSubscriber := &ccStateSubscriber{} internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) r.UpdateState(resolver.State{Addresses: addrs}) From 34da793eaa2bb22c9a9fe24ce430c2c581841668 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Fri, 6 Sep 2024 16:12:27 +0530 Subject: [PATCH 44/62] Address review comments --- balancer/pickfirstleaf/pickfirstleaf.go | 6 ++ .../pickfirstleaf/test/pickfirstleaf_test.go | 56 +++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/balancer/pickfirstleaf/pickfirstleaf.go b/balancer/pickfirstleaf/pickfirstleaf.go index bbfdc5a436cf..2de92e82ccd4 100644 --- a/balancer/pickfirstleaf/pickfirstleaf.go +++ b/balancer/pickfirstleaf/pickfirstleaf.go @@ -159,8 +159,14 @@ func (b *pickfirstBalancer) ResolverError(err error) { }, func() { close(doneCh) }) + <-doneCh } +// resolverError is called by the ClientConn when the name resolver producers an +// an error or when pickfirst determined the resolver update to be invalid. +// If the resolver returns an error before sending the first update, +// it is handled by the gracefulswitch balancer (which is always the top-level +// LB policy on any channel), so we don't need to handle that here. // Only executed in the context of a serializer callback. func (b *pickfirstBalancer) resolverError(err error) { if b.logger.V(2) { diff --git a/balancer/pickfirstleaf/test/pickfirstleaf_test.go b/balancer/pickfirstleaf/test/pickfirstleaf_test.go index 2c09360c36ec..b2cd54bd9dd2 100644 --- a/balancer/pickfirstleaf/test/pickfirstleaf_test.go +++ b/balancer/pickfirstleaf/test/pickfirstleaf_test.go @@ -840,6 +840,62 @@ func (s) TestPickFirstLeaf_EmptyAddressList(t *testing.T) { } } +// TestPickFirstLeaf_InitialResolverError verifies the behaviour when a resolver +// returns an error before any valid configuration. +func (s) TestPickFirstLeaf_InitialResolverError(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + r := manual.NewBuilderWithScheme("whatever") + balChan := make(chan *stateStoringBalancer, 1) + balancer.Register(&stateStoringBalancerBuilder{balancer: balChan}) + + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(stateStoringServiceConfig), + } + cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.NewClient() failed: %v", err) + } + + ccClosed := false + defer func() { + if !ccClosed { + cc.Close() + } + }() + + stateSubscriber := &ccStateSubscriber{} + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) + // At this point, the resolver has not returned any addresses to the channel. + // This RPC must block until the context expires. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("EmptyCall() = %s, want %s", status.Code(err), codes.DeadlineExceeded) + } + + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + r.CC.ReportError(fmt.Errorf("test error")) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Close the clientconn to flush the connectivity state manager. + cc.Close() + ccClosed = true + + /// The clientconn should never transition to CONNECTING. + wantTransitions := []connectivity.State{ + connectivity.TransientFailure, + connectivity.Shutdown, + } + + if diff := cmp.Diff(wantTransitions, stateSubscriber.transitions); diff != "" { + t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) + } +} + // stateStoringBalancer stores the state of the subconns being created. type stateStoringBalancer struct { balancer.Balancer From 6a7720e22e739509222875aa4e8ea7a1d63a8c47 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Tue, 10 Sep 2024 00:05:41 +0530 Subject: [PATCH 45/62] Update picker less frequently --- balancer/pickfirstleaf/pickfirstleaf.go | 13 ++-- balancer/pickfirstleaf/pickfirstleaf_test.go | 69 ++++++++++++++++++++ 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/balancer/pickfirstleaf/pickfirstleaf.go b/balancer/pickfirstleaf/pickfirstleaf.go index 2de92e82ccd4..0db53aa7ed69 100644 --- a/balancer/pickfirstleaf/pickfirstleaf.go +++ b/balancer/pickfirstleaf/pickfirstleaf.go @@ -149,6 +149,7 @@ type pickfirstBalancer struct { subConns *resolver.AddressMap // scData for active subonns mapped by address. addressList addressList firstPass bool + numTF int } func (b *pickfirstBalancer) ResolverError(err error) { @@ -529,11 +530,14 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.Sub // We have finished the first pass, keep re-connecting failing subconns. switch newState.ConnectivityState { case connectivity.TransientFailure: + b.numTF++ sd.lastErr = newState.ConnectionError - b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.TransientFailure, - Picker: &picker{err: newState.ConnectionError}, - }) + if b.numTF%b.subConns.Len() == 0 { + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: newState.ConnectionError}, + }) + } // We don't need to request re-resolution since the subconn already does // that before reporting TRANSIENT_FAILURE. // TODO: #7534 - Move re-resolution requests from subconn into pick_first. @@ -545,6 +549,7 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.Sub // Only executed in the context of a serializer callback. func (b *pickfirstBalancer) endFirstPass(lastErr error) { b.firstPass = false + b.numTF = 0 b.state = connectivity.TransientFailure b.cc.UpdateState(balancer.State{ diff --git a/balancer/pickfirstleaf/pickfirstleaf_test.go b/balancer/pickfirstleaf/pickfirstleaf_test.go index 69cb3994c2c7..d2c1a712cb47 100644 --- a/balancer/pickfirstleaf/pickfirstleaf_test.go +++ b/balancer/pickfirstleaf/pickfirstleaf_test.go @@ -19,13 +19,24 @@ package pickfirstleaf import ( + "fmt" "testing" + "time" "google.golang.org/grpc/attributes" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/resolver" ) +const ( + // Default short timeout, to be used when waiting for events which are not + // expected to happen. + defaultTestShortTimeout = 100 * time.Millisecond +) + type s struct { grpctest.Tester } @@ -191,3 +202,61 @@ func (s) TestAddressList_SeekTo(t *testing.T) { t.Errorf("addressList.increment() = %t, want %t", got, want) } } + +// TestPickFirstLeaf_TFPickerUpdate sends TRANSIENT_FAILURE subconn state updates +// for each subconn managed by a pickfirst balancer. It verifies that the picker +// is updated with the expected frequency. +func (s) TestPickFirstLeaf_TFPickerUpdate(t *testing.T) { + cc := testutils.NewBalancerClientConn(t) + bal := pickfirstBuilder{}.Build(cc, balancer.BuildOptions{}) + defer bal.Close() + bal.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "1.1.1.1:1"}}}, + {Addresses: []resolver.Address{{Addr: "2.2.2.2:2"}}}, + }, + }, + }) + + // PF should report TRANSIENT_FAILURE only once all the sunbconns have failed + // once. + tfErr := fmt.Errorf("test err: connection refused") + sc0 := <-cc.NewSubConnCh + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: tfErr}) + + p := <-cc.NewPickerCh + _, err := p.Pick(balancer.PickInfo{}) + if want, got := balancer.ErrNoSubConnAvailable, err; got != want { + t.Fatalf("picker.Pick() = %v, want %v", got, want) + } + + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: tfErr}) + + p = <-cc.NewPickerCh + _, err = p.Pick(balancer.PickInfo{}) + if want, got := tfErr, err; got != want { + t.Fatalf("picker.Pick() = %v, want %v", got, want) + } + + // Subsequent TRANSIENT_FAILUREs should be reported only after seeing "# of subconns" + // TRANSIENT_FAILUREs. + newTfErr := fmt.Errorf("test err: unreachable") + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: newTfErr}) + select { + case <-time.After(defaultTestShortTimeout): + case p = <-cc.NewPickerCh: + sc, err := p.Pick(balancer.PickInfo{}) + t.Fatalf("unexpected picker update: %v, %v", sc, err) + } + + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: newTfErr}) + p = <-cc.NewPickerCh + _, err = p.Pick(balancer.PickInfo{}) + if want, got := newTfErr, err; got != want { + t.Fatalf("picker.Pick() = %v, want %v", got, want) + } +} From 191444553300a80da072f0694ac693063082cd2b Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Tue, 10 Sep 2024 13:20:10 +0530 Subject: [PATCH 46/62] Fix typos --- balancer/pickfirstleaf/pickfirstleaf.go | 2 +- balancer/pickfirstleaf/pickfirstleaf_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/balancer/pickfirstleaf/pickfirstleaf.go b/balancer/pickfirstleaf/pickfirstleaf.go index 0db53aa7ed69..f9e17b674709 100644 --- a/balancer/pickfirstleaf/pickfirstleaf.go +++ b/balancer/pickfirstleaf/pickfirstleaf.go @@ -163,7 +163,7 @@ func (b *pickfirstBalancer) ResolverError(err error) { <-doneCh } -// resolverError is called by the ClientConn when the name resolver producers an +// resolverError is called by the ClientConn when the name resolver produces // an error or when pickfirst determined the resolver update to be invalid. // If the resolver returns an error before sending the first update, // it is handled by the gracefulswitch balancer (which is always the top-level diff --git a/balancer/pickfirstleaf/pickfirstleaf_test.go b/balancer/pickfirstleaf/pickfirstleaf_test.go index d2c1a712cb47..68a677fc9416 100644 --- a/balancer/pickfirstleaf/pickfirstleaf_test.go +++ b/balancer/pickfirstleaf/pickfirstleaf_test.go @@ -250,7 +250,7 @@ func (s) TestPickFirstLeaf_TFPickerUpdate(t *testing.T) { case <-time.After(defaultTestShortTimeout): case p = <-cc.NewPickerCh: sc, err := p.Pick(balancer.PickInfo{}) - t.Fatalf("unexpected picker update: %v, %v", sc, err) + t.Fatalf("Unexpected picker update: %v, %v", sc, err) } sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: newTfErr}) From ef1af599c3aa72e3d711e76d319fd72c4c4430b3 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Wed, 18 Sep 2024 02:24:37 +0530 Subject: [PATCH 47/62] address review comments --- balancer/pickfirst/pickfirst.go | 2 +- balancer/pickfirstleaf/pickfirstleaf.go | 37 ++++++++++--------------- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/balancer/pickfirst/pickfirst.go b/balancer/pickfirst/pickfirst.go index 8c7ee1a85360..971adb586911 100644 --- a/balancer/pickfirst/pickfirst.go +++ b/balancer/pickfirst/pickfirst.go @@ -39,11 +39,11 @@ import ( ) func init() { + internal.ShuffleAddressListForTesting = func(n int, swap func(i, j int)) { rand.Shuffle(n, swap) } if envconfig.NewPickFirstEnabled { return } balancer.Register(pickfirstBuilder{}) - internal.ShuffleAddressListForTesting = func(n int, swap func(i, j int)) { rand.Shuffle(n, swap) } } var logger = grpclog.Component("pick-first-lb") diff --git a/balancer/pickfirstleaf/pickfirstleaf.go b/balancer/pickfirstleaf/pickfirstleaf.go index f9e17b674709..495e65f2131b 100644 --- a/balancer/pickfirstleaf/pickfirstleaf.go +++ b/balancer/pickfirstleaf/pickfirstleaf.go @@ -18,6 +18,11 @@ // Package pickfirstleaf contains the pick_first load balancing policy which // will be the universal leaf policy after dualstack changes are implemented. +// +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. package pickfirstleaf import ( @@ -25,7 +30,6 @@ import ( "encoding/json" "errors" "fmt" - "math/rand" "google.golang.org/grpc/balancer" "google.golang.org/grpc/connectivity" @@ -41,7 +45,6 @@ import ( func init() { if envconfig.NewPickFirstEnabled { - internal.ShuffleAddressListForTesting = func(n int, swap func(i, j int)) { rand.Shuffle(n, swap) } // Register as the default pick_first balancer. Name = "pick_first" } @@ -173,9 +176,6 @@ func (b *pickfirstBalancer) resolverError(err error) { if b.logger.V(2) { b.logger.Infof("Received error from the name resolver: %v", err) } - if b.state == connectivity.Shutdown { - return - } // The picker will not change since the balancer does not currently // report an error. if b.state != connectivity.TransientFailure { @@ -207,9 +207,6 @@ func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState // updateClientConnState handles clientConn state changes. // Only executed in the context of a serializer callback. func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState) error { - if b.state == connectivity.Shutdown { - return errBalancerClosed - } if len(state.ResolverState.Addresses) == 0 && len(state.ResolverState.Endpoints) == 0 { // Cleanup state pertaining to the previous resolver state. // Treat an empty address list like an error by calling b.ResolverError. @@ -265,7 +262,7 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState } b.reconcileSubConns(newEndpoints) - // If its the first resolver update or the balancer was already READY + // If it's the first resolver update or the balancer was already READY // (but the new address list does not contain the ready subconn) or // CONNECTING, enter CONNECTING. // We may be in TRANSIENT_FAILURE due to a previous empty address list, @@ -382,9 +379,7 @@ func (b *pickfirstBalancer) shutdownRemaining(selected *scData) { sd.subConn.Shutdown() } } - for _, k := range b.subConns.Keys() { - b.subConns.Delete(k) - } + b.subConns = resolver.NewAddressMap() b.subConns.Set(selected.addr, selected) } @@ -394,7 +389,7 @@ func (b *pickfirstBalancer) shutdownRemaining(selected *scData) { // attempted. // Only executed in the context of a serializer callback. func (b *pickfirstBalancer) requestConnection() { - if !b.addressList.isValid() || b.state == connectivity.Shutdown { + if !b.addressList.isValid() { return } curAddr := b.addressList.currentAddress() @@ -408,14 +403,7 @@ func (b *pickfirstBalancer) requestConnection() { // This should never happen, unless the clientConn is being shut // down. b.logger.Warningf("Failed to create a subConn for address %v: %v", curAddr.String(), err) - // The LB policy remains in TRANSIENT_FAILURE until a new resolver - // update is received. - b.state = connectivity.TransientFailure - b.addressList.reset() - b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.TransientFailure, - Picker: &picker{err: fmt.Errorf("failed to create a new subConn: %v", err)}, - }) + // Do nothing, the LB policy will be closed soon. return } b.subConns.Set(curAddr, sd) @@ -434,6 +422,11 @@ func (b *pickfirstBalancer) requestConnection() { case connectivity.Ready: // Should never happen. b.logger.Errorf("Requesting a connection even though we have a READY subconn") + case connectivity.Shutdown: + // Should never happen. + b.logger.Errorf("SubConn with state SHUTDOWN present in subconns map") + case connectivity.Connecting: + // Wait for the subconn to report success or failure. } } @@ -530,7 +523,7 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.Sub // We have finished the first pass, keep re-connecting failing subconns. switch newState.ConnectivityState { case connectivity.TransientFailure: - b.numTF++ + b.numTF = (b.numTF + 1) % b.subConns.Len() sd.lastErr = newState.ConnectionError if b.numTF%b.subConns.Len() == 0 { b.cc.UpdateState(balancer.State{ From efa343b06a2f720c9be1cfbc18849f0f028ca4c6 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Wed, 18 Sep 2024 02:46:35 +0530 Subject: [PATCH 48/62] Fix test breakage due to #7613 --- balancer/pickfirstleaf/test/pickfirstleaf_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/balancer/pickfirstleaf/test/pickfirstleaf_test.go b/balancer/pickfirstleaf/test/pickfirstleaf_test.go index b2cd54bd9dd2..36794df48d98 100644 --- a/balancer/pickfirstleaf/test/pickfirstleaf_test.go +++ b/balancer/pickfirstleaf/test/pickfirstleaf_test.go @@ -266,6 +266,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_DisjointLists(t *testing.T) { internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) bm.backends[0].S.Stop() + bm.backends[0].S = nil r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[1]}}) var bal *stateStoringBalancer select { @@ -288,6 +289,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_DisjointLists(t *testing.T) { } bm.backends[2].S.Stop() + bm.backends[2].S = nil r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[2], addrs[3]}}) if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[3]); err != nil { @@ -327,6 +329,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_ActiveBackendInUpdatedList(t *testing internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) bm.backends[0].S.Stop() + bm.backends[0].S = nil r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[1]}}) var bal *stateStoringBalancer select { @@ -349,6 +352,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_ActiveBackendInUpdatedList(t *testing } bm.backends[2].S.Stop() + bm.backends[2].S = nil r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[2], addrs[1]}}) // Verify that the ClientConn stays in READY. @@ -389,6 +393,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_InActiveBackendInUpdatedList(t *testi internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) bm.backends[0].S.Stop() + bm.backends[0].S = nil r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[1]}}) var bal *stateStoringBalancer select { @@ -411,6 +416,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_InActiveBackendInUpdatedList(t *testi } bm.backends[2].S.Stop() + bm.backends[2].S = nil if err := bm.backends[0].StartServer(); err != nil { t.Fatalf("Failed to re-start test backend: %v", err) } @@ -452,6 +458,7 @@ func (s) TestPickFirstLeaf_ResolverUpdates_IdenticalLists(t *testing.T) { internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) bm.backends[0].S.Stop() + bm.backends[0].S = nil r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[1]}}) var bal *stateStoringBalancer select { @@ -549,6 +556,7 @@ func (s) TestPickFirstLeaf_StopConnectedServer_FirstServerRestart(t *testing.T) // Shut down the connected server. bm.backends[0].S.Stop() + bm.backends[0].S = nil testutils.AwaitState(ctx, t, cc, connectivity.Idle) // Start the new target server. @@ -614,6 +622,7 @@ func (s) TestPickFirstLeaf_StopConnectedServer_SecondServerRestart(t *testing.T) // Shut down the connected server. bm.backends[1].S.Stop() + bm.backends[1].S = nil testutils.AwaitState(ctx, t, cc, connectivity.Idle) // Start the new target server. @@ -685,6 +694,7 @@ func (s) TestPickFirstLeaf_StopConnectedServer_SecondServerToFirst(t *testing.T) // Shut down the connected server. bm.backends[1].S.Stop() + bm.backends[1].S = nil testutils.AwaitState(ctx, t, cc, connectivity.Idle) // Start the new target server. @@ -755,6 +765,7 @@ func (s) TestPickFirstLeaf_StopConnectedServer_FirstServerToSecond(t *testing.T) // Shut down the connected server. bm.backends[0].S.Stop() + bm.backends[0].S = nil testutils.AwaitState(ctx, t, cc, connectivity.Idle) // Start the new target server. @@ -978,6 +989,7 @@ func (b *backendManager) stopAllExcept(index int) { for idx, b := range b.backends { if idx != index { b.S.Stop() + b.S = nil } } } From 60e14d01c749e5d83c3b6f352ca188c83b3b7376 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Wed, 18 Sep 2024 16:04:47 +0530 Subject: [PATCH 49/62] Replace the callback serializer with a mutex --- balancer/pickfirstleaf/pickfirstleaf.go | 143 +++++++++--------------- 1 file changed, 54 insertions(+), 89 deletions(-) diff --git a/balancer/pickfirstleaf/pickfirstleaf.go b/balancer/pickfirstleaf/pickfirstleaf.go index 495e65f2131b..3ec6d177787d 100644 --- a/balancer/pickfirstleaf/pickfirstleaf.go +++ b/balancer/pickfirstleaf/pickfirstleaf.go @@ -26,10 +26,10 @@ package pickfirstleaf import ( - "context" "encoding/json" "errors" "fmt" + "sync" "google.golang.org/grpc/balancer" "google.golang.org/grpc/connectivity" @@ -37,7 +37,6 @@ import ( "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/envconfig" internalgrpclog "google.golang.org/grpc/internal/grpclog" - "google.golang.org/grpc/internal/grpcsync" "google.golang.org/grpc/internal/pretty" "google.golang.org/grpc/resolver" "google.golang.org/grpc/serviceconfig" @@ -65,14 +64,12 @@ const logPrefix = "[pick-first-leaf-lb %p] " type pickfirstBuilder struct{} func (pickfirstBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer { - ctx, cancel := context.WithCancel(context.Background()) b := &pickfirstBalancer{ - cc: cc, - addressList: addressList{}, - subConns: resolver.NewAddressMap(), - serializer: grpcsync.NewCallbackSerializer(ctx), - serializerCancel: cancel, - state: connectivity.Connecting, + cc: cc, + addressList: addressList{}, + subConns: resolver.NewAddressMap(), + state: connectivity.Connecting, + mu: sync.Mutex{}, } b.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(logPrefix, b)) return b @@ -100,14 +97,13 @@ type pfConfig struct { } // scData keeps track of the current state of the subConn. +// It is not safe for concurrent access. type scData struct { // The following fields are initialized at build time and read-only after // that. subConn balancer.SubConn addr resolver.Address - // The following fields should only be accessed from a serializer callback - // to ensure synchronization. state connectivity.State lastErr error } @@ -119,11 +115,7 @@ func (b *pickfirstBalancer) newSCData(addr resolver.Address) (*scData, error) { } sc, err := b.cc.NewSubConn([]resolver.Address{addr}, balancer.NewSubConnOptions{ StateListener: func(state balancer.SubConnState) { - // Store the state and delegate. - b.serializer.TrySchedule(func(_ context.Context) { - sd.state = state.ConnectivityState - b.updateSubConnState(sd, state) - }) + b.updateSubConnState(sd, state) }, }) if err != nil { @@ -139,40 +131,29 @@ type pickfirstBalancer struct { logger *internalgrpclog.PrefixLogger cc balancer.ClientConn - // The serializer and its cancel func are initialized at build time, and the - // rest of the fields here are only accessed from serializer callbacks (or - // from balancer.Balancer methods, which themselves are guaranteed to be - // mutually exclusive) and hence do not need to be guarded by a mutex. - // The serializer is used to ensure synchronization of updates triggered + // The mutex is used to ensure synchronization of updates triggered // from the idle picker and the already serialized resolver, // subconn state updates. - serializer *grpcsync.CallbackSerializer - serializerCancel func() - state connectivity.State - subConns *resolver.AddressMap // scData for active subonns mapped by address. - addressList addressList - firstPass bool - numTF int -} - -func (b *pickfirstBalancer) ResolverError(err error) { - doneCh := make(chan error, 1) - b.serializer.ScheduleOr(func(_ context.Context) { - b.resolverError(err) - close(doneCh) - }, func() { - close(doneCh) - }) - <-doneCh + mu sync.Mutex + state connectivity.State + subConns *resolver.AddressMap // scData for active subonns mapped by address. + addressList addressList + firstPass bool + numTF int } -// resolverError is called by the ClientConn when the name resolver produces +// ResolverError is called by the ClientConn when the name resolver produces // an error or when pickfirst determined the resolver update to be invalid. // If the resolver returns an error before sending the first update, // it is handled by the gracefulswitch balancer (which is always the top-level // LB policy on any channel), so we don't need to handle that here. -// Only executed in the context of a serializer callback. -func (b *pickfirstBalancer) resolverError(err error) { +func (b *pickfirstBalancer) ResolverError(err error) { + b.mu.Lock() + defer b.mu.Unlock() + b.resolverErrorLocked(err) +} + +func (b *pickfirstBalancer) resolverErrorLocked(err error) { if b.logger.V(2) { b.logger.Infof("Received error from the name resolver: %v", err) } @@ -185,7 +166,7 @@ func (b *pickfirstBalancer) resolverError(err error) { return } - b.closeSubConns() + b.closeSubConnsLocked() b.addressList.updateEndpointList(nil) b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.TransientFailure, @@ -194,24 +175,13 @@ func (b *pickfirstBalancer) resolverError(err error) { } func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState) error { - errCh := make(chan error, 1) - b.serializer.ScheduleOr(func(_ context.Context) { - err := b.updateClientConnState(state) - errCh <- err - }, func() { - errCh <- errBalancerClosed - }) - return <-errCh -} - -// updateClientConnState handles clientConn state changes. -// Only executed in the context of a serializer callback. -func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState) error { + b.mu.Lock() + defer b.mu.Unlock() if len(state.ResolverState.Addresses) == 0 && len(state.ResolverState.Endpoints) == 0 { // Cleanup state pertaining to the previous resolver state. // Treat an empty address list like an error by calling b.ResolverError. b.state = connectivity.TransientFailure - b.resolverError(errors.New("produced zero addresses")) + b.resolverErrorLocked(errors.New("produced zero addresses")) return balancer.ErrBadResolverState } cfg, ok := state.BalancerConfig.(pfConfig) @@ -261,7 +231,7 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState return nil } - b.reconcileSubConns(newEndpoints) + b.reconcileSubConnsLocked(newEndpoints) // If it's the first resolver update or the balancer was already READY // (but the new address list does not contain the ready subconn) or // CONNECTING, enter CONNECTING. @@ -276,11 +246,11 @@ func (b *pickfirstBalancer) updateClientConnState(state balancer.ClientConnState ConnectivityState: connectivity.Connecting, Picker: &picker{err: balancer.ErrNoSubConnAvailable}, }) - b.requestConnection() + b.requestConnectionLocked() } else if b.state == connectivity.TransientFailure { // If we're in TRANSIENT_FAILURE, we stay in TRANSIENT_FAILURE until // we're READY. See A62. - b.requestConnection() + b.requestConnectionLocked() } return nil } @@ -292,26 +262,23 @@ func (b *pickfirstBalancer) UpdateSubConnState(subConn balancer.SubConn, state b } func (b *pickfirstBalancer) Close() { - b.serializer.TrySchedule(func(_ context.Context) { - b.closeSubConns() - b.state = connectivity.Shutdown - }) - b.serializerCancel() - <-b.serializer.Done() + b.mu.Lock() + defer b.mu.Unlock() + b.closeSubConnsLocked() + b.state = connectivity.Shutdown } // ExitIdle moves the balancer out of idle state. It can be called concurrently // by the idlePicker and clientConn so access to variables should be synchronized. func (b *pickfirstBalancer) ExitIdle() { - b.serializer.TrySchedule(func(_ context.Context) { - if b.state == connectivity.Idle { - b.requestConnection() - } - }) + b.mu.Lock() + defer b.mu.Unlock() + if b.state == connectivity.Idle { + b.requestConnectionLocked() + } } -// Only executed in the context of a serializer callback. -func (b *pickfirstBalancer) closeSubConns() { +func (b *pickfirstBalancer) closeSubConnsLocked() { for _, sd := range b.subConns.Values() { sd.(*scData).subConn.Shutdown() } @@ -342,8 +309,7 @@ func deDupAddresses(endpoints []resolver.Endpoint) []resolver.Endpoint { return newEndpoints } -// Only executed in the context of a serializer callback. -func (b *pickfirstBalancer) reconcileSubConns(newEndpoints []resolver.Endpoint) { +func (b *pickfirstBalancer) reconcileSubConnsLocked(newEndpoints []resolver.Endpoint) { // Remove old subConns that were not in new address list. oldAddrs := resolver.NewAddressMap() for _, k := range b.subConns.Keys() { @@ -369,10 +335,9 @@ func (b *pickfirstBalancer) reconcileSubConns(newEndpoints []resolver.Endpoint) } } -// shutdownRemaining shuts down remaining subConns. Called when a subConn +// shutdownRemainingLocked shuts down remaining subConns. Called when a subConn // becomes ready, which means that all other subConn must be shutdown. -// Only executed in the context of a serializer callback. -func (b *pickfirstBalancer) shutdownRemaining(selected *scData) { +func (b *pickfirstBalancer) shutdownRemainingLocked(selected *scData) { for _, v := range b.subConns.Values() { sd := v.(*scData) if sd.subConn != selected.subConn { @@ -383,12 +348,11 @@ func (b *pickfirstBalancer) shutdownRemaining(selected *scData) { b.subConns.Set(selected.addr, selected) } -// requestConnection starts connecting on the subchannel corresponding to the +// requestConnectionLocked starts connecting on the subchannel corresponding to the // current address. If no subchannel exists, one is created. If the current // subchannel is in TransientFailure, a connection to the next address is // attempted. -// Only executed in the context of a serializer callback. -func (b *pickfirstBalancer) requestConnection() { +func (b *pickfirstBalancer) requestConnectionLocked() { if !b.addressList.isValid() { return } @@ -415,10 +379,10 @@ func (b *pickfirstBalancer) requestConnection() { scd.subConn.Connect() case connectivity.TransientFailure: if !b.addressList.increment() { - b.endFirstPass(scd.lastErr) + b.endFirstPassLocked(scd.lastErr) return } - b.requestConnection() + b.requestConnectionLocked() case connectivity.Ready: // Should never happen. b.logger.Errorf("Requesting a connection even though we have a READY subconn") @@ -430,8 +394,10 @@ func (b *pickfirstBalancer) requestConnection() { } } -// Only executed in the context of a serializer callback. func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.SubConnState) { + b.mu.Lock() + defer b.mu.Unlock() + sd.state = newState.ConnectivityState // Previously relevant subconns can still callback with state updates. // To prevent pickers from returning these obsolete subconns, this logic // is included to check if the current list of active subconns includes this @@ -444,7 +410,7 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.Sub } if newState.ConnectivityState == connectivity.Ready { - b.shutdownRemaining(sd) + b.shutdownRemainingLocked(sd) if !b.addressList.seekTo(sd.addr) { // This should not fail as we should have only one subconn after // entering READY. The subconn should be present in the addressList. @@ -497,11 +463,11 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.Sub return } if b.addressList.increment() { - b.requestConnection() + b.requestConnectionLocked() return } // End of the first pass. - b.endFirstPass(newState.ConnectionError) + b.endFirstPassLocked(newState.ConnectionError) case connectivity.Idle: // A subconn can transition from CONNECTING directly to IDLE when // a transport is successfully created, but the connection fails before @@ -539,8 +505,7 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.Sub } } -// Only executed in the context of a serializer callback. -func (b *pickfirstBalancer) endFirstPass(lastErr error) { +func (b *pickfirstBalancer) endFirstPassLocked(lastErr error) { b.firstPass = false b.numTF = 0 b.state = connectivity.TransientFailure From 88fbabedbb9ab07b0b9c38149a17c8640c6bda26 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Wed, 18 Sep 2024 16:33:22 +0530 Subject: [PATCH 50/62] Convert requestConnectionLocked to iteration from recursion --- balancer/pickfirstleaf/pickfirstleaf.go | 73 ++++++++++++++----------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/balancer/pickfirstleaf/pickfirstleaf.go b/balancer/pickfirstleaf/pickfirstleaf.go index 3ec6d177787d..d778beacd592 100644 --- a/balancer/pickfirstleaf/pickfirstleaf.go +++ b/balancer/pickfirstleaf/pickfirstleaf.go @@ -351,47 +351,54 @@ func (b *pickfirstBalancer) shutdownRemainingLocked(selected *scData) { // requestConnectionLocked starts connecting on the subchannel corresponding to the // current address. If no subchannel exists, one is created. If the current // subchannel is in TransientFailure, a connection to the next address is -// attempted. +// attempted until a subchannel is found. func (b *pickfirstBalancer) requestConnectionLocked() { if !b.addressList.isValid() { return } - curAddr := b.addressList.currentAddress() - sd, ok := b.subConns.Get(curAddr) - if !ok { - var err error - // We want to assign the new scData to sd from the outer scope, hence - // we can't use := below. - sd, err = b.newSCData(curAddr) - if err != nil { - // This should never happen, unless the clientConn is being shut - // down. - b.logger.Warningf("Failed to create a subConn for address %v: %v", curAddr.String(), err) - // Do nothing, the LB policy will be closed soon. - return + var lastErr error + for valid := true; valid; valid = b.addressList.increment() { + curAddr := b.addressList.currentAddress() + sd, ok := b.subConns.Get(curAddr) + if !ok { + var err error + // We want to assign the new scData to sd from the outer scope, hence + // we can't use := below. + sd, err = b.newSCData(curAddr) + if err != nil { + // This should never happen, unless the clientConn is being shut + // down. + if b.logger.V(2) { + b.logger.Infof("Failed to create a subConn for address %v: %v", curAddr.String(), err) + } + // Do nothing, the LB policy will be closed soon. + return + } + b.subConns.Set(curAddr, sd) } - b.subConns.Set(curAddr, sd) - } - scd := sd.(*scData) - switch scd.state { - case connectivity.Idle: - scd.subConn.Connect() - case connectivity.TransientFailure: - if !b.addressList.increment() { - b.endFirstPassLocked(scd.lastErr) - return + scd := sd.(*scData) + switch scd.state { + case connectivity.Idle: + scd.subConn.Connect() + case connectivity.TransientFailure: + // Try the next address. + lastErr = scd.lastErr + continue + case connectivity.Ready: + // Should never happen. + b.logger.Errorf("Requesting a connection even though we have a READY subconn") + case connectivity.Shutdown: + // Should never happen. + b.logger.Errorf("SubConn with state SHUTDOWN present in subconns map") + case connectivity.Connecting: + // Wait for the subconn to report success or failure. } - b.requestConnectionLocked() - case connectivity.Ready: - // Should never happen. - b.logger.Errorf("Requesting a connection even though we have a READY subconn") - case connectivity.Shutdown: - // Should never happen. - b.logger.Errorf("SubConn with state SHUTDOWN present in subconns map") - case connectivity.Connecting: - // Wait for the subconn to report success or failure. + return } + // All the remaining addresses in the list are in TRANSIENT_FAILURE, end the + // first pass. + b.endFirstPassLocked(lastErr) } func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.SubConnState) { From 30256a5c1e3b6291f9f8869c309358cd4ccc59c7 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Wed, 18 Sep 2024 16:44:47 +0530 Subject: [PATCH 51/62] Fix lint --- balancer/pickfirstleaf/pickfirstleaf.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/balancer/pickfirstleaf/pickfirstleaf.go b/balancer/pickfirstleaf/pickfirstleaf.go index d778beacd592..f1bbc5fe6282 100644 --- a/balancer/pickfirstleaf/pickfirstleaf.go +++ b/balancer/pickfirstleaf/pickfirstleaf.go @@ -51,8 +51,7 @@ func init() { } var ( - logger = grpclog.Component("pick-first-leaf-lb") - errBalancerClosed = fmt.Errorf("pickfirst: LB policy is closed") + logger = grpclog.Component("pick-first-leaf-lb") // Name is the name of the pick_first_leaf balancer. // Can be changed in init() if this balancer is to be registered as the default // pickfirst. From 6daf9cc7f6f4050f806694e382339edd1ca0eafe Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Thu, 19 Sep 2024 14:53:02 +0530 Subject: [PATCH 52/62] Report TF on resolver error --- balancer/pickfirstleaf/pickfirstleaf.go | 8 +-- balancer/pickfirstleaf/pickfirstleaf_test.go | 56 ++++++++++++++----- .../pickfirstleaf/test/pickfirstleaf_test.go | 56 ------------------- 3 files changed, 46 insertions(+), 74 deletions(-) diff --git a/balancer/pickfirstleaf/pickfirstleaf.go b/balancer/pickfirstleaf/pickfirstleaf.go index f1bbc5fe6282..7abea38f8a12 100644 --- a/balancer/pickfirstleaf/pickfirstleaf.go +++ b/balancer/pickfirstleaf/pickfirstleaf.go @@ -143,9 +143,6 @@ type pickfirstBalancer struct { // ResolverError is called by the ClientConn when the name resolver produces // an error or when pickfirst determined the resolver update to be invalid. -// If the resolver returns an error before sending the first update, -// it is handled by the gracefulswitch balancer (which is always the top-level -// LB policy on any channel), so we don't need to handle that here. func (b *pickfirstBalancer) ResolverError(err error) { b.mu.Lock() defer b.mu.Unlock() @@ -157,8 +154,9 @@ func (b *pickfirstBalancer) resolverErrorLocked(err error) { b.logger.Infof("Received error from the name resolver: %v", err) } // The picker will not change since the balancer does not currently - // report an error. - if b.state != connectivity.TransientFailure { + // report an error. If the balancer hasn't received a single good resolver + // update yet, transition to TRANSIENT_FAILURE. + if b.state != connectivity.TransientFailure && b.addressList.size() > 0 { if b.logger.V(2) { b.logger.Infof("Ignoring resolver error because balancer is using a previous good update.") } diff --git a/balancer/pickfirstleaf/pickfirstleaf_test.go b/balancer/pickfirstleaf/pickfirstleaf_test.go index 68a677fc9416..53146039bff6 100644 --- a/balancer/pickfirstleaf/pickfirstleaf_test.go +++ b/balancer/pickfirstleaf/pickfirstleaf_test.go @@ -19,6 +19,8 @@ package pickfirstleaf import ( + "context" + "errors" "fmt" "testing" "time" @@ -32,6 +34,8 @@ import ( ) const ( + // Default timeout for tests in this package. + defaultTestTimeout = 10 * time.Second // Default short timeout, to be used when waiting for events which are not // expected to happen. defaultTestShortTimeout = 100 * time.Millisecond @@ -207,6 +211,8 @@ func (s) TestAddressList_SeekTo(t *testing.T) { // for each subconn managed by a pickfirst balancer. It verifies that the picker // is updated with the expected frequency. func (s) TestPickFirstLeaf_TFPickerUpdate(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() cc := testutils.NewBalancerClientConn(t) bal := pickfirstBuilder{}.Build(cc, balancer.BuildOptions{}) defer bal.Close() @@ -226,20 +232,16 @@ func (s) TestPickFirstLeaf_TFPickerUpdate(t *testing.T) { sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: tfErr}) - p := <-cc.NewPickerCh - _, err := p.Pick(balancer.PickInfo{}) - if want, got := balancer.ErrNoSubConnAvailable, err; got != want { - t.Fatalf("picker.Pick() = %v, want %v", got, want) + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatalf("cc.WaitForPickerWithErr(%v) returned error: %v", tfErr, err) } sc1 := <-cc.NewSubConnCh sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: tfErr}) - p = <-cc.NewPickerCh - _, err = p.Pick(balancer.PickInfo{}) - if want, got := tfErr, err; got != want { - t.Fatalf("picker.Pick() = %v, want %v", got, want) + if err := cc.WaitForPickerWithErr(ctx, tfErr); err != nil { + t.Fatalf("cc.WaitForPickerWithErr(%v) returned error: %v", tfErr, err) } // Subsequent TRANSIENT_FAILUREs should be reported only after seeing "# of subconns" @@ -248,15 +250,43 @@ func (s) TestPickFirstLeaf_TFPickerUpdate(t *testing.T) { sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: newTfErr}) select { case <-time.After(defaultTestShortTimeout): - case p = <-cc.NewPickerCh: + case p := <-cc.NewPickerCh: sc, err := p.Pick(balancer.PickInfo{}) t.Fatalf("Unexpected picker update: %v, %v", sc, err) } sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: newTfErr}) - p = <-cc.NewPickerCh - _, err = p.Pick(balancer.PickInfo{}) - if want, got := newTfErr, err; got != want { - t.Fatalf("picker.Pick() = %v, want %v", got, want) + if err := cc.WaitForPickerWithErr(ctx, newTfErr); err != nil { + t.Fatalf("cc.WaitForPickerWithErr(%v) returned error: %v", tfErr, err) + } +} + +// TestPickFirstLeaf_InitialResolverError sends a resolver error to the balancer +// before a valid resolver update. It verifies that the clientconn state is +// updated to TRANSIENT_FAILURE. +func (s) TestPickFirstLeaf_InitialResolverError(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc := testutils.NewBalancerClientConn(t) + bal := pickfirstBuilder{}.Build(cc, balancer.BuildOptions{}) + defer bal.Close() + bal.ResolverError(errors.New("resolution failed: test error")) + + if err := cc.WaitForConnectivityState(ctx, connectivity.TransientFailure); err != nil { + t.Fatalf("cc.WaitForConnectivityState(%v) returned error: %v", connectivity.TransientFailure, err) + } + + // After sending a valid update, the LB policy should report CONNECTING. + bal.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "1.1.1.1:1"}}}, + {Addresses: []resolver.Address{{Addr: "2.2.2.2:2"}}}, + }, + }, + }) + + if err := cc.WaitForConnectivityState(ctx, connectivity.Connecting); err != nil { + t.Fatalf("cc.WaitForConnectivityState(%v) returned error: %v", connectivity.Connecting, err) } } diff --git a/balancer/pickfirstleaf/test/pickfirstleaf_test.go b/balancer/pickfirstleaf/test/pickfirstleaf_test.go index 36794df48d98..0070b272faa3 100644 --- a/balancer/pickfirstleaf/test/pickfirstleaf_test.go +++ b/balancer/pickfirstleaf/test/pickfirstleaf_test.go @@ -851,62 +851,6 @@ func (s) TestPickFirstLeaf_EmptyAddressList(t *testing.T) { } } -// TestPickFirstLeaf_InitialResolverError verifies the behaviour when a resolver -// returns an error before any valid configuration. -func (s) TestPickFirstLeaf_InitialResolverError(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - r := manual.NewBuilderWithScheme("whatever") - balChan := make(chan *stateStoringBalancer, 1) - balancer.Register(&stateStoringBalancerBuilder{balancer: balChan}) - - dopts := []grpc.DialOption{ - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithResolvers(r), - grpc.WithDefaultServiceConfig(stateStoringServiceConfig), - } - cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...) - if err != nil { - t.Fatalf("grpc.NewClient() failed: %v", err) - } - - ccClosed := false - defer func() { - if !ccClosed { - cc.Close() - } - }() - - stateSubscriber := &ccStateSubscriber{} - internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, stateSubscriber) - // At this point, the resolver has not returned any addresses to the channel. - // This RPC must block until the context expires. - sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) - defer sCancel() - client := testgrpc.NewTestServiceClient(cc) - if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("EmptyCall() = %s, want %s", status.Code(err), codes.DeadlineExceeded) - } - - testutils.AwaitState(ctx, t, cc, connectivity.Idle) - r.CC.ReportError(fmt.Errorf("test error")) - testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) - - // Close the clientconn to flush the connectivity state manager. - cc.Close() - ccClosed = true - - /// The clientconn should never transition to CONNECTING. - wantTransitions := []connectivity.State{ - connectivity.TransientFailure, - connectivity.Shutdown, - } - - if diff := cmp.Diff(wantTransitions, stateSubscriber.transitions); diff != "" { - t.Errorf("ClientConn states mismatch (-want +got):\n%s", diff) - } -} - // stateStoringBalancer stores the state of the subconns being created. type stateStoringBalancer struct { balancer.Balancer From 1f03ef90043de5b2a9b683c248d730e2ad1c3e90 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Wed, 25 Sep 2024 22:56:29 +0530 Subject: [PATCH 53/62] Comment wrapping, s/subcon/SubCon/g and flatten the endpoint list early in UpdateClientConnState --- balancer/pickfirstleaf/pickfirstleaf.go | 218 +++++++++---------- balancer/pickfirstleaf/pickfirstleaf_test.go | 22 +- 2 files changed, 111 insertions(+), 129 deletions(-) diff --git a/balancer/pickfirstleaf/pickfirstleaf.go b/balancer/pickfirstleaf/pickfirstleaf.go index 7abea38f8a12..0cb0f295d1a7 100644 --- a/balancer/pickfirstleaf/pickfirstleaf.go +++ b/balancer/pickfirstleaf/pickfirstleaf.go @@ -53,8 +53,8 @@ func init() { var ( logger = grpclog.Component("pick-first-leaf-lb") // Name is the name of the pick_first_leaf balancer. - // Can be changed in init() if this balancer is to be registered as the default - // pickfirst. + // Can be changed in init() if this balancer is to be registered as the + // default pickfirst. Name = "pick_first_leaf" ) @@ -132,10 +132,11 @@ type pickfirstBalancer struct { // The mutex is used to ensure synchronization of updates triggered // from the idle picker and the already serialized resolver, - // subconn state updates. - mu sync.Mutex - state connectivity.State - subConns *resolver.AddressMap // scData for active subonns mapped by address. + // SubConn state updates. + mu sync.Mutex + state connectivity.State + // scData for active subonns mapped by address. + subConns *resolver.AddressMap addressList addressList firstPass bool numTF int @@ -164,7 +165,7 @@ func (b *pickfirstBalancer) resolverErrorLocked(err error) { } b.closeSubConnsLocked() - b.addressList.updateEndpointList(nil) + b.addressList.updateAddrs(nil) b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.TransientFailure, Picker: &picker{err: fmt.Errorf("name resolver error: %v", err)}, @@ -190,52 +191,65 @@ func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState b.logger.Infof("Received new config %s, resolver state %s", pretty.ToJSON(cfg), pretty.ToJSON(state.ResolverState)) } - newEndpoints := state.ResolverState.Endpoints - if len(newEndpoints) == 0 { - newEndpoints = make([]resolver.Endpoint, len(state.ResolverState.Addresses)) - // Convert addresses to endpoints. - for i, a := range state.ResolverState.Addresses { - newEndpoints[i].Attributes = a.BalancerAttributes - newEndpoints[i].Addresses = []resolver.Address{a} - // We can't remove balancer attributes here since xds packages use - // them to store locality metadata. + var newAddrs []resolver.Address + if endpoints := state.ResolverState.Endpoints; len(endpoints) != 0 { + // Perform the optional shuffling described in gRFC A62. The shuffling + // will change the order of endpoints but not touch the order of the + // addresses within each endpoint. - A61 + if cfg.ShuffleAddressList { + endpoints = append([]resolver.Endpoint{}, endpoints...) + internal.ShuffleAddressListForTesting.(func(int, func(int, int)))(len(endpoints), func(i, j int) { endpoints[i], endpoints[j] = endpoints[j], endpoints[i] }) + } + + // "Flatten the list by concatenating the ordered list of addresses for + // each of the endpoints, in order." - A61 + for _, endpoint := range endpoints { + // "In the flattened list, interleave addresses from the two address + // families, as per RFC-8305 section 4." - A61 + // TODO: support the above language. + newAddrs = append(newAddrs, endpoint.Addresses...) + } + } else { + // Endpoints not set, process addresses until we migrate resolver + // emissions fully to Endpoints. The top channel does wrap emitted + // addresses with endpoints, however some balancers such as weighted + // target do not forward the corresponding correct endpoints down/split + // endpoints properly. Once all balancers correctly forward endpoints + // down, can delete this else conditional. + newAddrs = state.ResolverState.Addresses + if cfg.ShuffleAddressList { + newAddrs = append([]resolver.Address{}, newAddrs...) + internal.ShuffleAddressListForTesting.(func(int, func(int, int)))(len(endpoints), func(i, j int) { endpoints[i], endpoints[j] = endpoints[j], endpoints[i] }) } } + // If an address appears in multiple endpoints or in the same endpoint + // multiple times, we keep it only once. We will create only one SubConn + // for the address because an AddressMap is used to store SubConns. + // Not de-duplicating would result in attempting to connect to the same + // SubConn multiple times in the same pass. We don't want this + newAddrs = deDupAddresses(newAddrs) + // Since we have a new set of addresses, we are again at first pass. b.firstPass = true - // If multiple endpoints have the same address, they would use the same - // subconn because an AddressMap is used to store subconns. - // This would result in attempting to connect to the same subconn multiple - // times in the same pass. We don't want this, so we ensure each address is - // present in only one endpoint before moving further. - newEndpoints = deDupAddresses(newEndpoints) - - // Perform the optional shuffling described in gRFC A62. The shuffling will - // change the order of endpoints but not touch the order of the addresses - // within each endpoint. - A61 - if cfg.ShuffleAddressList { - newEndpoints = append([]resolver.Endpoint{}, newEndpoints...) - internal.ShuffleAddressListForTesting.(func(int, func(int, int)))(len(newEndpoints), func(i, j int) { newEndpoints[i], newEndpoints[j] = newEndpoints[j], newEndpoints[i] }) - } - - // If the previous ready subconn exists in new address list, - // keep this connection and don't create new subconns. + + // If the previous ready SubConn exists in new address list, + // keep this connection and don't create new SubConns. prevAddr := b.addressList.currentAddress() prevAddrsCount := b.addressList.size() - b.addressList.updateEndpointList(newEndpoints) + b.addressList.updateAddrs(newAddrs) if b.state == connectivity.Ready && b.addressList.seekTo(prevAddr) { return nil } - b.reconcileSubConnsLocked(newEndpoints) + b.reconcileSubConnsLocked(newAddrs) // If it's the first resolver update or the balancer was already READY - // (but the new address list does not contain the ready subconn) or + // (but the new address list does not contain the ready SubConn) or // CONNECTING, enter CONNECTING. // We may be in TRANSIENT_FAILURE due to a previous empty address list, - // we should still enter CONNECTING because the sticky TF behaviour mentioned - // in A62 applies only when the TRANSIENT_FAILURE is reported due to connectivity - // failures. + // we should still enter CONNECTING because the sticky TF behaviour + // mentioned in A62 applies only when the TRANSIENT_FAILURE is reported + // due to connectivity failures. if b.state == connectivity.Ready || b.state == connectivity.Connecting || prevAddrsCount == 0 { // Start connection attempt at first address. b.state = connectivity.Connecting @@ -266,7 +280,8 @@ func (b *pickfirstBalancer) Close() { } // ExitIdle moves the balancer out of idle state. It can be called concurrently -// by the idlePicker and clientConn so access to variables should be synchronized. +// by the idlePicker and clientConn so access to variables should be +// synchronized. func (b *pickfirstBalancer) ExitIdle() { b.mu.Lock() defer b.mu.Unlock() @@ -282,48 +297,36 @@ func (b *pickfirstBalancer) closeSubConnsLocked() { b.subConns = resolver.NewAddressMap() } -// deDupAddresses ensures that each address belongs to only one endpoint. -func deDupAddresses(endpoints []resolver.Endpoint) []resolver.Endpoint { +// deDupAddresses ensures that each address appears only once in the slice. +func deDupAddresses(addrs []resolver.Address) []resolver.Address { seenAddrs := resolver.NewAddressMap() - newEndpoints := []resolver.Endpoint{} + retAddrs := []resolver.Address{} - for _, ep := range endpoints { - addrs := []resolver.Address{} - for _, addr := range ep.Addresses { - if _, ok := seenAddrs.Get(addr); ok { - continue - } - addrs = append(addrs, addr) - } - if len(addrs) == 0 { + for _, addr := range addrs { + if _, ok := seenAddrs.Get(addr); ok { continue } - newEndpoints = append(newEndpoints, resolver.Endpoint{ - Addresses: addrs, - Attributes: ep.Attributes, - }) + retAddrs = append(retAddrs, addr) } - return newEndpoints + return retAddrs } -func (b *pickfirstBalancer) reconcileSubConnsLocked(newEndpoints []resolver.Endpoint) { +func (b *pickfirstBalancer) reconcileSubConnsLocked(newAddrs []resolver.Address) { // Remove old subConns that were not in new address list. - oldAddrs := resolver.NewAddressMap() + oldAddrsMap := resolver.NewAddressMap() for _, k := range b.subConns.Keys() { - oldAddrs.Set(k, true) + oldAddrsMap.Set(k, true) } // Flatten the new endpoint addresses. - newAddrs := resolver.NewAddressMap() - for _, endpoint := range newEndpoints { - for _, addr := range endpoint.Addresses { - newAddrs.Set(addr, true) - } + newAddrsMap := resolver.NewAddressMap() + for _, addr := range newAddrs { + newAddrsMap.Set(addr, true) } // Shut them down and remove them. - for _, oldAddr := range oldAddrs.Keys() { - if _, ok := newAddrs.Get(oldAddr); ok { + for _, oldAddr := range oldAddrsMap.Keys() { + if _, ok := newAddrsMap.Get(oldAddr); ok { continue } val, _ := b.subConns.Get(oldAddr) @@ -345,8 +348,8 @@ func (b *pickfirstBalancer) shutdownRemainingLocked(selected *scData) { b.subConns.Set(selected.addr, selected) } -// requestConnectionLocked starts connecting on the subchannel corresponding to the -// current address. If no subchannel exists, one is created. If the current +// requestConnectionLocked starts connecting on the subchannel corresponding to +// the current address. If no subchannel exists, one is created. If the current // subchannel is in TransientFailure, a connection to the next address is // attempted until a subchannel is found. func (b *pickfirstBalancer) requestConnectionLocked() { @@ -359,8 +362,8 @@ func (b *pickfirstBalancer) requestConnectionLocked() { sd, ok := b.subConns.Get(curAddr) if !ok { var err error - // We want to assign the new scData to sd from the outer scope, hence - // we can't use := below. + // We want to assign the new scData to sd from the outer scope, + // hence we can't use := below. sd, err = b.newSCData(curAddr) if err != nil { // This should never happen, unless the clientConn is being shut @@ -384,12 +387,12 @@ func (b *pickfirstBalancer) requestConnectionLocked() { continue case connectivity.Ready: // Should never happen. - b.logger.Errorf("Requesting a connection even though we have a READY subconn") + b.logger.Errorf("Requesting a connection even though we have a READY SubConn") case connectivity.Shutdown: // Should never happen. - b.logger.Errorf("SubConn with state SHUTDOWN present in subconns map") + b.logger.Errorf("SubConn with state SHUTDOWN present in SubConns map") case connectivity.Connecting: - // Wait for the subconn to report success or failure. + // Wait for the SubConn to report success or failure. } return } @@ -402,10 +405,10 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.Sub b.mu.Lock() defer b.mu.Unlock() sd.state = newState.ConnectivityState - // Previously relevant subconns can still callback with state updates. - // To prevent pickers from returning these obsolete subconns, this logic - // is included to check if the current list of active subconns includes this - // subconn. + // Previously relevant SubConns can still callback with state updates. + // To prevent pickers from returning these obsolete SubConns, this logic + // is included to check if the current list of active SubConns includes this + // SubConn. if activeSD, found := b.subConns.Get(sd.addr); !found || activeSD != sd { return } @@ -416,8 +419,8 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.Sub if newState.ConnectivityState == connectivity.Ready { b.shutdownRemainingLocked(sd) if !b.addressList.seekTo(sd.addr) { - // This should not fail as we should have only one subconn after - // entering READY. The subconn should be present in the addressList. + // This should not fail as we should have only one SubConn after + // entering READY. The SubConn should be present in the addressList. b.logger.Errorf("Address %q not found address list in %v", sd.addr, b.addressList.addresses) return } @@ -446,9 +449,9 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.Sub if b.firstPass { switch newState.ConnectivityState { case connectivity.Connecting: - // The balancer can be in either IDLE, CONNECTING or TRANSIENT_FAILURE. - // If it's in TRANSIENT_FAILURE, stay in TRANSIENT_FAILURE until - // it's READY. See A62. + // The balancer can be in either IDLE, CONNECTING or + // TRANSIENT_FAILURE. If it's in TRANSIENT_FAILURE, stay in + // TRANSIENT_FAILURE until it's READY. See A62. // If the balancer is already in CONNECTING, no update is needed. if b.state == connectivity.Idle { b.state = connectivity.Connecting @@ -459,9 +462,9 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.Sub } case connectivity.TransientFailure: sd.lastErr = newState.ConnectionError - // Since we're re-using common subconns while handling resolver updates, - // we could receive an out of turn TRANSIENT_FAILURE from a pass - // over the previous address list. We ignore such updates. + // Since we're re-using common SubConns while handling resolver + // updates, we could receive an out of turn TRANSIENT_FAILURE from + // a pass over the previous address list. We ignore such updates. if curAddr := b.addressList.currentAddress(); !equalAddressIgnoringBalAttributes(&curAddr, &sd.addr) { return @@ -473,10 +476,10 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.Sub // End of the first pass. b.endFirstPassLocked(newState.ConnectionError) case connectivity.Idle: - // A subconn can transition from CONNECTING directly to IDLE when - // a transport is successfully created, but the connection fails before - // the subconn can send the notification for READY. We treat this - // as a successful connection and transition to IDLE. + // A SubConn can transition from CONNECTING directly to IDLE when + // a transport is successfully created, but the connection fails + // before the SubConn can send the notification for READY. We treat + // this as a successful connection and transition to IDLE. if curAddr := b.addressList.currentAddress(); !equalAddressIgnoringBalAttributes(&sd.addr, &curAddr) { return } @@ -490,7 +493,7 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.Sub return } - // We have finished the first pass, keep re-connecting failing subconns. + // We have finished the first pass, keep re-connecting failing SubConns. switch newState.ConnectivityState { case connectivity.TransientFailure: b.numTF = (b.numTF + 1) % b.subConns.Len() @@ -501,9 +504,10 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.Sub Picker: &picker{err: newState.ConnectionError}, }) } - // We don't need to request re-resolution since the subconn already does - // that before reporting TRANSIENT_FAILURE. - // TODO: #7534 - Move re-resolution requests from subconn into pick_first. + // We don't need to request re-resolution since the SubConn already + // does that before reporting TRANSIENT_FAILURE. + // TODO: #7534 - Move re-resolution requests from SubConn into + // pick_first. case connectivity.Idle: sd.subConn.Connect() } @@ -518,7 +522,7 @@ func (b *pickfirstBalancer) endFirstPassLocked(lastErr error) { ConnectivityState: connectivity.TransientFailure, Picker: &picker{err: lastErr}, }) - // Start re-connecting all the subconns that are already in IDLE. + // Start re-connecting all the SubConns that are already in IDLE. for _, v := range b.subConns.Values() { sd := v.(*scData) if sd.state == connectivity.Idle { @@ -547,9 +551,9 @@ func (i *idlePicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { return balancer.PickResult{}, balancer.ErrNoSubConnAvailable } -// addressList manages sequentially iterating over addresses present in a list of -// endpoints. It provides a 1 dimensional view of the addresses present in the -// endpoints. +// addressList manages sequentially iterating over addresses present in a list +// of endpoints. It provides a 1 dimensional view of the addresses present in +// the endpoints. // This type is not safe for concurrent access. type addressList struct { addresses []resolver.Address @@ -587,17 +591,13 @@ func (al *addressList) reset() { al.idx = 0 } -func (al *addressList) updateEndpointList(endpoints []resolver.Endpoint) { - // Flatten the addresses. - addrs := []resolver.Address{} - for _, e := range endpoints { - addrs = append(addrs, e.Addresses...) - } +func (al *addressList) updateAddrs(addrs []resolver.Address) { al.addresses = addrs al.reset() } -// seekTo returns false if the needle was not found and the current index was left unchanged. +// seekTo returns false if the needle was not found and the current index was +// left unchanged. func (al *addressList) seekTo(needle resolver.Address) bool { for ai, addr := range al.addresses { if !equalAddressIgnoringBalAttributes(&addr, &needle) { @@ -609,10 +609,10 @@ func (al *addressList) seekTo(needle resolver.Address) bool { return false } -// equalAddressIgnoringBalAttributes returns true is a and b are considered equal. -// This is different from the Equal method on the resolver.Address type which -// considers all fields to determine equality. Here, we only consider fields -// that are meaningful to the subconn. +// equalAddressIgnoringBalAttributes returns true is a and b are considered +// equal. This is different from the Equal method on the resolver.Address type +// which considers all fields to determine equality. Here, we only consider +// fields that are meaningful to the SubConn. func equalAddressIgnoringBalAttributes(a, b *resolver.Address) bool { return a.Addr == b.Addr && a.ServerName == b.ServerName && a.Attributes.Equal(b.Attributes) && diff --git a/balancer/pickfirstleaf/pickfirstleaf_test.go b/balancer/pickfirstleaf/pickfirstleaf_test.go index 53146039bff6..c052f6bf1040 100644 --- a/balancer/pickfirstleaf/pickfirstleaf_test.go +++ b/balancer/pickfirstleaf/pickfirstleaf_test.go @@ -73,17 +73,8 @@ func (s) TestAddressList_Iteration(t *testing.T) { }, } - endpoints := []resolver.Endpoint{ - { - Addresses: []resolver.Address{addrs[0], addrs[1]}, - }, - { - Addresses: []resolver.Address{addrs[2]}, - }, - } - addressList := addressList{} - addressList.updateEndpointList(endpoints) + addressList.updateAddrs(addrs) for i := 0; i < len(addrs); i++ { if got, want := addressList.isValid(), true; got != want { @@ -147,17 +138,8 @@ func (s) TestAddressList_SeekTo(t *testing.T) { }, } - endpoints := []resolver.Endpoint{ - { - Addresses: []resolver.Address{addrs[0], addrs[1]}, - }, - { - Addresses: []resolver.Address{addrs[2]}, - }, - } - addressList := addressList{} - addressList.updateEndpointList(endpoints) + addressList.updateAddrs(addrs) // Try finding an address in the list. key := resolver.Address{ From 61c48163948443da1fa4076885a8cbeef784679d Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Thu, 26 Sep 2024 01:43:30 +0530 Subject: [PATCH 54/62] Don't close SubConns in resolverErrorLocked and fix comments --- balancer/pickfirstleaf/pickfirstleaf.go | 10 ++- balancer/pickfirstleaf/pickfirstleaf_test.go | 86 ++++++++++++++++---- 2 files changed, 77 insertions(+), 19 deletions(-) diff --git a/balancer/pickfirstleaf/pickfirstleaf.go b/balancer/pickfirstleaf/pickfirstleaf.go index 0cb0f295d1a7..73e6997a24d2 100644 --- a/balancer/pickfirstleaf/pickfirstleaf.go +++ b/balancer/pickfirstleaf/pickfirstleaf.go @@ -53,11 +53,12 @@ func init() { var ( logger = grpclog.Component("pick-first-leaf-lb") // Name is the name of the pick_first_leaf balancer. - // Can be changed in init() if this balancer is to be registered as the - // default pickfirst. + // It is changed to "pick_first" in init() if this balancer is to be + // registered as the default pickfirst. Name = "pick_first_leaf" ) +// TODO: change to pick-first when this becomes the default pick_first policy. const logPrefix = "[pick-first-leaf-lb %p] " type pickfirstBuilder struct{} @@ -154,6 +155,7 @@ func (b *pickfirstBalancer) resolverErrorLocked(err error) { if b.logger.V(2) { b.logger.Infof("Received error from the name resolver: %v", err) } + // The picker will not change since the balancer does not currently // report an error. If the balancer hasn't received a single good resolver // update yet, transition to TRANSIENT_FAILURE. @@ -164,8 +166,6 @@ func (b *pickfirstBalancer) resolverErrorLocked(err error) { return } - b.closeSubConnsLocked() - b.addressList.updateAddrs(nil) b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.TransientFailure, Picker: &picker{err: fmt.Errorf("name resolver error: %v", err)}, @@ -179,6 +179,8 @@ func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState // Cleanup state pertaining to the previous resolver state. // Treat an empty address list like an error by calling b.ResolverError. b.state = connectivity.TransientFailure + b.closeSubConnsLocked() + b.addressList.updateAddrs(nil) b.resolverErrorLocked(errors.New("produced zero addresses")) return balancer.ErrBadResolverState } diff --git a/balancer/pickfirstleaf/pickfirstleaf_test.go b/balancer/pickfirstleaf/pickfirstleaf_test.go index c052f6bf1040..f653358d7c4f 100644 --- a/balancer/pickfirstleaf/pickfirstleaf_test.go +++ b/balancer/pickfirstleaf/pickfirstleaf_test.go @@ -189,8 +189,8 @@ func (s) TestAddressList_SeekTo(t *testing.T) { } } -// TestPickFirstLeaf_TFPickerUpdate sends TRANSIENT_FAILURE subconn state updates -// for each subconn managed by a pickfirst balancer. It verifies that the picker +// TestPickFirstLeaf_TFPickerUpdate sends TRANSIENT_FAILURE SubConn state updates +// for each SubConn managed by a pickfirst balancer. It verifies that the picker // is updated with the expected frequency. func (s) TestPickFirstLeaf_TFPickerUpdate(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) @@ -198,38 +198,41 @@ func (s) TestPickFirstLeaf_TFPickerUpdate(t *testing.T) { cc := testutils.NewBalancerClientConn(t) bal := pickfirstBuilder{}.Build(cc, balancer.BuildOptions{}) defer bal.Close() - bal.UpdateClientConnState(balancer.ClientConnState{ + ccState := balancer.ClientConnState{ ResolverState: resolver.State{ Endpoints: []resolver.Endpoint{ {Addresses: []resolver.Address{{Addr: "1.1.1.1:1"}}}, {Addresses: []resolver.Address{{Addr: "2.2.2.2:2"}}}, }, }, - }) + } + if err := bal.UpdateClientConnState(ccState); err != nil { + t.Fatalf("UpdateClientConnState(%v) returned error: %v", ccState, err) + } // PF should report TRANSIENT_FAILURE only once all the sunbconns have failed // once. tfErr := fmt.Errorf("test err: connection refused") - sc0 := <-cc.NewSubConnCh - sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: tfErr}) + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: tfErr}) if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { t.Fatalf("cc.WaitForPickerWithErr(%v) returned error: %v", tfErr, err) } - sc1 := <-cc.NewSubConnCh - sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: tfErr}) + sc2 := <-cc.NewSubConnCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: tfErr}) if err := cc.WaitForPickerWithErr(ctx, tfErr); err != nil { t.Fatalf("cc.WaitForPickerWithErr(%v) returned error: %v", tfErr, err) } - // Subsequent TRANSIENT_FAILUREs should be reported only after seeing "# of subconns" + // Subsequent TRANSIENT_FAILUREs should be reported only after seeing "# of SubConns" // TRANSIENT_FAILUREs. newTfErr := fmt.Errorf("test err: unreachable") - sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: newTfErr}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: newTfErr}) select { case <-time.After(defaultTestShortTimeout): case p := <-cc.NewPickerCh: @@ -237,7 +240,7 @@ func (s) TestPickFirstLeaf_TFPickerUpdate(t *testing.T) { t.Fatalf("Unexpected picker update: %v, %v", sc, err) } - sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: newTfErr}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: newTfErr}) if err := cc.WaitForPickerWithErr(ctx, newTfErr); err != nil { t.Fatalf("cc.WaitForPickerWithErr(%v) returned error: %v", tfErr, err) } @@ -259,16 +262,69 @@ func (s) TestPickFirstLeaf_InitialResolverError(t *testing.T) { } // After sending a valid update, the LB policy should report CONNECTING. - bal.UpdateClientConnState(balancer.ClientConnState{ + ccState := balancer.ClientConnState{ ResolverState: resolver.State{ Endpoints: []resolver.Endpoint{ {Addresses: []resolver.Address{{Addr: "1.1.1.1:1"}}}, {Addresses: []resolver.Address{{Addr: "2.2.2.2:2"}}}, }, }, - }) + } + if err := bal.UpdateClientConnState(ccState); err != nil { + t.Fatalf("UpdateClientConnState(%v) returned error: %v", ccState, err) + } if err := cc.WaitForConnectivityState(ctx, connectivity.Connecting); err != nil { t.Fatalf("cc.WaitForConnectivityState(%v) returned error: %v", connectivity.Connecting, err) } } + +// TestPickFirstLeaf_ResolverErrorinTF sends a resolver error to the balancer +// before when it's attempting to connect to a SubConn TRANSIENT_FAILURE. It +// verifies that the picker is updated and the SubConn is not closed. +func (s) TestPickFirstLeaf_ResolverErrorinTF(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc := testutils.NewBalancerClientConn(t) + bal := pickfirstBuilder{}.Build(cc, balancer.BuildOptions{}) + defer bal.Close() + + // After sending a valid update, the LB policy should report CONNECTING. + ccState := balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "1.1.1.1:1"}}}, + }, + }, + } + + if err := bal.UpdateClientConnState(ccState); err != nil { + t.Fatalf("UpdateClientConnState(%v) returned error: %v", ccState, err) + } + + sc1 := <-cc.NewSubConnCh + if err := cc.WaitForConnectivityState(ctx, connectivity.Connecting); err != nil { + t.Fatalf("cc.WaitForConnectivityState(%v) returned error: %v", connectivity.Connecting, err) + } + + scErr := fmt.Errorf("test error: connection refused") + sc1.UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + ConnectionError: scErr, + }) + + if err := cc.WaitForPickerWithErr(ctx, scErr); err != nil { + t.Fatalf("cc.WaitForPickerWithErr(%v) returned error: %v", scErr, err) + } + + bal.ResolverError(errors.New("resolution failed: test error")) + if err := cc.WaitForErrPicker(ctx); err != nil { + t.Fatalf("cc.WaitForPickerWithErr() returned error: %v", err) + } + + select { + case <-time.After(defaultTestShortTimeout): + case sc := <-cc.ShutdownSubConnCh: + t.Fatalf("Unexpected SubConn shutdown: %v", sc) + } +} From 2e9f8739b58de20180840346b475f25caf9c3890 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Thu, 26 Sep 2024 01:50:39 +0530 Subject: [PATCH 55/62] Move common tests to pickfirst_test.go --- balancer/pickfirst/pickfirst_test.go | 132 +++++++++++++++++++ balancer/pickfirstleaf/pickfirstleaf_test.go | 84 ------------ 2 files changed, 132 insertions(+), 84 deletions(-) create mode 100644 balancer/pickfirst/pickfirst_test.go diff --git a/balancer/pickfirst/pickfirst_test.go b/balancer/pickfirst/pickfirst_test.go new file mode 100644 index 000000000000..43d8b20df3e7 --- /dev/null +++ b/balancer/pickfirst/pickfirst_test.go @@ -0,0 +1,132 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package pickfirst + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" +) + +const ( + // Default timeout for tests in this package. + defaultTestTimeout = 10 * time.Second + // Default short timeout, to be used when waiting for events which are not + // expected to happen. + defaultTestShortTimeout = 100 * time.Millisecond +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestPickFirstLeaf_InitialResolverError sends a resolver error to the balancer +// before a valid resolver update. It verifies that the clientconn state is +// updated to TRANSIENT_FAILURE. +func (s) TestPickFirstLeaf_InitialResolverError(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc := testutils.NewBalancerClientConn(t) + bal := pickfirstBuilder{}.Build(cc, balancer.BuildOptions{}) + defer bal.Close() + bal.ResolverError(errors.New("resolution failed: test error")) + + if err := cc.WaitForConnectivityState(ctx, connectivity.TransientFailure); err != nil { + t.Fatalf("cc.WaitForConnectivityState(%v) returned error: %v", connectivity.TransientFailure, err) + } + + // After sending a valid update, the LB policy should report CONNECTING. + ccState := balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "1.1.1.1:1"}}}, + {Addresses: []resolver.Address{{Addr: "2.2.2.2:2"}}}, + }, + }, + } + if err := bal.UpdateClientConnState(ccState); err != nil { + t.Fatalf("UpdateClientConnState(%v) returned error: %v", ccState, err) + } + + if err := cc.WaitForConnectivityState(ctx, connectivity.Connecting); err != nil { + t.Fatalf("cc.WaitForConnectivityState(%v) returned error: %v", connectivity.Connecting, err) + } +} + +// TestPickFirstLeaf_ResolverErrorinTF sends a resolver error to the balancer +// before when it's attempting to connect to a SubConn TRANSIENT_FAILURE. It +// verifies that the picker is updated and the SubConn is not closed. +func (s) TestPickFirstLeaf_ResolverErrorinTF(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc := testutils.NewBalancerClientConn(t) + bal := pickfirstBuilder{}.Build(cc, balancer.BuildOptions{}) + defer bal.Close() + + // After sending a valid update, the LB policy should report CONNECTING. + ccState := balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "1.1.1.1:1"}}}, + }, + }, + } + + if err := bal.UpdateClientConnState(ccState); err != nil { + t.Fatalf("UpdateClientConnState(%v) returned error: %v", ccState, err) + } + + sc1 := <-cc.NewSubConnCh + if err := cc.WaitForConnectivityState(ctx, connectivity.Connecting); err != nil { + t.Fatalf("cc.WaitForConnectivityState(%v) returned error: %v", connectivity.Connecting, err) + } + + scErr := fmt.Errorf("test error: connection refused") + sc1.UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + ConnectionError: scErr, + }) + + if err := cc.WaitForPickerWithErr(ctx, scErr); err != nil { + t.Fatalf("cc.WaitForPickerWithErr(%v) returned error: %v", scErr, err) + } + + bal.ResolverError(errors.New("resolution failed: test error")) + if err := cc.WaitForErrPicker(ctx); err != nil { + t.Fatalf("cc.WaitForPickerWithErr() returned error: %v", err) + } + + select { + case <-time.After(defaultTestShortTimeout): + case sc := <-cc.ShutdownSubConnCh: + t.Fatalf("Unexpected SubConn shutdown: %v", sc) + } +} diff --git a/balancer/pickfirstleaf/pickfirstleaf_test.go b/balancer/pickfirstleaf/pickfirstleaf_test.go index f653358d7c4f..7dfe6f9d0227 100644 --- a/balancer/pickfirstleaf/pickfirstleaf_test.go +++ b/balancer/pickfirstleaf/pickfirstleaf_test.go @@ -20,7 +20,6 @@ package pickfirstleaf import ( "context" - "errors" "fmt" "testing" "time" @@ -245,86 +244,3 @@ func (s) TestPickFirstLeaf_TFPickerUpdate(t *testing.T) { t.Fatalf("cc.WaitForPickerWithErr(%v) returned error: %v", tfErr, err) } } - -// TestPickFirstLeaf_InitialResolverError sends a resolver error to the balancer -// before a valid resolver update. It verifies that the clientconn state is -// updated to TRANSIENT_FAILURE. -func (s) TestPickFirstLeaf_InitialResolverError(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - cc := testutils.NewBalancerClientConn(t) - bal := pickfirstBuilder{}.Build(cc, balancer.BuildOptions{}) - defer bal.Close() - bal.ResolverError(errors.New("resolution failed: test error")) - - if err := cc.WaitForConnectivityState(ctx, connectivity.TransientFailure); err != nil { - t.Fatalf("cc.WaitForConnectivityState(%v) returned error: %v", connectivity.TransientFailure, err) - } - - // After sending a valid update, the LB policy should report CONNECTING. - ccState := balancer.ClientConnState{ - ResolverState: resolver.State{ - Endpoints: []resolver.Endpoint{ - {Addresses: []resolver.Address{{Addr: "1.1.1.1:1"}}}, - {Addresses: []resolver.Address{{Addr: "2.2.2.2:2"}}}, - }, - }, - } - if err := bal.UpdateClientConnState(ccState); err != nil { - t.Fatalf("UpdateClientConnState(%v) returned error: %v", ccState, err) - } - - if err := cc.WaitForConnectivityState(ctx, connectivity.Connecting); err != nil { - t.Fatalf("cc.WaitForConnectivityState(%v) returned error: %v", connectivity.Connecting, err) - } -} - -// TestPickFirstLeaf_ResolverErrorinTF sends a resolver error to the balancer -// before when it's attempting to connect to a SubConn TRANSIENT_FAILURE. It -// verifies that the picker is updated and the SubConn is not closed. -func (s) TestPickFirstLeaf_ResolverErrorinTF(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - cc := testutils.NewBalancerClientConn(t) - bal := pickfirstBuilder{}.Build(cc, balancer.BuildOptions{}) - defer bal.Close() - - // After sending a valid update, the LB policy should report CONNECTING. - ccState := balancer.ClientConnState{ - ResolverState: resolver.State{ - Endpoints: []resolver.Endpoint{ - {Addresses: []resolver.Address{{Addr: "1.1.1.1:1"}}}, - }, - }, - } - - if err := bal.UpdateClientConnState(ccState); err != nil { - t.Fatalf("UpdateClientConnState(%v) returned error: %v", ccState, err) - } - - sc1 := <-cc.NewSubConnCh - if err := cc.WaitForConnectivityState(ctx, connectivity.Connecting); err != nil { - t.Fatalf("cc.WaitForConnectivityState(%v) returned error: %v", connectivity.Connecting, err) - } - - scErr := fmt.Errorf("test error: connection refused") - sc1.UpdateState(balancer.SubConnState{ - ConnectivityState: connectivity.TransientFailure, - ConnectionError: scErr, - }) - - if err := cc.WaitForPickerWithErr(ctx, scErr); err != nil { - t.Fatalf("cc.WaitForPickerWithErr(%v) returned error: %v", scErr, err) - } - - bal.ResolverError(errors.New("resolution failed: test error")) - if err := cc.WaitForErrPicker(ctx); err != nil { - t.Fatalf("cc.WaitForPickerWithErr() returned error: %v", err) - } - - select { - case <-time.After(defaultTestShortTimeout): - case sc := <-cc.ShutdownSubConnCh: - t.Fatalf("Unexpected SubConn shutdown: %v", sc) - } -} From c4b4aa49d15419f7c968eebb14804dd2d4fd8fcc Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Mon, 7 Oct 2024 12:51:37 +0530 Subject: [PATCH 56/62] Set first pass when exiting idle --- balancer/pickfirstleaf/pickfirstleaf.go | 1 + 1 file changed, 1 insertion(+) diff --git a/balancer/pickfirstleaf/pickfirstleaf.go b/balancer/pickfirstleaf/pickfirstleaf.go index 73e6997a24d2..5b842f409922 100644 --- a/balancer/pickfirstleaf/pickfirstleaf.go +++ b/balancer/pickfirstleaf/pickfirstleaf.go @@ -288,6 +288,7 @@ func (b *pickfirstBalancer) ExitIdle() { b.mu.Lock() defer b.mu.Unlock() if b.state == connectivity.Idle { + b.firstPass = true b.requestConnectionLocked() } } From f0e479e22ca2f9b87822eadf96b26c0e1644c93d Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Mon, 7 Oct 2024 14:06:12 +0530 Subject: [PATCH 57/62] Ensure exitIdle doesn't increment the address list multiple times --- balancer/pickfirstleaf/pickfirstleaf.go | 19 ++++++++++++++++--- balancer/pickfirstleaf/pickfirstleaf_test.go | 13 +++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/balancer/pickfirstleaf/pickfirstleaf.go b/balancer/pickfirstleaf/pickfirstleaf.go index 5b842f409922..cfd10d1cddc9 100644 --- a/balancer/pickfirstleaf/pickfirstleaf.go +++ b/balancer/pickfirstleaf/pickfirstleaf.go @@ -30,6 +30,7 @@ import ( "errors" "fmt" "sync" + "sync/atomic" "google.golang.org/grpc/balancer" "google.golang.org/grpc/connectivity" @@ -287,7 +288,7 @@ func (b *pickfirstBalancer) Close() { func (b *pickfirstBalancer) ExitIdle() { b.mu.Lock() defer b.mu.Unlock() - if b.state == connectivity.Idle { + if b.state == connectivity.Idle && b.addressList.currentAddress() == b.addressList.first() { b.firstPass = true b.requestConnectionLocked() } @@ -546,11 +547,14 @@ func (p *picker) Pick(balancer.PickInfo) (balancer.PickResult, error) { // idlePicker is used when the SubConn is IDLE and kicks the SubConn into // CONNECTING when Pick is called. type idlePicker struct { - exitIdle func() + connectionRequested atomic.Bool + exitIdle func() } func (i *idlePicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { - i.exitIdle() + if i.connectionRequested.CompareAndSwap(false, true) { + i.exitIdle() + } return balancer.PickResult{}, balancer.ErrNoSubConnAvailable } @@ -590,6 +594,15 @@ func (al *addressList) currentAddress() resolver.Address { return al.addresses[al.idx] } +// first returns the first address in the list. If the list is empty, it returns +// an empty address instead. +func (al *addressList) first() resolver.Address { + if len(al.addresses) == 0 { + return resolver.Address{} + } + return al.addresses[0] +} + func (al *addressList) reset() { al.idx = 0 } diff --git a/balancer/pickfirstleaf/pickfirstleaf_test.go b/balancer/pickfirstleaf/pickfirstleaf_test.go index 7dfe6f9d0227..dc62c45f4719 100644 --- a/balancer/pickfirstleaf/pickfirstleaf_test.go +++ b/balancer/pickfirstleaf/pickfirstleaf_test.go @@ -73,8 +73,21 @@ func (s) TestAddressList_Iteration(t *testing.T) { } addressList := addressList{} + emptyAddress := resolver.Address{} + if got, want := addressList.first(), emptyAddress; got != want { + t.Fatalf("addressList.first() = %v, want %v", got, want) + } + addressList.updateAddrs(addrs) + if got, want := addressList.first(), addressList.currentAddress(); got != want { + t.Fatalf("addressList.first() = %v, want %v", got, want) + } + + if got, want := addressList.first(), addrs[0]; got != want { + t.Fatalf("addressList.first() = %v, want %v", got, want) + } + for i := 0; i < len(addrs); i++ { if got, want := addressList.isValid(), true; got != want { t.Fatalf("addressList.isValid() = %t, want %t", got, want) From 58e2ff7429d361ea02f1c79b90f9f6169060a6a3 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Mon, 7 Oct 2024 18:07:52 +0530 Subject: [PATCH 58/62] Move handling of CONNECITN to IDLE subchannels to outer block --- balancer/pickfirstleaf/pickfirstleaf.go | 28 ++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/balancer/pickfirstleaf/pickfirstleaf.go b/balancer/pickfirstleaf/pickfirstleaf.go index cfd10d1cddc9..7f95e38442de 100644 --- a/balancer/pickfirstleaf/pickfirstleaf.go +++ b/balancer/pickfirstleaf/pickfirstleaf.go @@ -408,6 +408,7 @@ func (b *pickfirstBalancer) requestConnectionLocked() { func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.SubConnState) { b.mu.Lock() defer b.mu.Unlock() + oldState := sd.state sd.state = newState.ConnectivityState // Previously relevant SubConns can still callback with state updates. // To prevent pickers from returning these obsolete SubConns, this logic @@ -449,6 +450,19 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.Sub }) return } + if oldState == connectivity.Connecting && newState.ConnectivityState == connectivity.Idle { + // A SubConn can transition from CONNECTING directly to IDLE when + // a transport is successfully created, but the connection fails + // before the SubConn can send the notification for READY. We treat + // this as a successful connection and transition to IDLE. + b.shutdownRemainingLocked(sd) + b.state = connectivity.Idle + b.addressList.reset() + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.Idle, + Picker: &idlePicker{exitIdle: b.ExitIdle}, + }) + } if b.firstPass { switch newState.ConnectivityState { @@ -479,20 +493,6 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.Sub } // End of the first pass. b.endFirstPassLocked(newState.ConnectionError) - case connectivity.Idle: - // A SubConn can transition from CONNECTING directly to IDLE when - // a transport is successfully created, but the connection fails - // before the SubConn can send the notification for READY. We treat - // this as a successful connection and transition to IDLE. - if curAddr := b.addressList.currentAddress(); !equalAddressIgnoringBalAttributes(&sd.addr, &curAddr) { - return - } - b.state = connectivity.Idle - b.addressList.reset() - b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.Idle, - Picker: &idlePicker{exitIdle: b.ExitIdle}, - }) } return } From 3103efe1441c5c00542bad9cbe00ae7870d7651c Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Tue, 8 Oct 2024 09:28:22 +0530 Subject: [PATCH 59/62] Fix test log statement --- balancer/pickfirstleaf/pickfirstleaf_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/balancer/pickfirstleaf/pickfirstleaf_test.go b/balancer/pickfirstleaf/pickfirstleaf_test.go index dc62c45f4719..84b3cb65bed4 100644 --- a/balancer/pickfirstleaf/pickfirstleaf_test.go +++ b/balancer/pickfirstleaf/pickfirstleaf_test.go @@ -230,7 +230,7 @@ func (s) TestPickFirstLeaf_TFPickerUpdate(t *testing.T) { sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: tfErr}) if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { - t.Fatalf("cc.WaitForPickerWithErr(%v) returned error: %v", tfErr, err) + t.Fatalf("cc.WaitForPickerWithErr(%v) returned error: %v", balancer.ErrNoSubConnAvailable, err) } sc2 := <-cc.NewSubConnCh @@ -254,6 +254,6 @@ func (s) TestPickFirstLeaf_TFPickerUpdate(t *testing.T) { sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure, ConnectionError: newTfErr}) if err := cc.WaitForPickerWithErr(ctx, newTfErr); err != nil { - t.Fatalf("cc.WaitForPickerWithErr(%v) returned error: %v", tfErr, err) + t.Fatalf("cc.WaitForPickerWithErr(%v) returned error: %v", newTfErr, err) } } From e98bb802e2f2061034895dc154e9633017067096 Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Wed, 9 Oct 2024 15:54:04 +0530 Subject: [PATCH 60/62] Organize packages based on review comments --- balancer/pickfirst/pickfirst.go | 2 +- balancer/{ => pickfirst}/pickfirstleaf/pickfirstleaf.go | 6 +++--- .../pickfirstleaf/pickfirstleaf_ext_test.go} | 4 ++-- .../{ => pickfirst}/pickfirstleaf/pickfirstleaf_test.go | 0 4 files changed, 6 insertions(+), 6 deletions(-) rename balancer/{ => pickfirst}/pickfirstleaf/pickfirstleaf.go (98%) rename balancer/{pickfirstleaf/test/pickfirstleaf_test.go => pickfirst/pickfirstleaf/pickfirstleaf_ext_test.go} (99%) rename balancer/{ => pickfirst}/pickfirstleaf/pickfirstleaf_test.go (100%) diff --git a/balancer/pickfirst/pickfirst.go b/balancer/pickfirst/pickfirst.go index 4dbb5976899d..e069346a7565 100644 --- a/balancer/pickfirst/pickfirst.go +++ b/balancer/pickfirst/pickfirst.go @@ -35,7 +35,7 @@ import ( "google.golang.org/grpc/resolver" "google.golang.org/grpc/serviceconfig" - _ "google.golang.org/grpc/balancer/pickfirstleaf" // For automatically registering the new pickfirst if required. + _ "google.golang.org/grpc/balancer/pickfirst/pickfirstleaf" // For automatically registering the new pickfirst if required. ) func init() { diff --git a/balancer/pickfirstleaf/pickfirstleaf.go b/balancer/pickfirst/pickfirstleaf/pickfirstleaf.go similarity index 98% rename from balancer/pickfirstleaf/pickfirstleaf.go rename to balancer/pickfirst/pickfirstleaf/pickfirstleaf.go index 7f95e38442de..8abe1d3eb3d0 100644 --- a/balancer/pickfirstleaf/pickfirstleaf.go +++ b/balancer/pickfirst/pickfirstleaf/pickfirstleaf.go @@ -33,9 +33,9 @@ import ( "sync/atomic" "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/pickfirst/internal" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/grpclog" - "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/envconfig" internalgrpclog "google.golang.org/grpc/internal/grpclog" "google.golang.org/grpc/internal/pretty" @@ -201,7 +201,7 @@ func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState // addresses within each endpoint. - A61 if cfg.ShuffleAddressList { endpoints = append([]resolver.Endpoint{}, endpoints...) - internal.ShuffleAddressListForTesting.(func(int, func(int, int)))(len(endpoints), func(i, j int) { endpoints[i], endpoints[j] = endpoints[j], endpoints[i] }) + internal.RandShuffle(len(endpoints), func(i, j int) { endpoints[i], endpoints[j] = endpoints[j], endpoints[i] }) } // "Flatten the list by concatenating the ordered list of addresses for @@ -222,7 +222,7 @@ func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState newAddrs = state.ResolverState.Addresses if cfg.ShuffleAddressList { newAddrs = append([]resolver.Address{}, newAddrs...) - internal.ShuffleAddressListForTesting.(func(int, func(int, int)))(len(endpoints), func(i, j int) { endpoints[i], endpoints[j] = endpoints[j], endpoints[i] }) + internal.RandShuffle(len(endpoints), func(i, j int) { endpoints[i], endpoints[j] = endpoints[j], endpoints[i] }) } } diff --git a/balancer/pickfirstleaf/test/pickfirstleaf_test.go b/balancer/pickfirst/pickfirstleaf/pickfirstleaf_ext_test.go similarity index 99% rename from balancer/pickfirstleaf/test/pickfirstleaf_test.go rename to balancer/pickfirst/pickfirstleaf/pickfirstleaf_ext_test.go index 0070b272faa3..2ab40ef1615a 100644 --- a/balancer/pickfirstleaf/test/pickfirstleaf_test.go +++ b/balancer/pickfirst/pickfirstleaf/pickfirstleaf_ext_test.go @@ -16,7 +16,7 @@ * */ -package test +package pickfirstleaf_test import ( "context" @@ -29,7 +29,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/pickfirstleaf" + "google.golang.org/grpc/balancer/pickfirst/pickfirstleaf" "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" diff --git a/balancer/pickfirstleaf/pickfirstleaf_test.go b/balancer/pickfirst/pickfirstleaf/pickfirstleaf_test.go similarity index 100% rename from balancer/pickfirstleaf/pickfirstleaf_test.go rename to balancer/pickfirst/pickfirstleaf/pickfirstleaf_test.go From a5b55e45c5ac5e27578193ce82438b9732f44a3d Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Wed, 9 Oct 2024 23:29:24 +0530 Subject: [PATCH 61/62] use onceFunc in idlePicker and reduce duplication when entering IDLE --- .../pickfirst/pickfirstleaf/pickfirstleaf.go | 30 ++++++------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/balancer/pickfirst/pickfirstleaf/pickfirstleaf.go b/balancer/pickfirst/pickfirstleaf/pickfirstleaf.go index 8abe1d3eb3d0..74f7e8eb23b3 100644 --- a/balancer/pickfirst/pickfirstleaf/pickfirstleaf.go +++ b/balancer/pickfirst/pickfirstleaf/pickfirstleaf.go @@ -30,7 +30,6 @@ import ( "errors" "fmt" "sync" - "sync/atomic" "google.golang.org/grpc/balancer" "google.golang.org/grpc/balancer/pickfirst/internal" @@ -439,29 +438,21 @@ func (b *pickfirstBalancer) updateSubConnState(sd *scData, newState balancer.Sub // If the LB policy is READY, and it receives a subchannel state change, // it means that the READY subchannel has failed. - if b.state == connectivity.Ready && newState.ConnectivityState != connectivity.Ready { + // A SubConn can also transition from CONNECTING directly to IDLE when + // a transport is successfully created, but the connection fails + // before the SubConn can send the notification for READY. We treat + // this as a successful connection and transition to IDLE. + if (b.state == connectivity.Ready && newState.ConnectivityState != connectivity.Ready) || (oldState == connectivity.Connecting && newState.ConnectivityState == connectivity.Idle) { // Once a transport fails, the balancer enters IDLE and starts from // the first address when the picker is used. - b.state = connectivity.Idle - b.addressList.reset() - b.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.Idle, - Picker: &idlePicker{exitIdle: b.ExitIdle}, - }) - return - } - if oldState == connectivity.Connecting && newState.ConnectivityState == connectivity.Idle { - // A SubConn can transition from CONNECTING directly to IDLE when - // a transport is successfully created, but the connection fails - // before the SubConn can send the notification for READY. We treat - // this as a successful connection and transition to IDLE. b.shutdownRemainingLocked(sd) b.state = connectivity.Idle b.addressList.reset() b.cc.UpdateState(balancer.State{ ConnectivityState: connectivity.Idle, - Picker: &idlePicker{exitIdle: b.ExitIdle}, + Picker: &idlePicker{exitIdle: sync.OnceFunc(b.ExitIdle)}, }) + return } if b.firstPass { @@ -547,14 +538,11 @@ func (p *picker) Pick(balancer.PickInfo) (balancer.PickResult, error) { // idlePicker is used when the SubConn is IDLE and kicks the SubConn into // CONNECTING when Pick is called. type idlePicker struct { - connectionRequested atomic.Bool - exitIdle func() + exitIdle func() } func (i *idlePicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { - if i.connectionRequested.CompareAndSwap(false, true) { - i.exitIdle() - } + i.exitIdle() return balancer.PickResult{}, balancer.ErrNoSubConnAvailable } From eb5a09a185e6b23ed89096cdcde4dc33ff2aab7f Mon Sep 17 00:00:00 2001 From: Arjan Bal Date: Thu, 10 Oct 2024 00:04:42 +0530 Subject: [PATCH 62/62] Fix comments --- balancer/pickfirst/pickfirstleaf/pickfirstleaf.go | 2 +- test/clientconn_state_transition_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/balancer/pickfirst/pickfirstleaf/pickfirstleaf.go b/balancer/pickfirst/pickfirstleaf/pickfirstleaf.go index 74f7e8eb23b3..48ce8c50e5c1 100644 --- a/balancer/pickfirst/pickfirstleaf/pickfirstleaf.go +++ b/balancer/pickfirst/pickfirstleaf/pickfirstleaf.go @@ -229,7 +229,7 @@ func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState // multiple times, we keep it only once. We will create only one SubConn // for the address because an AddressMap is used to store SubConns. // Not de-duplicating would result in attempting to connect to the same - // SubConn multiple times in the same pass. We don't want this + // SubConn multiple times in the same pass. We don't want this. newAddrs = deDupAddresses(newAddrs) // Since we have a new set of addresses, we are again at first pass. diff --git a/test/clientconn_state_transition_test.go b/test/clientconn_state_transition_test.go index 89e40d9d124c..56ebafaa9308 100644 --- a/test/clientconn_state_transition_test.go +++ b/test/clientconn_state_transition_test.go @@ -326,7 +326,7 @@ func (s) TestStateTransitions_TriesAllAddrsBeforeTransientFailure(t *testing.T) grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateRecordingBalancerName)), grpc.WithConnectParams(grpc.ConnectParams{ // Set a really long back-off delay to ensure the first subConn does - // not enter ready before the second subConn connects. + // not enter IDLE before the second subConn connects. Backoff: backoff.Config{ BaseDelay: 1 * time.Hour, },