sql/logictest: TestParallel failed

[cockroach-teamcity]

sql/logictest.TestParallel failed with artifacts on master @ 8b696d17241b2eb2d333e23750cf6fdc433897d8:

F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !goroutine 2745290 [runnable]:
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !sort.Reverse(...)
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !	/usr/local/go/src/sort/sort.go:265
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !github.com/cockroachdb/cockroach/pkg/kv/kvserver.rankedCandidateListForRebalancing({0x5176938, 0xc00a918d50}, {{0xc002f95200, 0x5, 0x8}, {0x4014000000000000, 0x404db33333333333}, {0x4014000000000000, 0x4021333333333333}, {0x4014000000000000, ...}, ...}, ...)
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/allocator_scorer.go:1023 +0xd5c
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !github.com/cockroachdb/cockroach/pkg/kv/kvserver.Allocator.rebalanceTarget({0xc00c0726c0, 0xc00c5737f0, {0xc003e236c0, 0xc00d2e9b90}, 0x0}, {0x5176938, 0xc00a918d50}, {0x4000, 0x8000, {0x15f90}, ...}, ...)
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/allocator.go:1064 +0x954
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !github.com/cockroachdb/cockroach/pkg/kv/kvserver.Allocator.RebalanceVoter({0xc00c0726c0, 0xc00c5737f0, {0xc003e236c0, 0xc00d2e9b90}, 0x0}, {0x5176938, 0xc00a918d50}, {0x4000, 0x8000, {0x15f90}, ...}, ...)
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/allocator.go:1201 +0x15a
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicateQueue).shouldQueue(0xc001c91290, {0x5176938, 0xc00a918d50}, {0x100000001, 0x1, 0x0}, 0xc0041e2a80, {0x0, 0xc007508150})
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replicate_queue.go:247 +0x40c
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).maybeAdd(0xc00bf40c80, {0x5176938, 0xc00a093440}, {0x5210170, 0xc0041e2a80}, {0x16bdd20503d45bdd, 0x0, 0x0})
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:661 +0x3df
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !github.com/cockroachdb/cockroach/pkg/kv/kvserver.baseQueueHelper.MaybeAdd({0xc001de4000}, {0x5176938, 0xc00a093440}, {0x5210170, 0xc0041e2a80}, {0x7e2990b0103d3794, 0x108c6320, 0xc0})
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:542 +0x2d
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).MaybeAddAsync.func1({0x5176938, 0xc00a093440}, {0x511c6b0, 0xc00bf40c80})
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:593 +0x4f
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).Async.func1({0x5176938, 0xc00a093440})
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:580 +0x32
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:491 +0x12a
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:481 +0x445
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !****************************************************************************
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !This node experienced a fatal error (printed above), and as a result the
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !process is terminating.
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !Fatal errors can occur due to faulty hardware (disks, memory, clocks) or a
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !problem in CockroachDB. With your help, the support team at Cockroach Labs
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !will try to determine the root cause, recommend next steps, and we can
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !improve CockroachDB based on your report.
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !Please submit a crash report by following the instructions here:
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !    https://github.com/cockroachdb/cockroach/issues/new/choose
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !If you would rather not post publicly, please contact us directly at:
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !    support@cockroachlabs.com
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 !The Cockroach Labs team appreciates your feedback.


ERROR: exit status 7

85 runs completed, 1 failures, over 15m4s
context canceled

Help

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM)Parameters in this failure:

• TAGS=deadlock

• GOFLAGS=-parallel=2

/cc @cockroachdb/sql-queries

This test on roachdash | Improve this report!

[cucaroach]

Error context:

I211205 09:32:38.431664 2738233 kv/kvserver/replica_command.go:1823  [n1,replicate,s1,r70/1:/System/tsd/cr.node.{sy…-tx…}] 10568  failed to rollback LEARNER n3,s3, abandoning it for the replicate queue: change replicas of r70 failed: fetching current range descriptor value: failed to send RPC: sending to all replicas failed; last error: failed to connect to n5 at 127.0.0.1:38079: context canceled
I211205 09:32:38.431837 2738233 kv/kvserver/queue.go:582  [n1,replicate,s1,r70/1:/System/tsd/cr.node.{sy…-tx…}] 10569  rate limited in MaybeAdd (replicate): node unavailable; try another peer
I211205 09:32:38.431999 2738233 kv/kvserver/replicate_queue.go:314  [n1,replicate,s1,r70/1:/System/tsd/cr.node.{sy…-tx…}] 10570  (n3,s3):4LEARNER: remote failed to apply snapshot: rpc error: code = Canceled desc = grpc: the client connection is closing
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571  while applying snapshot: while applying snapshot: error rate limiting bulk io write: context canceled
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +(1) attached stack trace
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  -- stack trace:
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | 	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_raft.go:621
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | [...repeated from below...]
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +Wraps: (2) while applying snapshot
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +Wraps: (3) attached stack trace
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  -- stack trace:
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | github.com/cockroachdb/cockroach/pkg/kv/kvserver.limitBulkIOWrite
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | 	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/syncing_write.go:54
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*SSTSnapshotStorageFile).Write
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | 	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_sst_snapshot_storage.go:183
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*SSTSnapshotStorageScratch).WriteSST
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | 	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_sst_snapshot_storage.go:118
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).applySnapshot
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | 	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_raftstorage.go:867
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | 	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_raft.go:619
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processRaftSnapshotRequest.func1
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | 	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_raft.go:340
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).withReplicaForRequest
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | 	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_raft.go:218
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processRaftSnapshotRequest
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | 	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_raft.go:278
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).receiveSnapshot
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | 	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_snapshot.go:663
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot.func1
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | 	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_raft.go:82
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | 	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:347
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | 	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_raft.go:72
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftSnapshot.func1.1
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | 	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:433
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftSnapshot.func1
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | 	/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:434
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | 	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:491
F211205 09:32:38.421971 2744786 kv/kvserver/store_raft.go:348  [n3,s3,r70/4:{-}] 10571 +  | runtime.goexit

[nvb]

Fixed by #73484.