HBASE-27897 ConnectionImplementation#locateRegionInMeta should pause and retry when taking user region lock failed (#5258)

Signed-off-by: Wellington Chevreuil <wchevreuil@apache.org>

Author: sunhelly

hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java

@@ -963,8 +963,12 @@ private RegionLocations locateRegionInMeta(TableName tableName, byte[] row, bool
       // Query the meta region
       long pauseBase = this.pause;
       takeUserRegionLock();
-      final long lockStartTime = EnvironmentEdgeManager.currentTime();
+      long lockStartTime = 0;
+      boolean lockedUserRegion = false;
       try {
+        takeUserRegionLock();
+        lockStartTime = EnvironmentEdgeManager.currentTime();
+        lockedUserRegion = true;
         // We don't need to check if useCache is enabled or not. Even if useCache is false
         // we already cleared the cache for this row before acquiring userRegion lock so if this
         // row is present in cache that means some other thread has populated it while we were
@@ -1063,10 +1067,12 @@ rpcControllerFactory, getMetaLookupPool(), metaReplicaCallTimeoutScanInMicroSeco
         relocateMeta =
           !(e instanceof RegionOfflineException || e instanceof NoServerForRegionException);
       } finally {
-        userRegionLock.unlock();
-        // update duration of the lock being held
-        if (metrics != null) {
-          metrics.updateUserRegionLockHeld(EnvironmentEdgeManager.currentTime() - lockStartTime);
+        if (lockedUserRegion) {
+          userRegionLock.unlock();
+          // update duration of the lock being held
+          if (metrics != null) {
+            metrics.updateUserRegionLockHeld(EnvironmentEdgeManager.currentTime() - lockStartTime);
+          }
         }
       }
       try {

hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java

@@ -592,21 +592,21 @@ public void testUserRegionLockThrowsException() throws IOException, InterruptedE
       // obtain the client metrics
       MetricsConnection metrics = conn.getConnectionMetrics();
       long queueCount = metrics.userRegionLockQueueHist.getCount();
-      assertEquals("Queue of userRegionLock should be updated twice. queueCount: " + queueCount,
-        queueCount, 2);
+      assertEquals("Queue of userRegionLock should be updated twice. queueCount: " + queueCount, 2,
+        queueCount);
 
       long timeoutCount = metrics.userRegionLockTimeoutCount.getCount();
-      assertEquals("Timeout of userRegionLock should happen once. timeoutCount: " + timeoutCount,
-        timeoutCount, 1);
+      assertEquals("Timeout of userRegionLock should happen once. timeoutCount: " + timeoutCount, 1,
+        timeoutCount);
 
       long waitingTimerCount = metrics.userRegionLockWaitingTimer.getCount();
       assertEquals("userRegionLock should be grabbed successfully once. waitingTimerCount: "
-        + waitingTimerCount, waitingTimerCount, 1);
+        + waitingTimerCount, 1, waitingTimerCount);
 
       long heldTimerCount = metrics.userRegionLockHeldTimer.getCount();
       assertEquals(
-        "userRegionLock should be held successfully once. heldTimerCount: " + heldTimerCount,
-        heldTimerCount, 1);
+        "userRegionLock should be held successfully once. heldTimerCount: " + heldTimerCount, 1,
+        heldTimerCount);
       double heldTime = metrics.userRegionLockHeldTimer.getSnapshot().getMax();
       assertTrue("Max held time should be greater than 2 seconds. heldTime: " + heldTime,
         heldTime >= 2E9);