HADOOP-17377: ABFS: MsiTokenProvider doesn't retry HTTP 429/410 from the Instance Metadata Service

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java

@@ -151,7 +151,7 @@ public Writable call(
 
     // Nothing should be logged for a suppressed exception.
     server.logException(logger, new TestException1(), dummyCall);
-    verifyZeroInteractions(logger);
+    verifyNoInteractions(logger);
 
     // No stack trace should be logged for a terse exception.
     server.logException(logger, new TestException2(), dummyCall);

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java

@@ -58,8 +58,7 @@ public void testSameOrigin() throws ServletException, IOException {
     CrossOriginFilter filter = new CrossOriginFilter();
     filter.init(filterConfig);
     filter.doFilter(mockReq, mockRes, mockChain);
-
-    Mockito.verifyZeroInteractions(mockRes);
+    Mockito.verifyNoInteractions(mockRes);
     Mockito.verify(mockChain).doFilter(mockReq, mockRes);
   }
 
@@ -224,7 +223,7 @@ public void testDisallowedOrigin() throws ServletException, IOException {
     filter.init(filterConfig);
     filter.doFilter(mockReq, mockRes, mockChain);
 
-    Mockito.verifyZeroInteractions(mockRes);
+    Mockito.verifyNoInteractions(mockRes);
     Mockito.verify(mockChain).doFilter(mockReq, mockRes);
   }
 
@@ -252,7 +251,7 @@ public void testDisallowedMethod() throws ServletException, IOException {
     filter.init(filterConfig);
     filter.doFilter(mockReq, mockRes, mockChain);
 
-    Mockito.verifyZeroInteractions(mockRes);
+    Mockito.verifyNoInteractions(mockRes);
     Mockito.verify(mockChain).doFilter(mockReq, mockRes);
   }
 
@@ -283,7 +282,7 @@ public void testDisallowedHeader() throws ServletException, IOException {
     filter.init(filterConfig);
     filter.doFilter(mockReq, mockRes, mockChain);
 
-    Mockito.verifyZeroInteractions(mockRes);
+    Mockito.verifyNoInteractions(mockRes);
     Mockito.verify(mockChain).doFilter(mockReq, mockRes);
   }
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestRestCsrfPreventionFilter.java

@@ -75,7 +75,7 @@ public void testNoHeaderDefaultConfigBadRequest()
 
     verify(mockRes, atLeastOnce()).sendError(
         HttpServletResponse.SC_BAD_REQUEST, EXPECTED_MESSAGE);
-    Mockito.verifyZeroInteractions(mockChain);
+    Mockito.verifyNoInteractions(mockChain);
   }
 
   @Test
@@ -110,7 +110,7 @@ public void testNoHeaderCustomAgentConfigBadRequest()
 
     verify(mockRes, atLeastOnce()).sendError(
         HttpServletResponse.SC_BAD_REQUEST, EXPECTED_MESSAGE);
-    Mockito.verifyZeroInteractions(mockChain);
+    Mockito.verifyNoInteractions(mockChain);
   }
 
   @Test
@@ -228,7 +228,7 @@ public void testMissingHeaderWithCustomHeaderConfigBadRequest()
     filter.init(filterConfig);
     filter.doFilter(mockReq, mockRes, mockChain);
 
-    Mockito.verifyZeroInteractions(mockChain);
+    Mockito.verifyNoInteractions(mockChain);
   }
 
   @Test
@@ -260,7 +260,7 @@ public void testMissingHeaderNoMethodsToIgnoreConfigBadRequest()
     filter.init(filterConfig);
     filter.doFilter(mockReq, mockRes, mockChain);
 
-    Mockito.verifyZeroInteractions(mockChain);
+    Mockito.verifyNoInteractions(mockChain);
   }
 
   @Test
@@ -356,6 +356,6 @@ public void testMissingHeaderMultipleIgnoreMethodsConfigBadRequest()
     filter.init(filterConfig);
     filter.doFilter(mockReq, mockRes, mockChain);
 
-    Mockito.verifyZeroInteractions(mockChain);
+    Mockito.verifyNoInteractions(mockChain);
   }
 }

hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml

@@ -177,6 +177,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <artifactId>junit-jupiter-params</artifactId>
       <scope>test</scope>
     </dependency>
+      <dependency>
+          <groupId>org.mockito</groupId>
+          <artifactId>mockito-core</artifactId>
+          <scope>test</scope>
+      </dependency>
   </dependencies>
 
   <build>

hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java

@@ -25,6 +25,7 @@
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
+import java.lang.reflect.Field;
 import java.security.PrivilegedExceptionAction;
 import java.util.Collections;
 import java.util.HashMap;
@@ -62,13 +63,13 @@
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.test.LambdaTestUtils;
 import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.Time;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.mockito.Mockito;
-import org.mockito.internal.util.reflection.FieldSetter;
 
 /**
  * The administrator interface of the {@link Router} implemented by
@@ -118,18 +119,19 @@ public static void globalSetUp() throws Exception {
    * @throws IOException
    * @throws NoSuchFieldException
    */
-  private static void setUpMocks() throws IOException, NoSuchFieldException {
+  private static void setUpMocks() throws IOException, NoSuchFieldException, IllegalAccessException {
     RouterRpcServer spyRpcServer =
         Mockito.spy(routerContext.getRouter().createRpcServer());
-    FieldSetter.setField(routerContext.getRouter(),
-        Router.class.getDeclaredField("rpcServer"), spyRpcServer);
+    Field rpcServerField = Router.class.getDeclaredField("rpcServer");
+    rpcServerField.setAccessible(true);
+    rpcServerField.set(routerContext.getRouter(), spyRpcServer);
     Mockito.doReturn(null).when(spyRpcServer).getFileInfo(Mockito.anyString());
 
     // mock rpc client for destination check when editing mount tables.
     mockRpcClient = Mockito.spy(spyRpcServer.getRPCClient());
-    FieldSetter.setField(spyRpcServer,
-        RouterRpcServer.class.getDeclaredField("rpcClient"),
-        mockRpcClient);
+    Field rpcClientField = RouterRpcServer.class.getDeclaredField("rpcClient");
+    rpcClientField.setAccessible(true);
+    rpcClientField.set(spyRpcServer, mockRpcClient);
     RemoteLocation remoteLocation0 =
         new RemoteLocation("ns0", "/testdir", null);
     RemoteLocation remoteLocation1 =

hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java

@@ -24,7 +24,6 @@
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
 import static org.apache.hadoop.test.Whitebox.getInternalState;
@@ -73,6 +72,7 @@
 import org.apache.hadoop.ipc.StandbyException;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Test;
+import org.mockito.Mockito;
 import org.slf4j.event.Level;
 
 /**
@@ -423,7 +423,7 @@ public void testSubclusterDown() throws Exception {
     FSNamesystem ns0 = nn0.getNamesystem();
     HAContext nn0haCtx = (HAContext)getInternalState(ns0, "haContext");
     HAContext mockCtx = mock(HAContext.class);
-    doThrow(new StandbyException("Mock")).when(mockCtx).checkOperation(any());
+    doThrow(new StandbyException("Mock")).when(mockCtx).checkOperation(Mockito.any());
     setInternalState(ns0, "haContext", mockCtx);
 
     // router0 should throw an exception

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java

@@ -1575,7 +1575,7 @@ public void testNoLookupsWhenNotUsed() throws Exception {
     CacheManager cm = cluster.getNamesystem().getCacheManager();
     LocatedBlocks locations = Mockito.mock(LocatedBlocks.class);
     cm.setCachedLocations(locations);
-    Mockito.verifyZeroInteractions(locations);
+    Mockito.verifyNoInteractions(locations);
   }
 
   @Test(timeout=120000)

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java

@@ -447,6 +447,6 @@ public void testShortCircuitSnapshotSearch() throws SnapshotException {
     INodesInPath iip = Mockito.mock(INodesInPath.class);
     List<INodeDirectory> snapDirs = new ArrayList<>();
     FSDirSnapshotOp.checkSnapshot(fsn.getFSDirectory(), iip, snapDirs);
-    Mockito.verifyZeroInteractions(iip);
+    Mockito.verifyNoInteractions(iip);
   }
 }

hadoop-project/pom.xml

@@ -1308,7 +1308,20 @@
       <dependency>
         <groupId>org.mockito</groupId>
         <artifactId>mockito-core</artifactId>
-        <version>2.28.2</version>
+        <version>4.11.0</version>
+        <scope>compile</scope>
+      </dependency>
+      <dependency>
+        <groupId>org.mockito</groupId>
+        <artifactId>mockito-inline</artifactId>
+        <version>4.11.0</version>
+        <scope>test</scope>
+        <exclusions>
+          <exclusion>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+          </exclusion>
+        </exclusions>
       </dependency>
       <dependency>
         <groupId>org.mockito</groupId>

hadoop-tools/hadoop-azure/pom.xml

@@ -323,6 +323,13 @@
       <artifactId>mockito-core</artifactId>
       <scope>test</scope>
     </dependency>
+
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-inline</artifactId>
+      <scope>test</scope>
+    </dependency>
+
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-minikdc</artifactId>

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java

@@ -111,6 +111,12 @@ public final class AbfsHttpConstants {
   // The HTTP 100 Continue informational status response code indicates that everything so far
   // is OK and that the client should continue with the request or ignore it if it is already finished.
   public static final String HUNDRED_CONTINUE = "100-continue";
+  /**
+   * HTTP status code indicating that the server has received too many requests and the client should
+   * qualify for retrying the operation, as described in the Microsoft Azure documentation.
+   * {@link "https://learn.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-use-vm-token#error-handling"}.
+   */
+  public static final int HTTP_TOO_MANY_REQUESTS = 429;
 
   public static final char CHAR_FORWARD_SLASH = '/';
   public static final char CHAR_EXCLAMATION_POINT = '!';

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java

@@ -213,7 +213,19 @@ public String getRequestId() {
       return this.requestId;
     }
 
-    protected HttpException(
+    /**
+     Constructs an instance of HttpException with detailed information about an HTTP error response.
+     This exception is designed to encapsulate details of an HTTP error response, providing context about the error
+     encountered during an HTTP operation. It includes the HTTP error code, the associated request ID, an error message,
+     the URL that triggered the error, the content type of the response, and the response body.
+     @param httpErrorCode The HTTP error code indicating the nature of the encountered error.
+     @param requestId The unique identifier associated with the corresponding HTTP request.
+     @param message A descriptive error message providing additional information about the encountered error.
+     @param url The URL that resulted in the HTTP error response.
+     @param contentType The content type of the HTTP response.
+     @param body The body of the HTTP response, containing more details about the error.
+     */
+    public HttpException(
         final int httpErrorCode,
         final String requestId,
         final String message,
@@ -341,7 +353,20 @@ private static boolean isRecoverableFailure(IOException e) {
         || e instanceof FileNotFoundException);
   }
 
-  private static AzureADToken getTokenSingleCall(String authEndpoint,
+/**
+ Retrieves an Azure OAuth token for authentication through a single API call.
+ This method facilitates the acquisition of an OAuth token from Azure Active Directory
+ to enable secure authentication for various services. It supports both Managed Service Identity (MSI)
+ tokens and non-MSI tokens based on the provided parameters.
+ @param authEndpoint The URL endpoint for OAuth token retrieval.
+ @param payload The payload to be included in the token request. This typically contains grant type and
+ any required parameters for token acquisition.
+ @param headers A Hashtable containing additional HTTP headers to be included in the token request.
+ @param httpMethod The HTTP method to be used for the token request (e.g., GET, POST).
+ @param isMsi A boolean flag indicating whether to request a Managed Service Identity (MSI) token or not.
+ @return An AzureADToken object containing the acquired OAuth token and associated metadata.
+ */
+  public static AzureADToken getTokenSingleCall(String authEndpoint,
       String payload, Hashtable<String, String> headers, String httpMethod,
       boolean isMsi)
           throws IOException {

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java

@@ -25,6 +25,7 @@
 import org.apache.hadoop.classification.VisibleForTesting;
 
 import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_CONTINUE;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_TOO_MANY_REQUESTS;
 
 /**
  * Retry policy used by AbfsClient.
@@ -123,6 +124,9 @@ public ExponentialRetryPolicy(final int retryCount, final int minBackoff, final
    * and the current strategy. The valid http status code lies in the range of 1xx-5xx.
    * But an invalid status code might be set due to network or timeout kind of issues.
    * Such invalid status code also qualify for retry.
+   * HTTP status code 410 qualifies for retry based on
+   * https://docs.microsoft.com/en-in/azure/virtual-machines/linux/
+   * instance-metadata-service?tabs=windows#errors-and-debugging
    *
    * @param retryCount The current retry attempt count.
    * @param statusCode The status code of the response, or -1 for socket error.
@@ -132,6 +136,8 @@ public boolean shouldRetry(final int retryCount, final int statusCode) {
     return retryCount < this.retryCount
         && (statusCode < HTTP_CONTINUE
         || statusCode == HttpURLConnection.HTTP_CLIENT_TIMEOUT
+        || statusCode == HttpURLConnection.HTTP_GONE
+        || statusCode == HTTP_TOO_MANY_REQUESTS
         || (statusCode >= HttpURLConnection.HTTP_INTERNAL_ERROR
             && statusCode != HttpURLConnection.HTTP_NOT_IMPLEMENTED
             && statusCode != HttpURLConnection.HTTP_VERSION));