HDDS-11708. Recon ListKeys API should return a proper http response status code if NSSummary rebuild is in progress. (apache#7437)

Author: devmadhuu

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java

@@ -80,6 +80,7 @@
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.recon.api.ServiceNotReadyException;
 import org.apache.hadoop.ozone.recon.api.types.NSSummary;
 import org.apache.hadoop.ozone.recon.api.types.DUResponse;
 import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
@@ -356,16 +357,14 @@ public static StringBuilder constructFullPathPrefix(long initialParentId, String
       if (nsSummary == null) {
         log.warn("NSSummary tree is currently being rebuilt or the directory could be in the progress of " +
             "deletion, returning empty string for path construction.");
-        fullPath.setLength(0);
-        return fullPath;
+        throw new ServiceNotReadyException("Service is initializing. Please try again later.");
       }
       if (nsSummary.getParentId() == -1) {
         if (rebuildTriggered.compareAndSet(false, true)) {
           triggerRebuild(reconNamespaceSummaryManager, omMetadataManager);
         }
         log.warn("NSSummary tree is currently being rebuilt, returning empty string for path construction.");
-        fullPath.setLength(0);
-        return fullPath;
+        throw new ServiceNotReadyException("Service is initializing. Please try again later.");
       }
       // On the last pass, dir-name will be empty and parent will be zero, indicating the loop should end.
       if (!nsSummary.getDirName().isEmpty()) {

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java

@@ -976,7 +976,7 @@ public Response listKeys(@QueryParam("replicationType") String replicationType,
     ListKeysResponse listKeysResponse = new ListKeysResponse();
     if (!ReconUtils.isInitializationComplete(omMetadataManager)) {
       listKeysResponse.setStatus(ResponseStatus.INITIALIZING);
-      return Response.ok(listKeysResponse).build();
+      return Response.status(Response.Status.SERVICE_UNAVAILABLE).entity(listKeysResponse).build();
     }
     ParamInfo paramInfo = new ParamInfo(replicationType, creationDate, keySize, startPrefix, prevKey,
         limit, false, "");
@@ -997,9 +997,9 @@ public Response listKeys(@QueryParam("replicationType") String replicationType,
   }
 
   private Response getListKeysResponse(ParamInfo paramInfo) {
+    ListKeysResponse listKeysResponse = new ListKeysResponse();
     try {
       paramInfo.setLimit(Math.max(0, paramInfo.getLimit())); // Ensure limit is non-negative
-      ListKeysResponse listKeysResponse = new ListKeysResponse();
       listKeysResponse.setPath(paramInfo.getStartPrefix());
       long replicatedTotal = 0;
       long unreplicatedTotal = 0;
@@ -1009,7 +1009,6 @@ private Response getListKeysResponse(ParamInfo paramInfo) {
           omMetadataManager.getKeyTableLite(BucketLayout.LEGACY);
       retrieveKeysFromTable(keyTable, paramInfo, listKeysResponse.getKeys());
 
-
       // Search keys from FSO layout.
       searchKeysInFSO(paramInfo, listKeysResponse.getKeys());
 
@@ -1029,6 +1028,10 @@ private Response getListKeysResponse(ParamInfo paramInfo) {
 
       return Response.ok(listKeysResponse).build();
     } catch (RuntimeException e) {
+      if (e instanceof ServiceNotReadyException) {
+        listKeysResponse.setStatus(ResponseStatus.INITIALIZING);
+        return Response.status(Response.Status.SERVICE_UNAVAILABLE).entity(listKeysResponse).build();
+      }
       LOG.error("Error generating listKeys response", e);
       return ReconResponseUtils.createInternalServerErrorResponse(
           "Unexpected runtime error while searching keys in OM DB: " + e.getMessage());

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ServiceNotReadyException.java

@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.api;
+
+/**
+ * This exception being thrown when Rest API service is still initializing and not yet ready.
+ */
+public class ServiceNotReadyException extends RuntimeException {
+  public ServiceNotReadyException(String message) {
+    super(message);
+  }
+}
+

hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java

@@ -62,10 +62,12 @@
 import org.apache.hadoop.ozone.recon.scm.ReconPipelineManager;
 import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade;
 import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager;
+import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager;
 import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
 import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl;
 import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl;
 import org.apache.hadoop.ozone.recon.tasks.ContainerKeyMapperTask;
+import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithFSO;
 import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition.UnHealthyContainerStates;
 import org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers;
 import org.junit.jupiter.api.BeforeEach;
@@ -121,6 +123,7 @@ public class TestContainerEndpoint {
       LoggerFactory.getLogger(TestContainerEndpoint.class);
 
   private OzoneStorageContainerManager ozoneStorageContainerManager;
+  private ReconNamespaceSummaryManager reconNamespaceSummaryManager;
   private ReconContainerManager reconContainerManager;
   private ContainerStateManager containerStateManager;
   private ReconPipelineManager reconPipelineManager;
@@ -198,6 +201,8 @@ private void initializeInjector() throws Exception {
     containerEndpoint = reconTestInjector.getInstance(ContainerEndpoint.class);
     containerHealthSchemaManager =
         reconTestInjector.getInstance(ContainerHealthSchemaManager.class);
+    this.reconNamespaceSummaryManager =
+        reconTestInjector.getInstance(ReconNamespaceSummaryManager.class);
 
     pipeline = getRandomPipeline();
     pipelineID = pipeline.getId();
@@ -472,6 +477,10 @@ public void testGetKeysForContainer() throws IOException {
     // Now to check if the ContainerEndpoint also reads the File table
     // Set up test data for FSO keys
     setUpFSOData();
+    NSSummaryTaskWithFSO nSSummaryTaskWithFso =
+        new NSSummaryTaskWithFSO(reconNamespaceSummaryManager,
+            reconOMMetadataManager, new OzoneConfiguration());
+    nSSummaryTaskWithFso.reprocessWithFSO(reconOMMetadataManager);
     // Reprocess the container key mapper to ensure the latest mapping is used
     reprocessContainerKeyMapper();
     response = containerEndpoint.getKeysForContainer(20L, -1, "");
@@ -556,6 +565,10 @@ public void testGetKeysForContainerWithPrevKey() throws IOException {
     setUpFSOData();
     // Reprocess the container key mapper to ensure the latest mapping is used
     reprocessContainerKeyMapper();
+    NSSummaryTaskWithFSO nSSummaryTaskWithFso =
+        new NSSummaryTaskWithFSO(reconNamespaceSummaryManager,
+            reconOMMetadataManager, new OzoneConfiguration());
+    nSSummaryTaskWithFso.reprocessWithFSO(reconOMMetadataManager);
     response = containerEndpoint.getKeysForContainer(20L, -1, "/0/1/2/file7");
 
     // Ensure that the expected number of keys is returned

hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java

@@ -89,6 +89,7 @@
 import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD;
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 import static org.mockito.Mockito.anyLong;
@@ -791,8 +792,9 @@ public void testConstructFullPath() throws IOException {
         .setParentObjectID(DIR_TWO_OBJECT_ID)
         .build();
     // Call constructFullPath and verify the result
-    fullPath = ReconUtils.constructFullPath(keyInfo,
-        reconNamespaceSummaryManager, reconOMMetadataManager);
+    OmKeyInfo finalKeyInfo = keyInfo;
+    assertThrows(ServiceNotReadyException.class, () -> ReconUtils.constructFullPath(finalKeyInfo,
+        reconNamespaceSummaryManager, reconOMMetadataManager));
   }
 
   @Test
@@ -813,8 +815,8 @@ public void testConstructFullPathWithNegativeParentIdTriggersRebuild() throws IO
         .setParentObjectID(dirOneObjectId)
         .build();
 
-    String result = ReconUtils.constructFullPath(keyInfo, mockSummaryManager, mockMetadataManager);
-    assertEquals("", result, "Expected an empty string return due to rebuild trigger");
+    assertThrows(ServiceNotReadyException.class, () ->
+        ReconUtils.constructFullPath(keyInfo, mockSummaryManager, mockMetadataManager));
   }
 
   @Test
@@ -836,7 +838,8 @@ public void testLoggingWhenParentIdIsNegative() throws IOException {
         .setParentObjectID(1L)
         .build();
 
-    ReconUtils.constructFullPath(keyInfo, mockManager, null);
+    assertThrows(ServiceNotReadyException.class, () ->
+        ReconUtils.constructFullPath(keyInfo, mockManager, null));
 
     // Assert
     ArgumentCaptor<String> logCaptor = ArgumentCaptor.forClass(String.class);

hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOmDBInsightEndPoint.java

@@ -41,6 +41,7 @@
 import org.apache.hadoop.ozone.recon.api.types.KeyInsightInfoResponse;
 import org.apache.hadoop.ozone.recon.api.types.ListKeysResponse;
 import org.apache.hadoop.ozone.recon.api.types.NSSummary;
+import org.apache.hadoop.ozone.recon.api.types.ResponseStatus;
 import org.apache.hadoop.ozone.recon.persistence.AbstractReconSqlDBTest;
 import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager;
 import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
@@ -217,6 +218,7 @@ public class TestOmDBInsightEndPoint extends AbstractReconSqlDBTest {
   private static final long KEY_TWENTY_TWO_OBJECT_ID = 37L;
   private static final long KEY_TWENTY_THREE_OBJECT_ID = 38L;
   private static final long KEY_TWENTY_FOUR_OBJECT_ID = 39L;
+  private static final long KEY_TWENTY_FIVE_OBJECT_ID = 42L;
 
   private static final long EMPTY_OBS_BUCKET_OBJECT_ID = 40L;
   private static final long EMPTY_FSO_BUCKET_OBJECT_ID = 41L;
@@ -242,6 +244,7 @@ public class TestOmDBInsightEndPoint extends AbstractReconSqlDBTest {
   private static final long KEY_SEVENTEEN_SIZE = 2 * OzoneConsts.KB + 1; // bin 2
   private static final long KEY_EIGHTEEN_SIZE = OzoneConsts.KB + 1; // bin 1
   private static final long KEY_NINETEEN_SIZE = 2 * OzoneConsts.KB + 1; // bin 2
+  private static final long KEY_TWENTY_SIZE = OzoneConsts.KB + 1; // bin 1
 
   private static final String OBS_BUCKET_PATH = "/volume1/obs-bucket";
   private static final String FSO_BUCKET_PATH = "/volume1/fso-bucket";
@@ -1940,6 +1943,18 @@ public void testListKeysForEmptyOBSBucket() {
     assertEquals("", listKeysResponse.getLastKey());
   }
 
+  @Test
+  public void testListKeysWhenNSSummaryNotInitialized() throws Exception {
+    reconNamespaceSummaryManager.clearNSSummaryTable();
+    // bucket level DU
+    Response bucketResponse =
+        omdbInsightEndpoint.listKeys("RATIS", "", 0, FSO_BUCKET_TWO_PATH,
+            "", 1000);
+    ListKeysResponse listKeysResponse = (ListKeysResponse) bucketResponse.getEntity();
+    assertEquals(ResponseStatus.INITIALIZING, listKeysResponse.getStatus());
+    assertEquals(Response.Status.SERVICE_UNAVAILABLE.getStatusCode(), bucketResponse.getStatus());
+  }
+
   @Test
   public void testListKeysForEmptyFSOBucket() {
     Response bucketResponse = omdbInsightEndpoint.listKeys("RATIS", "", 0, EMPTY_FSO_BUCKET_PATH,