HDDS-10574. Improve TestObjectPut (apache#6426)

Author: adoroszlai

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java

@@ -482,7 +482,7 @@ public OzoneFileStatus getOzoneFileStatus(String volumeName,
   @Override
   public void createDirectory(String volumeName, String bucketName,
                               String keyName) throws IOException {
-
+    getBucket(volumeName, bucketName).createDirectory(keyName);
   }
 
   @Override

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java

@@ -36,12 +36,11 @@
 
 import javax.xml.bind.DatatypeConverter;
 import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.hdds.client.RatisReplicationConfig;
 import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.client.ReplicationFactor;
 import org.apache.hadoop.hdds.client.ReplicationType;
-import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.storage.ByteBufferStreamOutput;
 import org.apache.hadoop.ozone.OzoneAcl;
@@ -55,6 +54,8 @@
 import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
 import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
 import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import static org.apache.hadoop.ozone.OzoneConsts.ETAG;
 import static org.apache.hadoop.ozone.OzoneConsts.MD5_HASH;
@@ -63,7 +64,9 @@
 /**
  * In-memory ozone bucket for testing.
  */
-public class OzoneBucketStub extends OzoneBucket {
+public final class OzoneBucketStub extends OzoneBucket {
+
+  private static final Logger LOG = LoggerFactory.getLogger(OzoneBucketStub.class);
 
   private Map<String, OzoneKeyDetails> keyDetails = new HashMap<>();
 
@@ -80,7 +83,7 @@ public static Builder newBuilder() {
     return new Builder();
   }
 
-  public OzoneBucketStub(Builder b) {
+  private OzoneBucketStub(Builder b) {
     super(b);
     this.replicationConfig = super.getReplicationConfig();
   }
@@ -93,43 +96,6 @@ public static final class Builder extends OzoneBucket.Builder {
     private Builder() {
     }
 
-    @Override
-    public Builder setVolumeName(String volumeName) {
-      super.setVolumeName(volumeName);
-      return this;
-    }
-
-    @Override
-    public Builder setName(String name) {
-      super.setName(name);
-      return this;
-    }
-
-    @Override
-    public Builder setDefaultReplicationConfig(
-        DefaultReplicationConfig defaultReplicationConfig) {
-      super.setDefaultReplicationConfig(defaultReplicationConfig);
-      return this;
-    }
-
-    @Override
-    public Builder setStorageType(StorageType storageType) {
-      super.setStorageType(storageType);
-      return this;
-    }
-
-    @Override
-    public Builder setVersioning(Boolean versioning) {
-      super.setVersioning(versioning);
-      return this;
-    }
-
-    @Override
-    public Builder setCreationTime(long creationTime) {
-      super.setCreationTime(creationTime);
-      return this;
-    }
-
     @Override
     public OzoneBucketStub build() {
       return new OzoneBucketStub(this);
@@ -149,31 +115,16 @@ public OzoneOutputStream createKey(String key, long size,
                                      ReplicationFactor factor,
                                      Map<String, String> metadata)
       throws IOException {
-    ByteArrayOutputStream byteArrayOutputStream =
-        new ByteArrayOutputStream((int) size) {
-          @Override
-          public void close() throws IOException {
-            keyContents.put(key, toByteArray());
-            keyDetails.put(key, new OzoneKeyDetails(
-                getVolumeName(),
-                getName(),
-                key,
-                size,
-                System.currentTimeMillis(),
-                System.currentTimeMillis(),
-                new ArrayList<>(), replicationConfig, metadata, null,
-                () -> readKey(key), true
-            ));
-            super.close();
-          }
-        };
-    return new OzoneOutputStream(byteArrayOutputStream, null);
+    ReplicationConfig replication = ReplicationConfig.fromTypeAndFactor(type, factor);
+    return createKey(key, size, replication, metadata);
   }
 
   @Override
   public OzoneOutputStream createKey(String key, long size,
       ReplicationConfig rConfig, Map<String, String> metadata)
       throws IOException {
+    assertDoesNotExist(key + "/");
+
     final ReplicationConfig repConfig;
     if (rConfig == null) {
       repConfig = getReplicationConfig();
@@ -208,6 +159,8 @@ public OzoneDataStreamOutput createStreamKey(String key, long size,
                                                ReplicationConfig rConfig,
                                                Map<String, String> keyMetadata)
       throws IOException {
+    assertDoesNotExist(key + "/");
+
     ByteBufferStreamOutput byteBufferStreamOutput =
         new KeyMetadataAwareByteBufferStreamOutput(keyMetadata) {
 
@@ -611,6 +564,9 @@ public ReplicationConfig getReplicationConfig() {
 
   @Override
   public void createDirectory(String keyName) throws IOException {
+    assertDoesNotExist(StringUtils.stripEnd(keyName, "/"));
+
+    LOG.info("createDirectory({})", keyName);
     keyDetails.put(keyName, new OzoneKeyDetails(
         getVolumeName(),
         getName(),
@@ -622,6 +578,12 @@ public void createDirectory(String keyName) throws IOException {
         () -> readKey(keyName), false));
   }
 
+  private void assertDoesNotExist(String keyName) throws OMException {
+    if (keyDetails.get(keyName) != null) {
+      throw new OMException("already exists", ResultCodes.FILE_ALREADY_EXISTS);
+    }
+  }
+
   /**
    * ByteArrayOutputStream stub with metadata.
    */

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java

@@ -38,17 +38,17 @@
 /**
  * Ozone volume with in-memory state for testing.
  */
-public class OzoneVolumeStub extends OzoneVolume {
+public final class OzoneVolumeStub extends OzoneVolume {
 
-  private Map<String, OzoneBucketStub> buckets = new HashMap<>();
+  private final Map<String, OzoneBucket> buckets = new HashMap<>();
 
-  private ArrayList<OzoneAcl> aclList = new ArrayList<>();
+  private final ArrayList<OzoneAcl> aclList = new ArrayList<>();
 
   public static Builder newBuilder() {
     return new Builder();
   }
 
-  public OzoneVolumeStub(Builder b) {
+  private OzoneVolumeStub(Builder b) {
     super(b);
   }
 
@@ -124,6 +124,7 @@ public void createBucket(String bucketName, BucketArgs bucketArgs) {
         .setDefaultReplicationConfig(new DefaultReplicationConfig(
             RatisReplicationConfig.getInstance(
                 HddsProtos.ReplicationFactor.THREE)))
+        .setBucketLayout(bucketArgs.getBucketLayout())
         .setStorageType(bucketArgs.getStorageType())
         .setVersioning(bucketArgs.getVersioning())
         .setCreationTime(Time.now())