Skip to content

Commit 589eeef

Browse files
peterxclipeterxcli
authored
HDDS-12371. Duplicated key scanning on multipartInfo table when listing multipart uploads (#7937)
1 parent 6e766bf commit 589eeef

7 files changed

Lines changed: 104 additions & 192 deletions

File tree

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/MultipartUploadKeys.java‎

Lines changed: 0 additions & 92 deletions
This file was deleted.

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartUploadList.java‎

Lines changed: 46 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,22 +22,22 @@
2222
/**
2323
* List of in-flight MPU uploads.
2424
*/
25-
public class OmMultipartUploadList {
25+
public final class OmMultipartUploadList {
2626

2727
private List<OmMultipartUpload> uploads;
2828
private String nextKeyMarker;
2929
private String nextUploadIdMarker;
3030
private boolean isTruncated;
3131

32-
public OmMultipartUploadList(
33-
List<OmMultipartUpload> uploads,
34-
String nextKeyMarker,
35-
String nextUploadIdMarker,
36-
boolean isTruncated) {
37-
this.uploads = uploads;
38-
this.nextKeyMarker = nextKeyMarker;
39-
this.nextUploadIdMarker = nextUploadIdMarker;
40-
this.isTruncated = isTruncated;
32+
private OmMultipartUploadList(Builder builder) {
33+
this.uploads = builder.uploads;
34+
this.nextKeyMarker = builder.nextKeyMarker;
35+
this.nextUploadIdMarker = builder.nextUploadIdMarker;
36+
this.isTruncated = builder.isTruncated;
37+
}
38+
39+
public static Builder newBuilder() {
40+
return new Builder();
4141
}
4242

4343
public List<OmMultipartUpload> getUploads() {
@@ -61,4 +61,40 @@ public boolean isTruncated() {
6161
return isTruncated;
6262
}
6363

64+
/**
65+
* Builder class for OmMultipartUploadList.
66+
*/
67+
public static class Builder {
68+
private List<OmMultipartUpload> uploads;
69+
private String nextKeyMarker = "";
70+
private String nextUploadIdMarker = "";
71+
private boolean isTruncated;
72+
73+
public Builder() {
74+
}
75+
76+
public Builder setUploads(List<OmMultipartUpload> uploads) {
77+
this.uploads = uploads;
78+
return this;
79+
}
80+
81+
public Builder setNextKeyMarker(String nextKeyMarker) {
82+
this.nextKeyMarker = nextKeyMarker;
83+
return this;
84+
}
85+
86+
public Builder setNextUploadIdMarker(String nextUploadIdMarker) {
87+
this.nextUploadIdMarker = nextUploadIdMarker;
88+
return this;
89+
}
90+
91+
public Builder setIsTruncated(boolean isTruncated) {
92+
this.isTruncated = isTruncated;
93+
return this;
94+
}
95+
96+
public OmMultipartUploadList build() {
97+
return new OmMultipartUploadList(this);
98+
}
99+
}
64100
}

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1842,10 +1842,12 @@ public OmMultipartUploadList listMultipartUploads(String volumeName,
18421842
))
18431843
.collect(Collectors.toList());
18441844

1845-
OmMultipartUploadList response = new OmMultipartUploadList(uploadList,
1846-
listMultipartUploadsResponse.getNextKeyMarker(),
1847-
listMultipartUploadsResponse.getNextUploadIdMarker(),
1848-
listMultipartUploadsResponse.getIsTruncated());
1845+
OmMultipartUploadList response = OmMultipartUploadList.newBuilder()
1846+
.setUploads(uploadList)
1847+
.setNextKeyMarker(listMultipartUploadsResponse.getNextKeyMarker())
1848+
.setNextUploadIdMarker(listMultipartUploadsResponse.getNextUploadIdMarker())
1849+
.setIsTruncated(listMultipartUploadsResponse.getIsTruncated())
1850+
.build();
18491851

18501852
return response;
18511853
}

‎hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,14 +38,14 @@
3838
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
3939
import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
4040
import org.apache.hadoop.ozone.om.helpers.ListOpenFilesResult;
41-
import org.apache.hadoop.ozone.om.helpers.MultipartUploadKeys;
4241
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
4342
import org.apache.hadoop.ozone.om.helpers.OmDBAccessIdInfo;
4443
import org.apache.hadoop.ozone.om.helpers.OmDBTenantState;
4544
import org.apache.hadoop.ozone.om.helpers.OmDBUserPrincipalInfo;
4645
import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
4746
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
4847
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
48+
import org.apache.hadoop.ozone.om.helpers.OmMultipartUpload;
4949
import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo;
5050
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
5151
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
@@ -494,8 +494,11 @@ <KEY, VALUE> long countEstimatedRowsInTable(Table<KEY, VALUE> table)
494494
/**
495495
* Return the existing upload keys which includes volumeName, bucketName,
496496
* keyName.
497+
* @param noPagination if true, returns all keys; if false, applies pagination
498+
* @return When paginated, returns up to maxUploads + 1 entries, where the
499+
* extra entry is used to determine the next page markers
497500
*/
498-
MultipartUploadKeys getMultipartUploadKeys(String volumeName,
501+
List<OmMultipartUpload> getMultipartUploadKeys(String volumeName,
499502
String bucketName, String prefix, String keyMarker, String uploadIdMarker, int maxUploads,
500503
boolean noPagination) throws IOException;
501504

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java‎

Lines changed: 15 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,6 @@
7979
import java.security.GeneralSecurityException;
8080
import java.security.PrivilegedExceptionAction;
8181
import java.time.Duration;
82-
import java.time.Instant;
8382
import java.util.ArrayList;
8483
import java.util.Collection;
8584
import java.util.Collections;
@@ -132,7 +131,6 @@
132131
import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo;
133132
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
134133
import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
135-
import org.apache.hadoop.ozone.om.helpers.MultipartUploadKeys;
136134
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
137135
import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
138136
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
@@ -826,38 +824,25 @@ public OmMultipartUploadList listMultipartUploads(String volumeName,
826824
metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName,
827825
bucketName);
828826
try {
829-
830-
MultipartUploadKeys multipartUploadKeys = metadataManager
827+
List<OmMultipartUpload> multipartUploadKeys = metadataManager
831828
.getMultipartUploadKeys(volumeName, bucketName, prefix, keyMarker, uploadIdMarker, maxUploads,
832829
!withPagination);
830+
OmMultipartUploadList.Builder resultBuilder = OmMultipartUploadList.newBuilder();
833831

834-
List<OmMultipartUpload> collect = multipartUploadKeys.getKeys().stream()
835-
.map(OmMultipartUpload::from)
836-
.peek(upload -> {
837-
try {
838-
Table<String, OmMultipartKeyInfo> keyInfoTable =
839-
metadataManager.getMultipartInfoTable();
840-
841-
OmMultipartKeyInfo multipartKeyInfo =
842-
keyInfoTable.get(upload.getDbKey());
843-
844-
upload.setCreationTime(
845-
Instant.ofEpochMilli(multipartKeyInfo.getCreationTime()));
846-
upload.setReplicationConfig(
847-
multipartKeyInfo.getReplicationConfig());
848-
} catch (IOException e) {
849-
LOG.warn(
850-
"Open key entry for multipart upload record can't be read {}",
851-
metadataManager.getOzoneKey(upload.getVolumeName(),
852-
upload.getBucketName(), upload.getKeyName()));
853-
}
854-
})
855-
.collect(Collectors.toList());
832+
if (withPagination && multipartUploadKeys.size() == maxUploads + 1) {
833+
int lastIndex = multipartUploadKeys.size() - 1;
834+
OmMultipartUpload lastUpload = multipartUploadKeys.get(lastIndex);
835+
resultBuilder.setNextKeyMarker(lastUpload.getKeyName())
836+
.setNextUploadIdMarker(lastUpload.getUploadId())
837+
.setIsTruncated(true);
838+
839+
// remove next upload from the list
840+
multipartUploadKeys.remove(lastIndex);
841+
}
856842

857-
return new OmMultipartUploadList(collect,
858-
multipartUploadKeys.getNextKeyMarker(),
859-
multipartUploadKeys.getNextUploadIdMarker(),
860-
multipartUploadKeys.isTruncated());
843+
return resultBuilder
844+
.setUploads(multipartUploadKeys)
845+
.build();
861846

862847
} catch (IOException ex) {
863848
LOG.error("List Multipart Uploads Failed: volume: " + volumeName +

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java‎

Lines changed: 19 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import java.nio.file.Path;
4646
import java.nio.file.Paths;
4747
import java.time.Duration;
48+
import java.time.Instant;
4849
import java.util.ArrayList;
4950
import java.util.Collection;
5051
import java.util.HashMap;
@@ -56,9 +57,8 @@
5657
import java.util.Objects;
5758
import java.util.Optional;
5859
import java.util.Set;
59-
import java.util.SortedSet;
60+
import java.util.SortedMap;
6061
import java.util.TreeMap;
61-
import java.util.TreeSet;
6262
import java.util.concurrent.TimeUnit;
6363
import java.util.stream.Collectors;
6464
import java.util.stream.Stream;
@@ -89,7 +89,6 @@
8989
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
9090
import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
9191
import org.apache.hadoop.ozone.om.helpers.ListOpenFilesResult;
92-
import org.apache.hadoop.ozone.om.helpers.MultipartUploadKeys;
9392
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
9493
import org.apache.hadoop.ozone.om.helpers.OmDBAccessIdInfo;
9594
import org.apache.hadoop.ozone.om.helpers.OmDBTenantState;
@@ -1936,12 +1935,11 @@ public <KEY, VALUE> long countEstimatedRowsInTable(Table<KEY, VALUE> table)
19361935
}
19371936

19381937
@Override
1939-
public MultipartUploadKeys getMultipartUploadKeys(
1938+
public List<OmMultipartUpload> getMultipartUploadKeys(
19401939
String volumeName, String bucketName, String prefix, String keyMarker,
19411940
String uploadIdMarker, int maxUploads, boolean noPagination) throws IOException {
19421941

1943-
MultipartUploadKeys.Builder resultBuilder = MultipartUploadKeys.newBuilder();
1944-
SortedSet<String> responseKeys = new TreeSet<>();
1942+
SortedMap<String, OmMultipartKeyInfo> responseKeys = new TreeMap<>();
19451943
Set<String> aborted = new HashSet<>();
19461944

19471945
String prefixKey =
@@ -1964,9 +1962,10 @@ public MultipartUploadKeys getMultipartUploadKeys(
19641962
String cacheKey = cacheEntry.getKey().getCacheKey();
19651963
if (cacheKey.startsWith(prefixKey)) {
19661964
// Check if it is marked for delete, due to abort mpu
1967-
if (cacheEntry.getValue().getCacheValue() != null &&
1965+
OmMultipartKeyInfo multipartKeyInfo = cacheEntry.getValue().getCacheValue();
1966+
if (multipartKeyInfo != null &&
19681967
cacheKey.compareTo(seekKey) >= 0) {
1969-
responseKeys.add(cacheKey);
1968+
responseKeys.put(cacheKey, multipartKeyInfo);
19701969
} else {
19711970
aborted.add(cacheKey);
19721971
}
@@ -1984,35 +1983,26 @@ public MultipartUploadKeys getMultipartUploadKeys(
19841983
KeyValue<String, OmMultipartKeyInfo> entry = iterator.next();
19851984
// If it is marked for abort, skip it.
19861985
if (!aborted.contains(entry.getKey())) {
1987-
responseKeys.add(entry.getKey());
1986+
responseKeys.put(entry.getKey(), entry.getValue());
19881987
dbKeysCount++;
19891988
}
19901989
}
19911990
}
19921991

1993-
if (noPagination) {
1994-
resultBuilder.setKeys(responseKeys);
1995-
return resultBuilder.build();
1996-
}
1992+
List<OmMultipartUpload> result = new ArrayList<>(responseKeys.size());
1993+
1994+
for (Map.Entry<String, OmMultipartKeyInfo> entry : responseKeys.entrySet()) {
1995+
OmMultipartUpload multipartUpload = OmMultipartUpload.from(entry.getKey());
1996+
1997+
multipartUpload.setCreationTime(
1998+
Instant.ofEpochMilli(entry.getValue().getCreationTime()));
1999+
multipartUpload.setReplicationConfig(
2000+
entry.getValue().getReplicationConfig());
19972001

1998-
String lastKey = responseKeys.stream()
1999-
.skip(maxUploads)
2000-
.findFirst()
2001-
.orElse(null);
2002-
if (lastKey != null) {
2003-
// implies the keyset size is greater than maxUploads
2004-
OmMultipartUpload lastKeyMultipartUpload = OmMultipartUpload.from(lastKey);
2005-
resultBuilder.setNextKeyMarker(lastKeyMultipartUpload.getKeyName())
2006-
.setNextUploadIdMarker(lastKeyMultipartUpload.getUploadId())
2007-
.setIsTruncated(true);
2008-
2009-
// keep the [0, maxUploads] keys
2010-
responseKeys = responseKeys.subSet(responseKeys.first(), lastKey);
2002+
result.add(multipartUpload);
20112003
}
20122004

2013-
return resultBuilder
2014-
.setKeys(responseKeys)
2015-
.build();
2005+
return noPagination || result.size() <= maxUploads ? result : result.subList(0, maxUploads + 1);
20162006
}
20172007

20182008
@Override

0 commit comments

Comments
 (0)