Skip to content

Commit a31755a

Browse files
adoroszlaiadoroszlai
authored
HDDS-12315. Speed up some Freon integration tests (apache#7870)
1 parent d8f3149 commit a31755a

15 files changed

Lines changed: 557 additions & 725 deletions

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_LISTING_PAGE_SIZE;
2121
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
2222
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE;
23+
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
24+
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
2325
import static org.assertj.core.api.Assertions.assertThat;
2426
import static org.junit.jupiter.api.Assertions.assertEquals;
2527

@@ -33,17 +35,30 @@
3335
import org.apache.hadoop.fs.Path;
3436
import org.apache.hadoop.fs.RemoteIterator;
3537
import org.apache.hadoop.fs.contract.ContractTestUtils;
38+
import org.apache.hadoop.hdds.conf.ConfigurationTarget;
3639
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
40+
import org.apache.ratis.util.Preconditions;
3741

3842
/**
3943
* Common test cases for Ozone file systems.
4044
*/
41-
final class OzoneFileSystemTests {
45+
public final class OzoneFileSystemTests {
4246

4347
private OzoneFileSystemTests() {
4448
// no instances
4549
}
4650

51+
/**
52+
* Set file system listing page size. Also disable the file system cache to
53+
* ensure new {@link FileSystem} instance reflects the updated page size.
54+
*/
55+
public static void setPageSize(ConfigurationTarget conf, int pageSize) {
56+
Preconditions.assertTrue(pageSize > 0, () -> "pageSize=" + pageSize + " <= 0");
57+
conf.setInt(OZONE_FS_LISTING_PAGE_SIZE, pageSize);
58+
conf.setBoolean(String.format("fs.%s.impl.disable.cache", OZONE_URI_SCHEME), true);
59+
conf.setBoolean(String.format("fs.%s.impl.disable.cache", OZONE_OFS_URI_SCHEME), true);
60+
}
61+
4762
/**
4863
* Tests listStatusIterator operation on directory with different
4964
* numbers of child directories.
@@ -60,10 +75,8 @@ public static void listStatusIteratorOnPageSize(OzoneConfiguration conf,
6075
pageSize + pageSize
6176
};
6277
OzoneConfiguration config = new OzoneConfiguration(conf);
63-
config.setInt(OZONE_FS_LISTING_PAGE_SIZE, pageSize);
78+
setPageSize(config, pageSize);
6479
URI uri = FileSystem.getDefaultUri(config);
65-
config.setBoolean(
66-
String.format("fs.%s.impl.disable.cache", uri.getScheme()), true);
6780
try (FileSystem subject = FileSystem.get(uri, config)) {
6881
Path dir = new Path(Objects.requireNonNull(rootPath),
6982
"listStatusIterator");

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGenerator.java‎

Lines changed: 57 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -17,148 +17,110 @@
1717

1818
package org.apache.hadoop.ozone.freon;
1919

20+
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
21+
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
2022
import static org.assertj.core.api.Assertions.assertThat;
2123
import static org.junit.jupiter.api.Assertions.assertEquals;
2224

23-
import java.io.File;
24-
import java.io.FileOutputStream;
2525
import java.io.IOException;
2626
import java.net.URI;
2727
import java.util.ArrayList;
2828
import java.util.Arrays;
2929
import java.util.Collections;
3030
import java.util.List;
31-
import org.apache.commons.io.FileUtils;
31+
import java.util.UUID;
3232
import org.apache.hadoop.fs.FileStatus;
3333
import org.apache.hadoop.fs.FileSystem;
3434
import org.apache.hadoop.fs.Path;
35+
import org.apache.hadoop.fs.ozone.OzoneFileSystemTests;
3536
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
3637
import org.apache.hadoop.hdds.conf.StorageSize;
3738
import org.apache.hadoop.hdds.conf.StorageUnit;
3839
import org.apache.hadoop.hdds.utils.IOUtils;
39-
import org.apache.hadoop.ozone.MiniOzoneCluster;
40+
import org.apache.hadoop.ozone.client.BucketArgs;
4041
import org.apache.hadoop.ozone.client.ObjectStore;
4142
import org.apache.hadoop.ozone.client.OzoneClient;
42-
import org.apache.hadoop.ozone.client.OzoneClientFactory;
4343
import org.apache.hadoop.ozone.client.OzoneVolume;
44-
import org.apache.hadoop.ozone.om.OMConfigKeys;
4544
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
46-
import org.apache.ozone.test.GenericTestUtils;
47-
import org.apache.ratis.server.RaftServer;
48-
import org.apache.ratis.server.raftlog.RaftLog;
45+
import org.apache.ozone.test.NonHATests;
46+
import org.junit.jupiter.api.AfterEach;
4947
import org.junit.jupiter.api.BeforeEach;
50-
import org.junit.jupiter.api.Test;
51-
import org.junit.jupiter.api.io.TempDir;
48+
import org.junit.jupiter.params.ParameterizedTest;
49+
import org.junit.jupiter.params.provider.EnumSource;
5250
import org.slf4j.Logger;
5351
import org.slf4j.LoggerFactory;
54-
import org.slf4j.event.Level;
5552

5653
/**
5754
* Test for HadoopDirTreeGenerator.
5855
*/
59-
public class TestHadoopDirTreeGenerator {
60-
@TempDir
61-
private java.nio.file.Path path;
62-
private OzoneConfiguration conf = null;
63-
private MiniOzoneCluster cluster = null;
56+
public abstract class TestHadoopDirTreeGenerator implements NonHATests.TestCase {
57+
58+
private static final int PAGE_SIZE = 10;
59+
6460
private ObjectStore store = null;
6561
private static final Logger LOG =
6662
LoggerFactory.getLogger(TestHadoopDirTreeGenerator.class);
6763
private OzoneClient client;
6864

6965
@BeforeEach
70-
public void setup() {
71-
GenericTestUtils.setLogLevel(RaftLog.LOG, Level.DEBUG);
72-
GenericTestUtils.setLogLevel(RaftServer.LOG, Level.DEBUG);
73-
}
74-
75-
/**
76-
* Shutdown MiniDFSCluster.
77-
*/
78-
private void shutdown() throws IOException {
79-
IOUtils.closeQuietly(client);
80-
if (cluster != null) {
81-
cluster.shutdown();
82-
}
83-
}
84-
85-
/**
86-
* Create a MiniDFSCluster for testing.
87-
*
88-
* @throws IOException
89-
*/
90-
private void startCluster() throws Exception {
91-
conf = getOzoneConfiguration();
92-
conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
93-
BucketLayout.LEGACY.name());
94-
cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build();
95-
cluster.waitForClusterToBeReady();
96-
cluster.waitTobeOutOfSafeMode();
97-
98-
client = OzoneClientFactory.getRpcClient(conf);
66+
void setup() throws Exception {
67+
client = cluster().newClient();
9968
store = client.getObjectStore();
10069
}
10170

102-
protected OzoneConfiguration getOzoneConfiguration() {
103-
return new OzoneConfiguration();
71+
@AfterEach
72+
void cleanup() {
73+
IOUtils.closeQuietly(client);
10474
}
10575

106-
@Test
107-
public void testNestedDirTreeGeneration() throws Exception {
108-
try {
109-
startCluster();
110-
FileOutputStream out = FileUtils.openOutputStream(new File(path.toString(),
111-
"conf"));
112-
cluster.getConf().writeXml(out);
113-
out.getFD().sync();
114-
out.close();
115-
116-
verifyDirTree("vol1", "bucket1", 1,
117-
1, 1, "0");
118-
verifyDirTree("vol2", "bucket1", 1,
119-
5, 1, "5B");
120-
verifyDirTree("vol3", "bucket1", 2,
121-
5, 3, "1B");
122-
verifyDirTree("vol4", "bucket1", 3,
123-
2, 4, "2B");
124-
verifyDirTree("vol5", "bucket1", 5,
125-
4, 1, "0");
126-
// default page size is Constants.LISTING_PAGE_SIZE = 1024
127-
verifyDirTree("vol6", "bucket1", 2,
128-
1, 1100, "0");
129-
} finally {
130-
shutdown();
131-
}
76+
@ParameterizedTest
77+
@EnumSource(names = {"FILE_SYSTEM_OPTIMIZED", "LEGACY"})
78+
public void testNestedDirTreeGeneration(BucketLayout layout) throws Exception {
79+
String uuid = UUID.randomUUID().toString();
80+
verifyDirTree("vol1-" + uuid, "bucket1", 1, 1, 1, "0", layout);
81+
verifyDirTree("vol2-" + uuid, "bucket1", 1, 5, 1, "5B", layout);
82+
verifyDirTree("vol3-" + uuid, "bucket1", 2, 5, 3, "1B", layout);
83+
verifyDirTree("vol4-" + uuid, "bucket1", 3, 2, 4, "2B", layout);
84+
verifyDirTree("vol5-" + uuid, "bucket1", 5, 4, 1, "0", layout);
85+
verifyDirTree("vol6-" + uuid, "bucket1", 2, 1, PAGE_SIZE + PAGE_SIZE / 2, "0", layout);
13286
}
13387

13488
private void verifyDirTree(String volumeName, String bucketName, int depth,
135-
int span, int fileCount, String perFileSize)
89+
int span, int fileCount, String perFileSize, BucketLayout layout)
13690
throws IOException {
13791

13892
store.createVolume(volumeName);
13993
OzoneVolume volume = store.getVolume(volumeName);
140-
volume.createBucket(bucketName);
141-
String rootPath = "o3fs://" + bucketName + "." + volumeName;
142-
String confPath = new File(path.toString(), "conf").getAbsolutePath();
143-
new Freon().execute(
144-
new String[]{"-conf", confPath, "dtsg", "-d", depth + "", "-c",
145-
fileCount + "", "-s", span + "", "-n", "1", "-r", rootPath,
146-
"-g", perFileSize});
94+
volume.createBucket(bucketName, BucketArgs.newBuilder().setBucketLayout(layout).build());
95+
String rootPath = OZONE_URI_SCHEME + "://" + bucketName + "." + volumeName;
96+
String om = cluster().getConf().get(OZONE_OM_ADDRESS_KEY);
97+
new Freon().getCmd().execute(
98+
"-D", OZONE_OM_ADDRESS_KEY + "=" + om,
99+
"dtsg",
100+
"-c", String.valueOf(fileCount),
101+
"-d", String.valueOf(depth),
102+
"-g", perFileSize,
103+
"-n", "1",
104+
"-r", rootPath,
105+
"-s", String.valueOf(span)
106+
);
147107
// verify the directory structure
148108
LOG.info("Started verifying the directory structure...");
149-
FileSystem fileSystem = FileSystem.get(URI.create(rootPath),
150-
conf);
151-
Path rootDir = new Path(rootPath.concat("/"));
152-
// verify root path details
153-
FileStatus[] fileStatuses = fileSystem.listStatus(rootDir);
154-
// verify the num of peer directories, expected span count is 1
155-
// as it has only one dir at root.
156-
verifyActualSpan(1, Arrays.asList(fileStatuses));
157-
for (FileStatus fileStatus : fileStatuses) {
158-
int actualDepth =
159-
traverseToLeaf(fileSystem, fileStatus.getPath(), 1, depth, span,
160-
fileCount, StorageSize.parse(perFileSize, StorageUnit.BYTES));
161-
assertEquals(depth, actualDepth, "Mismatch depth in a path");
109+
OzoneConfiguration conf = new OzoneConfiguration(cluster().getConf());
110+
OzoneFileSystemTests.setPageSize(conf, PAGE_SIZE);
111+
try (FileSystem fileSystem = FileSystem.get(URI.create(rootPath), conf)) {
112+
Path rootDir = new Path(rootPath.concat("/"));
113+
// verify root path details
114+
FileStatus[] fileStatuses = fileSystem.listStatus(rootDir);
115+
// verify the num of peer directories, expected span count is 1
116+
// as it has only one dir at root.
117+
verifyActualSpan(1, Arrays.asList(fileStatuses));
118+
for (FileStatus fileStatus : fileStatuses) {
119+
int actualDepth =
120+
traverseToLeaf(fileSystem, fileStatus.getPath(), 1, depth, span,
121+
fileCount, StorageSize.parse(perFileSize, StorageUnit.BYTES));
122+
assertEquals(depth, actualDepth, "Mismatch depth in a path");
123+
}
162124
}
163125
}
164126

0 commit comments

Comments
 (0)