Skip to content

Commit 40e7639

Browse files
author
Alexey Kudinkin
committed
Fixing tests w/ invalid configuration
1 parent 6c7cf0d commit 40e7639

9 files changed

Lines changed: 70 additions & 24 deletions

File tree

‎hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.apache.hudi.common.table.view.TableFileSystemView.BaseFileOnlyView;
3737
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
3838
import org.apache.hudi.common.testutils.Transformations;
39+
import org.apache.hudi.common.util.CollectionUtils;
3940
import org.apache.hudi.common.util.Option;
4041
import org.apache.hudi.config.HoodieClusteringConfig;
4142
import org.apache.hudi.config.HoodieWriteConfig;
@@ -61,7 +62,6 @@
6162
import org.apache.spark.sql.Dataset;
6263
import org.apache.spark.sql.Row;
6364
import org.apache.spark.storage.StorageLevel;
64-
import org.junit.jupiter.api.BeforeEach;
6565
import org.junit.jupiter.api.Test;
6666
import org.junit.jupiter.params.ParameterizedTest;
6767
import org.junit.jupiter.params.provider.ValueSource;
@@ -87,9 +87,8 @@ public class TestHoodieMergeOnReadTable extends SparkClientFunctionalTestHarness
8787
private HoodieTableMetaClient metaClient;
8888
private HoodieTestDataGenerator dataGen;
8989

90-
@BeforeEach
91-
void setUp() throws IOException {
92-
Properties properties = new Properties();
90+
void setUp(Properties props) throws IOException {
91+
Properties properties = CollectionUtils.copy(props);
9392
properties.setProperty(HoodieTableConfig.BASE_FILE_FORMAT.key(), HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().toString());
9493
metaClient = getHoodieMetaClient(HoodieTableType.MERGE_ON_READ, properties);
9594
dataGen = new HoodieTestDataGenerator();
@@ -99,6 +98,9 @@ void setUp() throws IOException {
9998
@Test
10099
public void testMetadataAggregateFromWriteStatus() throws Exception {
101100
HoodieWriteConfig cfg = getConfigBuilder(false).withWriteStatusClass(MetadataMergeWriteStatus.class).build();
101+
102+
setUp(cfg.getProps());
103+
102104
try (SparkRDDWriteClient client = getHoodieWriteClient(cfg);) {
103105

104106
String newCommitTime = "001";
@@ -125,6 +127,9 @@ public void testUpsertPartitioner(boolean populateMetaFields) throws Exception {
125127
HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(true);
126128
addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
127129
HoodieWriteConfig cfg = cfgBuilder.build();
130+
131+
setUp(cfg.getProps());
132+
128133
try (SparkRDDWriteClient client = getHoodieWriteClient(cfg);) {
129134

130135
/**
@@ -213,6 +218,8 @@ public void testLogFileCountsAfterCompaction(boolean preserveCommitMeta) throws
213218
addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
214219
HoodieWriteConfig config = cfgBuilder.build();
215220

221+
setUp(config.getProps());
222+
216223
try (SparkRDDWriteClient writeClient = getHoodieWriteClient(config);) {
217224
String newCommitTime = "100";
218225
writeClient.startCommitWithTime(newCommitTime);
@@ -302,6 +309,8 @@ public void testMetadataStatsOnCommit(Boolean rollbackUsingMarkers) throws Excep
302309
HoodieWriteConfig cfg = getConfigBuilder(false, rollbackUsingMarkers, IndexType.INMEMORY)
303310
.withAutoCommit(false).build();
304311

312+
setUp(cfg.getProps());
313+
305314
try (SparkRDDWriteClient client = getHoodieWriteClient(cfg);) {
306315
HoodieTable table = HoodieSparkTable.create(cfg, context(), metaClient);
307316

@@ -381,6 +390,9 @@ public void testMetadataStatsOnCommit(Boolean rollbackUsingMarkers) throws Excep
381390
@Test
382391
public void testRollingStatsWithSmallFileHandling() throws Exception {
383392
HoodieWriteConfig cfg = getConfigBuilder(false, IndexType.INMEMORY).withAutoCommit(false).build();
393+
394+
setUp(cfg.getProps());
395+
384396
try (SparkRDDWriteClient client = getHoodieWriteClient(cfg);) {
385397
Map<String, Long> fileIdToInsertsMap = new HashMap<>();
386398
Map<String, Long> fileIdToUpsertsMap = new HashMap<>();
@@ -497,6 +509,9 @@ public void testRollingStatsWithSmallFileHandling() throws Exception {
497509
@Test
498510
public void testHandleUpdateWithMultiplePartitions() throws Exception {
499511
HoodieWriteConfig cfg = getConfig(true);
512+
513+
setUp(cfg.getProps());
514+
500515
try (SparkRDDWriteClient client = getHoodieWriteClient(cfg);) {
501516

502517
/**
@@ -578,6 +593,9 @@ public void testReleaseResource() throws Exception {
578593
HoodieWriteConfig.Builder builder = getConfigBuilder(true);
579594
builder.withReleaseResourceEnabled(true);
580595
builder.withAutoCommit(false);
596+
597+
setUp(builder.build().getProps());
598+
581599
/**
582600
* Write 1 (test when RELEASE_RESOURCE_ENABLE is true)
583601
*/

‎hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableCompaction.java‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
import java.nio.file.Paths;
5555
import java.util.Arrays;
5656
import java.util.List;
57+
import java.util.Properties;
5758
import java.util.stream.Collectors;
5859
import java.util.stream.Stream;
5960

@@ -98,8 +99,13 @@ public void testWriteDuringCompaction() throws IOException {
9899
.withLayoutConfig(HoodieLayoutConfig.newBuilder()
99100
.withLayoutType(HoodieStorageLayout.LayoutType.BUCKET.name())
100101
.withLayoutPartitioner(SparkBucketIndexPartitioner.class.getName()).build())
101-
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BUCKET).withBucketNum("1").build()).build();
102-
metaClient = getHoodieMetaClient(HoodieTableType.MERGE_ON_READ, config.getProps());
102+
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BUCKET).withBucketNum("1").build())
103+
.build();
104+
105+
Properties props = getPropertiesForKeyGen(true);
106+
props.putAll(config.getProps());
107+
108+
metaClient = getHoodieMetaClient(HoodieTableType.MERGE_ON_READ, props);
103109
client = getHoodieWriteClient(config);
104110

105111
// write data and commit
@@ -138,8 +144,13 @@ public void testWriteLogDuringCompaction(boolean enableMetadataTable, boolean en
138144
.withLayoutConfig(HoodieLayoutConfig.newBuilder()
139145
.withLayoutType(HoodieStorageLayout.LayoutType.BUCKET.name())
140146
.withLayoutPartitioner(SparkBucketIndexPartitioner.class.getName()).build())
141-
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BUCKET).withBucketNum("1").build()).build();
142-
metaClient = getHoodieMetaClient(HoodieTableType.MERGE_ON_READ, config.getProps());
147+
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BUCKET).withBucketNum("1").build())
148+
.build();
149+
150+
Properties props = getPropertiesForKeyGen(true);
151+
props.putAll(config.getProps());
152+
153+
metaClient = getHoodieMetaClient(HoodieTableType.MERGE_ON_READ, props);
143154
client = getHoodieWriteClient(config);
144155

145156
final List<HoodieRecord> records = dataGen.generateInserts("001", 100);

‎hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableIncrementalRead.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ void setUp() {
8282
public void testIncrementalReadsWithCompaction() throws Exception {
8383
final String partitionPath = "2020/02/20"; // use only one partition for this test
8484
final HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator(new String[] { partitionPath });
85-
Properties props = new Properties();
85+
Properties props = getPropertiesForKeyGen(true);
8686
props.setProperty(HoodieTableConfig.BASE_FILE_FORMAT.key(), HoodieFileFormat.PARQUET.toString());
8787
HoodieTableMetaClient metaClient = getHoodieMetaClient(HoodieTableType.MERGE_ON_READ, props);
8888
HoodieWriteConfig cfg = getConfigBuilder(true).build();

‎hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableRollback.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import org.apache.hudi.common.table.view.TableFileSystemView;
4242
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
4343
import org.apache.hudi.common.testutils.HoodieTestTable;
44+
import org.apache.hudi.common.util.CollectionUtils;
4445
import org.apache.hudi.common.util.Option;
4546
import org.apache.hudi.common.util.collection.Pair;
4647
import org.apache.hudi.config.HoodieCleanConfig;
@@ -155,7 +156,7 @@ void testRollbackWithDeltaAndCompactionCommit(boolean rollbackUsingMarkers) thro
155156
addConfigsForPopulateMetaFields(cfgBuilder, true);
156157
HoodieWriteConfig cfg = cfgBuilder.build();
157158

158-
Properties properties = new Properties();
159+
Properties properties = CollectionUtils.copy(cfg.getProps());
159160
properties.setProperty(HoodieTableConfig.BASE_FILE_FORMAT.key(), HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().toString());
160161
HoodieTableMetaClient metaClient = getHoodieMetaClient(HoodieTableType.MERGE_ON_READ, properties);
161162

@@ -327,7 +328,7 @@ void testMultiRollbackWithDeltaAndCompactionCommit() throws Exception {
327328
addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
328329
HoodieWriteConfig cfg = cfgBuilder.build();
329330

330-
Properties properties = populateMetaFields ? new Properties() : getPropertiesForKeyGen();
331+
Properties properties = getPropertiesForKeyGen(populateMetaFields);
331332
properties.setProperty(HoodieTableConfig.BASE_FILE_FORMAT.key(), HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().toString());
332333
HoodieTableMetaClient metaClient = getHoodieMetaClient(HoodieTableType.MERGE_ON_READ, properties);
333334

@@ -606,8 +607,10 @@ void testMORTableRestore(boolean restoreAfterCompaction) throws Exception {
606607
.withMarkersType(MarkerType.DIRECT.name());
607608
HoodieWriteConfig cfg = cfgBuilder.build();
608609

609-
Properties properties = new Properties();
610+
Properties properties = getPropertiesForKeyGen(true);
611+
properties.putAll(cfg.getProps());
610612
properties.setProperty(HoodieTableConfig.BASE_FILE_FORMAT.key(), HoodieTableConfig.BASE_FILE_FORMAT.defaultValue().toString());
613+
611614
HoodieTableMetaClient metaClient = getHoodieMetaClient(HoodieTableType.MERGE_ON_READ, properties);
612615

613616
try (final SparkRDDWriteClient client = getHoodieWriteClient(cfg)) {

‎hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestHarness.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,7 @@ protected Properties getPropertiesForKeyGen() {
346346

347347
protected Properties getPropertiesForKeyGen(boolean populateMetaFields) {
348348
Properties properties = new Properties();
349-
properties.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), populateMetaFields);
349+
properties.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), String.valueOf(populateMetaFields));
350350
properties.put("hoodie.datasource.write.recordkey.field", "_row_key");
351351
properties.put("hoodie.datasource.write.partitionpath.field", "partition_path");
352352
properties.put(HoodieTableConfig.RECORDKEY_FIELDS.key(), "_row_key");

‎hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/SparkClientFunctionalTestHarness.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ public HoodieTableMetaClient getHoodieMetaClient(Configuration hadoopConf, Strin
160160
}
161161

162162
public HoodieTableMetaClient getHoodieMetaClient(Configuration hadoopConf, String basePath) throws IOException {
163-
return getHoodieMetaClient(hadoopConf, basePath, new Properties());
163+
return getHoodieMetaClient(hadoopConf, basePath, getPropertiesForKeyGen(true));
164164
}
165165

166166
@Override
@@ -310,8 +310,12 @@ protected FileStatus[] listAllBaseFilesInPath(HoodieTable table) throws IOExcept
310310
}
311311

312312
protected Properties getPropertiesForKeyGen() {
313+
return getPropertiesForKeyGen(false);
314+
}
315+
316+
protected Properties getPropertiesForKeyGen(boolean populateMetaFields) {
313317
Properties properties = new Properties();
314-
properties.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false");
318+
properties.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), String.valueOf(populateMetaFields));
315319
properties.put("hoodie.datasource.write.recordkey.field", "_row_key");
316320
properties.put("hoodie.datasource.write.partitionpath.field", "partition_path");
317321
properties.put(HoodieTableConfig.RECORDKEY_FIELDS.key(), "_row_key");
@@ -321,9 +325,9 @@ protected Properties getPropertiesForKeyGen() {
321325
}
322326

323327
protected void addConfigsForPopulateMetaFields(HoodieWriteConfig.Builder configBuilder, boolean populateMetaFields) {
328+
configBuilder.withProperties(getPropertiesForKeyGen(populateMetaFields));
324329
if (!populateMetaFields) {
325-
configBuilder.withProperties(getPropertiesForKeyGen())
326-
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.SIMPLE).build());
330+
configBuilder.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.SIMPLE).build());
327331
}
328332
}
329333

‎hudi-common/src/main/java/org/apache/hudi/common/util/CollectionUtils.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,15 @@ public static boolean nonEmpty(Collection<?> c) {
5454
return !isNullOrEmpty(c);
5555
}
5656

57+
/**
58+
* Makes a copy of provided {@link Properties} object
59+
*/
60+
public static Properties copy(Properties props) {
61+
Properties copy = new Properties();
62+
copy.putAll(props);
63+
return copy;
64+
}
65+
5766
/**
5867
* Returns last element of the array of {@code T}
5968
*/

‎hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -127,11 +127,12 @@ public static HoodieTableMetaClient init(Configuration hadoopConf, String basePa
127127
Properties properties)
128128
throws IOException {
129129
properties = HoodieTableMetaClient.withPropertyBuilder()
130-
.setTableName(RAW_TRIPS_TEST_NAME)
131-
.setTableType(tableType)
132-
.setPayloadClass(HoodieAvroPayload.class)
133-
.fromProperties(properties)
134-
.build();
130+
.setTableName(RAW_TRIPS_TEST_NAME)
131+
.setTableType(tableType)
132+
.setPayloadClass(HoodieAvroPayload.class)
133+
.setPartitionFields("some_nonexistent_field")
134+
.fromProperties(properties)
135+
.build();
135136
return HoodieTableMetaClient.initTableAndGetMetaClient(hadoopConf, basePath, properties);
136137
}
137138

‎hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestHoodieIncrSource.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,9 +114,9 @@ private void readAndAssert(IncrSourceHelper.MissingCheckpointStrategy missingChe
114114
if (expectedCount == 0) {
115115
assertFalse(batchCheckPoint.getKey().isPresent());
116116
} else {
117-
assertEquals(batchCheckPoint.getKey().get().count(), expectedCount);
117+
assertEquals(expectedCount, batchCheckPoint.getKey().get().count());
118118
}
119-
Assertions.assertEquals(batchCheckPoint.getRight(), expectedCheckpoint);
119+
Assertions.assertEquals(expectedCheckpoint, batchCheckPoint.getRight());
120120
}
121121

122122
private Pair<String, List<HoodieRecord>> writeRecords(SparkRDDWriteClient writeClient, boolean insert, List<HoodieRecord> insertRecords, String commit) throws IOException {

0 commit comments

Comments
 (0)