Skip to content

Commit 4749922

Browse files
authored
[HUDI-5038] Increase default num_instants to fetch for incremental source (#6955)
1 parent 49b9ba0 commit 4749922

2 files changed

Lines changed: 4 additions & 1 deletion

File tree

hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ public static class Config {
5252
* {@value #NUM_INSTANTS_PER_FETCH} allows the max number of instants whose changes can be incrementally fetched.
5353
*/
5454
static final String NUM_INSTANTS_PER_FETCH = "hoodie.deltastreamer.source.hoodieincr.num_instants";
55-
static final Integer DEFAULT_NUM_INSTANTS_PER_FETCH = 1;
55+
static final Integer DEFAULT_NUM_INSTANTS_PER_FETCH = 5;
5656

5757
/**
5858
* {@value #HOODIE_SRC_PARTITION_FIELDS} specifies partition fields that needs to be added to source table after

hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2101,6 +2101,7 @@ public void testHoodieIncrFallback() throws Exception {
21012101
HoodieDeltaStreamer.Config downstreamCfg =
21022102
TestHelpers.makeConfigForHudiIncrSrc(tableBasePath, downstreamTableBasePath,
21032103
WriteOperationType.BULK_INSERT, true, null);
2104+
downstreamCfg.configs.add("hoodie.deltastreamer.source.hoodieincr.num_instants=1");
21042105
new HoodieDeltaStreamer(downstreamCfg, jsc).sync();
21052106

21062107
insertInTable(tableBasePath, 9, WriteOperationType.UPSERT);
@@ -2112,6 +2113,8 @@ public void testHoodieIncrFallback() throws Exception {
21122113
downstreamCfg.configs = new ArrayList<>();
21132114
}
21142115

2116+
// Remove source.hoodieincr.num_instants config
2117+
downstreamCfg.configs.remove(downstreamCfg.configs.size() - 1);
21152118
downstreamCfg.configs.add(DataSourceReadOptions.INCREMENTAL_FALLBACK_TO_FULL_TABLE_SCAN_FOR_NON_EXISTING_FILES().key() + "=true");
21162119
//Adding this conf to make testing easier :)
21172120
downstreamCfg.configs.add("hoodie.deltastreamer.source.hoodieincr.num_instants=10");

0 commit comments

Comments
 (0)