diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java index a6b979fcbf1a7..3c162a0522f00 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java @@ -52,7 +52,7 @@ public static class Config { * {@value #NUM_INSTANTS_PER_FETCH} allows the max number of instants whose changes can be incrementally fetched. */ static final String NUM_INSTANTS_PER_FETCH = "hoodie.deltastreamer.source.hoodieincr.num_instants"; - static final Integer DEFAULT_NUM_INSTANTS_PER_FETCH = 1; + static final Integer DEFAULT_NUM_INSTANTS_PER_FETCH = 5; /** * {@value #HOODIE_SRC_PARTITION_FIELDS} specifies partition fields that needs to be added to source table after diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java index ec163aec33b9a..8218bcd00e82f 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java @@ -2101,6 +2101,7 @@ public void testHoodieIncrFallback() throws Exception { HoodieDeltaStreamer.Config downstreamCfg = TestHelpers.makeConfigForHudiIncrSrc(tableBasePath, downstreamTableBasePath, WriteOperationType.BULK_INSERT, true, null); + downstreamCfg.configs.add("hoodie.deltastreamer.source.hoodieincr.num_instants=1"); new HoodieDeltaStreamer(downstreamCfg, jsc).sync(); insertInTable(tableBasePath, 9, WriteOperationType.UPSERT); @@ -2112,6 +2113,8 @@ public void testHoodieIncrFallback() throws Exception { downstreamCfg.configs = new ArrayList<>(); } + // Remove source.hoodieincr.num_instants config + downstreamCfg.configs.remove(downstreamCfg.configs.size() - 1); downstreamCfg.configs.add(DataSourceReadOptions.INCREMENTAL_FALLBACK_TO_FULL_TABLE_SCAN_FOR_NON_EXISTING_FILES().key() + "=true"); //Adding this conf to make testing easier :) downstreamCfg.configs.add("hoodie.deltastreamer.source.hoodieincr.num_instants=10");