Skip to content

Commit f524c4f

Browse files
YannByronfengjian
authored andcommitted
[HUDI-5082] Improve the cdc log file name format (apache#7042)
1 parent aa862fe commit f524c4f

3 files changed

Lines changed: 5 additions & 4 deletions

File tree

hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ public class FSUtils {
8080
// Log files are of this pattern - .b5068208-e1a4-11e6-bf01-fe55135034f3_20170101134598.log.1_1-0-1
8181
// Archive log files are of this pattern - .commits_.archive.1_1-0-1
8282
private static final Pattern LOG_FILE_PATTERN =
83-
Pattern.compile("\\.(.+)_(.*)\\.(.+)\\.(\\d+)(_((\\d+)-(\\d+)-(\\d+))(-cdc)?)?");
83+
Pattern.compile("\\.(.+)_(.*)\\.(.+)\\.(\\d+)(_((\\d+)-(\\d+)-(\\d+))(.cdc)?)?");
8484
private static final String LOG_FILE_PREFIX = ".";
8585
private static final int MAX_ATTEMPTS_RECOVER_LEASE = 10;
8686
private static final long MIN_CLEAN_TO_KEEP = 10;

hudi-common/src/main/java/org/apache/hudi/common/table/cdc/HoodieCDCUtils.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
*/
3535
public class HoodieCDCUtils {
3636

37-
public static final String CDC_LOGFILE_SUFFIX = "-cdc";
37+
public static final String CDC_LOGFILE_SUFFIX = ".cdc";
3838

3939
/* the `op` column represents how a record is changed. */
4040
public static final String CDC_OPERATION_TYPE = "op";

hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/HoodieCDCTestBase.scala

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import org.apache.hudi.DataSourceReadOptions._
2121
import org.apache.hudi.DataSourceWriteOptions._
2222
import org.apache.hudi.common.config.HoodieMetadataConfig
2323
import org.apache.hudi.common.model.{HoodieCommitMetadata, HoodieKey, HoodieLogFile, HoodieRecord}
24-
import org.apache.hudi.common.table.cdc.HoodieCDCOperation
24+
import org.apache.hudi.common.table.cdc.{HoodieCDCOperation, HoodieCDCUtils}
2525
import org.apache.hudi.common.table.HoodieTableConfig
2626
import org.apache.hudi.common.table.log.HoodieLogFormat
2727
import org.apache.hudi.common.table.log.block.HoodieDataBlock
@@ -101,7 +101,8 @@ abstract class HoodieCDCTestBase extends HoodieClientTestBase {
101101
val hoodieWriteStats = commitMetadata.getWriteStats.asScala
102102
hoodieWriteStats.exists { hoodieWriteStat =>
103103
val cdcPaths = hoodieWriteStat.getCdcStats
104-
cdcPaths != null && cdcPaths.nonEmpty
104+
cdcPaths != null && cdcPaths.nonEmpty &&
105+
cdcPaths.keySet().asScala.forall(_.endsWith(HoodieCDCUtils.CDC_LOGFILE_SUFFIX))
105106
}
106107
}
107108

0 commit comments

Comments
 (0)