adding a new config to assist in bypassing the default partition validation

Author: nsivabalan

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java

@@ -480,6 +480,13 @@ public class HoodieWriteConfig extends HoodieConfig {
       .sinceVersion("0.11.0")
       .withDocumentation("Auto adjust lock configurations when metadata table is enabled and for async table services.");
 
+  public static final ConfigProperty<Boolean> SKIP_DEFAULT_PARTITION_VALIDATION = ConfigProperty
+      .key("hoodie.skip.default.partition.validation")
+      .defaultValue(false)
+      .sinceVersion("0.12.0")
+      .withDocumentation("When table is upgraded from pre 0.12 to 0.12, we check for \"default\" partition and fail if found one. " +
+          "Users are expected to rewrite the data in those partitions. Enabling this config will bypass this validation");
+
   private ConsistencyGuardConfig consistencyGuardConfig;
   private FileSystemRetryConfig fileSystemRetryConfig;
 
@@ -2038,6 +2045,11 @@ public WriteConcurrencyMode getWriteConcurrencyMode() {
     return WriteConcurrencyMode.fromValue(getString(WRITE_CONCURRENCY_MODE));
   }
 
+  // misc configs
+  public Boolean doSkipDefaultPartitionValidation() {
+    return getBoolean(SKIP_DEFAULT_PARTITION_VALIDATION);
+  }
+
   /**
    * Are any table services configured to run inline for both scheduling and execution?
    *
@@ -2517,6 +2529,11 @@ public Builder withAutoAdjustLockConfigs(boolean autoAdjustLockConfigs) {
       return this;
     }
 
+    public Builder doSkipDefaultPartitionValidation(boolean skipDefaultPartitionValidation) {
+      writeConfig.setValue(SKIP_DEFAULT_PARTITION_VALIDATION, String.valueOf(skipDefaultPartitionValidation));
+      return this;
+    }
+
     protected void setDefaults() {
       writeConfig.setDefaultValue(MARKERS_TYPE, getDefaultMarkersType(engineType));
       // Check for mandatory properties

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/FourToFiveUpgradeHandler.java

@@ -47,7 +47,7 @@ public class FourToFiveUpgradeHandler implements UpgradeHandler {
   public Map<ConfigProperty, String> upgrade(HoodieWriteConfig config, HoodieEngineContext context, String instantTime, SupportsUpgradeDowngrade upgradeDowngradeHelper) {
     try {
       FileSystem fs = new Path(config.getBasePath()).getFileSystem(context.getHadoopConf().get());
-      if (fs.exists(new Path(config.getBasePath() + "/" + DEPRECATED_DEFAULT_PARTITION_PATH))) {
+      if (!config.doSkipDefaultPartitionValidation() && fs.exists(new Path(config.getBasePath() + "/" + DEPRECATED_DEFAULT_PARTITION_PATH))) {
         LOG.error(String.format("\"%s\" partition detected. From 0.12, we are changing the default partition in hudi to %s "
                 + " Please read and write back the data in \"%s\" partition in hudi to new partition path \"%s\". \"\n"
                 + " Sample spark command to use to re-write the data: \n\n"

hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java

@@ -331,15 +331,20 @@ public void testUpgradeDowngradeBetweenThreeAndCurrentVersion() throws IOExcepti
 
   @Test
   public void testUpgradeFourtoFive() throws Exception {
-    testUpgradeFourToFiveInternal(false);
+    testUpgradeFourToFiveInternal(false, false);
   }
 
   @Test
   public void testUpgradeFourtoFiveWithDefaultPartition() throws Exception {
-    testUpgradeFourToFiveInternal(true);
+    testUpgradeFourToFiveInternal(true, false);
   }
 
-  private void testUpgradeFourToFiveInternal(boolean assertDefaultPartition) throws Exception {
+  @Test
+  public void testUpgradeFourtoFiveWithDefaultPartitionWithSkipValidation() throws Exception {
+    testUpgradeFourToFiveInternal(true, true);
+  }
+
+  private void testUpgradeFourToFiveInternal(boolean assertDefaultPartition, boolean skipDefaultPartitionValidation) throws Exception {
     String tableName = metaClient.getTableConfig().getTableName();
     // clean up and re instantiate meta client w/ right table props
     cleanUp();
@@ -354,7 +359,8 @@ private void testUpgradeFourToFiveInternal(boolean assertDefaultPartition) throw
 
     initMetaClient(getTableType(), properties);
     // init config, table and client.
-    HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).withRollbackUsingMarkers(false).withProps(params).build();
+    HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).withRollbackUsingMarkers(false)
+        .doSkipDefaultPartitionValidation(skipDefaultPartitionValidation).withProps(params).build();
     SparkRDDWriteClient client = getHoodieWriteClient(cfg);
     // Write inserts
     doInsert(client);
@@ -367,7 +373,11 @@ private void testUpgradeFourToFiveInternal(boolean assertDefaultPartition) throw
     downgradeTableConfigsFromFiveToFour(cfg);
 
     // perform upgrade
-    if (!assertDefaultPartition) {
+    if (assertDefaultPartition && !skipDefaultPartitionValidation) {
+      // if "default" partition is present, upgrade should fail
+      assertThrows(HoodieException.class, () -> new UpgradeDowngrade(metaClient, cfg, context, SparkUpgradeDowngradeHelper.getInstance())
+          .run(HoodieTableVersion.FIVE, null), "Upgrade from 4 to 5 is expected to fail if \"default\" partition is present.");
+    } else {
       new UpgradeDowngrade(metaClient, cfg, context, SparkUpgradeDowngradeHelper.getInstance())
           .run(HoodieTableVersion.FIVE, null);
 
@@ -378,10 +388,6 @@ private void testUpgradeFourToFiveInternal(boolean assertDefaultPartition) throw
 
       // verify table props
       assertTableProps(cfg);
-    } else {
-      // if "default" partition is present, upgrade should fail
-      assertThrows(HoodieException.class, () -> new UpgradeDowngrade(metaClient, cfg, context, SparkUpgradeDowngradeHelper.getInstance())
-          .run(HoodieTableVersion.FIVE, null), "Upgrade from 4 to 5 is expected to fail if \"default\" partition is present.");
     }
   }