Fix test due to encoding slashes

Author: yihua

docker/demo/sparksql-bootstrap-prep-source.commands

@@ -18,5 +18,7 @@
 import org.apache.spark.sql.functions.col
 
 val df = spark.read.format("org.apache.hudi").load("/user/hive/warehouse/stock_ticks_cow/*/*/*").drop("_hoodie_commit_time", "_hoodie_record_key", "_hoodie_file_name", "_hoodie_commit_seqno", "_hoodie_partition_path")
-df.write.format("parquet").save("/user/hive/warehouse/stock_ticks_cow_bs_src/2018/08/31/")
+// TODO(HUDI-4944): fix the test to use a partition column with slashes (`/`) included
+//  in the value.  Currently it fails the tests due to slash encoding.
+df.write.format("parquet").partitionBy("symbol").save("/user/hive/warehouse/stock_ticks_cow_bs_src")
 System.exit(0)

hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java

@@ -231,33 +231,35 @@ private void ingestFirstBatchAndHiveSync() throws Exception {
     executeSparkSQLCommand(SPARKSQL_BS_PREP_COMMANDS, true);
     List<String> bootstrapCmds = CollectionUtils.createImmutableList(
         "spark-submit --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " + HUDI_UTILITIES_BUNDLE
-        + " --table-type COPY_ON_WRITE "
-        + " --run-bootstrap "
-        + " --source-class org.apache.hudi.utilities.sources.JsonDFSSource --source-ordering-field ts "
-        + " --target-base-path " + COW_BOOTSTRAPPED_BASE_PATH + " --target-table " + COW_BOOTSTRAPPED_TABLE_NAME
-        + " --props /var/demo/config/dfs-source.properties"
-        + " --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider "
-        + " --initial-checkpoint-provider"
-        + " org.apache.hudi.utilities.checkpointing.InitialCheckpointFromAnotherHoodieTimelineProvider"
-        + " --hoodie-conf hoodie.bootstrap.base.path=" + BOOTSTRAPPED_SRC_PATH
-        + " --hoodie-conf hoodie.deltastreamer.checkpoint.provider.path=" + COW_BASE_PATH
-        + " --hoodie-conf hoodie.bootstrap.parallelism=2 "
-        + " --hoodie-conf hoodie.bootstrap.keygen.class=" + SimpleKeyGenerator.class.getName()
-        + String.format(HIVE_SYNC_CMD_FMT, "dt", COW_BOOTSTRAPPED_TABLE_NAME),
+            + " --table-type COPY_ON_WRITE "
+            + " --run-bootstrap "
+            + " --source-class org.apache.hudi.utilities.sources.JsonDFSSource --source-ordering-field ts "
+            + " --target-base-path " + COW_BOOTSTRAPPED_BASE_PATH + " --target-table " + COW_BOOTSTRAPPED_TABLE_NAME
+            + " --props /var/demo/config/dfs-source.properties"
+            + " --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider "
+            + " --initial-checkpoint-provider"
+            + " org.apache.hudi.utilities.checkpointing.InitialCheckpointFromAnotherHoodieTimelineProvider"
+            + " --hoodie-conf hoodie.datasource.write.partitionpath.field=symbol"
+            + " --hoodie-conf hoodie.bootstrap.base.path=" + BOOTSTRAPPED_SRC_PATH
+            + " --hoodie-conf hoodie.deltastreamer.checkpoint.provider.path=" + COW_BASE_PATH
+            + " --hoodie-conf hoodie.bootstrap.parallelism=2 "
+            + " --hoodie-conf hoodie.bootstrap.keygen.class=" + SimpleKeyGenerator.class.getName()
+            + String.format(HIVE_SYNC_CMD_FMT, "dt", COW_BOOTSTRAPPED_TABLE_NAME),
         "spark-submit --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " + HUDI_UTILITIES_BUNDLE
-        + " --table-type MERGE_ON_READ "
-        + " --run-bootstrap "
-        + " --source-class org.apache.hudi.utilities.sources.JsonDFSSource --source-ordering-field ts "
-        + " --target-base-path " + MOR_BOOTSTRAPPED_BASE_PATH + " --target-table " + MOR_BOOTSTRAPPED_TABLE_NAME
-        + " --props /var/demo/config/dfs-source.properties"
-        + " --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider "
-        + " --initial-checkpoint-provider"
-        + " org.apache.hudi.utilities.checkpointing.InitialCheckpointFromAnotherHoodieTimelineProvider"
-        + " --hoodie-conf hoodie.bootstrap.base.path=" + BOOTSTRAPPED_SRC_PATH
-        + " --hoodie-conf hoodie.deltastreamer.checkpoint.provider.path=" + COW_BASE_PATH
-        + " --hoodie-conf hoodie.bootstrap.parallelism=2 "
-        + " --hoodie-conf hoodie.bootstrap.keygen.class=" + SimpleKeyGenerator.class.getName()
-        + String.format(HIVE_SYNC_CMD_FMT, "dt", MOR_BOOTSTRAPPED_TABLE_NAME));
+            + " --table-type MERGE_ON_READ "
+            + " --run-bootstrap "
+            + " --source-class org.apache.hudi.utilities.sources.JsonDFSSource --source-ordering-field ts "
+            + " --target-base-path " + MOR_BOOTSTRAPPED_BASE_PATH + " --target-table " + MOR_BOOTSTRAPPED_TABLE_NAME
+            + " --props /var/demo/config/dfs-source.properties"
+            + " --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider "
+            + " --initial-checkpoint-provider"
+            + " org.apache.hudi.utilities.checkpointing.InitialCheckpointFromAnotherHoodieTimelineProvider"
+            + " --hoodie-conf hoodie.datasource.write.partitionpath.field=symbol"
+            + " --hoodie-conf hoodie.bootstrap.base.path=" + BOOTSTRAPPED_SRC_PATH
+            + " --hoodie-conf hoodie.deltastreamer.checkpoint.provider.path=" + COW_BASE_PATH
+            + " --hoodie-conf hoodie.bootstrap.parallelism=2 "
+            + " --hoodie-conf hoodie.bootstrap.keygen.class=" + SimpleKeyGenerator.class.getName()
+            + String.format(HIVE_SYNC_CMD_FMT, "dt", MOR_BOOTSTRAPPED_TABLE_NAME));
     executeCommandStringsInDocker(ADHOC_1_CONTAINER, bootstrapCmds);
   }
 

hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java

@@ -56,7 +56,6 @@
 import org.apache.hudi.exception.TableNotFoundException;
 import org.apache.hudi.hive.HiveSyncConfig;
 import org.apache.hudi.hive.HoodieHiveSyncClient;
-import org.apache.hudi.keygen.NonpartitionedKeyGenerator;
 import org.apache.hudi.keygen.SimpleKeyGenerator;
 import org.apache.hudi.metrics.Metrics;
 import org.apache.hudi.utilities.DummySchemaProvider;
@@ -640,12 +639,15 @@ public void testBulkInsertsAndUpsertsWithBootstrap() throws Exception {
     Dataset<Row> sourceDf = sqlContext.read()
         .format("org.apache.hudi")
         .load(tableBasePath);
-    sourceDf.write().format("parquet").save(bootstrapSourcePath);
+    // TODO(HUDI-4944): fix the test to use a partition column with slashes (`/`) included
+    //  in the value.  Currently it fails the tests due to slash encoding.
+    sourceDf.write().format("parquet").partitionBy("rider").save(bootstrapSourcePath);
 
     String newDatasetBasePath = dfsBasePath + "/test_dataset_bootstrapped";
     cfg.runBootstrap = true;
     cfg.configs.add(String.format("hoodie.bootstrap.base.path=%s", bootstrapSourcePath));
-    cfg.configs.add(String.format("hoodie.bootstrap.keygen.class=%s", NonpartitionedKeyGenerator.class.getName()));
+    cfg.configs.add(String.format("%s=%s", DataSourceWriteOptions.PARTITIONPATH_FIELD().key(), "rider"));
+    cfg.configs.add(String.format("hoodie.bootstrap.keygen.class=%s", SimpleKeyGenerator.class.getName()));
     cfg.configs.add("hoodie.bootstrap.parallelism=5");
     cfg.targetBasePath = newDatasetBasePath;
     new HoodieDeltaStreamer(cfg, jsc).sync();