Skip to content

Commit d0636da

Browse files
Merge branch 'fix-clean-command' into 'release-0.11.0' (merge request !56)
fix RunCleanProcedure's ArrayIndexOutOfBoundsException
2 parents 2fa06c4 + 8bc3048 commit d0636da

1 file changed

Lines changed: 14 additions & 9 deletions

File tree

  • hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures

hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunCleanProcedure.scala

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.spark.sql.hudi.command.procedures
1919

20+
import java.util.function.Supplier
2021
import org.apache.hudi.HoodieCLIUtils
2122
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline
2223
import org.apache.hudi.common.util.JsonUtils
@@ -25,7 +26,7 @@ import org.apache.spark.internal.Logging
2526
import org.apache.spark.sql.Row
2627
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
2728

28-
import java.util.function.Supplier
29+
import java.util
2930

3031
class RunCleanProcedure extends BaseProcedure with ProcedureBuilder with Logging {
3132

@@ -74,16 +75,20 @@ class RunCleanProcedure extends BaseProcedure with ProcedureBuilder with Logging
7475
if (cleanPolicy.isDefined) {
7576
props += (HoodieCleanConfig.CLEANER_POLICY.key() -> String.valueOf(cleanPolicy.get))
7677
}
77-
val client = HoodieCLIUtils.createHoodieClientFromPath(sparkSession, basePath, props, tableName)
78+
val client = HoodieCLIUtils.createHoodieClientFromPath(sparkSession, basePath, props)
7879
val hoodieCleanMeta = client.clean(cleanInstantTime, scheduleInLine, skipLocking)
80+
val rows = new util.ArrayList[Row]
81+
82+
if (hoodieCleanMeta != null) {
83+
rows.add(Row(hoodieCleanMeta.getStartCleanTime,
84+
hoodieCleanMeta.getTimeTakenInMillis,
85+
hoodieCleanMeta.getTotalFilesDeleted,
86+
hoodieCleanMeta.getEarliestCommitToRetain,
87+
JsonUtils.getObjectMapper.writeValueAsString(hoodieCleanMeta.getBootstrapPartitionMetadata),
88+
hoodieCleanMeta.getVersion))
89+
}
7990

80-
if (hoodieCleanMeta == null) Seq(Row.empty)
81-
else Seq(Row(hoodieCleanMeta.getStartCleanTime,
82-
hoodieCleanMeta.getTimeTakenInMillis,
83-
hoodieCleanMeta.getTotalFilesDeleted,
84-
hoodieCleanMeta.getEarliestCommitToRetain,
85-
JsonUtils.getObjectMapper.writeValueAsString(hoodieCleanMeta.getBootstrapPartitionMetadata),
86-
hoodieCleanMeta.getVersion))
91+
rows.stream().toArray().map(r => r.asInstanceOf[Row]).toList
8792
}
8893
}
8994

0 commit comments

Comments
 (0)