Skip to content

Commit e0f919b

Browse files
committed
Only transformUp the given plan once.
1 parent a65766b commit e0f919b

1 file changed

Lines changed: 20 additions & 17 deletions

File tree

sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala

Lines changed: 20 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -430,33 +430,36 @@ private[hive] class HiveMetastoreCatalog(hive: HiveContext) extends Catalog with
430430
hive.convertMetastoreParquet &&
431431
hive.conf.parquetUseDataSourceApi &&
432432
relation.tableDesc.getSerdeClassName.toLowerCase.contains("parquet") =>
433-
relation
433+
val parquetRelation = convertToParquetRelation(relation)
434+
val attributedRewrites = relation.output.zip(parquetRelation.output)
435+
(relation, parquetRelation, attributedRewrites)
434436

435437
// Read path
436438
case p @ PhysicalOperation(_, _, relation: MetastoreRelation)
437439
if hive.convertMetastoreParquet &&
438440
hive.conf.parquetUseDataSourceApi &&
439441
relation.tableDesc.getSerdeClassName.toLowerCase.contains("parquet") =>
440-
relation
442+
val parquetRelation = convertToParquetRelation(relation)
443+
val attributedRewrites = relation.output.zip(parquetRelation.output)
444+
(relation, parquetRelation, attributedRewrites)
441445
}
442446

447+
val relationMap = toBeReplaced.map(r => (r._1, r._2)).toMap
448+
val attributedRewrites = AttributeMap(toBeReplaced.map(_._3).fold(Nil)(_ ++: _))
449+
443450
// Replaces all `MetastoreRelation`s with corresponding `ParquetRelation2`s, and fixes
444451
// attribute IDs referenced in other nodes.
445-
toBeReplaced.distinct.foldLeft(plan) { (lastPlan, relation) =>
446-
val parquetRelation = convertToParquetRelation(relation)
447-
val attributedRewrites = AttributeMap(relation.output.zip(parquetRelation.output))
448-
449-
lastPlan.transformUp {
450-
case r: MetastoreRelation if r == relation => {
451-
val withAlias =
452-
r.alias.map(a => Subquery(a, parquetRelation)).getOrElse(
453-
Subquery(r.tableName, parquetRelation))
454-
455-
withAlias
456-
}
457-
case other => other.transformExpressions {
458-
case a: Attribute if a.resolved => attributedRewrites.getOrElse(a, a)
459-
}
452+
plan.transformUp {
453+
case r: MetastoreRelation => {
454+
val parquetRelation = relationMap(r)
455+
val withAlias =
456+
r.alias.map(a => Subquery(a, parquetRelation)).getOrElse(
457+
Subquery(r.tableName, parquetRelation))
458+
459+
withAlias
460+
}
461+
case other => other.transformExpressions {
462+
case a: Attribute if a.resolved => attributedRewrites.getOrElse(a, a)
460463
}
461464
}
462465
}

0 commit comments

Comments
 (0)