Skip to content

[SPARK-28685][SQL][TEST] Test HMS 2.0.0+ in VersionsSuite/HiveClientSuites on JDK 11 #25405

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wangyum wants to merge 1 commit into from
Closed

[SPARK-28685][SQL][TEST] Test HMS 2.0.0+ in VersionsSuite/HiveClientSuites on JDK 11 #25405

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientVersions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,13 @@ package org.apache.spark.sql.hive.client

import scala.collection.immutable.IndexedSeq

import org.apache.spark.SparkFunSuite
import org.apache.commons.lang3.{JavaVersion, SystemUtils}

private[client] trait HiveClientVersions {
protected val versions =
protected val versions = if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9)) {
IndexedSeq("2.0", "2.1", "2.2", "2.3", "3.0", "3.1")
} else {
IndexedSeq("0.12", "0.13", "0.14", "1.0", "1.1", "1.2", "2.0", "2.1", "2.2", "2.3", "3.0",
"3.1")
}
}
12 changes: 9 additions & 3 deletions sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package org.apache.spark.sql.hive.client
import java.io.{ByteArrayOutputStream, File, PrintStream, PrintWriter}
import java.net.URI

import org.apache.commons.lang3.{JavaVersion, SystemUtils}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hive.common.StatsSetupConst
import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
Expand Down Expand Up @@ -102,8 +103,11 @@ class VersionsSuite extends SparkFunSuite with Logging {
assert(getNestedMessages(e) contains "Unknown column 'A0.OWNER_NAME' in 'field list'")
}

private val versions =
private val versions = if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9)) {
Seq("2.0", "2.1", "2.2", "2.3", "3.0", "3.1")
} else {
Seq("0.12", "0.13", "0.14", "1.0", "1.1", "1.2", "2.0", "2.1", "2.2", "2.3", "3.0", "3.1")
}

private var client: HiveClient = null

Expand Down Expand Up @@ -323,7 +327,8 @@ class VersionsSuite extends SparkFunSuite with Logging {
}

test(s"$version: dropTable") {
val versionsWithoutPurge = versions.takeWhile(_ != "0.14")
val versionsWithoutPurge =
if (versions.contains("0.14")) versions.takeWhile(_ != "0.14") else Nil
// First try with the purge option set. This should fail if the version is < 0.14, in which
// case we check the version and try without it.
try {
Expand Down Expand Up @@ -478,7 +483,8 @@ class VersionsSuite extends SparkFunSuite with Logging {

test(s"$version: dropPartitions") {
val spec = Map("key1" -> "1", "key2" -> "3")
val versionsWithoutPurge = versions.takeWhile(_ != "1.2")
val versionsWithoutPurge =
if (versions.contains("1.2")) versions.takeWhile(_ != "1.2") else Nil
// Similar to dropTable; try with purge set, and if it fails, make sure we're running
// with a version that is older than the minimum (1.2 in this case).
try {
Expand Down