Skip to content

[SPARK-14435][BUILD] Shade Kryo in our custom Hive 1.2.1 fork #12215

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
JoshRosen wants to merge 6 commits into from
Closed

[SPARK-14435][BUILD] Shade Kryo in our custom Hive 1.2.1 fork #12215

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
8f74c74
[SPARK-14435] Shade Kryo in our custom Hive 1.2.1 fork
JoshRosen Apr 6, 2016
50e8779
Add staging repository for new Hive release.
JoshRosen Apr 7, 2016
cdf266b
Also add staging repo to SBT.
JoshRosen Apr 7, 2016
e2237bd
Update ClasspathDependenciesSuite tests.
JoshRosen Apr 8, 2016
88ab812
Revert "Also add staging repo to SBT."
JoshRosen Apr 8, 2016
30eb58d
Revert "Add staging repository for new Hive release."
JoshRosen Apr 8, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@
<curator.version>2.4.0</curator.version>
<hive.group>org.spark-project.hive</hive.group>
<!-- Version used in Maven Hive dependency -->
<hive.version>1.2.1.spark</hive.version>
<hive.version>1.2.1.spark2</hive.version>
<!-- Version used for internal directory structure -->
<hive.version.short>1.2.1</hive.version.short>
<derby.version>10.10.1.1</derby.version>
Expand Down
4 changes: 2 additions & 2 deletions sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,6 @@ import scala.collection.JavaConverters._
import scala.language.implicitConversions
import scala.reflect.ClassTag

import com.esotericsoftware.kryo.Kryo
import com.esotericsoftware.kryo.io.{Input, Output}
import com.google.common.base.Objects
import org.apache.avro.Schema
import org.apache.hadoop.conf.Configuration
Expand All @@ -37,6 +35,8 @@ import org.apache.hadoop.hive.serde2.ColumnProjectionUtils
import org.apache.hadoop.hive.serde2.avro.{AvroGenericRecordWritable, AvroSerdeUtils}
import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector
import org.apache.hadoop.io.Writable
import org.apache.hive.com.esotericsoftware.kryo.Kryo
import org.apache.hive.com.esotericsoftware.kryo.io.{Input, Output}

import org.apache.spark.internal.Logging
import org.apache.spark.sql.types.Decimal
Expand Down
41 changes: 9 additions & 32 deletions sql/hive/src/test/scala/org/apache/spark/sql/hive/ClasspathDependenciesSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,7 @@ import org.apache.spark.SparkFunSuite
/**
* Verify that some classes load and that others are not found on the classpath.
*
*
* This is used to detect classpath and shading conflict, especially between
* Spark's required Kryo version and that which can be found in some Hive versions.
* This is used to detect classpath and shading conflicts.
*/
class ClasspathDependenciesSuite extends SparkFunSuite {
private val classloader = this.getClass.getClassLoader
Expand All @@ -40,10 +38,6 @@ class ClasspathDependenciesSuite extends SparkFunSuite {
classloader.loadClass(classname)
}

private def assertLoads(classes: String*): Unit = {
classes.foreach(assertLoads)
}

private def findResource(classname: String): URL = {
val resource = resourceName(classname)
classloader.getResource(resource)
Expand All @@ -63,17 +57,12 @@ class ClasspathDependenciesSuite extends SparkFunSuite {
}
}

private def assertClassNotFound(classes: String*): Unit = {
classes.foreach(assertClassNotFound)
test("shaded Protobuf") {
assertLoads("org.apache.hive.com.google.protobuf.ServiceException")
}

private val KRYO = "com.esotericsoftware.kryo.Kryo"

private val SPARK_HIVE = "org.apache.hive."
private val SPARK_SHADED = "org.spark-project.hive.shaded."

test("shaded Protobuf") {
assertLoads(SPARK_SHADED + "com.google.protobuf.ServiceException")
test("shaded Kryo") {
assertLoads("org.apache.hive.com.esotericsoftware.kryo.Kryo")
}

test("hive-common") {
Expand All @@ -86,25 +75,13 @@ class ClasspathDependenciesSuite extends SparkFunSuite {

private val STD_INSTANTIATOR = "org.objenesis.strategy.StdInstantiatorStrategy"

test("unshaded kryo") {
assertLoads(KRYO, STD_INSTANTIATOR)
}

test("Forbidden Dependencies") {
assertClassNotFound(
SPARK_HIVE + KRYO,
SPARK_SHADED + KRYO,
"org.apache.hive." + KRYO,
"com.esotericsoftware.shaded." + STD_INSTANTIATOR,
SPARK_HIVE + "com.esotericsoftware.shaded." + STD_INSTANTIATOR,
"org.apache.hive.com.esotericsoftware.shaded." + STD_INSTANTIATOR
)
assertClassNotFound("com.esotericsoftware.shaded." + STD_INSTANTIATOR)
assertClassNotFound("org.apache.hive.com.esotericsoftware.shaded." + STD_INSTANTIATOR)
}

test("parquet-hadoop-bundle") {
assertLoads(
"parquet.hadoop.ParquetOutputFormat",
"parquet.hadoop.ParquetInputFormat"
)
assertLoads("parquet.hadoop.ParquetOutputFormat")
assertLoads("parquet.hadoop.ParquetInputFormat")
}
}