From 598ca06d38b08a1d751396ad776ed2b10492dd55 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Fri, 30 Mar 2018 03:19:17 +0100 Subject: [PATCH 1/9] jackson bump --- dev/deps/spark-deps-hadoop-palantir | 26 +++++++++++++------------- pom.xml | 9 ++------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-palantir b/dev/deps/spark-deps-hadoop-palantir index c2cf638c0332..e7517c3230c5 100644 --- a/dev/deps/spark-deps-hadoop-palantir +++ b/dev/deps/spark-deps-hadoop-palantir @@ -89,21 +89,21 @@ httpclient-4.5.4.jar httpcore-4.4.8.jar ion-java-1.0.1.jar ivy-2.4.0.jar -jackson-annotations-2.6.7.jar -jackson-core-2.6.7.jar +jackson-annotations-2.9.5.jar +jackson-core-2.9.5.jar jackson-core-asl-1.9.13.jar -jackson-databind-2.6.7.1.jar -jackson-dataformat-cbor-2.6.7.jar -jackson-dataformat-yaml-2.6.7.jar -jackson-datatype-guava-2.6.7.jar -jackson-datatype-jdk7-2.6.7.jar -jackson-datatype-joda-2.6.7.jar +jackson-databind-2.9.5.jar +jackson-dataformat-cbor-2.9.5.jar +jackson-dataformat-yaml-2.9.5.jar +jackson-datatype-guava-2.9.5.jar +jackson-datatype-jdk7-2.6.3.jar +jackson-datatype-joda-2.9.5.jar jackson-jaxrs-1.9.13.jar jackson-mapper-asl-1.9.13.jar -jackson-module-afterburner-2.6.7.jar -jackson-module-jaxb-annotations-2.6.7.jar -jackson-module-paranamer-2.7.9.jar -jackson-module-scala_2.11-2.6.7.1.jar +jackson-module-afterburner-2.9.5.jar +jackson-module-jaxb-annotations-2.9.5.jar +jackson-module-paranamer-2.9.5.jar +jackson-module-scala_2.11-2.9.5.jar jackson-xc-1.9.13.jar janino-3.0.8.jar java-xmlbuilder-1.1.jar @@ -185,7 +185,7 @@ scala-xml_2.11-1.0.5.jar shapeless_2.11-2.3.2.jar slf4j-api-1.7.25.jar slf4j-log4j12-1.7.25.jar -snakeyaml-1.15.jar +snakeyaml-1.18.jar snappy-java-1.1.7.1.jar spire-macros_2.11-0.13.0.jar spire_2.11-0.13.0.jar diff --git a/pom.xml b/pom.xml index b759193e6835..f73d7cb74e6a 100644 --- a/pom.xml +++ b/pom.xml @@ -168,8 +168,8 @@ 2.11.8 2.11 1.9.13 - 2.6.7 - 2.6.7.1 + 2.9.5 + 2.9.5 1.1.7.1 1.1.2 1.2.0-incubating @@ -846,11 +846,6 @@ jackson-datatype-guava ${fasterxml.jackson.version} - - com.fasterxml.jackson.datatype - jackson-datatype-jdk7 - ${fasterxml.jackson.version} - com.fasterxml.jackson.datatype jackson-datatype-joda From d60f6df31a06f7368393ebf7da79827bd06e1cc3 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Sun, 8 Apr 2018 17:18:10 -0700 Subject: [PATCH 2/9] json4s --- dev/deps/spark-deps-hadoop-palantir | 8 ++++---- pom.xml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-palantir b/dev/deps/spark-deps-hadoop-palantir index e7517c3230c5..9e5121ac7761 100644 --- a/dev/deps/spark-deps-hadoop-palantir +++ b/dev/deps/spark-deps-hadoop-palantir @@ -132,10 +132,10 @@ jetty-util-6.1.26.jar jmespath-java-1.0.jar joda-time-2.9.9.jar json-smart-1.1.1.jar -json4s-ast_2.11-3.5.3.jar -json4s-core_2.11-3.5.3.jar -json4s-jackson_2.11-3.5.3.jar -json4s-scalap_2.11-3.5.3.jar +json4s-ast_2.11-3.6.0-M2.jar +json4s-core_2.11-3.6.0-M2.jar +json4s-jackson_2.11-3.6.0-M2.jar +json4s-scalap_2.11-3.6.0-M2.jar jsp-api-2.1.jar jsr305-3.0.1.jar jtransforms-2.4.0.jar diff --git a/pom.xml b/pom.xml index f73d7cb74e6a..065ee0ad3bd0 100644 --- a/pom.xml +++ b/pom.xml @@ -921,7 +921,7 @@ org.json4s json4s-jackson_${scala.binary.version} - 3.5.3 + 3.6.0-M2 com.fasterxml.jackson.core From e59208c6a0e882eaa40fc9bf5dadbd3aff25fa67 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Sun, 8 Apr 2018 20:11:26 -0700 Subject: [PATCH 3/9] empty commit From 20895ed24695618dc40bf4afeff84154bb59d5a7 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Sun, 8 Apr 2018 20:37:36 -0700 Subject: [PATCH 4/9] cache --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4e78897454b3..e261070c9814 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -201,11 +201,11 @@ jobs: <<: *defaults steps: # Saves us from recompiling every time... - - restore_cache: - keys: - - v1-build-sbt-{{ .Branch }}-{{ .Revision }} - - v1-build-sbt-{{ .Branch }}- - - v1-build-sbt-master- + #- restore_cache: + #keys: + #- v1-build-sbt-{{ .Branch }}-{{ .Revision }} + #- v1-build-sbt-{{ .Branch }}- + #- v1-build-sbt-master- - *checkout-code - run: name: Hard link cache contents into current build directory From 3a0c88bb657ce1f6de6a899241918f67599b93c0 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Sun, 8 Apr 2018 22:58:25 -0700 Subject: [PATCH 5/9] annotations --- .../rest/SubmitRestProtocolMessage.scala | 2 +- .../apache/spark/rdd/RDDOperationScope.scala | 2 +- .../status/api/v1/JacksonMessageWriter.scala | 2 +- .../sql/execution/streaming/OffsetSeq.scala | 5 ++-- .../streaming/OffsetSeqLogSuite.scala | 23 +++++++++++-------- 5 files changed, 20 insertions(+), 14 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala b/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala index ef5a7e35ad56..97b689cdadd5 100644 --- a/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala @@ -36,7 +36,7 @@ import org.apache.spark.util.Utils * (2) the Spark version of the client / server * (3) an optional message */ -@JsonInclude(Include.NON_NULL) +@JsonInclude(Include.NON_ABSENT) @JsonAutoDetect(getterVisibility = Visibility.ANY, setterVisibility = Visibility.ANY) @JsonPropertyOrder(alphabetic = true) private[rest] abstract class SubmitRestProtocolMessage { diff --git a/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala b/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala index 53d69ba26811..3abb2d8a11f3 100644 --- a/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala +++ b/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala @@ -41,7 +41,7 @@ import org.apache.spark.internal.Logging * There is no particular relationship between an operation scope and a stage or a job. * A scope may live inside one stage (e.g. map) or span across multiple jobs (e.g. take). */ -@JsonInclude(Include.NON_NULL) +@JsonInclude(Include.NON_ABSENT) @JsonPropertyOrder(Array("id", "name", "parent")) private[spark] class RDDOperationScope( val name: String, diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala b/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala index 76af33c1a18d..61305a0770d9 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala @@ -49,7 +49,7 @@ private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{ } mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule) mapper.enable(SerializationFeature.INDENT_OUTPUT) - mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL) + mapper.setSerializationInclusion(JsonInclude.Include.NON_ABSENT) mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat) override def isWriteable( diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/OffsetSeq.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/OffsetSeq.scala index 73945b39b896..91caa6062cf8 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/OffsetSeq.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/OffsetSeq.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.execution.streaming import org.json4s.NoTypeHints import org.json4s.jackson.Serialization +import org.json4s.reflect.PrimaryConstructor import org.apache.spark.internal.Logging import org.apache.spark.sql.RuntimeConfig @@ -60,7 +61,7 @@ object OffsetSeq { * `nulls` in the sequence are converted to `None`s. */ def fill(metadata: Option[String], offsets: Offset*): OffsetSeq = { - OffsetSeq(offsets.map(Option(_)), metadata.map(OffsetSeqMetadata.apply)) + OffsetSeq(offsets.map(Option(_)), metadata.map(OffsetSeqMetadata.fromJson)) } } @@ -86,7 +87,7 @@ object OffsetSeqMetadata extends Logging { private implicit val format = Serialization.formats(NoTypeHints) private val relevantSQLConfs = Seq(SHUFFLE_PARTITIONS, STATE_STORE_PROVIDER_CLASS) - def apply(json: String): OffsetSeqMetadata = Serialization.read[OffsetSeqMetadata](json) + def fromJson(json: String): OffsetSeqMetadata = Serialization.read[OffsetSeqMetadata](json) def apply( batchWatermarkMs: Long, diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLogSuite.scala index e6cdc063c4e9..d35138b70417 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLogSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLogSuite.scala @@ -19,6 +19,9 @@ package org.apache.spark.sql.execution.streaming import java.io.File +import org.json4s.{JValue, StringInput} +import org.json4s.jackson.JsonMethods + import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.util.stringToFile import org.apache.spark.sql.internal.SQLConf @@ -36,32 +39,34 @@ class OffsetSeqLogSuite extends SparkFunSuite with SharedSQLContext { Map(key -> shufflePartitions.toString) } + val json: JValue = JsonMethods.parse(StringInput("""{}""")) // None set - assert(new OffsetSeqMetadata(0, 0, Map.empty) === OffsetSeqMetadata("""{}""")) + assert(new OffsetSeqMetadata(0, 0, Map.empty) === OffsetSeqMetadata.fromJson("""{}""")) // One set assert(new OffsetSeqMetadata(1, 0, Map.empty) === - OffsetSeqMetadata("""{"batchWatermarkMs":1}""")) + OffsetSeqMetadata.fromJson("""{"batchWatermarkMs":1}""")) assert(new OffsetSeqMetadata(0, 2, Map.empty) === - OffsetSeqMetadata("""{"batchTimestampMs":2}""")) + OffsetSeqMetadata.fromJson("""{"batchTimestampMs":2}""")) assert(OffsetSeqMetadata(0, 0, getConfWith(shufflePartitions = 2)) === - OffsetSeqMetadata(s"""{"conf": {"$key":2}}""")) + OffsetSeqMetadata.fromJson(s"""{"conf": {"$key":2}}""")) // Two set assert(new OffsetSeqMetadata(1, 2, Map.empty) === - OffsetSeqMetadata("""{"batchWatermarkMs":1,"batchTimestampMs":2}""")) + OffsetSeqMetadata.fromJson("""{"batchWatermarkMs":1,"batchTimestampMs":2}""")) assert(OffsetSeqMetadata(1, 0, getConfWith(shufflePartitions = 3)) === - OffsetSeqMetadata(s"""{"batchWatermarkMs":1,"conf": {"$key":3}}""")) + OffsetSeqMetadata.fromJson(s"""{"batchWatermarkMs":1,"conf": {"$key":3}}""")) assert(OffsetSeqMetadata(0, 2, getConfWith(shufflePartitions = 3)) === - OffsetSeqMetadata(s"""{"batchTimestampMs":2,"conf": {"$key":3}}""")) + OffsetSeqMetadata.fromJson(s"""{"batchTimestampMs":2,"conf": {"$key":3}}""")) // All set assert(OffsetSeqMetadata(1, 2, getConfWith(shufflePartitions = 3)) === - OffsetSeqMetadata(s"""{"batchWatermarkMs":1,"batchTimestampMs":2,"conf": {"$key":3}}""")) + OffsetSeqMetadata.fromJson( + s"""{"batchWatermarkMs":1,"batchTimestampMs":2,"conf": {"$key":3}}""")) // Drop unknown fields assert(OffsetSeqMetadata(1, 2, getConfWith(shufflePartitions = 3)) === - OffsetSeqMetadata( + OffsetSeqMetadata.fromJson( s"""{"batchWatermarkMs":1,"batchTimestampMs":2,"conf": {"$key":3}},"unknown":1""")) } From 09759020f8cd9e550037a6cbba4403e82c25e05a Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Sun, 8 Apr 2018 22:59:03 -0700 Subject: [PATCH 6/9] no dummies --- .../spark/sql/execution/streaming/OffsetSeqLogSuite.scala | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLogSuite.scala index d35138b70417..59c7142c0bed 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLogSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLogSuite.scala @@ -19,9 +19,6 @@ package org.apache.spark.sql.execution.streaming import java.io.File -import org.json4s.{JValue, StringInput} -import org.json4s.jackson.JsonMethods - import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.util.stringToFile import org.apache.spark.sql.internal.SQLConf @@ -39,7 +36,6 @@ class OffsetSeqLogSuite extends SparkFunSuite with SharedSQLContext { Map(key -> shufflePartitions.toString) } - val json: JValue = JsonMethods.parse(StringInput("""{}""")) // None set assert(new OffsetSeqMetadata(0, 0, Map.empty) === OffsetSeqMetadata.fromJson("""{}""")) From 39500ec4d72d3dd78b2fece84ff9f01f9ef8274b Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Sun, 15 Apr 2018 21:43:33 -0700 Subject: [PATCH 7/9] no json4s --- .circleci/config.yml | 10 +++++----- dev/deps/spark-deps-hadoop-palantir | 8 ++++---- pom.xml | 2 +- .../sql/execution/streaming/OffsetSeq.scala | 5 ++--- .../streaming/OffsetSeqLogSuite.scala | 19 +++++++++---------- 5 files changed, 21 insertions(+), 23 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e261070c9814..4e78897454b3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -201,11 +201,11 @@ jobs: <<: *defaults steps: # Saves us from recompiling every time... - #- restore_cache: - #keys: - #- v1-build-sbt-{{ .Branch }}-{{ .Revision }} - #- v1-build-sbt-{{ .Branch }}- - #- v1-build-sbt-master- + - restore_cache: + keys: + - v1-build-sbt-{{ .Branch }}-{{ .Revision }} + - v1-build-sbt-{{ .Branch }}- + - v1-build-sbt-master- - *checkout-code - run: name: Hard link cache contents into current build directory diff --git a/dev/deps/spark-deps-hadoop-palantir b/dev/deps/spark-deps-hadoop-palantir index 9e5121ac7761..e7517c3230c5 100644 --- a/dev/deps/spark-deps-hadoop-palantir +++ b/dev/deps/spark-deps-hadoop-palantir @@ -132,10 +132,10 @@ jetty-util-6.1.26.jar jmespath-java-1.0.jar joda-time-2.9.9.jar json-smart-1.1.1.jar -json4s-ast_2.11-3.6.0-M2.jar -json4s-core_2.11-3.6.0-M2.jar -json4s-jackson_2.11-3.6.0-M2.jar -json4s-scalap_2.11-3.6.0-M2.jar +json4s-ast_2.11-3.5.3.jar +json4s-core_2.11-3.5.3.jar +json4s-jackson_2.11-3.5.3.jar +json4s-scalap_2.11-3.5.3.jar jsp-api-2.1.jar jsr305-3.0.1.jar jtransforms-2.4.0.jar diff --git a/pom.xml b/pom.xml index 065ee0ad3bd0..f73d7cb74e6a 100644 --- a/pom.xml +++ b/pom.xml @@ -921,7 +921,7 @@ org.json4s json4s-jackson_${scala.binary.version} - 3.6.0-M2 + 3.5.3 com.fasterxml.jackson.core diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/OffsetSeq.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/OffsetSeq.scala index 91caa6062cf8..73945b39b896 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/OffsetSeq.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/OffsetSeq.scala @@ -19,7 +19,6 @@ package org.apache.spark.sql.execution.streaming import org.json4s.NoTypeHints import org.json4s.jackson.Serialization -import org.json4s.reflect.PrimaryConstructor import org.apache.spark.internal.Logging import org.apache.spark.sql.RuntimeConfig @@ -61,7 +60,7 @@ object OffsetSeq { * `nulls` in the sequence are converted to `None`s. */ def fill(metadata: Option[String], offsets: Offset*): OffsetSeq = { - OffsetSeq(offsets.map(Option(_)), metadata.map(OffsetSeqMetadata.fromJson)) + OffsetSeq(offsets.map(Option(_)), metadata.map(OffsetSeqMetadata.apply)) } } @@ -87,7 +86,7 @@ object OffsetSeqMetadata extends Logging { private implicit val format = Serialization.formats(NoTypeHints) private val relevantSQLConfs = Seq(SHUFFLE_PARTITIONS, STATE_STORE_PROVIDER_CLASS) - def fromJson(json: String): OffsetSeqMetadata = Serialization.read[OffsetSeqMetadata](json) + def apply(json: String): OffsetSeqMetadata = Serialization.read[OffsetSeqMetadata](json) def apply( batchWatermarkMs: Long, diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLogSuite.scala index 59c7142c0bed..e6cdc063c4e9 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLogSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/OffsetSeqLogSuite.scala @@ -37,32 +37,31 @@ class OffsetSeqLogSuite extends SparkFunSuite with SharedSQLContext { } // None set - assert(new OffsetSeqMetadata(0, 0, Map.empty) === OffsetSeqMetadata.fromJson("""{}""")) + assert(new OffsetSeqMetadata(0, 0, Map.empty) === OffsetSeqMetadata("""{}""")) // One set assert(new OffsetSeqMetadata(1, 0, Map.empty) === - OffsetSeqMetadata.fromJson("""{"batchWatermarkMs":1}""")) + OffsetSeqMetadata("""{"batchWatermarkMs":1}""")) assert(new OffsetSeqMetadata(0, 2, Map.empty) === - OffsetSeqMetadata.fromJson("""{"batchTimestampMs":2}""")) + OffsetSeqMetadata("""{"batchTimestampMs":2}""")) assert(OffsetSeqMetadata(0, 0, getConfWith(shufflePartitions = 2)) === - OffsetSeqMetadata.fromJson(s"""{"conf": {"$key":2}}""")) + OffsetSeqMetadata(s"""{"conf": {"$key":2}}""")) // Two set assert(new OffsetSeqMetadata(1, 2, Map.empty) === - OffsetSeqMetadata.fromJson("""{"batchWatermarkMs":1,"batchTimestampMs":2}""")) + OffsetSeqMetadata("""{"batchWatermarkMs":1,"batchTimestampMs":2}""")) assert(OffsetSeqMetadata(1, 0, getConfWith(shufflePartitions = 3)) === - OffsetSeqMetadata.fromJson(s"""{"batchWatermarkMs":1,"conf": {"$key":3}}""")) + OffsetSeqMetadata(s"""{"batchWatermarkMs":1,"conf": {"$key":3}}""")) assert(OffsetSeqMetadata(0, 2, getConfWith(shufflePartitions = 3)) === - OffsetSeqMetadata.fromJson(s"""{"batchTimestampMs":2,"conf": {"$key":3}}""")) + OffsetSeqMetadata(s"""{"batchTimestampMs":2,"conf": {"$key":3}}""")) // All set assert(OffsetSeqMetadata(1, 2, getConfWith(shufflePartitions = 3)) === - OffsetSeqMetadata.fromJson( - s"""{"batchWatermarkMs":1,"batchTimestampMs":2,"conf": {"$key":3}}""")) + OffsetSeqMetadata(s"""{"batchWatermarkMs":1,"batchTimestampMs":2,"conf": {"$key":3}}""")) // Drop unknown fields assert(OffsetSeqMetadata(1, 2, getConfWith(shufflePartitions = 3)) === - OffsetSeqMetadata.fromJson( + OffsetSeqMetadata( s"""{"batchWatermarkMs":1,"batchTimestampMs":2,"conf": {"$key":3}},"unknown":1""")) } From 9ce86e0cdf59b229c7f7f7a9e4d1c88a59036a7c Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Fri, 20 Apr 2018 12:31:51 +0100 Subject: [PATCH 8/9] upgrade influx sink --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f73d7cb74e6a..403247707900 100644 --- a/pom.xml +++ b/pom.xml @@ -146,7 +146,7 @@ 2.0.8 3.2.5 0.9.2 - 0.3.12 + 0.4.0 1.8.1 hadoop2 0.9.4 From 5dd3b2740a96a8ad27cdbdebcf6e1ec0b0b8aa2c Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Fri, 20 Apr 2018 13:19:05 +0100 Subject: [PATCH 9/9] influx sink --- dev/deps/spark-deps-hadoop-palantir | 20 +++++++++++--------- pom.xml | 12 +++++++++++- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-palantir b/dev/deps/spark-deps-hadoop-palantir index e7517c3230c5..141092eb2173 100644 --- a/dev/deps/spark-deps-hadoop-palantir +++ b/dev/deps/spark-deps-hadoop-palantir @@ -49,12 +49,12 @@ core-1.1.2.jar curator-client-2.7.1.jar curator-framework-2.7.1.jar curator-recipes-2.7.1.jar -dropwizard-jackson-0.9.2.jar -dropwizard-lifecycle-0.9.2.jar -dropwizard-metrics-0.9.2.jar -dropwizard-metrics-influxdb-1.1.8.jar -dropwizard-util-0.9.2.jar -dropwizard-validation-0.9.2.jar +dropwizard-jackson-1.0.0.jar +dropwizard-lifecycle-1.0.0.jar +dropwizard-metrics-1.0.0.jar +dropwizard-metrics-influxdb-1.2.2.jar +dropwizard-util-1.0.0.jar +dropwizard-validation-1.0.0.jar fastutil-7.0.13.jar flatbuffers-1.2.0-3f79e055.jar generex-1.0.1.jar @@ -79,7 +79,7 @@ hadoop-yarn-client-2.8.2-palantir.2.jar hadoop-yarn-common-2.8.2-palantir.2.jar hadoop-yarn-server-common-2.8.2-palantir.2.jar hadoop-yarn-server-web-proxy-2.8.2-palantir.2.jar -hibernate-validator-5.2.2.Final.jar +hibernate-validator-5.2.4.Final.jar hk2-api-2.5.0-b32.jar hk2-locator-2.5.0-b32.jar hk2-utils-2.5.0-b32.jar @@ -96,8 +96,9 @@ jackson-databind-2.9.5.jar jackson-dataformat-cbor-2.9.5.jar jackson-dataformat-yaml-2.9.5.jar jackson-datatype-guava-2.9.5.jar -jackson-datatype-jdk7-2.6.3.jar +jackson-datatype-jdk8-2.9.5.jar jackson-datatype-joda-2.9.5.jar +jackson-datatype-jsr310-2.9.5.jar jackson-jaxrs-1.9.13.jar jackson-mapper-asl-1.9.13.jar jackson-module-afterburner-2.9.5.jar @@ -140,6 +141,7 @@ jsp-api-2.1.jar jsr305-3.0.1.jar jtransforms-2.4.0.jar jul-to-slf4j-1.7.25.jar +kafka-clients-0.11.0.0.jar kryo-shaded-3.0.3.jar kubernetes-client-3.0.0.jar kubernetes-model-2.0.0.jar @@ -151,7 +153,7 @@ machinist_2.11-0.6.1.jar macro-compat_2.11-1.1.1.jar metrics-core-3.2.5.jar metrics-graphite-3.2.5.jar -metrics-influxdb-1.1.8.jar +metrics-influxdb-1.2.2.jar metrics-json-3.2.5.jar metrics-jvm-3.2.5.jar minlog-1.3.0.jar diff --git a/pom.xml b/pom.xml index 403247707900..1a5146b64da2 100644 --- a/pom.xml +++ b/pom.xml @@ -145,7 +145,7 @@ 2.4.0 2.0.8 3.2.5 - 0.9.2 + 1.0.0 0.4.0 1.8.1 hadoop2 @@ -786,6 +786,11 @@ com.fasterxml.jackson.core jackson-annotations + + + net.jpountz.lz4 + lz4 + @@ -846,6 +851,11 @@ jackson-datatype-guava ${fasterxml.jackson.version} + + com.fasterxml.jackson.datatype + jackson-datatype-jdk8 + ${fasterxml.jackson.version} + com.fasterxml.jackson.datatype jackson-datatype-joda