Skip to content

Commit 988b698

Browse files
srowenpdeyhim
srowen
authored and
pdeyhim
committed
SPARK-1387. Update build plugins, avoid plugin version warning, centralize versions
Another handful of small build changes to organize and standardize a bit, and avoid warnings: - Update Maven plugin versions for good measure - Since plugins need maven 3.0.4 already, require it explicitly (<3.0.4 had some bugs anyway) - Use variables to define versions across dependencies where they should move in lock step - ... and make this consistent between Maven/SBT OK, I also updated the JIRA URL while I was at it here. Author: Sean Owen <[email protected]> Closes apache#291 from srowen/SPARK-1387 and squashes the following commits: 461eca1 [Sean Owen] Couldn't resist also updating JIRA location to new one c2d5cc5 [Sean Owen] Update plugins and Maven version; use variables consistently across Maven/SBT to define dependency versions that should stay in step.
1 parent 9eda545 commit 988b698

File tree

10 files changed

+57
-50
lines changed

10 files changed

+57
-50
lines changed

‎assembly/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,7 @@
208208
<plugin>
209209
<groupId>org.codehaus.mojo</groupId>
210210
<artifactId>buildnumber-maven-plugin</artifactId>
211-
<version>1.1</version>
211+
<version>1.2</version>
212212
<executions>
213213
<execution>
214214
<phase>validate</phase>

‎core/pom.xml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,12 +117,10 @@
117117
<dependency>
118118
<groupId>com.twitter</groupId>
119119
<artifactId>chill_${scala.binary.version}</artifactId>
120-
<version>0.3.1</version>
121120
</dependency>
122121
<dependency>
123122
<groupId>com.twitter</groupId>
124123
<artifactId>chill-java</artifactId>
125-
<version>0.3.1</version>
126124
</dependency>
127125
<dependency>
128126
<groupId>commons-net</groupId>

‎dev/audit-release/maven_app_core/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
<plugins>
5050
<plugin>
5151
<artifactId>maven-compiler-plugin</artifactId>
52-
<version>2.3.2</version>
52+
<version>3.1</version>
5353
</plugin>
5454
</plugins>
5555
</build>

‎docs/building-with-maven.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ title: Building Spark with Maven
66
* This will become a table of contents (this text will be scraped).
77
{:toc}
88

9-
Building Spark using Maven Requires Maven 3 (the build process is tested with Maven 3.0.4) and Java 1.6 or newer.
9+
Building Spark using Maven requires Maven 3.0.4 or newer and Java 1.6 or newer.
1010

1111

1212
## Setting up Maven's Memory Usage ##

‎examples/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@
110110
<dependency>
111111
<groupId>org.apache.hbase</groupId>
112112
<artifactId>hbase</artifactId>
113-
<version>0.94.6</version>
113+
<version>${hbase.version}</version>
114114
<exclusions>
115115
<exclusion>
116116
<groupId>asm</groupId>

‎graphx/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@
5454
<dependency>
5555
<groupId>org.jblas</groupId>
5656
<artifactId>jblas</artifactId>
57-
<version>1.2.3</version>
57+
<version>${jblas.version}</version>
5858
</dependency>
5959
<dependency>
6060
<groupId>org.eclipse.jetty</groupId>

‎mllib/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@
5858
<dependency>
5959
<groupId>org.jblas</groupId>
6060
<artifactId>jblas</artifactId>
61-
<version>1.2.3</version>
61+
<version>${jblas.version}</version>
6262
</dependency>
6363
<dependency>
6464
<groupId>org.scalanlp</groupId>

‎pom.xml‎

Lines changed: 23 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -54,11 +54,11 @@
5454
</developers>
5555
<issueManagement>
5656
<system>JIRA</system>
57-
<url>https://spark-project.atlassian.net/browse/SPARK</url>
57+
<url>https://issues.apache.org/jira/browse/SPARK</url>
5858
</issueManagement>
5959

6060
<prerequisites>
61-
<maven>3.0.0</maven>
61+
<maven>3.0.4</maven>
6262
</prerequisites>
6363

6464
<mailingLists>
@@ -123,6 +123,10 @@
123123
<hbase.version>0.94.6</hbase.version>
124124
<hive.version>0.12.0</hive.version>
125125
<parquet.version>1.3.2</parquet.version>
126+
<jblas.version>1.2.3</jblas.version>
127+
<jetty.version>8.1.14.v20131031</jetty.version>
128+
<chill.version>0.3.1</chill.version>
129+
<codahale.metrics.version>3.0.0</codahale.metrics.version>
126130

127131
<PermGen>64m</PermGen>
128132
<MaxPermGen>512m</MaxPermGen>
@@ -192,22 +196,22 @@
192196
<dependency>
193197
<groupId>org.eclipse.jetty</groupId>
194198
<artifactId>jetty-util</artifactId>
195-
<version>8.1.14.v20131031</version>
199+
<version>${jetty.version}</version>
196200
</dependency>
197201
<dependency>
198202
<groupId>org.eclipse.jetty</groupId>
199203
<artifactId>jetty-security</artifactId>
200-
<version>8.1.14.v20131031</version>
204+
<version>${jetty.version}</version>
201205
</dependency>
202206
<dependency>
203207
<groupId>org.eclipse.jetty</groupId>
204208
<artifactId>jetty-plus</artifactId>
205-
<version>8.1.14.v20131031</version>
209+
<version>${jetty.version}</version>
206210
</dependency>
207211
<dependency>
208212
<groupId>org.eclipse.jetty</groupId>
209213
<artifactId>jetty-server</artifactId>
210-
<version>8.1.14.v20131031</version>
214+
<version>${jetty.version}</version>
211215
</dependency>
212216
<dependency>
213217
<groupId>com.google.guava</groupId>
@@ -273,7 +277,7 @@
273277
<dependency>
274278
<groupId>com.twitter</groupId>
275279
<artifactId>chill_${scala.binary.version}</artifactId>
276-
<version>0.3.1</version>
280+
<version>${chill.version}</version>
277281
<exclusions>
278282
<exclusion>
279283
<groupId>org.ow2.asm</groupId>
@@ -288,7 +292,7 @@
288292
<dependency>
289293
<groupId>com.twitter</groupId>
290294
<artifactId>chill-java</artifactId>
291-
<version>0.3.1</version>
295+
<version>${chill.version}</version>
292296
<exclusions>
293297
<exclusion>
294298
<groupId>org.ow2.asm</groupId>
@@ -392,27 +396,27 @@
392396
<dependency>
393397
<groupId>com.codahale.metrics</groupId>
394398
<artifactId>metrics-core</artifactId>
395-
<version>3.0.0</version>
399+
<version>${codahale.metrics.version}</version>
396400
</dependency>
397401
<dependency>
398402
<groupId>com.codahale.metrics</groupId>
399403
<artifactId>metrics-jvm</artifactId>
400-
<version>3.0.0</version>
404+
<version>${codahale.metrics.version}</version>
401405
</dependency>
402406
<dependency>
403407
<groupId>com.codahale.metrics</groupId>
404408
<artifactId>metrics-json</artifactId>
405-
<version>3.0.0</version>
409+
<version>${codahale.metrics.version}</version>
406410
</dependency>
407411
<dependency>
408412
<groupId>com.codahale.metrics</groupId>
409413
<artifactId>metrics-ganglia</artifactId>
410-
<version>3.0.0</version>
414+
<version>${codahale.metrics.version}</version>
411415
</dependency>
412416
<dependency>
413417
<groupId>com.codahale.metrics</groupId>
414418
<artifactId>metrics-graphite</artifactId>
415-
<version>3.0.0</version>
419+
<version>${codahale.metrics.version}</version>
416420
</dependency>
417421
<dependency>
418422
<groupId>org.scala-lang</groupId>
@@ -585,7 +589,7 @@
585589
<plugin>
586590
<groupId>org.apache.maven.plugins</groupId>
587591
<artifactId>maven-enforcer-plugin</artifactId>
588-
<version>1.1.1</version>
592+
<version>1.3.1</version>
589593
<executions>
590594
<execution>
591595
<id>enforce-versions</id>
@@ -595,7 +599,7 @@
595599
<configuration>
596600
<rules>
597601
<requireMavenVersion>
598-
<version>3.0.0</version>
602+
<version>3.0.4</version>
599603
</requireMavenVersion>
600604
<requireJavaVersion>
601605
<version>${java.version}</version>
@@ -608,12 +612,12 @@
608612
<plugin>
609613
<groupId>org.codehaus.mojo</groupId>
610614
<artifactId>build-helper-maven-plugin</artifactId>
611-
<version>1.7</version>
615+
<version>1.8</version>
612616
</plugin>
613617
<plugin>
614618
<groupId>net.alchim31.maven</groupId>
615619
<artifactId>scala-maven-plugin</artifactId>
616-
<version>3.1.5</version>
620+
<version>3.1.6</version>
617621
<executions>
618622
<execution>
619623
<id>scala-compile-first</id>
@@ -674,7 +678,7 @@
674678
<plugin>
675679
<groupId>org.apache.maven.plugins</groupId>
676680
<artifactId>maven-surefire-plugin</artifactId>
677-
<version>2.12.4</version>
681+
<version>2.17</version>
678682
<configuration>
679683
<!-- Uses scalatest instead -->
680684
<skipTests>true</skipTests>
@@ -713,7 +717,7 @@
713717
<plugin>
714718
<groupId>org.apache.maven.plugins</groupId>
715719
<artifactId>maven-shade-plugin</artifactId>
716-
<version>2.0</version>
720+
<version>2.2</version>
717721
</plugin>
718722
<plugin>
719723
<groupId>org.apache.maven.plugins</groupId>
@@ -810,7 +814,6 @@
810814
<plugin>
811815
<groupId>org.apache.maven.plugins</groupId>
812816
<artifactId>maven-jar-plugin</artifactId>
813-
<version>2.4</version>
814817
<executions>
815818
<execution>
816819
<goals>

‎project/SparkBuild.scala‎

Lines changed: 28 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -251,10 +251,10 @@ object SparkBuild extends Build {
251251

252252
libraryDependencies ++= Seq(
253253
"io.netty" % "netty-all" % "4.0.17.Final",
254-
"org.eclipse.jetty" % "jetty-server" % "8.1.14.v20131031",
255-
"org.eclipse.jetty" % "jetty-util" % "8.1.14.v20131031",
256-
"org.eclipse.jetty" % "jetty-plus" % "8.1.14.v20131031",
257-
"org.eclipse.jetty" % "jetty-security" % "8.1.14.v20131031",
254+
"org.eclipse.jetty" % "jetty-server" % jettyVersion,
255+
"org.eclipse.jetty" % "jetty-util" % jettyVersion,
256+
"org.eclipse.jetty" % "jetty-plus" % jettyVersion,
257+
"org.eclipse.jetty" % "jetty-security" % jettyVersion,
258258
/** Workaround for SPARK-959. Dependency used by org.eclipse.jetty. Fixed in ivy 2.3.0. */
259259
"org.eclipse.jetty.orbit" % "javax.servlet" % "3.0.0.v201112011016" artifacts Artifact("javax.servlet", "jar", "jar"),
260260
"org.scalatest" %% "scalatest" % "1.9.1" % "test",
@@ -279,6 +279,13 @@ object SparkBuild extends Build {
279279
publishLocalBoth <<= Seq(publishLocal in MavenCompile, publishLocal).dependOn
280280
) ++ net.virtualvoid.sbt.graph.Plugin.graphSettings ++ ScalaStyleSettings
281281

282+
val akkaVersion = "2.2.3-shaded-protobuf"
283+
val chillVersion = "0.3.1"
284+
val codahaleMetricsVersion = "3.0.0"
285+
val jblasVersion = "1.2.3"
286+
val jettyVersion = "8.1.14.v20131031"
287+
val hiveVersion = "0.12.0"
288+
val parquetVersion = "1.3.2"
282289
val slf4jVersion = "1.7.5"
283290

284291
val excludeNetty = ExclusionRule(organization = "org.jboss.netty")
@@ -312,9 +319,9 @@ object SparkBuild extends Build {
312319
"commons-daemon" % "commons-daemon" % "1.0.10", // workaround for bug HADOOP-9407
313320
"com.ning" % "compress-lzf" % "1.0.0",
314321
"org.xerial.snappy" % "snappy-java" % "1.0.5",
315-
"org.spark-project.akka" %% "akka-remote" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty),
316-
"org.spark-project.akka" %% "akka-slf4j" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty),
317-
"org.spark-project.akka" %% "akka-testkit" % "2.2.3-shaded-protobuf" % "test",
322+
"org.spark-project.akka" %% "akka-remote" % akkaVersion excludeAll(excludeNetty),
323+
"org.spark-project.akka" %% "akka-slf4j" % akkaVersion excludeAll(excludeNetty),
324+
"org.spark-project.akka" %% "akka-testkit" % akkaVersion % "test",
318325
"org.json4s" %% "json4s-jackson" % "3.2.6" excludeAll(excludeScalap),
319326
"it.unimi.dsi" % "fastutil" % "6.4.4",
320327
"colt" % "colt" % "1.2.0",
@@ -324,12 +331,12 @@ object SparkBuild extends Build {
324331
"org.apache.derby" % "derby" % "10.4.2.0" % "test",
325332
"org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeCommonsLogging, excludeSLF4J, excludeOldAsm),
326333
"org.apache.curator" % "curator-recipes" % "2.4.0" excludeAll(excludeNetty),
327-
"com.codahale.metrics" % "metrics-core" % "3.0.0",
328-
"com.codahale.metrics" % "metrics-jvm" % "3.0.0",
329-
"com.codahale.metrics" % "metrics-json" % "3.0.0",
330-
"com.codahale.metrics" % "metrics-graphite" % "3.0.0",
331-
"com.twitter" %% "chill" % "0.3.1" excludeAll(excludeAsm),
332-
"com.twitter" % "chill-java" % "0.3.1" excludeAll(excludeAsm),
334+
"com.codahale.metrics" % "metrics-core" % codahaleMetricsVersion,
335+
"com.codahale.metrics" % "metrics-jvm" % codahaleMetricsVersion,
336+
"com.codahale.metrics" % "metrics-json" % codahaleMetricsVersion,
337+
"com.codahale.metrics" % "metrics-graphite" % codahaleMetricsVersion,
338+
"com.twitter" %% "chill" % chillVersion excludeAll(excludeAsm),
339+
"com.twitter" % "chill-java" % chillVersion excludeAll(excludeAsm),
333340
"org.tachyonproject" % "tachyon" % "0.4.1-thrift" excludeAll(excludeHadoop, excludeCurator, excludeEclipseJetty, excludePowermock),
334341
"com.clearspring.analytics" % "stream" % "2.5.1"
335342
),
@@ -373,7 +380,7 @@ object SparkBuild extends Build {
373380
name := "spark-graphx",
374381
previousArtifact := sparkPreviousArtifact("spark-graphx"),
375382
libraryDependencies ++= Seq(
376-
"org.jblas" % "jblas" % "1.2.3"
383+
"org.jblas" % "jblas" % jblasVersion
377384
)
378385
)
379386

@@ -386,7 +393,7 @@ object SparkBuild extends Build {
386393
name := "spark-mllib",
387394
previousArtifact := sparkPreviousArtifact("spark-mllib"),
388395
libraryDependencies ++= Seq(
389-
"org.jblas" % "jblas" % "1.2.3",
396+
"org.jblas" % "jblas" % jblasVersion,
390397
"org.scalanlp" %% "breeze" % "0.7"
391398
)
392399
)
@@ -406,8 +413,8 @@ object SparkBuild extends Build {
406413
def sqlCoreSettings = sharedSettings ++ Seq(
407414
name := "spark-sql",
408415
libraryDependencies ++= Seq(
409-
"com.twitter" % "parquet-column" % "1.3.2",
410-
"com.twitter" % "parquet-hadoop" % "1.3.2"
416+
"com.twitter" % "parquet-column" % parquetVersion,
417+
"com.twitter" % "parquet-hadoop" % parquetVersion
411418
)
412419
)
413420

@@ -419,9 +426,9 @@ object SparkBuild extends Build {
419426
jarName in packageDependency <<= version map { v => "spark-hive-assembly-" + v + "-hadoop" + hadoopVersion + "-deps.jar" },
420427
javaOptions += "-XX:MaxPermSize=1g",
421428
libraryDependencies ++= Seq(
422-
"org.apache.hive" % "hive-metastore" % "0.12.0",
423-
"org.apache.hive" % "hive-exec" % "0.12.0",
424-
"org.apache.hive" % "hive-serde" % "0.12.0"
429+
"org.apache.hive" % "hive-metastore" % hiveVersion,
430+
"org.apache.hive" % "hive-exec" % hiveVersion,
431+
"org.apache.hive" % "hive-serde" % hiveVersion
425432
),
426433
// Multiple queries rely on the TestHive singleton. See comments there for more details.
427434
parallelExecution in Test := false,
@@ -560,7 +567,7 @@ object SparkBuild extends Build {
560567
name := "spark-streaming-zeromq",
561568
previousArtifact := sparkPreviousArtifact("spark-streaming-zeromq"),
562569
libraryDependencies ++= Seq(
563-
"org.spark-project.akka" %% "akka-zeromq" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty)
570+
"org.spark-project.akka" %% "akka-zeromq" % akkaVersion excludeAll(excludeNetty)
564571
)
565572
)
566573

‎streaming/pom.xml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,6 @@
9696
<plugin>
9797
<groupId>org.apache.maven.plugins</groupId>
9898
<artifactId>maven-jar-plugin</artifactId>
99-
<version>2.2</version>
10099
<executions>
101100
<execution>
102101
<goals>

0 commit comments

Comments
 (0)