Skip to content

Commit 83d9368

Browse files
steveloughransteveloughran
committed
SPARK-7481 cloud pom is still JAR (not pom). works against Hadoop 2.6 as well as 2.7, keeping azure the 2.7.x dependency. All dependencies are scoped @ hadoop.scope
Change-Id: I80bd95fd48e21cf2eb4d94907ac99081cd3bd375
1 parent aa4ea89 commit 83d9368

2 files changed

Lines changed: 109 additions & 155 deletions

File tree

‎cloud/pom.xml‎

Lines changed: 43 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -50,111 +50,68 @@
5050

5151
<dependencies>
5252
<dependency>
53-
<groupId>org.apache.spark</groupId>
54-
<artifactId>spark-core_${scala.binary.version}</artifactId>
55-
<version>${project.version}</version>
53+
<groupId>org.apache.hadoop</groupId>
54+
<artifactId>hadoop-aws</artifactId>
55+
<scope>${hadoop.deps.scope}</scope>
5656
</dependency>
5757

58-
<!--Used for test classes -->
5958
<dependency>
60-
<groupId>org.apache.spark</groupId>
61-
<artifactId>spark-core_${scala.binary.version}</artifactId>
62-
<version>${project.version}</version>
63-
<type>test-jar</type>
64-
<scope>test</scope>
59+
<groupId>org.apache.hadoop</groupId>
60+
<artifactId>hadoop-openstack</artifactId>
61+
<scope>${hadoop.deps.scope}</scope>
6562
</dependency>
66-
67-
<!-- Jets3t is needed for s3n and s3 classic to work-->
63+
<!--
64+
Add joda time to ensure that anything downstream which doesn't pull in spark-hive
65+
gets the correct joda time artifact, so doesn't have auth failures on later Java 8 JVMs
66+
-->
6867
<dependency>
69-
<groupId>net.java.dev.jets3t</groupId>
70-
<artifactId>jets3t</artifactId>
68+
<groupId>joda-time</groupId>
69+
<artifactId>joda-time</artifactId>
70+
<scope>${hadoop.deps.scope}</scope>
7171
</dependency>
72-
73-
<!-- Explicit listing of transitive deps that are shaded. Otherwise, odd compiler crashes. -->
72+
<!-- explicitly declare the jackson artifacts desired -->
73+
<dependency>
74+
<groupId>com.fasterxml.jackson.core</groupId>
75+
<artifactId>jackson-databind</artifactId>
76+
<scope>${hadoop.deps.scope}</scope>
77+
</dependency>
78+
<dependency>
79+
<groupId>com.fasterxml.jackson.core</groupId>
80+
<artifactId>jackson-annotations</artifactId>
81+
<scope>${hadoop.deps.scope}</scope>
82+
</dependency>
83+
<dependency>
84+
<groupId>com.fasterxml.jackson.dataformat</groupId>
85+
<artifactId>jackson-dataformat-cbor</artifactId>
86+
<scope>${hadoop.deps.scope}</scope>
87+
</dependency>
88+
<!--Explicit declaration to force in Spark version into transitive dependencies -->
7489
<dependency>
75-
<groupId>com.google.guava</groupId>
76-
<artifactId>guava</artifactId>
90+
<groupId>org.apache.httpcomponents</groupId>
91+
<artifactId>httpclient</artifactId>
92+
<scope>${hadoop.deps.scope}</scope>
93+
</dependency>
94+
<!--Explicit declaration to force in Spark version into transitive dependencies -->
95+
<dependency>
96+
<groupId>org.apache.httpcomponents</groupId>
97+
<artifactId>httpcore</artifactId>
98+
<scope>${hadoop.deps.scope}</scope>
7799
</dependency>
78-
<!-- End of shaded deps. -->
79100
</dependencies>
80101

81-
<build>
82-
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
83-
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
84-
</build>
85-
86102
<profiles>
87103

88-
<!--
89-
This profile is enabled automatically by the sbt build. It changes the scope for the guava
90-
dependency, since we don't shade it in the artifacts generated by the sbt build.
91-
-->
92104
<profile>
93-
<id>sbt</id>
105+
<id>hadoop-2.7</id>
94106
<dependencies>
95107
<dependency>
96-
<groupId>com.google.guava</groupId>
97-
<artifactId>guava</artifactId>
98-
<scope>compile</scope>
108+
<groupId>org.apache.hadoop</groupId>
109+
<artifactId>hadoop-azure</artifactId>
110+
<scope>${hadoop.deps.scope}</scope>
99111
</dependency>
100112
</dependencies>
101113
</profile>
102114

103-
<profile>
104-
<id>hadoop-2.7</id>
105-
<dependencies>
106-
<dependency>
107-
<groupId>org.apache.hadoop</groupId>
108-
<artifactId>hadoop-aws</artifactId>
109-
<scope>${hadoop.deps.scope}</scope>
110-
</dependency>
111-
<dependency>
112-
<groupId>org.apache.hadoop</groupId>
113-
<artifactId>hadoop-azure</artifactId>
114-
<scope>${hadoop.deps.scope}</scope>
115-
</dependency>
116-
<dependency>
117-
<groupId>org.apache.hadoop</groupId>
118-
<artifactId>hadoop-openstack</artifactId>
119-
<scope>${hadoop.deps.scope}</scope>
120-
</dependency>
121-
<!--
122-
Add joda time to ensure that anything downstream which doesn't pull in spark-hive
123-
gets the correct joda time artifact, so doesn't have auth failures on later Java 8 JVMs
124-
-->
125-
<dependency>
126-
<groupId>joda-time</groupId>
127-
<artifactId>joda-time</artifactId>
128-
</dependency>
129-
<!-- explicitly declare the jackson artifacts desired -->
130-
<dependency>
131-
<groupId>com.fasterxml.jackson.core</groupId>
132-
<artifactId>jackson-databind</artifactId>
133-
<scope>${hadoop.deps.scope}</scope>
134-
</dependency>
135-
<dependency>
136-
<groupId>com.fasterxml.jackson.core</groupId>
137-
<artifactId>jackson-annotations</artifactId>
138-
<scope>${hadoop.deps.scope}</scope>
139-
</dependency>
140-
<dependency>
141-
<groupId>com.fasterxml.jackson.dataformat</groupId>
142-
<artifactId>jackson-dataformat-cbor</artifactId>
143-
<scope>${hadoop.deps.scope}</scope>
144-
</dependency>
145-
<!--Explicit declaration to force in Spark version into transitive dependencies -->
146-
<dependency>
147-
<groupId>org.apache.httpcomponents</groupId>
148-
<artifactId>httpclient</artifactId>
149-
</dependency>
150-
<!--Explicit declaration to force in Spark version into transitive dependencies -->
151-
<dependency>
152-
<groupId>org.apache.httpcomponents</groupId>
153-
<artifactId>httpcore</artifactId>
154-
</dependency>
155-
</dependencies>
156-
</profile>
157-
158115
</profiles>
159116

160117
</project>

‎pom.xml‎

Lines changed: 66 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -1150,6 +1150,70 @@
11501150
</exclusion>
11511151
</exclusions>
11521152
</dependency>
1153+
<!--
1154+
the AWS module pulls in jackson; its transitive dependencies can create
1155+
intra-jackson-module version problems.
1156+
-->
1157+
<dependency>
1158+
<groupId>org.apache.hadoop</groupId>
1159+
<artifactId>hadoop-aws</artifactId>
1160+
<version>${hadoop.version}</version>
1161+
<scope>${hadoop.deps.scope}</scope>
1162+
<exclusions>
1163+
<exclusion>
1164+
<groupId>org.apache.hadoop</groupId>
1165+
<artifactId>hadoop-common</artifactId>
1166+
</exclusion>
1167+
<exclusion>
1168+
<groupId>commons-logging</groupId>
1169+
<artifactId>commons-logging</artifactId>
1170+
</exclusion>
1171+
<exclusion>
1172+
<groupId>org.codehaus.jackson</groupId>
1173+
<artifactId>jackson-mapper-asl</artifactId>
1174+
</exclusion>
1175+
<exclusion>
1176+
<groupId>org.codehaus.jackson</groupId>
1177+
<artifactId>jackson-core-asl</artifactId>
1178+
</exclusion>
1179+
<exclusion>
1180+
<groupId>com.fasterxml.jackson.core</groupId>
1181+
<artifactId>jackson-core</artifactId>
1182+
</exclusion>
1183+
<exclusion>
1184+
<groupId>com.fasterxml.jackson.core</groupId>
1185+
<artifactId>jackson-databind</artifactId>
1186+
</exclusion>
1187+
<exclusion>
1188+
<groupId>com.fasterxml.jackson.core</groupId>
1189+
<artifactId>jackson-annotations</artifactId>
1190+
</exclusion>
1191+
</exclusions>
1192+
</dependency>
1193+
<dependency>
1194+
<groupId>org.apache.hadoop</groupId>
1195+
<artifactId>hadoop-openstack</artifactId>
1196+
<version>${hadoop.version}</version>
1197+
<scope>${hadoop.deps.scope}</scope>
1198+
<exclusions>
1199+
<exclusion>
1200+
<groupId>org.apache.hadoop</groupId>
1201+
<artifactId>hadoop-common</artifactId>
1202+
</exclusion>
1203+
<exclusion>
1204+
<groupId>commons-logging</groupId>
1205+
<artifactId>commons-logging</artifactId>
1206+
</exclusion>
1207+
<exclusion>
1208+
<groupId>junit</groupId>
1209+
<artifactId>junit</artifactId>
1210+
</exclusion>
1211+
<exclusion>
1212+
<groupId>org.mockito</groupId>
1213+
<artifactId>mockito-all</artifactId>
1214+
</exclusion>
1215+
</exclusions>
1216+
</dependency>
11531217
<dependency>
11541218
<groupId>org.apache.zookeeper</groupId>
11551219
<artifactId>zookeeper</artifactId>
@@ -2533,75 +2597,10 @@
25332597
</properties>
25342598
<dependencyManagement>
25352599
<dependencies>
2536-
<!--
2537-
the AWS module pulls in jackson; its transitive dependencies can create
2538-
intra-jackson-module version problems.
2539-
-->
2540-
<dependency>
2541-
<groupId>org.apache.hadoop</groupId>
2542-
<artifactId>hadoop-aws</artifactId>
2543-
<version>${hadoop.version}</version>
2544-
<scope>${hadoop.deps.scope}</scope>
2545-
<exclusions>
2546-
<exclusion>
2547-
<groupId>org.apache.hadoop</groupId>
2548-
<artifactId>hadoop-common</artifactId>
2549-
</exclusion>
2550-
<exclusion>
2551-
<groupId>commons-logging</groupId>
2552-
<artifactId>commons-logging</artifactId>
2553-
</exclusion>
2554-
<exclusion>
2555-
<groupId>org.codehaus.jackson</groupId>
2556-
<artifactId>jackson-mapper-asl</artifactId>
2557-
</exclusion>
2558-
<exclusion>
2559-
<groupId>org.codehaus.jackson</groupId>
2560-
<artifactId>jackson-core-asl</artifactId>
2561-
</exclusion>
2562-
<exclusion>
2563-
<groupId>com.fasterxml.jackson.core</groupId>
2564-
<artifactId>jackson-core</artifactId>
2565-
</exclusion>
2566-
<exclusion>
2567-
<groupId>com.fasterxml.jackson.core</groupId>
2568-
<artifactId>jackson-databind</artifactId>
2569-
</exclusion>
2570-
<exclusion>
2571-
<groupId>com.fasterxml.jackson.core</groupId>
2572-
<artifactId>jackson-annotations</artifactId>
2573-
</exclusion>
2574-
</exclusions>
2575-
</dependency>
2576-
2577-
<dependency>
2578-
<groupId>org.apache.hadoop</groupId>
2579-
<artifactId>hadoop-openstack</artifactId>
2580-
<version>${hadoop.version}</version>
2581-
<scope>${hadoop.deps.scope}</scope>
2582-
<exclusions>
2583-
<exclusion>
2584-
<groupId>org.apache.hadoop</groupId>
2585-
<artifactId>hadoop-common</artifactId>
2586-
</exclusion>
2587-
<exclusion>
2588-
<groupId>commons-logging</groupId>
2589-
<artifactId>commons-logging</artifactId>
2590-
</exclusion>
2591-
<exclusion>
2592-
<groupId>junit</groupId>
2593-
<artifactId>junit</artifactId>
2594-
</exclusion>
2595-
<exclusion>
2596-
<groupId>org.mockito</groupId>
2597-
<artifactId>mockito-all</artifactId>
2598-
</exclusion>
2599-
</exclusions>
2600-
</dependency>
26012600

26022601
<!--
2603-
Azure's Jackson dependencies need to be cut to avoid version inconsistencies
2604-
within the jackson-* JARs
2602+
Hadoop-Azure isn't in Hadoop 2.6, so this has to be made
2603+
a 2.7+ moodule.
26052604
-->
26062605
<dependency>
26072606
<groupId>org.apache.hadoop</groupId>
@@ -2657,8 +2656,6 @@
26572656
hadoop versions to declare different include/exclude
26582657
rules (especially transient dependencies).
26592658

2660-
To use this profile, the hadoop-2.7 profile must also
2661-
be declared
26622659
-->
26632660
<profile>
26642661
<id>cloud</id>

0 commit comments

Comments
 (0)