Skip to content

Commit 8bf5394

Browse files
steveloughransteveloughran
committed
HADOOP-19385. S3A: add a hadoop-format-testing module for testing
Pom with dependencies Change-Id: I9c01a724b085f8c409ca22fb8e79de484b0273f8
1 parent d2095fa commit 8bf5394

File tree

4 files changed

+204
-1
lines changed

4 files changed

+204
-1
lines changed

‎hadoop-cloud-storage-project/hadoop-format-testing/pom.xml‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. See accompanying LICENSE file.
14+
-->
15+
<project xmlns="http://maven.apache.org/POM/4.0.0"
16+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
17+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
18+
https://maven.apache.org/xsd/maven-4.0.0.xsd">
19+
<modelVersion>4.0.0</modelVersion>
20+
<parent>
21+
<groupId>org.apache.hadoop</groupId>
22+
<artifactId>hadoop-project</artifactId>
23+
<version>3.5.0-SNAPSHOT</version>
24+
<relativePath>../../hadoop-project</relativePath>
25+
</parent>
26+
<artifactId>hadoop-format-testing</artifactId>
27+
<version>3.5.0-SNAPSHOT</version>
28+
<name>Apache Hadoop Format Regrssion Testing</name>
29+
<description>
30+
This module is exclusively for use in regression testing
31+
integration between various file formats and
32+
cloud connectors.
33+
34+
It MUST NOT be declared as a dependency by
35+
any external component.
36+
Consider it's status: Private/unstable.
37+
38+
For everyone who does a full stack build of the file
39+
format libraries as well as hadoop, this adds more loops
40+
to the build. However, because it is not a public distributable,
41+
it is safe to use a previous build's artifacts...all that is
42+
lost is the immediate regression testing.
43+
44+
Note that all test dependencies are declared as of the
45+
production JAR, not the test one.
46+
47+
This is to simplify use in testing in cloud modules;
48+
import this JAR and get all the transient dependencies
49+
tuned for these tests.
50+
</description>
51+
<packaging>jar</packaging>
52+
53+
<properties>
54+
<file.encoding>UTF-8</file.encoding>
55+
<downloadSources>true</downloadSources>
56+
<parquet.version>1.15.0</parquet.version>
57+
<iceberg.version>1.7.1</iceberg.version>
58+
</properties>
59+
60+
<dependencies>
61+
<dependency>
62+
<groupId>junit</groupId>
63+
<artifactId>junit</artifactId>
64+
</dependency>
65+
<dependency>
66+
<groupId>org.apache.hadoop</groupId>
67+
<artifactId>hadoop-common</artifactId>
68+
<scope>provided</scope>
69+
<exclusions>
70+
<exclusion>
71+
<groupId>com.google.guava</groupId>
72+
<artifactId>guava</artifactId>
73+
</exclusion>
74+
</exclusions>
75+
</dependency>
76+
<dependency>
77+
<groupId>org.apache.hadoop</groupId>
78+
<artifactId>hadoop-annotations</artifactId>
79+
<scope>provided</scope>
80+
</dependency>
81+
<dependency>
82+
<groupId>org.apache.hadoop</groupId>
83+
<artifactId>hadoop-common</artifactId>
84+
<type>test-jar</type>
85+
<scope>provided</scope>
86+
</dependency>
87+
88+
<dependency>
89+
<groupId>org.assertj</groupId>
90+
<artifactId>assertj-core</artifactId>
91+
</dependency>
92+
<dependency>
93+
<groupId>org.hamcrest</groupId>
94+
<artifactId>hamcrest-library</artifactId>
95+
</dependency>
96+
<dependency>
97+
<groupId>org.apache.parquet</groupId>
98+
<artifactId>parquet-hadoop</artifactId>
99+
<version>${parquet.version}</version>
100+
</dependency>
101+
102+
<dependency>
103+
<groupId>org.apache.iceberg</groupId>
104+
<artifactId>iceberg-core</artifactId>
105+
<version>${iceberg.version}</version>
106+
<exclusions>
107+
<exclusion>
108+
<groupId>org.apache.httpcomponents.core5</groupId>
109+
<artifactId>*</artifactId>
110+
</exclusion>
111+
<exclusion>
112+
<groupId>org.apache.httpcomponents.client5</groupId>
113+
<artifactId>*</artifactId>
114+
</exclusion>
115+
<exclusion>
116+
<groupId>com.fasterxml.jackson.core</groupId>
117+
<artifactId>*</artifactId>
118+
</exclusion>
119+
<exclusion>
120+
<groupId>io.airlift</groupId>
121+
<artifactId>aircompressor</artifactId>
122+
</exclusion>
123+
</exclusions>
124+
</dependency>
125+
126+
</dependencies>
127+
128+
<build>
129+
<resources>
130+
<resource>
131+
<directory>src/main/resources</directory>
132+
<filtering>true</filtering>
133+
</resource>
134+
</resources>
135+
<testResources>
136+
<testResource>
137+
<directory>src/test/resources</directory>
138+
<filtering>true</filtering>
139+
</testResource>
140+
</testResources>
141+
<plugins>
142+
143+
<plugin>
144+
<artifactId>maven-dependency-plugin</artifactId>
145+
<executions>
146+
<execution>
147+
<phase>package</phase>
148+
<goals>
149+
<goal>copy-dependencies</goal>
150+
</goals>
151+
<configuration>
152+
<outputDirectory>${project.build.directory}/lib</outputDirectory>
153+
</configuration>
154+
</execution>
155+
<execution>
156+
<id>deplist</id>
157+
<phase>compile</phase>
158+
<goals>
159+
<goal>list</goal>
160+
</goals>
161+
<configuration>
162+
<!-- referenced by a built-in command -->
163+
<outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt</outputFile>
164+
</configuration>
165+
</execution>
166+
</executions>
167+
</plugin>
168+
169+
<plugin>
170+
<groupId>org.apache.maven.plugins</groupId>
171+
<artifactId>maven-source-plugin</artifactId>
172+
<configuration>
173+
<attach>true</attach>
174+
</configuration>
175+
<executions>
176+
<execution>
177+
<goals>
178+
<goal>jar</goal>
179+
</goals>
180+
</execution>
181+
</executions>
182+
</plugin>
183+
</plugins>
184+
</build>
185+
</project>

‎hadoop-cloud-storage-project/pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
<modules>
3333
<module>hadoop-cloud-storage</module>
3434
<module>hadoop-cos</module>
35+
<module>hadoop-format-testing</module>
3536
<module>hadoop-huaweicloud</module>
3637
</modules>
3738

‎hadoop-project/pom.xml‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -787,7 +787,17 @@
787787
<version>${hadoop.version}</version>
788788
<type>test-jar</type>
789789
</dependency>
790-
790+
<dependency>
791+
<groupId>org.apache.hadoop</groupId>
792+
<artifactId>hadoop-format-testing</artifactId>
793+
<version>${hadoop.version}</version>
794+
</dependency>
795+
<dependency>
796+
<groupId>org.apache.hadoop</groupId>
797+
<artifactId>hadoop-format-testing</artifactId>
798+
<version>${hadoop.version}</version>
799+
<type>test-jar</type>
800+
</dependency>
791801
<dependency>
792802
<groupId>com.google.guava</groupId>
793803
<artifactId>guava</artifactId>

‎hadoop-tools/hadoop-aws/pom.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -581,5 +581,12 @@
581581
<artifactId>bcpkix-jdk18on</artifactId>
582582
<scope>test</scope>
583583
</dependency>
584+
585+
<dependency>
586+
<groupId>org.apache.hadoop</groupId>
587+
<artifactId>hadoop-format-testing</artifactId>
588+
<type>test-jar</type>
589+
<scope>test</scope>
590+
</dependency>
584591
</dependencies>
585592
</project>

0 commit comments

Comments
 (0)