Skip to content

Commit da2530b

Browse files
XuQianJin-Starsxushiyan
authored andcommitted
[HUDI-3563] Make quickstart examples covered by CI tests
1 parent f20c986 commit da2530b

38 files changed

Lines changed: 2980 additions & 225 deletions

File tree

.github/workflows/bot.yml

Lines changed: 44 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -14,20 +14,39 @@ jobs:
1414
build:
1515
runs-on: ubuntu-latest
1616
strategy:
17+
max-parallel: 7
1718
matrix:
1819
include:
19-
- scala: "scala-2.11"
20-
spark: "spark2"
21-
- scala: "scala-2.11"
22-
spark: "spark2,spark-shade-unbundle-avro"
23-
- scala: "scala-2.12"
24-
spark: "spark3.1.x"
25-
- scala: "scala-2.12"
26-
spark: "spark3.1.x,spark-shade-unbundle-avro"
27-
- scala: "scala-2.12"
28-
spark: "spark3"
29-
- scala: "scala-2.12"
30-
spark: "spark3,spark-shade-unbundle-avro"
20+
# Spark 2.4.4
21+
- scalaProfile: "scala-2.11"
22+
sparkProfile: "spark2"
23+
sparkVersion: "2.4.4"
24+
25+
# Spark 3.1.x
26+
- scalaProfile: "scala-2.12"
27+
sparkProfile: "spark3.1.x"
28+
sparkVersion: "3.1.0"
29+
30+
- scalaProfile: "scala-2.12"
31+
sparkProfile: "spark3.1.x"
32+
sparkVersion: "3.1.1"
33+
34+
- scalaProfile: "scala-2.12"
35+
sparkProfile: "spark3.1.x"
36+
sparkVersion: "3.1.2"
37+
38+
- scalaProfile: "scala-2.12"
39+
sparkProfile: "spark3.1.x"
40+
sparkVersion: "3.1.3"
41+
42+
# Spark 3.2.x
43+
- scalaProfile: "scala-2.12"
44+
sparkProfile: "spark3.2.0"
45+
sparkVersion: "3.2.0"
46+
47+
- scalaProfile: "scala-2.12"
48+
sparkProfile: "spark3"
49+
sparkVersion: "3.2.1"
3150
steps:
3251
- uses: actions/checkout@v2
3352
- name: Set up JDK 8
@@ -38,6 +57,16 @@ jobs:
3857
architecture: x64
3958
- name: Build Project
4059
env:
41-
SCALA_PROFILE: ${{ matrix.scala }}
42-
SPARK_PROFILE: ${{ matrix.spark }}
43-
run: mvn install -P "$SCALA_PROFILE,$SPARK_PROFILE" -DskipTests=true -Dmaven.javadoc.skip=true -B -V
60+
SCALA_PROFILE: ${{ matrix.scalaProfile }}
61+
SPARK_PROFILE: ${{ matrix.sparkProfile }}
62+
SPARK_VERSION: ${{ matrix.sparkVersion }}
63+
run:
64+
mvn -T 2.5C clean install -P "$SCALA_PROFILE,$SPARK_PROFILE" -Dspark.version="$SPARK_VERSION" -DskipTests=true -Dmaven.javadoc.skip=true -B -V
65+
- name: Quickstart Test
66+
env:
67+
SCALA_PROFILE: ${{ matrix.scalaProfile }}
68+
SPARK_PROFILE: ${{ matrix.sparkProfile }}
69+
SPARK_VERSION: ${{ matrix.sparkVersion }}
70+
if: ${{ !startsWith(env.SPARK_VERSION, '3.2.') }} # skip test spark 3.2 before hadoop upgrade to 3.x
71+
run:
72+
mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -Dspark.version="$SPARK_VERSION" -DfailIfNoTests=false -pl hudi-examples/hudi-examples-flink,hudi-examples/hudi-examples-java,hudi-examples/hudi-examples-spark

azure-pipelines.yml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ variables:
2626
SPARK_VERSION: '2.4.4'
2727
HADOOP_VERSION: '2.7'
2828
SPARK_ARCHIVE: spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)
29+
EXCLUDE_TESTED_MODULES: '!hudi-examples/hudi-examples-common,!hudi-examples/hudi-examples-flink,!hudi-examples/hudi-examples-java,!hudi-examples/hudi-examples-spark,!hudi-common,!hudi-flink-datasource/hudi-flink,!hudi-client/hudi-spark-client,!hudi-client/hudi-client-common,!hudi-client/hudi-flink-client,!hudi-client/hudi-java-client,!hudi-cli,!hudi-utilities,!hudi-sync/hudi-hive-sync'
2930

3031
stages:
3132
- stage: test
@@ -132,7 +133,7 @@ stages:
132133
inputs:
133134
mavenPomFile: 'pom.xml'
134135
goals: 'test'
135-
options: -Punit-tests -pl !hudi-common,!hudi-flink-datasource/hudi-flink,!hudi-client/hudi-spark-client,!hudi-client/hudi-client-common,!hudi-client/hudi-flink-client,!hudi-client/hudi-java-client,!hudi-cli,!hudi-utilities,!hudi-sync/hudi-hive-sync
136+
options: -Punit-tests -pl $(EXCLUDE_TESTED_MODULES)
136137
publishJUnitResults: false
137138
jdkVersionOption: '1.8'
138139
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
@@ -141,7 +142,7 @@ stages:
141142
inputs:
142143
mavenPomFile: 'pom.xml'
143144
goals: 'test'
144-
options: -Pfunctional-tests -pl !hudi-common,!hudi-flink-datasource/hudi-flink,!hudi-client/hudi-spark-client,!hudi-client/hudi-client-common,!hudi-client/hudi-flink-client,!hudi-client/hudi-java-client,!hudi-cli,!hudi-utilities,!hudi-sync/hudi-hive-sync
145+
options: -Pfunctional-tests -pl $(EXCLUDE_TESTED_MODULES)
145146
publishJUnitResults: false
146147
jdkVersionOption: '1.8'
147148
mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one or more
4+
contributor license agreements. See the NOTICE file distributed with
5+
this work for additional information regarding copyright ownership.
6+
The ASF licenses this file to You under the Apache License, Version 2.0
7+
(the "License"); you may not use this file except in compliance with
8+
the License. You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
-->
18+
<project xmlns="http://maven.apache.org/POM/4.0.0"
19+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
21+
<parent>
22+
<artifactId>hudi-examples</artifactId>
23+
<groupId>org.apache.hudi</groupId>
24+
<version>0.11.0-SNAPSHOT</version>
25+
</parent>
26+
<modelVersion>4.0.0</modelVersion>
27+
28+
<artifactId>hudi-examples-common</artifactId>
29+
30+
<properties>
31+
<main.basedir>${project.parent.basedir}</main.basedir>
32+
<checkstyle.skip>true</checkstyle.skip>
33+
</properties>
34+
35+
<build>
36+
<resources>
37+
<resource>
38+
<directory>src/main/resources</directory>
39+
</resource>
40+
</resources>
41+
42+
<plugins>
43+
<plugin>
44+
<groupId>net.alchim31.maven</groupId>
45+
<artifactId>scala-maven-plugin</artifactId>
46+
<executions>
47+
<execution>
48+
<id>scala-compile-first</id>
49+
<phase>process-resources</phase>
50+
<goals>
51+
<goal>add-source</goal>
52+
<goal>compile</goal>
53+
</goals>
54+
</execution>
55+
</executions>
56+
</plugin>
57+
<plugin>
58+
<groupId>org.apache.maven.plugins</groupId>
59+
<artifactId>maven-compiler-plugin</artifactId>
60+
<executions>
61+
<execution>
62+
<phase>compile</phase>
63+
<goals>
64+
<goal>compile</goal>
65+
</goals>
66+
</execution>
67+
</executions>
68+
</plugin>
69+
<plugin>
70+
<groupId>org.apache.maven.plugins</groupId>
71+
<artifactId>maven-jar-plugin</artifactId>
72+
<executions>
73+
<execution>
74+
<goals>
75+
<goal>test-jar</goal>
76+
</goals>
77+
<phase>test-compile</phase>
78+
</execution>
79+
</executions>
80+
<configuration>
81+
<skip>false</skip>
82+
</configuration>
83+
</plugin>
84+
<plugin>
85+
<groupId>org.apache.rat</groupId>
86+
<artifactId>apache-rat-plugin</artifactId>
87+
</plugin>
88+
</plugins>
89+
</build>
90+
91+
<dependencies>
92+
<dependency>
93+
<groupId>org.apache.hudi</groupId>
94+
<artifactId>hudi-common</artifactId>
95+
<version>${project.version}</version>
96+
</dependency>
97+
98+
<!-- Avro -->
99+
<dependency>
100+
<groupId>org.apache.avro</groupId>
101+
<artifactId>avro</artifactId>
102+
</dependency>
103+
104+
<dependency>
105+
<groupId>org.apache.parquet</groupId>
106+
<artifactId>parquet-avro</artifactId>
107+
</dependency>
108+
</dependencies>
109+
</project>

hudi-examples/src/main/java/org/apache/hudi/examples/common/HoodieExampleDataGenerator.java renamed to hudi-examples/hudi-examples-common/src/main/java/org/apache/hudi/examples/common/HoodieExampleDataGenerator.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@
4343
import java.util.stream.IntStream;
4444
import java.util.stream.Stream;
4545

46-
4746
/**
4847
* Class to be used to generate test data.
4948
*/
@@ -63,7 +62,7 @@ public class HoodieExampleDataGenerator<T extends HoodieRecordPayload<T>> {
6362
+ "{\"name\":\"fare\",\"type\": \"double\"}]}";
6463
public static Schema avroSchema = new Schema.Parser().parse(TRIP_EXAMPLE_SCHEMA);
6564

66-
private static Random rand = new Random(46474747);
65+
private static final Random rand = new Random(46474747);
6766

6867
private final Map<Integer, KeyPartition> existingKeys;
6968
private final String[] partitionPaths;

0 commit comments

Comments
 (0)