merge from master

Author: yuezhang

.github/workflows/bot.yml

@@ -21,37 +21,45 @@ jobs:
           - scalaProfile: "scala-2.11"
             sparkProfile: "spark2.4"
             sparkVersion: "2.4.4"
+            flinkProfile: "flink1.13"
 
           # Spark 2.4.4, scala 2.12
           - scalaProfile: "scala-2.12"
             sparkProfile: "spark2.4"
             sparkVersion: "2.4.4"
+            flinkProfile: "flink1.14"
 
           # Spark 3.1.x
           - scalaProfile: "scala-2.12"
             sparkProfile: "spark3.1"
             sparkVersion: "3.1.0"
+            flinkProfile: "flink1.13"
 
           - scalaProfile: "scala-2.12"
             sparkProfile: "spark3.1"
             sparkVersion: "3.1.1"
+            flinkProfile: "flink1.13"
 
           - scalaProfile: "scala-2.12"
             sparkProfile: "spark3.1"
             sparkVersion: "3.1.2"
+            flinkProfile: "flink1.14"
 
           - scalaProfile: "scala-2.12"
             sparkProfile: "spark3.1"
             sparkVersion: "3.1.3"
+            flinkProfile: "flink1.14"
 
           # Spark 3.2.x
           - scalaProfile: "scala-2.12"
             sparkProfile: "spark3.2"
             sparkVersion: "3.2.0"
+            flinkProfile: "flink1.13"
 
           - scalaProfile: "scala-2.12"
             sparkProfile: "spark3.2"
             sparkVersion: "3.2.1"
+            flinkProfile: "flink1.14"
 
     steps:
       - uses: actions/checkout@v2
@@ -66,13 +74,15 @@ jobs:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
           SPARK_VERSION: ${{ matrix.sparkVersion }}
+          FLINK_PROFILE: ${{ matrix.flinkProfile }}
         run:
-          mvn -T 2.5C clean install -P "$SCALA_PROFILE,$SPARK_PROFILE" -Dspark.version="$SPARK_VERSION" -DskipTests=true -Dmaven.javadoc.skip=true -B -V
+          mvn -T 2.5C clean install -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -Dspark.version="$SPARK_VERSION" -DskipTests=true -Dmaven.javadoc.skip=true -B -V
       - name: Quickstart Test
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
           SPARK_VERSION: ${{ matrix.sparkVersion }}
+          FLINK_PROFILE: ${{ matrix.flinkProfile }}
         if: ${{ !startsWith(env.SPARK_VERSION, '3.2.') }} # skip test spark 3.2 before hadoop upgrade to 3.x
         run:
-          mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -Dspark.version="$SPARK_VERSION" -DfailIfNoTests=false -pl hudi-examples/hudi-examples-flink,hudi-examples/hudi-examples-java,hudi-examples/hudi-examples-spark
+          mvn test -P "unit-tests" -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -Dspark.version="$SPARK_VERSION" -DfailIfNoTests=false -pl hudi-examples/hudi-examples-flink,hudi-examples/hudi-examples-java,hudi-examples/hudi-examples-spark

docker/README.md

@@ -51,9 +51,19 @@ mvn clean pre-integration-test -DskipTests -Ddocker.compose.skip=true -Ddocker.b
 mvn clean pre-integration-test -DskipTests -Ddocker.compose.skip=true -Ddocker.build.skip=false -pl :hudi-hadoop-trinobase-docker -am
 ```
 
-Alternatively, you can use `docker` cli directly under `hoodie/hadoop`. Note that, you need to manually name your local
-image by using `-t` option to match the naming in the `pom.xml`, so that you can update the corresponding image
-repository in Docker Hub (detailed steps in the next section).
+Alternatively, you can use `docker` cli directly under `hoodie/hadoop` to build images in a faster way. If you use this
+approach, make sure you first build Hudi modules with `integration-tests` profile as below so that the latest Hudi jars
+built are copied to the corresponding Hudi docker folder, e.g., `$HUDI_DIR/docker/hoodie/hadoop/hive_base/target`, which
+is required to build each docker image. Otherwise, the `target/` folder can be missing and `docker` cli complains about
+that: `failed to compute cache key: "/target" not found: not found`.
+
+```shell
+mvn -Pintegration-tests clean package -DskipTests
+```
+
+Note that, to build the image with `docker` cli, you need to manually name your local image by using `-t` option to
+match the naming in the `pom.xml`, so that you can update the corresponding image repository in Docker Hub (detailed
+steps in the next section).
 
 ```shell
 # Run under hoodie/hadoop, the <tag> is optional, "latest" by default

docker/hoodie/hadoop/base/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>0.11.0-SNAPSHOT</version>
+    <version>0.12.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <packaging>pom</packaging>

docker/hoodie/hadoop/base_java11/pom.xml

@@ -20,7 +20,7 @@
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>0.11.0-SNAPSHOT</version>
+    <version>0.12.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <packaging>pom</packaging>

docker/hoodie/hadoop/datanode/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>0.11.0-SNAPSHOT</version>
+    <version>0.12.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <packaging>pom</packaging>

docker/hoodie/hadoop/historyserver/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>0.11.0-SNAPSHOT</version>
+    <version>0.12.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <packaging>pom</packaging>

docker/hoodie/hadoop/hive_base/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>0.11.0-SNAPSHOT</version>
+    <version>0.12.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <packaging>pom</packaging>

docker/hoodie/hadoop/namenode/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>0.11.0-SNAPSHOT</version>
+    <version>0.12.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <packaging>pom</packaging>

docker/hoodie/hadoop/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <artifactId>hudi</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>0.11.0-SNAPSHOT</version>
+    <version>0.12.0-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>

docker/hoodie/hadoop/prestobase/pom.xml

@@ -20,7 +20,7 @@
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>0.11.0-SNAPSHOT</version>
+    <version>0.12.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <packaging>pom</packaging>