Merge branch 'apache:master' into master

Author: beryllw

.dlc.json

@@ -10,7 +10,7 @@
       "pattern": "^{"
     },
     {
-      "pattern": "^https://repo1.maven.org/maven2/org/apache/flink.*SNAPSHOT.*"
+      "pattern": "^https://repo1.maven.org/maven2/org/apache/flink.*"
     },
     {
       "pattern": "^https://mvnrepository.com"

.github/workflows/close_stale.yml

@@ -53,4 +53,7 @@ jobs:
           # Auto-remove stale has conflicts with existing approve label workflows
           # and will result in Stale label missing.
           remove-pr-stale-when-updated: false
-          remove-issue-stale-when-updated: false
+          remove-issue-stale-when-updated: false
+          # Allow pinning issues and PRs that is not meant to be closed automatically
+          exempt-issue-labels: "pinned"
+          exempt-pr-labels: "pinned"

.github/workflows/flink_cdc_base.yml

@@ -19,18 +19,24 @@ on:
   workflow_call:
     inputs:
       java-version:
-        description: "Jdk version to test against."
+        description: "Jdk versions to test against."
         required: false
         type: string
-        default: "8"
+        default: "['8']"
       flink-version:
-        description: "Flink version to test against."
+        description: "Flink versions to test against."
         required: false
         type: string
+        default: "['generic']"
       module:
         description: "Flink CDC module to test against."
         required: true
         type: string
+      parallelism:
+        description: "Flink parallelism."
+        required: false
+        type: number
+        default: 4
       custom-maven-parameter:
         description: "Custom maven parameter."
         required: false
@@ -41,7 +47,8 @@ env:
   flink-cdc-cli,\
   flink-cdc-common,\
   flink-cdc-composer,\
-  flink-cdc-runtime"
+  flink-cdc-runtime,\
+  flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base"
 
   MODULES_PIPELINE_CONNECTORS: "\
   flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values,\
@@ -50,7 +57,8 @@ env:
   flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks,\
   flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka,\
   flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon,\
-  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch"
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase"
 
   MODULES_MYSQL: "\
   flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc,\
@@ -98,9 +106,12 @@ jobs:
   compile_and_test:
     runs-on: ubuntu-latest
     timeout-minutes: 120
+    strategy:
+      matrix:
+        java-version: ${{ fromJSON(inputs.java-version) }}
+        flink-version: ${{ fromJSON(inputs.flink-version) }}
     steps:
-      - run: echo "Running CI pipeline for JDK version ${{ inputs.java-version }}"
-
+      - run: echo "Running CI pipeline for JDK version ${{ matrix.java-version }}"
       - name: Clean up disk space
         run: |
           set -euo pipefail
@@ -126,7 +137,7 @@ jobs:
       - name: Set JDK
         uses: actions/setup-java@v4
         with:
-          java-version: ${{ inputs.java-version }}
+          java-version: ${{ matrix.java-version }}
           distribution: 'temurin'
           cache: 'maven'
 
@@ -199,13 +210,13 @@ jobs:
             build_maven_parameter="-DspecifiedMongoVersion=7.0.12"
           fi
           
-          if [ ! -z "${{ inputs.flink-version }}" ]; then
-            build_maven_parameter="${build_maven_parameter:+$build_maven_parameter }-DspecifiedFlinkVersion=${{ inputs.flink-version }}"
+          if [ ! -z "${{ matrix.flink-version }}" ]; then
+            build_maven_parameter="${build_maven_parameter:+$build_maven_parameter }-DspecifiedFlinkVersion=${{ matrix.flink-version }}"
           fi
           
           build_maven_parameter="${build_maven_parameter:+$build_maven_parameter }${{ inputs.custom-maven-parameter }}"
           
-          mvn --no-snapshot-updates -B -DskipTests -pl $compile_modules -am install && mvn --no-snapshot-updates -B $build_maven_parameter -pl $modules verify
+          mvn --no-snapshot-updates -B -DskipTests -pl $compile_modules -am install && mvn --no-snapshot-updates -B $build_maven_parameter -pl $modules -DspecifiedParallelism=${{ inputs.parallelism }} verify
 
       - name: Print JVM thread dumps when cancelled
         if: ${{ failure() }}

.github/workflows/flink_cdc.yml renamed to .github/workflows/flink_cdc_ci.yml

@@ -60,37 +60,33 @@ jobs:
   ut:
     strategy:
       matrix:
-        java-version: [ '8' ]
         module: [ 'core', 'pipeline_connectors', 'mysql', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase', 'db2', 'vitess' ]
     name: Unit Tests
     uses: ./.github/workflows/flink_cdc_base.yml
     with:
-      java-version: ${{ matrix.java-version }}
+      java-version: "[8]"
       module: ${{ matrix.module }}
   pipeline_e2e:
     strategy:
       matrix:
-        java-version: [ '8' ]
-        flink-version: ['1.17.2', '1.18.1', '1.19.1', '1.20.0']
-        module: [ 'pipeline_e2e' ]
-    name: Pipeline E2E Tests
+        parallelism: [ 1, 4 ]
+    name: Pipeline E2E Tests (${{ matrix.parallelism }}-Parallelism)
     uses: ./.github/workflows/flink_cdc_base.yml
     with:
-      java-version: ${{ matrix.java-version }}
-      flink-version: ${{ matrix.flink-version }}
-      module: ${{ matrix.module }}
+      java-version: "[8]"
+      flink-version: "['1.19.1', '1.20.0']"
+      module: pipeline_e2e
+      parallelism: ${{ matrix.parallelism }}
   source_e2e:
-    strategy:
-      matrix:
-        java-version: [ '8' ]
-        flink-version: ['1.16.3', '1.17.2', '1.18.1', '1.19.1', '1.20.0']
-        module: [ 'source_e2e' ]
     name: Source E2E Tests
     uses: ./.github/workflows/flink_cdc_base.yml
     with:
-      java-version: ${{ matrix.java-version }}
-      flink-version: ${{ matrix.flink-version }}
-      module: ${{ matrix.module }}
+      java-version: "[8]"
+      flink-version: "['1.19.1', '1.20.0']"
+      module: source_e2e
   migration_test:
     name: Migration Tests
-    uses: ./.github/workflows/flink_cdc_migration_test.yml
+    uses: ./.github/workflows/flink_cdc_migration_test_base.yml
+    with:
+      java-version: "[8]"
+      flink-version: "['1.19.1', '1.20.0']"

.github/workflows/flink_cdc_ci_nightly.yml

@@ -0,0 +1,81 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Flink CDC CI Nightly
+on:
+  schedule:
+    - cron: '0 0 * * *' # Deploy every day
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  license_check:
+    name: License Check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Set up Ruby environment
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.3'
+      - name: Set JDK
+        uses: actions/setup-java@v4
+        with:
+          java-version: '11'
+          distribution: 'temurin'
+          cache: 'maven'
+      - name: Compiling jar packages
+        run: mvn --no-snapshot-updates -B package -DskipTests
+      - name: Run license check
+        run: gem install rubyzip -v 2.3.0 && ./tools/ci/license_check.rb
+  ut:
+    strategy:
+      matrix:
+        module: [ 'core', 'pipeline_connectors', 'mysql', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase', 'db2', 'vitess' ]
+    name: Unit Tests
+    uses: ./.github/workflows/flink_cdc_base.yml
+    with:
+      java-version: "[11]"
+      module: ${{ matrix.module }}
+  pipeline_e2e:
+    strategy:
+      matrix:
+        parallelism: [ 1, 4 ]
+    name: Pipeline E2E Tests (${{ matrix.parallelism }} Parallelism)
+    uses: ./.github/workflows/flink_cdc_base.yml
+    with:
+      java-version: "[11]"
+      flink-version: "['1.19.1', '1.20.0']"
+      module: pipeline_e2e
+      parallelism: ${{ matrix.parallelism }}
+  source_e2e:
+    name: Source E2E Tests
+    uses: ./.github/workflows/flink_cdc_base.yml
+    with:
+      java-version: "[11]"
+      flink-version: "['1.19.1', '1.20.0']"
+      module: source_e2e
+  migration_test:
+    name: Migration Tests
+    uses: ./.github/workflows/flink_cdc_migration_test_base.yml
+    with:
+      java-version: "[11]"
+      flink-version: "['1.19.1', '1.20.0']"

.github/workflows/flink_cdc_migration_test.yml renamed to .github/workflows/flink_cdc_migration_test_base.yml

@@ -17,18 +17,32 @@ name: Migration Tests
 
 on:
   workflow_call:
+    inputs:
+      java-version:
+        description: "Jdk versions to test against, passed as a JSON array string."
+        required: false
+        type: string
+        default: "['8']"
+      flink-version:
+        description: "Flink versions to test against, passed as a JSON array string."
+        required: true
+        type: string
 
 jobs:
   migration_test_ut:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java-version: ${{ fromJSON(inputs.java-version) }}
+        flink-version: ${{ fromJSON(inputs.flink-version) }}
     steps:
       - name: Check out repository code
         uses: actions/checkout@v4
         with:
           submodules: true
       - uses: actions/setup-java@v4
         with:
-          java-version: 8
+          java-version: ${{ matrix.java-version }}
           distribution: temurin
       - name: Compile snapshot CDC version
         run: mvn --no-snapshot-updates -B install -DskipTests
@@ -39,9 +53,8 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        # '1.20.0' is excluded since FLINK-36105 has not been merged.
-        flink-version: [ '1.18.1', '1.19.1' ]
-
+        java-version: ${{ fromJSON(inputs.java-version) }}
+        flink-version: ${{ fromJSON(inputs.flink-version) }}
     steps:
       - uses: actions/checkout@v4
       - name: Set up Ruby
@@ -51,7 +64,7 @@ jobs:
           bundler-cache: true # runs 'bundle install' and caches installed gems automatically
       - uses: actions/setup-java@v4
         with:
-          java-version: 8
+          java-version: ${{ matrix.java-version }}
           distribution: temurin
           cache: maven
       - name: Install dependencies
@@ -79,7 +92,8 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        flink-version: [ '1.18.1', '1.19.1', '1.20.0' ]
+        java-version: ${{ fromJSON(inputs.java-version) }}
+        flink-version: [ '1.19.1', '1.20.0' ]
 
     steps:
       - uses: actions/checkout@v4
@@ -90,7 +104,7 @@ jobs:
           bundler-cache: true # runs 'bundle install' and caches installed gems automatically
       - uses: actions/setup-java@v4
         with:
-          java-version: 8
+          java-version: ${{ matrix.java-version }}
           distribution: temurin
           cache: maven
       - name: Install dependencies