e2e for multiple k8s

.github/workflows/auto-nightly-ci.yaml

@@ -8,15 +8,77 @@ on:
   workflow_dispatch:
 
 jobs:
-  call-ci:
+  call_ci_latest:
     uses: ./.github/workflows/auto-pr-ci.yaml
     secrets: inherit
+    with:
+      ipfamily: all
 
+  call_ci_v1_22_7:
+    # it races to upload images and artifacts when all jobs running , so just call_ci_latest build image
+    needs: [call_ci_latest]
+    uses: ./.github/workflows/auto-pr-ci.yaml
+    secrets: inherit
+    with:
+      kindNodeImage: v1.22.7
+      ipfamily: dual
+      justE2E: 'true'
+
+  call_ci_v1_23_5:
+    # it races to upload images and artifacts when all jobs running , so just call_ci_latest build image
+    needs: [call_ci_latest]
+    uses: ./.github/workflows/auto-pr-ci.yaml
+    secrets: inherit
+    with:
+      kindNodeImage: v1.23.5
+      ipfamily: dual
+      justE2E: 'true'
+
+  call_ci_v1_24_4:
+    # it races to upload images and artifacts when all jobs running , so just call_ci_latest build image
+    needs: [call_ci_latest]
+    uses: ./.github/workflows/auto-pr-ci.yaml
+    secrets: inherit
+    with:
+      kindNodeImage: v1.24.4
+      ipfamily: dual
+      justE2E: 'true'
+
+  call_ci_v1_25_3:
+    # it races to upload images and artifacts when all jobs running , so just call_ci_latest build image
+    needs: [call_ci_latest]
+    uses: ./.github/workflows/auto-pr-ci.yaml
+    secrets: inherit
+    with:
+      kindNodeImage: v1.25.3
+      ipfamily: dual
+      justE2E: 'true'
+
+  call_ci_v1_26_2:
+    # it races to upload images and artifacts when all jobs running , so just call_ci_latest build image
+    needs: [call_ci_latest]
+    uses: ./.github/workflows/auto-pr-ci.yaml
+    secrets: inherit
+    with:
+      kindNodeImage: v1.26.2
+      ipfamily: dual
+      justE2E: 'true'
+
+  call_ci_v1_27_1:
+    # it races to upload images and artifacts when all jobs running , so just call_ci_latest build image
+    needs: [call_ci_latest]
+    uses: ./.github/workflows/auto-pr-ci.yaml
+    secrets: inherit
+    with:
+      kindNodeImage: v1.27.1
+      ipfamily: dual
+      justE2E: 'true'
+
   creat_issue:
     runs-on: ubuntu-latest
-    needs: [call-ci]
+    needs: [call_ci_latest, call_ci_v1_22_7, call_ci_v1_23_5, call_ci_v1_24_4, call_ci_v1_25_3, call_ci_v1_26_2, call_ci_v1_27_1]
     # https://docs.github.com/en/actions/learn-github-actions/contexts#jobs-context
-    if: ${{ always() && needs.call-ci.result == 'failure' }}
+    if: ${{ always() && ( needs.call_ci_latest.result == 'failure' || needs.call_ci_v1_22_7.result == 'failure' || needs.call_ci_v1_23_5.result == 'failure' || needs.call_ci_v1_24_4.result == 'failure' || needs.call_ci_v1_25_3.result == 'failure' || needs.call_ci_v1_26_2.result == 'failure' || needs.call_ci_v1_27_1.result == 'failure' ) }}
     steps:
       - name: echo
         run: |

.github/workflows/auto-pr-ci.yaml

@@ -3,6 +3,18 @@ name: Auto PR CI
 permissions: write-all
 on:
   workflow_call:
+    inputs:
+      kindNodeImage:
+        required: false
+        type: string
+      ipfamily:
+        required: false
+        type: string
+        default: 'all'
+      justE2E:
+        required: false
+        type: string
+        default: 'false'
   pull_request_target:
     types:
       - opened
@@ -31,6 +43,9 @@ on:
           - ipv6
           - dual
           - all
+      kindNodeImage:
+        description: 'kind node image tag'
+        required: false
 
 jobs:
   prepare:
@@ -43,6 +58,8 @@ jobs:
       ipfamily_ipv4only_e2e: ${{ env.RUN_E2E_IPV4_ONLY }}
       ipfamily_ipv6only_e2e: ${{ env.RUN_E2E_IPV6_ONLY }}
       ipfamily_dual_e2e: ${{ env.RUN_E2E_DUAL_STACK }}
+      kindNodeImage: ${{ env.RUN_kindNodeImage }}
+      JustE2E: ${{ env.RUN_JustE2E }}
     steps:
       - name: Check Code Changes
         uses: dorny/paths-filter@v2.11.1
@@ -69,12 +86,17 @@ jobs:
         id: get_ref
         run: |
           echo "event ${{ github.event_name }} "
+          echo "RUN_kindNodeImage=" >> $GITHUB_ENV
+          echo "RUN_JustE2E=false" >> $GITHUB_ENV
           if ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.ipfamily != '' }}; then
               echo "call by self workflow_dispatch"
               echo "RUN_TAG=${{ github.event.inputs.ref }}" >> $GITHUB_ENV
               echo "RUN_E2E_LABEL=${{ github.event.inputs.e2e_labels }}" >> $GITHUB_ENV
               echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV
               echo "RUN_UNITEST_ENABLED=true" >> $GITHUB_ENV
+              if ${{ github.event.inputs.kindNodeImage != '' }}; then
+                  echo "RUN_kindNodeImage=${{ github.event.inputs.kindNodeImage }}" >> $GITHUB_ENV
+              fi
               if ${{ github.event.inputs.ipfamily == 'ipv4' }}; then
                 echo "RUN_E2E_IPV4_ONLY=true" >> $GITHUB_ENV
                 echo "RUN_E2E_IPV6_ONLY=false" >> $GITHUB_ENV
@@ -131,9 +153,33 @@ jobs:
               echo "RUN_E2E_LABEL=" >> $GITHUB_ENV
               echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV
               echo "RUN_UNITEST_ENABLED=true" >> $GITHUB_ENV
-              echo "RUN_E2E_IPV4_ONLY=true" >> $GITHUB_ENV
-              echo "RUN_E2E_IPV6_ONLY=true" >> $GITHUB_ENV
-              echo "RUN_E2E_DUAL_STACK=true" >> $GITHUB_ENV
+              if ${{ inputs.kindNodeImage != '' }}; then
+                echo "RUN_kindNodeImage=${{ inputs.kindNodeImage }}" >> $GITHUB_ENV
+              fi
+              if ${{ inputs.justE2E == 'true' }}; then
+                echo "RUN_JustE2E=true" >> $GITHUB_ENV
+              fi
+              if ${{ inputs.ipfamily == 'ipv4' }}; then
+                echo "RUN_E2E_IPV4_ONLY=true" >> $GITHUB_ENV
+                echo "RUN_E2E_IPV6_ONLY=false" >> $GITHUB_ENV
+                echo "RUN_E2E_DUAL_STACK=false" >> $GITHUB_ENV
+              elif ${{ inputs.ipfamily == 'ipv6' }}; then
+                echo "RUN_E2E_IPV4_ONLY=false" >> $GITHUB_ENV
+                echo "RUN_E2E_IPV6_ONLY=true" >> $GITHUB_ENV
+                echo "RUN_E2E_DUAL_STACK=false" >> $GITHUB_ENV
+              elif ${{ inputs.ipfamily == 'dual' }}; then
+                echo "RUN_E2E_IPV4_ONLY=false" >> $GITHUB_ENV
+                echo "RUN_E2E_IPV6_ONLY=false" >> $GITHUB_ENV
+                echo "RUN_E2E_DUAL_STACK=true" >> $GITHUB_ENV
+              elif ${{ inputs.ipfamily == 'all' }}; then
+                echo "RUN_E2E_IPV4_ONLY=true" >> $GITHUB_ENV
+                echo "RUN_E2E_IPV6_ONLY=true" >> $GITHUB_ENV
+                echo "RUN_E2E_DUAL_STACK=true" >> $GITHUB_ENV
+              else
+                echo "RUN_E2E_IPV4_ONLY=true" >> $GITHUB_ENV
+                echo "RUN_E2E_IPV6_ONLY=true" >> $GITHUB_ENV
+                echo "RUN_E2E_DUAL_STACK=true" >> $GITHUB_ENV
+              fi
           fi
 
       # some event, the tag is not sha, so checkout it and get sha
@@ -151,7 +197,7 @@ jobs:
 
   call_unitest:
     needs: prepare
-    if: ${{ needs.prepare.outputs.unitest_enabled == 'true' }}
+    if: ${{ needs.prepare.outputs.unitest_enabled == 'true' && needs.prepare.outputs.JustE2E == 'false' }}
     # forbid to specify version for local workflow, GITHUB_REF Same as the caller workflow
     uses: ./.github/workflows/lint-golang.yaml
     with:
@@ -160,7 +206,7 @@ jobs:
 
   call_build_ci_image:
     needs: prepare
-    if: ${{ needs.prepare.outputs.e2e_enabled == 'true' }}
+    if: ${{ needs.prepare.outputs.e2e_enabled == 'true' && needs.prepare.outputs.JustE2E == 'false' }}
     # get  image:${{ needs.prepare.outputs.ref }} and image-ci:${{ needs.prepare.outputs.ref }}
     uses: ./.github/workflows/build-image-ci.yaml
     with:
@@ -169,7 +215,7 @@ jobs:
 
   lint_chart_against_release_image:
     needs: prepare
-    if: ${{ needs.prepare.outputs.e2e_enabled == 'true' }}
+    if: ${{ needs.prepare.outputs.e2e_enabled == 'true' && needs.prepare.outputs.JustE2E == 'false' }}
     # forbid to specify version for local workflow, GITHUB_REF Same as the caller workflow
     uses: ./.github/workflows/call-lint-chart.yaml
     with:
@@ -178,37 +224,40 @@ jobs:
 
   e2e_dual:
     needs: [call_build_ci_image, prepare]
-    if: ${{ needs.prepare.outputs.e2e_enabled == 'true' && needs.prepare.outputs.ipfamily_dual_e2e == 'true' }}
+    if: ${{ always() && needs.prepare.outputs.e2e_enabled == 'true' && needs.prepare.outputs.ipfamily_dual_e2e == 'true' }}
     uses: ./.github/workflows/call-e2e.yaml
     with:
       ref: ${{ needs.prepare.outputs.ref }}
       ipfamily: dual
       e2e_labels: ${{ needs.prepare.outputs.e2e_labels }}
+      kind_node_image: ${{ needs.prepare.outputs.kindNodeImage }}
     secrets: inherit
 
   e2e_ipv4:
     needs: [call_build_ci_image, prepare]
-    if: ${{ needs.prepare.outputs.e2e_enabled == 'true' && needs.prepare.outputs.ipfamily_ipv4only_e2e == 'true' }}
+    if: ${{ always() && needs.prepare.outputs.e2e_enabled == 'true' && needs.prepare.outputs.ipfamily_ipv4only_e2e == 'true' }}
     uses: ./.github/workflows/call-e2e.yaml
     with:
       ref: ${{ needs.prepare.outputs.ref }}
       ipfamily: ipv4
       e2e_labels: ${{ needs.prepare.outputs.e2e_labels }}
+      kind_node_image: ${{ needs.prepare.outputs.kindNodeImage }}
     secrets: inherit
 
-
   e2e_ipv6:
     needs: [call_build_ci_image, prepare]
-    if: ${{ needs.prepare.outputs.e2e_enabled == 'true'  && needs.prepare.outputs.ipfamily_ipv6only_e2e == 'true' }}
+    if: ${{ always() && needs.prepare.outputs.e2e_enabled == 'true'  && needs.prepare.outputs.ipfamily_ipv6only_e2e == 'true' }}
     uses: ./.github/workflows/call-e2e.yaml
     with:
       ref: ${{ needs.prepare.outputs.ref }}
       ipfamily: ipv6
       e2e_labels: ${{ needs.prepare.outputs.e2e_labels }}
+      kind_node_image: ${{ needs.prepare.outputs.kindNodeImage }}
     secrets: inherit
 
   trivy_scan:
     needs: [call_build_ci_image, prepare]
+    if: ${{ needs.prepare.outputs.JustE2E == 'false' }}
     uses: ./.github/workflows/call-trivy.yaml
     with:
       ref: ${{ needs.prepare.outputs.ref }}

.github/workflows/call-e2e.yaml

@@ -22,7 +22,9 @@ on:
       e2e_labels:
         required: false
         type: string
-
+      kind_node_image:
+        required: false
+        type: string
 
 jobs:
   e2e_against_commit:
@@ -107,7 +109,8 @@ jobs:
             make e2e_init -e E2E_KIND_CLUSTER_NAME=${{ env.RUN_KIND_CLUSTER_NAME }} \
                 -e PROJECT_IMAGE_VERSION=${{ inputs.ref }} \
                 -e E2E_IP_FAMILY=${{ inputs.ipfamily }} \
-                -e PYROSCOPE_LOCAL_PORT=''
+                -e PYROSCOPE_LOCAL_PORT='' \
+                -e E2E_KIND_NODE_IMAGE=${{ inputs.kind_node_image }}
 
       - name: Run e2e Test
         continue-on-error: true
@@ -136,15 +139,15 @@ jobs:
       - name: Upload e2e cluster log
         uses: actions/upload-artifact@v3.1.1
         with:
-          name: ${{ inputs.ipfamily }}-debuglog.txt
+          name: ${{ inputs.ipfamily }}-debuglog-${{ inputs.kind_node_image }}.txt
           path: ${{ env.E2E_LOG_PATH }}
           retention-days: 30
 
       - name: Upload e2e ginkgo report
         if: ${{ env.RUN_UPLOAD_LOG == 'true' }}
         uses: actions/upload-artifact@v3.1.1
         with:
-          name: ${{ inputs.ipfamily }}-e2ereport.json
+          name: ${{ inputs.ipfamily }}-e2ereport-${{ inputs.kind_node_image }}.json
           path: ${{ env.E2E_GINKGO_REPORT_PATH }}
           retention-days: 30
 

Makefile.defs

@@ -158,6 +158,7 @@ E2E_KIND_KUBECONFIG_PATH := $(E2E_RUNTIME_DIR)/kubeconfig_$(E2E_KIND_CLUSTER_NAM
 E2E_LOG_FILE ?= $(E2E_RUNTIME_DIR)/e2edebugLog.txt
 E2E_GINKGO_REPORT_FILE ?= $(E2E_RUNTIME_DIR)/ginkgoreport.json
 E2E_MULTUS_CNI_CONFIG_DIR ?= $(E2E_RUNTIME_DIR)/multus
+E2E_KIND_NODE_IMAGE ?=
 
 E2E_GINKGO_LABELS ?=
 E2E_GINKGO_OPTION ?=

test/Makefile

@@ -34,7 +34,9 @@ init_one_kind: checkBin clean
 	- sysctl -w fs.inotify.max_user_watches=524288 || true
 	- sysctl -w fs.inotify.max_user_instances=8192  || true
 	- kind delete cluster --name  $(KIND_CLUSTER_NAME)
-	kind create cluster --name  $(KIND_CLUSTER_NAME) --config $(E2E_RUNTIME_DIR)/kind_config_$(KIND_CLUSTER_NAME).yaml --kubeconfig $(KIND_KUBECONFIG)
+	KIND_OPTION="" ; \
+		[ -n "$(E2E_KIND_NODE_IMAGE)" ] && KIND_OPTION=" --image $(E2E_KIND_NODE_IMAGE) " && echo "setup kind with E2E_KIND_NODE_IMAGE=$(E2E_KIND_NODE_IMAGE)"; \
+		kind create cluster --name  $(KIND_CLUSTER_NAME) --config $(E2E_RUNTIME_DIR)/kind_config_$(KIND_CLUSTER_NAME).yaml --kubeconfig $(KIND_KUBECONFIG) $${KIND_OPTION}
 	- kubectl --kubeconfig $(KIND_KUBECONFIG) taint nodes --all node-role.kubernetes.io/master- || true
 	- kubectl --kubeconfig $(KIND_KUBECONFIG) taint nodes --all node-role.kubernetes.io/control-plane- || true
 	@echo "===================== deploy prometheus CRD ========== "