🌱 Add quickstart e2e test with v1beta1 with ClusterClass and RuntimeSDK

.gitignore

@@ -14,6 +14,7 @@ hack/tools/bin
 test/e2e/data/infrastructure-docker/**/cluster-template*.yaml
 !test/e2e/data/infrastructure-docker/**/clusterclass-quick-start.yaml
 !test/e2e/data/infrastructure-docker/**/clusterclass-quick-start-runtimesdk.yaml
+!test/e2e/data/infrastructure-docker/**/clusterclass-quick-start-runtimesdk-v1beta1.yaml
 !test/e2e/data/infrastructure-docker/**/cluster-template-in-memory.yaml
 !test/e2e/data/infrastructure-docker/**/clusterclass-in-memory.yaml
 test/e2e/data/infrastructure-docker/**/clusterclass-*.yaml

Makefile

@@ -627,6 +627,7 @@ generate-e2e-templates-main: $(KUSTOMIZE)
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-dualstack-ipv6-primary --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-dualstack-ipv6-primary.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-dualstack-ipv4-primary --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-dualstack-ipv4-primary.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-no-workers --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-no-workers.yaml
+	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-runtimesdk-v1beta1 --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-runtimesdk-v1beta1.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-kcp-only --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-kcp-only.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-autoscaler --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-autoscaler.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology.yaml

test/e2e/config/docker.yaml

@@ -343,6 +343,7 @@ providers:
     - sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv6-primary.yaml"
     - sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-dualstack-ipv4-primary.yaml"
     - sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-no-workers.yaml"
+    - sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-runtimesdk-v1beta1.yaml"
     - sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-kcp-only.yaml"
     - sourcePath: "../data/infrastructure-docker/main/cluster-template-topology-autoscaler.yaml"
     - sourcePath: "../data/infrastructure-docker/main/cluster-template-topology.yaml"
@@ -351,6 +352,7 @@ providers:
     - sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start.yaml"
     - sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start-kcp-only.yaml"
     - sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start-runtimesdk.yaml"
+    - sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start-runtimesdk-v1beta1.yaml"
     - sourcePath: "../data/infrastructure-docker/main/clusterclass-in-memory.yaml"
     - sourcePath: "../data/shared/main/metadata.yaml"
 

test/e2e/data/infrastructure-docker/main/cluster-template-topology-runtimesdk-v1beta1/cluster-runtimesdk.yaml

@@ -0,0 +1,45 @@
+# Note: This file is intentionally using v1beta1 for v1beta1 test coverage.
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: '${CLUSTER_NAME}'
+  namespace: default
+  labels:
+    cni: "${CLUSTER_NAME}-crs-0"
+spec:
+  clusterNetwork:
+    services:
+      cidrBlocks: ['${DOCKER_SERVICE_CIDRS}']
+    pods:
+      cidrBlocks: ['${DOCKER_POD_CIDRS}']
+    serviceDomain: '${DOCKER_SERVICE_DOMAIN}'
+  topology:
+    class: "quick-start-runtimesdk-v1beta1"
+    classNamespace: '${CLUSTER_CLASS_NAMESPACE:-${NAMESPACE}}'
+    version: "${KUBERNETES_VERSION}"
+    controlPlane:
+      nodeDeletionTimeout: 30s
+      replicas: ${CONTROL_PLANE_MACHINE_COUNT}
+    workers:
+      machineDeployments:
+        - class: "default-worker"
+          name: "md-0"
+          nodeDeletionTimeout: 30s
+          nodeVolumeDetachTimeout: 300s
+          minReadySeconds: 5
+          replicas: ${WORKER_MACHINE_COUNT}
+          failureDomain: fd4
+      machinePools:
+        - class: "default-worker"
+          name: "mp-0"
+          nodeDeletionTimeout: 30s
+          nodeVolumeDetachTimeout: 300s
+          minReadySeconds: 5
+          replicas: ${WORKER_MACHINE_COUNT}
+          failureDomains:
+            - fd4
+    variables:
+      - name: kubeadmControlPlaneMaxSurge
+        value: "1"
+      - name: imageRepository
+        value: "kindest"

test/e2e/data/infrastructure-docker/main/cluster-template-topology-runtimesdk-v1beta1/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+  - ../bases/crs.yaml
+  - cluster-runtimesdk.yaml

test/e2e/data/infrastructure-docker/main/clusterclass-quick-start-runtimesdk-v1beta1.yaml

@@ -0,0 +1,153 @@
+# Note: This file is intentionally using v1beta1 for v1beta1 test coverage.
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: ClusterClass
+metadata:
+  name: quick-start-runtimesdk-v1beta1
+spec:
+  controlPlane:
+    ref:
+      apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+      kind: KubeadmControlPlaneTemplate
+      name: quick-start-control-plane
+    machineInfrastructure:
+      ref:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: DockerMachineTemplate
+        name: quick-start-control-plane
+    namingStrategy:
+      template: "{{ .cluster.name }}-cp-{{ .random }}"
+  infrastructure:
+    ref:
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+      kind: DockerClusterTemplate
+      name: quick-start-cluster
+  infrastructureNamingStrategy:
+      template: "{{ .cluster.name }}-infra-{{ .random }}"
+  workers:
+    machineDeployments:
+    - class: default-worker
+      namingStrategy:
+        template: "{{ .cluster.name }}-md-{{ .machineDeployment.topologyName }}-{{ .random }}"
+      template:
+        bootstrap:
+          ref:
+            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+            kind: KubeadmConfigTemplate
+            name: quick-start-default-worker-bootstraptemplate
+        infrastructure:
+          ref:
+            apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+            kind: DockerMachineTemplate
+            name: quick-start-default-worker-machinetemplate
+    machinePools:
+    - class: default-worker
+      namingStrategy:
+        template: "{{ .cluster.name }}-mp-{{ .machinePool.topologyName }}-{{ .random }}"
+      template:
+        bootstrap:
+          ref:
+            apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+            kind: KubeadmConfigTemplate
+            name: quick-start-default-worker-bootstraptemplate
+        infrastructure:
+          ref:
+            apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+            kind: DockerMachinePoolTemplate
+            name: quick-start-default-worker-machinepooltemplate
+  patches:
+  - name: test-patch
+    external:
+      generateExtension: generate-patches.${EXTENSION_CONFIG_NAME:-test-extension}
+      validateExtension: validate-topology.${EXTENSION_CONFIG_NAME:-test-extension}
+      discoverVariablesExtension: discover-variables.${EXTENSION_CONFIG_NAME:-test-extension}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerClusterTemplate
+metadata:
+  name: quick-start-cluster
+spec:
+  template:
+    spec:
+      failureDomains:
+        fd1:
+          controlPlane: true
+        fd2:
+          controlPlane: true
+        fd3:
+          controlPlane: true
+        fd4:
+          controlPlane: false
+        fd5:
+          controlPlane: false
+        fd6:
+          controlPlane: false
+        fd7:
+          controlPlane: false
+        fd8:
+          controlPlane: false
+---
+kind: KubeadmControlPlaneTemplate
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+metadata:
+  name: quick-start-control-plane
+spec:
+  template:
+    spec:
+      machineTemplate:
+        nodeDrainTimeout: 1s
+      kubeadmConfigSpec:
+        clusterConfiguration:
+          apiServer:
+            # host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied.
+            certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal]
+        initConfiguration:
+          nodeRegistration: {} # node registration parameters are automatically injected by CAPD according to the kindest/node image in use.
+        joinConfiguration:
+          nodeRegistration: {} # node registration parameters are automatically injected by CAPD according to the kindest/node image in use.
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachineTemplate
+metadata:
+  name: quick-start-control-plane
+spec:
+  template:
+    spec:
+      extraMounts:
+      - containerPath: "/var/run/docker.sock"
+        hostPath: "/var/run/docker.sock"
+      preLoadImages: ${DOCKER_PRELOAD_IMAGES:-[]}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachineTemplate
+metadata:
+  name: quick-start-default-worker-machinetemplate
+spec:
+  template:
+    spec:
+      extraMounts:
+      - containerPath: "/var/run/docker.sock"
+        hostPath: "/var/run/docker.sock"
+      preLoadImages: ${DOCKER_PRELOAD_IMAGES:-[]}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachinePoolTemplate
+metadata:
+  name: quick-start-default-worker-machinepooltemplate
+spec:
+  template:
+    spec:
+      template:
+        extraMounts:
+        - containerPath: "/var/run/docker.sock"
+          hostPath: "/var/run/docker.sock"
+        preLoadImages: ${DOCKER_PRELOAD_IMAGES:-[]}
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KubeadmConfigTemplate
+metadata:
+  name: quick-start-default-worker-bootstraptemplate
+spec:
+  template:
+    spec:
+      joinConfiguration:
+        nodeRegistration: {} # node registration parameters are automatically injected by CAPD according to the kindest/node image in use.

test/e2e/data/infrastructure-docker/main/clusterclass-quick-start-runtimesdk.yaml

@@ -54,9 +54,9 @@ spec:
   patches:
   - name: test-patch
     external:
-      generatePatchesExtension: generate-patches.${EXTENSION_CONFIG_NAME:-"k8s-upgrade-with-runtimesdk"}
-      validateTopologyExtension: validate-topology.${EXTENSION_CONFIG_NAME:-"k8s-upgrade-with-runtimesdk"}
-      discoverVariablesExtension: discover-variables.${EXTENSION_CONFIG_NAME:-"k8s-upgrade-with-runtimesdk"}
+      generatePatchesExtension: generate-patches.${EXTENSION_CONFIG_NAME:-k8s-upgrade-with-runtimesdk}
+      validateTopologyExtension: validate-topology.${EXTENSION_CONFIG_NAME:-k8s-upgrade-with-runtimesdk}
+      discoverVariablesExtension: discover-variables.${EXTENSION_CONFIG_NAME:-k8s-upgrade-with-runtimesdk}
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
 kind: DockerClusterTemplate

test/e2e/quick_start.go

@@ -22,6 +22,7 @@ import (
 	"maps"
 	"os"
 	"path/filepath"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -70,6 +71,19 @@ type QuickStartSpecInput struct {
 	// which unblocks CNI installation, and for the control plane machines to be ready (after CNI installation).
 	ControlPlaneWaiters clusterctl.ControlPlaneWaiters
 
+	// ExtensionConfigName is the name of the ExtensionConfig. Defaults to "quick-start".
+	// This value is provided to clusterctl as "EXTENSION_CONFIG_NAME" variable and can be used to template the
+	// name of the ExtensionConfig into the ClusterClass.
+	ExtensionConfigName string
+
+	// ExtensionServiceNamespace is the namespace where the service for the Runtime Extension is located.
+	// Note: This should only be set if a Runtime Extension is used.
+	ExtensionServiceNamespace string
+
+	// ExtensionServiceNamespace is the name where the service for the Runtime Extension is located.
+	// Note: This should only be set if a Runtime Extension is used.
+	ExtensionServiceName string
+
 	// Allows to inject a function to be run after test namespace is created.
 	// If not specified, this is a no-op.
 	PostNamespaceCreated func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string)
@@ -107,6 +121,12 @@ func QuickStartSpec(ctx context.Context, inputGetter func() QuickStartSpecInput)
 
 		Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion))
 
+		if input.ExtensionServiceNamespace != "" && input.ExtensionServiceName != "" {
+			if input.ExtensionConfigName == "" {
+				input.ExtensionConfigName = specName
+			}
+		}
+
 		// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
 		namespace, cancelWatches = framework.SetupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, input.PostNamespaceCreated)
 
@@ -146,7 +166,36 @@ func QuickStartSpec(ctx context.Context, inputGetter func() QuickStartSpecInput)
 			clusterName = *input.ClusterName
 		}
 
-		variables := map[string]string{}
+		if input.ExtensionServiceNamespace != "" && input.ExtensionServiceName != "" {
+			// NOTE: test extension is already deployed in the management cluster. If for any reason in future we want
+			// to make this test more self-contained this test should be modified in order to create an additional
+			// management cluster; also the E2E test configuration should be modified introducing something like
+			// optional:true allowing to define which providers should not be installed by default in
+			// a management cluster.
+			By("Deploy Test Extension ExtensionConfig")
+
+			// In this test we are defaulting all handlers to non-blocking because we don't expect the handlers to block the
+			// cluster lifecycle by default. Setting defaultAllHandlersToBlocking to false enforces that the test-extension
+			// automatically creates the ConfigMap with non-blocking preloaded responses.
+			defaultAllHandlersToBlocking := false
+			// select on the current namespace
+			// This is necessary so in CI this test doesn't influence other tests by enabling lifecycle hooks
+			// in other test namespaces.
+			namespaces := []string{namespace.Name}
+			if input.DeployClusterClassInSeparateNamespace {
+				// Add the ClusterClass namespace, if the ClusterClass is deployed in a separate namespace.
+				namespaces = append(namespaces, clusterClassNamespace.Name)
+			}
+			extensionConfig := extensionConfig(input.ExtensionConfigName, input.ExtensionServiceNamespace, input.ExtensionServiceName, defaultAllHandlersToBlocking, namespaces...)
+			Expect(input.BootstrapClusterProxy.GetClient().Create(ctx,
+				extensionConfig)).
+				To(Succeed(), "Failed to create the ExtensionConfig")
+		}
+
+		variables := map[string]string{
+			// This is used to template the name of the ExtensionConfig into the ClusterClass.
+			"EXTENSION_CONFIG_NAME": input.ExtensionConfigName,
+		}
 		maps.Copy(variables, input.ClusterctlVariables)
 
 		if input.DeployClusterClassInSeparateNamespace {
@@ -200,11 +249,18 @@ func QuickStartSpec(ctx context.Context, inputGetter func() QuickStartSpecInput)
 	AfterEach(func() {
 		// Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself.
 		framework.DumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ClusterctlConfigPath, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup)
-		if input.DeployClusterClassInSeparateNamespace && !input.SkipCleanup {
-			framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{
-				Deleter: input.BootstrapClusterProxy.GetClient(),
-				Name:    clusterClassNamespace.Name,
-			})
+		if !input.SkipCleanup {
+			if input.ExtensionServiceNamespace != "" && input.ExtensionServiceName != "" {
+				Eventually(func() error {
+					return input.BootstrapClusterProxy.GetClient().Delete(ctx, extensionConfig(input.ExtensionConfigName, input.ExtensionServiceNamespace, input.ExtensionServiceName, true))
+				}, 10*time.Second, 1*time.Second).Should(Succeed(), "Deleting ExtensionConfig failed")
+			}
+			if input.DeployClusterClassInSeparateNamespace {
+				framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{
+					Deleter: input.BootstrapClusterProxy.GetClient(),
+					Name:    clusterClassNamespace.Name,
+				})
+			}
 		}
 	})
 }

test/e2e/quick_start_test.go

@@ -122,6 +122,31 @@ var _ = Describe("When following the Cluster API quick-start with ClusterClass [
 	})
 })
 
+var _ = Describe("When following the Cluster API quick-start with v1beta1 ClusterClass [ClusterClass]", Label("ClusterClass"), func() {
+	QuickStartSpec(ctx, func() QuickStartSpecInput {
+		return QuickStartSpecInput{
+			E2EConfig:             e2eConfig,
+			ClusterctlConfigPath:  clusterctlConfigPath,
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			ArtifactFolder:        artifactFolder,
+			SkipCleanup:           skipCleanup,
+			Flavor:                ptr.To("topology-runtimesdk-v1beta1"),
+			// The runtime extension gets deployed to the test-extension-system namespace and is exposed
+			// by the test-extension-webhook-service.
+			// The below values are used when creating the cluster-wide ExtensionConfig to refer
+			// the actual service.
+			ExtensionServiceNamespace: "test-extension-system",
+			ExtensionServiceName:      "test-extension-webhook-service",
+			PostMachinesProvisioned: func(proxy framework.ClusterProxy, namespace, clusterName string) {
+				// This check ensures that the resourceVersions are stable, i.e. it verifies there are no
+				// continuous reconciles when everything should be stable.
+				By("Checking that resourceVersions are stable")
+				framework.ValidateResourceVersionStable(ctx, proxy, namespace, clusterctlcluster.FilterClusterObjectsWithNameFilter(clusterName))
+			},
+		}
+	})
+})
+
 // NOTE: This test requires an IPv6 management cluster (can be configured via IP_FAMILY=IPv6).
 var _ = Describe("When following the Cluster API quick-start with IPv6 [IPv6]", Label("IPv6"), func() {
 	QuickStartSpec(ctx, func() QuickStartSpecInput {