MLE-15924 Setup E2E test in Jenkins using Minikube (#42)

* MLE-15924 Jenkins with Minikube

* update versions

* update docker image version

* removed comments

* update comment

---------

Co-authored-by: Peng Zhou <[email protected]>

Author: pengzhouml

Jenkinsfile

@@ -125,8 +125,10 @@ void runTests() {
 }
 
 void runE2eTests() {
-    //TODO: this is just a place holder as kuttl needs a proper environment
-    sh "echo make e2e-tests dockerImage=${params.dockerImage}"
+    sh '''
+        make e2e-setup-minikube
+        make e2e-test
+    '''
 }
 
 pipeline {

Makefile

@@ -5,6 +5,13 @@
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
 VERSION ?= 0.0.1
 
+export E2E_DOCKER_IMAGE ?= $(IMG)
+export E2E_KUSTOMIZE_VERSION ?= $(KUSTOMIZE_VERSION)
+export E2E_CONTROLLER_TOOLS_VERSION ?= $(CONTROLLER_TOOLS_VERSION)
+export E2E_MARKLOGIC_IMAGE_VERSION ?= progressofficial/marklogic-db:11.3.1-ubi-rootless-2.1.0
+export E2E_KUBERNETES_VERSION ?= v1.30.4
+
+
 # CHANNELS define the bundle channels used in the bundle.
 # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")
 # To re-generate a bundle for other specific channels without changing the standard setup, you can:
@@ -117,8 +124,15 @@ test: manifests generate fmt vet envtest ## Run tests.
 
 # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
 .PHONY: e2e-test  # Run the e2e tests against a Kind k8s instance that is spun up.
-e2e-test:
+e2e-test: 
 	go test -v -count=1 ./test/e2e
+	minikube delete || true
+
+e2e-setup-minikube: kustomize controller-gen build docker-build
+	minikube delete || true
+	minikube start --driver=docker --kubernetes-version=$(E2E_KUBERNETES_VERSION) --memory=8192 --cpus=2
+	minikube addons enable ingress
+	minikube image load $(IMG)
 
 GOLANGCI_LINT = $(shell pwd)/bin/golangci-lint
 GOLANGCI_LINT_VERSION ?= v1.54.2
@@ -215,11 +229,6 @@ ENVTEST ?= $(LOCALBIN)/setup-envtest
 KUSTOMIZE_VERSION ?= v5.5.0
 CONTROLLER_TOOLS_VERSION ?= v0.16.4
 
-export E2E_DOCKER_IMAGE ?= $(IMG)
-export E2E_KUSTOMIZE_VERSION ?= $(KUSTOMIZE_VERSION)
-export E2E_CONTROLLER_TOOLS_VERSION ?= $(CONTROLLER_TOOLS_VERSION)
-export E2E_MARKLOGIC_IMAGE_VERSION ?= marklogicdb/marklogic-db:11.2.0-ubi-rootless
-export E2E_KUBERNETES_VERSION ?= v1.30.4
 
 
 .PHONY: kustomize

config/manager/kustomization.yaml

@@ -5,4 +5,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: ml-marklogic-operator-dev.bed-artifactory.bedford.progress.com/marklogic-kubernetes-operator
-  newTag: 0.0.2
+  newTag: 1.0.0-ea2

test/e2e/2_marklogic_cluster_test.go

@@ -85,7 +85,7 @@ func TestMarklogicCluster(t *testing.T) {
 		client := c.Client()
 
 		podName := "dnode-0"
-		err := utils.WaitForPod(ctx, t, client, mlNamespace, podName, 60*time.Second)
+		err := utils.WaitForPod(ctx, t, client, mlNamespace, podName, 90*time.Second)
 		if err != nil {
 			t.Fatalf("Failed to wait for pod creation: %v", err)
 		}

test/e2e/main_test.go

@@ -11,32 +11,35 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/e2e-framework/klient/conf"
 	"sigs.k8s.io/e2e-framework/klient/wait"
 	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
 	"sigs.k8s.io/e2e-framework/pkg/env"
 	"sigs.k8s.io/e2e-framework/pkg/envconf"
 	"sigs.k8s.io/e2e-framework/pkg/envfuncs"
-	"sigs.k8s.io/e2e-framework/support/kind"
 	"sigs.k8s.io/e2e-framework/support/utils"
 )
 
 var (
-	testEnv env.Environment
-	dockerImage    = os.Getenv("E2E_DOCKER_IMAGE") //"ml-marklogic-operator-dev.bed-artifactory.bedford.progress.com/marklogic-kubernetes-operator:0.0.2"
-	kustomizeVer   = os.Getenv("E2E_KUSTOMIZE_VERSION") 
+	testEnv        env.Environment
+	dockerImage    = os.Getenv("E2E_DOCKER_IMAGE")
+	kustomizeVer   = os.Getenv("E2E_KUSTOMIZE_VERSION")
 	ctrlgenVer     = os.Getenv("E2E_CONTROLLER_TOOLS_VERSION")
 	marklogicImage = os.Getenv("E2E_MARKLOGIC_IMAGE_VERSION")
 	kubernetesVer  = os.Getenv("E2E_KUBERNETES_VERSION")
 )
 
 const (
-	namespace      = "marklogic-operator-system"
+	namespace = "marklogic-operator-system"
 )
 
 func TestMain(m *testing.M) {
 	testEnv = env.New()
-	kindClusterName := "test-cluster"
-	kindCluster := kind.NewCluster(kindClusterName)
+	path := conf.ResolveKubeConfigFile()
+	cfg := envconf.NewWithKubeConfig(path)
+	testEnv = env.NewWithConfig(cfg)
+
+	log.Printf("Running tests with the following configurations: path=%s", path)
 
 	log.Printf("Docker image: %s", dockerImage)
 	log.Printf("Kustomize version: %s", kustomizeVer)
@@ -46,7 +49,6 @@ func TestMain(m *testing.M) {
 
 	// Use Environment.Setup to configure pre-test setup
 	testEnv.Setup(
-		envfuncs.CreateClusterWithConfig(kindCluster, kindClusterName, "kind-config.yaml", kind.WithImage("kindest/node:"+kubernetesVer)),
 		envfuncs.CreateNamespace(namespace),
 
 		// install tool dependencies
@@ -83,43 +85,6 @@ func TestMain(m *testing.M) {
 			c := utils.RunCommand("controller-gen --version")
 			log.Printf("controller-gen: %s", c.Result())
 
-			// generate manifest files
-			log.Println("Generate manifests...")
-			wd, _ := os.Getwd()
-			log.Print(wd) // Output current working directory
-			if p := utils.RunCommand(`controller-gen rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases`); p.Err() != nil {
-				log.Printf("Failed to generate manifests: %s: %s", p.Err(), p.Result())
-				return ctx, p.Err()
-			}
-
-			// generate api objects
-			log.Println("Generate API objects...")
-			if p := utils.RunCommand(`controller-gen object:headerFile="hack/boilerplate.go.txt" paths="./..."`); p.Err() != nil {
-				log.Printf("Failed to generate API objects: %s: %s", p.Err(), p.Result())
-				return ctx, p.Err()
-			}
-
-			// Build docker image
-			log.Println("Building docker image...")
-			if p := utils.RunCommand(fmt.Sprintf("docker build -t %s .", dockerImage)); p.Err() != nil {
-				log.Printf("Failed to build docker image: %s: %s", p.Err(), p.Result())
-				return ctx, p.Err()
-			}
-
-			// Load docker image into kind
-			log.Println("Loading docker image into kind cluster...")
-			if err := kindCluster.LoadImage(ctx, dockerImage); err != nil {
-				log.Printf("Failed to load image into kind: %s", err)
-				return ctx, err
-			}
-
-			// Load MarkLogic image into kind
-			log.Println("Loading marklogic image into kind cluster...")
-			if err := kindCluster.LoadImage(ctx, marklogicImage); err != nil {
-				log.Printf("Failed to load image into kind: %s", err)
-				return ctx, err
-			}
-
 			// Deploy components
 			log.Println("Deploying controller-manager resources...")
 			p := utils.RunCommand(`kubectl version`)
@@ -160,8 +125,6 @@ func TestMain(m *testing.M) {
 			return ctx, nil
 		},
 		envfuncs.DeleteNamespace(namespace),
-		// envfuncs.ExportClusterLogs(kindClusterName, "../../logs"),
-		envfuncs.DestroyCluster(kindClusterName),
 	)
 
 	// Use Environment.Run to launch the test