e2e tests - use helm v3

Author: charles-edouard.breteche

test/gohelloworld/gohelloworld-chart/values.yaml

@@ -10,7 +10,7 @@ image:
 
 service:
   name: gohelloworld
-  type: LoadBalancer
+  type: ClusterIP
   externalPort: 8080
   internalPort: 8080
 

test/helm_task_test.go

@@ -20,9 +20,7 @@ package test
 
 import (
 	"fmt"
-	"net/http"
 	"testing"
-	"time"
 
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1"
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1"
@@ -31,9 +29,7 @@ import (
 	tb "github.com/tektoncd/pipeline/test/builder"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1beta1"
-	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/wait"
 	knativetest "knative.dev/pkg/test"
 )
 
@@ -42,14 +38,14 @@ const (
 	sourceImageName           = "go-helloworld-image"
 	createImageTaskName       = "create-image-task"
 	helmDeployTaskName        = "helm-deploy-task"
+	checkServiceTaskName      = "check-service-task"
 	helmDeployPipelineName    = "helm-deploy-pipeline"
 	helmDeployPipelineRunName = "helm-deploy-pipeline-run"
 	helmDeployServiceName     = "gohelloworld-chart"
 )
 
 var (
-	clusterRoleBindings  [3]*rbacv1.ClusterRoleBinding
-	tillerServiceAccount *corev1.ServiceAccount
+	clusterRoleBindings [1]*rbacv1.ClusterRoleBinding
 )
 
 // TestHelmDeployPipelineRun is an integration test that will verify a pipeline build an image
@@ -82,6 +78,11 @@ func TestHelmDeployPipelineRun(t *testing.T) {
 		t.Fatalf("Failed to create Task `%s`: %s", helmDeployTaskName, err)
 	}
 
+	t.Logf("Creating Task %s", checkServiceTaskName)
+	if _, err := c.TaskClient.Create(getCheckServiceTask(namespace)); err != nil {
+		t.Fatalf("Failed to create Task `%s`: %s", checkServiceTaskName, err)
+	}
+
 	t.Logf("Creating Pipeline %s", helmDeployPipelineName)
 	if _, err := c.PipelineClient.Create(getHelmDeployPipeline(namespace)); err != nil {
 		t.Fatalf("Failed to create Pipeline `%s`: %s", helmDeployPipelineName, err)
@@ -98,44 +99,9 @@ func TestHelmDeployPipelineRun(t *testing.T) {
 		t.Fatalf("PipelineRun execution failed; helm may or may not have been installed :(")
 	}
 
-	t.Log("Waiting for service to get external IP")
-	var serviceIP string
-	if err := WaitForServiceExternalIPState(c, namespace, helmDeployServiceName, func(svc *corev1.Service) (bool, error) {
-		ingress := svc.Status.LoadBalancer.Ingress
-		if ingress != nil {
-			if len(ingress) > 0 {
-				serviceIP = ingress[0].IP
-				return true, nil
-			}
-		}
-		return false, nil
-	}, "ServiceExternalIPisReady"); err != nil {
-		t.Errorf("Error waiting for Service %s to get an external IP: %s", helmDeployServiceName, err)
-	}
-
-	// cleanup task to remove helm from cluster, will not fail the test if it fails, just log
+	// cleanup task to remove helm releases from cluster and cluster role bindings, will not fail the test if it fails, just log
 	knativetest.CleanupOnInterrupt(func() { helmCleanup(c, t, namespace) }, t.Logf)
 	defer helmCleanup(c, t, namespace)
-
-	if serviceIP != "" {
-		t.Log("Polling service with external IP")
-		waitErr := wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) {
-			resp, err := http.Get(fmt.Sprintf("http://%s:8080", serviceIP))
-			if err != nil {
-				return false, nil
-			}
-			if resp != nil && resp.StatusCode != http.StatusOK {
-				return true, fmt.Errorf("expected 200 but received %d response code from service at http://%s:8080", resp.StatusCode, serviceIP)
-			}
-			return true, nil
-		})
-		if waitErr != nil {
-			t.Errorf("Error from pinging service IP %s : %s", serviceIP, waitErr)
-		}
-
-	} else {
-		t.Errorf("Service IP is empty.")
-	}
 }
 
 func getGoHelloworldGitResource(namespace string) *v1alpha1.PipelineResource {
@@ -197,16 +163,49 @@ func getHelmDeployTask(namespace string) *v1beta1.Task {
 				Name: "chartname", Type: v1beta1.ParamTypeString, Default: &empty,
 			}},
 			Steps: []v1beta1.Step{{Container: corev1.Container{
-				Image: "alpine/helm:2.14.0",
-				Args:  []string{"init", "--wait"},
-			}}, {Container: corev1.Container{
-				Image: "alpine/helm:2.14.0",
-				Args: []string{"install",
+				Image: "alpine/helm:3.1.2",
+				Args: []string{
+					"upgrade",
+					"--wait",
 					"--debug",
-					"--name=$(inputs.params.chartname)",
+					"--install",
+					"--namespace",
+					namespace,
+					"$(inputs.params.chartname)",
 					"$(inputs.params.pathToHelmCharts)",
 					"--set",
 					"image.repository=$(inputs.resources.image.url)",
+					"--set",
+					"service.type=ClusterIP",
+				},
+			}}, {Container: corev1.Container{
+				Image:   "lachlanevenson/k8s-kubectl",
+				Command: []string{"kubectl"},
+				Args: []string{
+					"get",
+					"all",
+					"--namespace",
+					namespace,
+				},
+			}}},
+		},
+	}
+}
+
+func getCheckServiceTask(namespace string) *v1beta1.Task {
+	return &v1beta1.Task{
+		ObjectMeta: metav1.ObjectMeta{Name: checkServiceTaskName, Namespace: namespace},
+		Spec: v1beta1.TaskSpec{
+			Params: []v1beta1.ParamSpec{{
+				Name: "serviceUrl", Type: v1beta1.ParamTypeString, Description: "Service url",
+			}},
+			Steps: []v1beta1.Step{{Container: corev1.Container{
+				Image: "jwilder/dockerize",
+				Args: []string{
+					"-wait",
+					"$(inputs.params.serviceUrl)",
+					"-timeout",
+					"1m",
 				},
 			}}},
 		},
@@ -251,6 +250,13 @@ func getHelmDeployPipeline(namespace string) *v1beta1.Pipeline {
 				}, {
 					Name: "chartname", Value: v1beta1.NewArrayOrString("$(params.chartname)"),
 				}},
+			}, {
+				Name:    "check-service",
+				TaskRef: &v1beta1.TaskRef{Name: checkServiceTaskName},
+				Params: []v1beta1.Param{{
+					Name: "serviceUrl", Value: v1beta1.NewArrayOrString(fmt.Sprintf("http://%s:8080", helmDeployServiceName)),
+				}},
+				RunAfter: []string{"helm-deploy"},
 			}},
 		},
 	}
@@ -274,37 +280,7 @@ func getHelmDeployPipelineRun(namespace string) *v1beta1.PipelineRun {
 }
 
 func setupClusterBindingForHelm(c *clients, t *testing.T, namespace string) {
-	tillerServiceAccount = &corev1.ServiceAccount{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "tiller",
-			Namespace: "kube-system",
-		},
-	}
-
-	t.Logf("Creating tiller service account")
-	if _, err := c.KubeClient.Kube.CoreV1().ServiceAccounts("kube-system").Create(tillerServiceAccount); err != nil {
-		if !errors.IsAlreadyExists(err) {
-			t.Fatalf("Failed to create default Service account for Helm %s", err)
-		}
-	}
-
 	clusterRoleBindings[0] = &rbacv1.ClusterRoleBinding{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: names.SimpleNameGenerator.RestrictLengthWithRandomSuffix("tiller"),
-		},
-		RoleRef: rbacv1.RoleRef{
-			APIGroup: "rbac.authorization.k8s.io",
-			Kind:     "ClusterRole",
-			Name:     "cluster-admin",
-		},
-		Subjects: []rbacv1.Subject{{
-			Kind:      "ServiceAccount",
-			Name:      "tiller",
-			Namespace: "kube-system",
-		}},
-	}
-
-	clusterRoleBindings[1] = &rbacv1.ClusterRoleBinding{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: names.SimpleNameGenerator.RestrictLengthWithRandomSuffix("default-tiller"),
 		},
@@ -320,22 +296,6 @@ func setupClusterBindingForHelm(c *clients, t *testing.T, namespace string) {
 		}},
 	}
 
-	clusterRoleBindings[2] = &rbacv1.ClusterRoleBinding{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: names.SimpleNameGenerator.RestrictLengthWithRandomSuffix("default-tiller"),
-		},
-		RoleRef: rbacv1.RoleRef{
-			APIGroup: "rbac.authorization.k8s.io",
-			Kind:     "ClusterRole",
-			Name:     "cluster-admin",
-		},
-		Subjects: []rbacv1.Subject{{
-			Kind:      "ServiceAccount",
-			Name:      "default",
-			Namespace: "kube-system",
-		}},
-	}
-
 	for _, crb := range clusterRoleBindings {
 		t.Logf("Creating Cluster Role binding %s for helm", crb.Name)
 		if _, err := c.KubeClient.Kube.RbacV1beta1().ClusterRoleBindings().Create(crb); err != nil {
@@ -348,12 +308,6 @@ func helmCleanup(c *clients, t *testing.T, namespace string) {
 	t.Logf("Cleaning up helm from cluster...")
 
 	removeAllHelmReleases(c, t, namespace)
-	removeHelmFromCluster(c, t, namespace)
-
-	t.Logf("Deleting tiller service account")
-	if err := c.KubeClient.Kube.CoreV1().ServiceAccounts("kube-system").Delete("tiller", &metav1.DeleteOptions{}); err != nil {
-		t.Fatalf("Failed to delete default Service account for Helm %s", err)
-	}
 
 	for _, crb := range clusterRoleBindings {
 		t.Logf("Deleting Cluster Role binding %s for helm", crb.Name)
@@ -370,9 +324,9 @@ func removeAllHelmReleases(c *clients, t *testing.T, namespace string) {
 		Spec: v1beta1.TaskSpec{
 			Steps: []v1beta1.Step{{Container: corev1.Container{
 				Name:    "helm-remove-all",
-				Image:   "alpine/helm:2.14.0",
+				Image:   "alpine/helm:3.1.2",
 				Command: []string{"/bin/sh"},
-				Args:    []string{"-c", "helm ls --short --all | xargs -n1 helm del --purge"},
+				Args:    []string{"-c", fmt.Sprintf("helm ls --short --all --namespace %s | xargs -n1 helm delete --namespace %s", namespace, namespace)},
 			}}},
 		},
 	}
@@ -400,39 +354,3 @@ func removeAllHelmReleases(c *clients, t *testing.T, namespace string) {
 		t.Logf("TaskRun %s failed to finish: %s", helmRemoveAllTaskRunName, err)
 	}
 }
-
-func removeHelmFromCluster(c *clients, t *testing.T, namespace string) {
-	helmResetTaskName := "helm-reset-task"
-	helmResetTask := &v1beta1.Task{
-		ObjectMeta: metav1.ObjectMeta{Name: helmResetTaskName, Namespace: namespace},
-		Spec: v1beta1.TaskSpec{
-			Steps: []v1beta1.Step{{Container: corev1.Container{
-				Image: "alpine/helm:2.14.0",
-				Args:  []string{"reset", "--force"},
-			}}},
-		},
-	}
-
-	helmResetTaskRunName := "helm-reset-taskrun"
-	helmResetTaskRun := &v1beta1.TaskRun{
-		ObjectMeta: metav1.ObjectMeta{Name: helmResetTaskRunName, Namespace: namespace},
-		Spec: v1beta1.TaskRunSpec{
-			TaskRef: &v1beta1.TaskRef{Name: helmResetTaskName},
-		},
-	}
-
-	t.Logf("Creating Task %s", helmResetTaskName)
-	if _, err := c.TaskClient.Create(helmResetTask); err != nil {
-		t.Fatalf("Failed to create Task `%s`: %s", helmResetTaskName, err)
-	}
-
-	t.Logf("Creating TaskRun %s", helmResetTaskRunName)
-	if _, err := c.TaskRunClient.Create(helmResetTaskRun); err != nil {
-		t.Fatalf("Failed to create TaskRun `%s`: %s", helmResetTaskRunName, err)
-	}
-
-	t.Logf("Waiting for TaskRun %s in namespace %s to complete", helmResetTaskRunName, namespace)
-	if err := WaitForTaskRunState(c, helmResetTaskRunName, TaskRunSucceed(helmResetTaskRunName), "TaskRunSuccess"); err != nil {
-		t.Logf("TaskRun %s failed to finish: %s", helmResetTaskRunName, err)
-	}
-}