Run e2e tests using GitHub actions

.github/workflows/go.yml

@@ -21,7 +21,61 @@ jobs:
 
     - name: Build
       env:
-        DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
+        REGISTRY: appscodeci
+        DOCKER_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+        USERNAME: 1gtm
       run: |
-        docker login docker.pkg.github.com --username stashed --password ${DOCKER_TOKEN}
-        make push REGISTRY=docker.pkg.github.com/stashed/stash
+        docker login --username ${USERNAME} --password ${DOCKER_TOKEN}
+        make push
+
+    - name: Create Kubernetes cluster
+      id: kind
+      uses: engineerd/setup-kind@v0.1.0
+      with:
+        config: hack/kubernetes/kind.yaml
+
+    - name: Prepare cluster for testing
+      id: local-path
+      env:
+        DOCKER_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+        USERNAME: 1gtm
+        REGISTRY_SECRET: regcred
+      run: |
+        export KUBECONFIG="$(kind get kubeconfig-path)"
+        echo
+        echo "waiting for nodes to be ready ..."
+        kubectl wait --for=condition=Ready nodes --all --timeout=5m
+        kubectl get nodes
+        echo
+        echo "installing local-path provisioner ..."
+        kubectl delete storageclass --all
+        kubectl apply -f https://github.com/rancher/local-path-provisioner/raw/v0.0.11/deploy/local-path-storage.yaml
+        kubectl wait --for=condition=Ready pods -n local-path-storage --all --timeout=5m
+        kubectl apply -f hack/kubernetes/storageclass/standard.yaml
+        echo
+        echo "create docker-registry secret"
+        kubectl create secret docker-registry ${REGISTRY_SECRET} --namespace=kube-system --docker-server=https://index.docker.io/v1/ --docker-username=${USERNAME} --docker-password=${DOCKER_TOKEN}
+
+    - name: Check out installer
+      run: |
+        cd ..
+        git clone https://github.com/stashed/installer.git
+        cd installer
+        git checkout master
+
+    - name: Run e2e tests
+      env:
+        REGISTRY: appscodeci
+        REGISTRY_SECRET: regcred
+        TEST_CREDENTIALS: ${{ secrets.TEST_CREDENTIALS }}
+        GOOGLE_SERVICE_ACCOUNT_JSON_KEY: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_JSON_KEY }}
+      run: |
+        export KUBECONFIG="$(kind get kubeconfig-path)"
+        echo "preparing test credentials"
+        mkdir -p hack/config
+        echo -n "$TEST_CREDENTIALS" > hack/config/.env
+        echo >> hack/config/.env
+        echo "GOOGLE_SERVICE_ACCOUNT_JSON_KEY=$(echo $GOOGLE_SERVICE_ACCOUNT_JSON_KEY)" >> hack/config/.env
+        make install
+        echo
+        make e2e-tests

Makefile

@@ -76,7 +76,7 @@ TAG              := $(VERSION)_$(OS)_$(ARCH)
 TAG_PROD         := $(TAG)
 TAG_DBG          := $(VERSION)-dbg_$(OS)_$(ARCH)
 
-GO_VERSION       ?= 1.12.7
+GO_VERSION       ?= 1.12.10
 BUILD_IMAGE      ?= appscode/golang-dev:$(GO_VERSION)-stretch
 
 OUTBIN = bin/$(OS)_$(ARCH)/$(BIN)
@@ -87,7 +87,10 @@ endif
 # Directories that we need created to build/test.
 BUILD_DIRS  := bin/$(OS)_$(ARCH)     \
                .go/bin/$(OS)_$(ARCH) \
-               .go/cache
+               .go/cache             \
+               $(HOME)/.credentials  \
+               $(HOME)/.kube         \
+               $(HOME)/.minikube
 
 DOCKERFILE_PROD  = Dockerfile.in
 DOCKERFILE_DBG   = Dockerfile.dbg
@@ -286,7 +289,7 @@ $(OUTBIN): .go/$(OUTBIN).stamp
 	    "
 	@if [ $(COMPRESS) = yes ] && [ $(OS) != darwin ]; then          \
 		echo "compressing $(OUTBIN)";                               \
-		docker run                                                  \
+		@docker run                                                 \
 		    -i                                                      \
 		    --rm                                                    \
 		    -u $$(id -u):$$(id -g)                                  \
@@ -359,7 +362,16 @@ unit-tests: $(BUILD_DIRS)
 	        ./hack/test.sh $(SRC_PKGS)                          \
 	    "
 
-STORAGE_CLASS ?= standard
+# - e2e-tests can hold both ginkgo args (as GINKGO_ARGS) and program/test args (as TEST_ARGS).
+#       make e2e-tests TEST_ARGS="--selfhosted-operator=false --storageclass=standard" GINKGO_ARGS="--flakeAttempts=2"
+#
+# - Minimalist:
+#       make e2e-tests
+#
+# NB: -t is used to catch ctrl-c interrupt from keyboard and -t will be problematic for CI.
+
+GINKGO_ARGS ?=
+TEST_ARGS   ?=
 
 .PHONY: e2e-tests
 e2e-tests: $(BUILD_DIRS)
@@ -370,12 +382,15 @@ e2e-tests: $(BUILD_DIRS)
 	    -v $$(pwd):/src                                         \
 	    -w /src                                                 \
 	    --net=host                                              \
-	    -v $(HOME)/.kube:/.kube                                \
+	    -v $(HOME)/.kube:/.kube                                 \
+	    -v $(HOME)/.minikube:$(HOME)/.minikube                  \
+	    -v $(HOME)/.credentials:$(HOME)/.credentials            \
 	    -v $$(pwd)/.go/bin/$(OS)_$(ARCH):/go/bin                \
 	    -v $$(pwd)/.go/bin/$(OS)_$(ARCH):/go/bin/$(OS)_$(ARCH)  \
 	    -v $$(pwd)/.go/cache:/.cache                            \
 	    --env HTTP_PROXY=$(HTTP_PROXY)                          \
 	    --env HTTPS_PROXY=$(HTTPS_PROXY)                        \
+	    --env KUBECONFIG=$(KUBECONFIG)                          \
 	    --env-file=$$(pwd)/hack/config/.env                     \
 	    $(BUILD_IMAGE)                                          \
 	    /bin/bash -c "                                          \
@@ -384,10 +399,16 @@ e2e-tests: $(BUILD_DIRS)
 	        VERSION=$(VERSION)                                  \
 	        DOCKER_REGISTRY=$(REGISTRY)                         \
 	        TAG=$(TAG)                                          \
-	        STORAGE_CLASS=$(STORAGE_CLASS)                      \
+	        KUBECONFIG=$${KUBECONFIG#$(HOME)}                   \
+	        GINKGO_ARGS='$(GINKGO_ARGS)'                        \
+	        TEST_ARGS='$(TEST_ARGS) --image-tag=$(TAG)'         \
 	        ./hack/e2e.sh                                       \
 	    "
 
+.PHONY: e2e-parallel
+e2e-parallel:
+	@$(MAKE) e2e-tests GINKGO_ARGS="-p -stream" --no-print-directory
+
 ADDTL_LINTERS   := goconst,gofmt,goimports,unparam
 
 .PHONY: lint
@@ -415,7 +436,7 @@ $(BUILD_DIRS):
 .PHONY: install
 install:
 	@cd ../installer; \
-	APPSCODE_ENV=dev  STASH_DOCKER_REGISTRY=$(REGISTRY) STASH_IMAGE_TAG=$(TAG) ./deploy/stash.sh
+	APPSCODE_ENV=dev  STASH_IMAGE_TAG=$(TAG) ./deploy/stash.sh --docker-registry=$(REGISTRY) --image-pull-secret=$(REGISTRY_SECRET)
 
 .PHONY: uninstall
 uninstall:

apis/util.go

@@ -21,3 +21,15 @@ const (
 	KindAppBinding            = "AppBinding"
 	KindDeploymentConfig      = "DeploymentConfig"
 )
+
+const (
+	ResourcePluralDeployment            = "deployments"
+	ResourcePluralReplicaSet            = "replicasets"
+	ResourcePluralReplicationController = "replicationcontrollers"
+	ResourcePluralStatefulSet           = "statefulsets"
+	ResourcePluralDaemonSet             = "daemonsets"
+	ResourcePluralPersistentVolumeClaim = "persistentvolumeclaims"
+	ResourcePluralAppBinding            = "appbindings"
+	ResourcePluralDeploymentConfig      = "deploymentconfigs"
+	ResourcePluralSecret                = "secrets"
+)

hack/e2e.sh

@@ -20,9 +20,10 @@ export CGO_ENABLED=0
 export GO111MODULE=on
 export GOFLAGS="-mod=vendor"
 
+GINKGO_ARGS=${GINKGO_ARGS:-}
+TEST_ARGS=${TEST_ARGS:-}
+DOCKER_REGISTRY=${DOCKER_REGISTRY:-}
+
 echo "Running e2e tests:"
-ginkgo -r --v --progress --trace test -- \
-    --docker-registry=${DOCKER_REGISTRY} \
-    --image-tag=${TAG}                   \
-    --storageclass=${STORAGE_CLASS}
-echo
+cmd="ginkgo -r --v --progress --trace --noisyPendings=false ${GINKGO_ARGS} test -- --docker-registry=${DOCKER_REGISTRY} ${TEST_ARGS}"
+echo $cmd; $cmd

hack/kubernetes/kind.yaml

@@ -0,0 +1,19 @@
+kind: Cluster
+apiVersion: kind.sigs.k8s.io/v1alpha3
+nodes:
+# the control plane node
+- role: control-plane
+  extraMounts:
+    - containerPath: /opt/local-path-provisioner
+      hostPath: /mnt
+      readOnly: false
+- role: worker
+  extraMounts:
+    - containerPath: /opt/local-path-provisioner
+      hostPath: /mnt
+      readOnly: false
+- role: worker
+  extraMounts:
+    - containerPath: /opt/local-path-provisioner
+      hostPath: /mnt
+      readOnly: false

hack/kubernetes/storageclass/standard.yaml

@@ -0,0 +1,9 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: standard
+  annotations:
+    storageclass.beta.kubernetes.io/is-default-class: "true"
+provisioner: rancher.io/local-path
+volumeBindingMode: WaitForFirstConsumer
+reclaimPolicy: Delete

test/e2e/e2e_suite_test.go

@@ -1,21 +1,20 @@
 package e2e_test
 
 import (
-	"path/filepath"
 	"strings"
 	"testing"
 	"time"
 
-	logs "github.com/appscode/go/log/golog"
 	. "github.com/onsi/ginkgo"
 	"github.com/onsi/ginkgo/reporters"
 	. "github.com/onsi/gomega"
 	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/dynamic"
 	clientsetscheme "k8s.io/client-go/kubernetes/scheme"
-	"k8s.io/client-go/util/homedir"
 	ka "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	discovery_util "kmodules.xyz/client-go/discovery"
+	"kmodules.xyz/client-go/logs"
 	"kmodules.xyz/client-go/tools/cli"
 	"kmodules.xyz/client-go/tools/clientcmd"
 	"stash.appscode.dev/stash/apis"
@@ -62,20 +61,16 @@ var _ = BeforeSuite(func() {
 	Expect(err).NotTo(HaveOccurred())
 
 	kaClient := ka.NewForConfigOrDie(clientConfig)
-	framework.StashProjectRoot = filepath.Join(homedir.HomeDir(), "go", "src", "github.com", "appscode", "stash")
+	dmClient := dynamic.NewForConfigOrDie(clientConfig)
 
-	root = framework.New(ctrlConfig.KubeClient, ctrlConfig.StashClient, kaClient, clientConfig, options.StorageClass)
+	root = framework.New(ctrlConfig.KubeClient, ctrlConfig.StashClient, kaClient, dmClient, clientConfig, options.StorageClass)
+	framework.RootFramework = root
 	err = root.CreateTestNamespace()
 	Expect(err).NotTo(HaveOccurred())
 	By("Using test namespace " + root.Namespace())
-
-	By("Starting the Stash Operator")
-	root.InstallStashOperator(options.KubeConfig, options.ExtraOptions)
 })
 
 var _ = AfterSuite(func() {
-	By("Deleting Stash Operator")
-	root.UninstallStashOperator()
 	By("Deleting namespace: " + root.Namespace())
 	err := root.DeleteNamespace(root.Namespace())
 	Expect(err).NotTo(HaveOccurred())

test/e2e/options_test.go

@@ -2,13 +2,13 @@ package e2e_test
 
 import (
 	"flag"
+	"os"
 	"path/filepath"
 
 	"github.com/appscode/go/flags"
 	logs "github.com/appscode/go/log/golog"
 	"k8s.io/client-go/util/homedir"
 	"stash.appscode.dev/stash/pkg/cmds/server"
-	opt "stash.appscode.dev/stash/test/e2e/framework"
 )
 
 type E2EOptions struct {
@@ -21,22 +21,24 @@ type E2EOptions struct {
 var (
 	options = &E2EOptions{
 		ExtraOptions: server.NewExtraOptions(),
-		KubeConfig:   filepath.Join(homedir.HomeDir(), ".kube", "config"),
+		KubeConfig: func() string {
+			kubecfg := os.Getenv("KUBECONFIG")
+			if kubecfg != "" {
+				return kubecfg
+			}
+			return filepath.Join(homedir.HomeDir(), ".kube", "config")
+		}(),
 	}
 )
 
 func init() {
-	//options.AddGoFlags(flag.CommandLine)
 	flag.StringVar(&options.DockerRegistry, "docker-registry", "", "Set Docker Registry")
 	flag.StringVar(&options.StashImageTag, "image-tag", "", "Set Stash Image Tag")
 	flag.StringVar(&options.KubeConfig, "kubeconfig", options.KubeConfig, "Path to kubeconfig file with authorization information (the master location is set by the master flag).")
 	flag.StringVar(&options.KubeContext, "kube-context", "", "Name of kube context")
 	flag.StringVar(&options.StorageClass, "storageclass", "standard", "Storageclass for PVC")
 	enableLogging()
 	flag.Parse()
-	opt.DockerRegistry = options.DockerRegistry
-	opt.DockerImageTag = options.StashImageTag
-
 }
 
 func enableLogging() {