Skip to content

Commit 01589bb

Browse files
enrichmanenrichman
authored
splitting tests (#461)
1 parent 30217df commit 01589bb

File tree

6 files changed

+307
-279
lines changed

6 files changed

+307
-279
lines changed

‎tests/cluster_certs_test.go‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
package k3k_test
2+
3+
import (
4+
"context"
5+
"os"
6+
"strings"
7+
8+
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
9+
10+
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
11+
12+
. "github.com/onsi/ginkgo/v2"
13+
. "github.com/onsi/gomega"
14+
)
15+
16+
var _ = When("a cluster with custom certificates is installed with individual cert secrets", Label("e2e"), func() {
17+
var virtualCluster *VirtualCluster
18+
19+
BeforeEach(func() {
20+
ctx := context.Background()
21+
22+
namespace := NewNamespace()
23+
24+
// create custom cert secret
25+
customCertDir := "testdata/customcerts/"
26+
27+
certList := []string{
28+
"server-ca",
29+
"client-ca",
30+
"request-header-ca",
31+
"service",
32+
"etcd-peer-ca",
33+
"etcd-server-ca",
34+
}
35+
36+
for _, certName := range certList {
37+
var cert, key []byte
38+
var err error
39+
filePathPrefix := ""
40+
certfile := certName
41+
if strings.HasPrefix(certName, "etcd") {
42+
filePathPrefix = "etcd/"
43+
certfile = strings.TrimPrefix(certName, "etcd-")
44+
}
45+
if !strings.Contains(certName, "service") {
46+
cert, err = os.ReadFile(customCertDir + filePathPrefix + certfile + ".crt")
47+
Expect(err).To(Not(HaveOccurred()))
48+
}
49+
key, err = os.ReadFile(customCertDir + filePathPrefix + certfile + ".key")
50+
Expect(err).To(Not(HaveOccurred()))
51+
52+
certSecret := caCertSecret(certName, namespace.Name, cert, key)
53+
err = k8sClient.Create(ctx, certSecret)
54+
Expect(err).To(Not(HaveOccurred()))
55+
}
56+
57+
cluster := NewCluster(namespace.Name)
58+
59+
cluster.Spec.CustomCAs = v1alpha1.CustomCAs{
60+
Enabled: true,
61+
Sources: v1alpha1.CredentialSources{
62+
ServerCA: v1alpha1.CredentialSource{
63+
SecretName: "server-ca",
64+
},
65+
ClientCA: v1alpha1.CredentialSource{
66+
SecretName: "client-ca",
67+
},
68+
ETCDServerCA: v1alpha1.CredentialSource{
69+
SecretName: "etcd-server-ca",
70+
},
71+
ETCDPeerCA: v1alpha1.CredentialSource{
72+
SecretName: "etcd-peer-ca",
73+
},
74+
RequestHeaderCA: v1alpha1.CredentialSource{
75+
SecretName: "request-header-ca",
76+
},
77+
ServiceAccountToken: v1alpha1.CredentialSource{
78+
SecretName: "service",
79+
},
80+
},
81+
}
82+
83+
CreateCluster(cluster)
84+
85+
client, restConfig := NewVirtualK8sClientAndConfig(cluster)
86+
87+
virtualCluster = &VirtualCluster{
88+
Cluster: cluster,
89+
RestConfig: restConfig,
90+
Client: client,
91+
}
92+
})
93+
94+
It("will load the custom certs in the server pod", func() {
95+
ctx := context.Background()
96+
97+
labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server"
98+
serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
99+
Expect(err).To(Not(HaveOccurred()))
100+
101+
Expect(len(serverPods.Items)).To(Equal(1))
102+
serverPod := serverPods.Items[0]
103+
104+
// check server-ca.crt
105+
serverCACrtPath := "/var/lib/rancher/k3s/server/tls/server-ca.crt"
106+
serverCACrt, err := readFileWithinPod(ctx, k8s, restcfg, serverPod.Name, serverPod.Namespace, serverCACrtPath)
107+
Expect(err).To(Not(HaveOccurred()))
108+
109+
serverCACrtTestFile, err := os.ReadFile("testdata/customcerts/server-ca.crt")
110+
Expect(err).To(Not(HaveOccurred()))
111+
Expect(serverCACrt).To(Equal(serverCACrtTestFile))
112+
})
113+
})

‎tests/cluster_persistence_test.go‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
package k3k_test
2+
3+
import (
4+
"context"
5+
"crypto/x509"
6+
"errors"
7+
"time"
8+
9+
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
10+
11+
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
12+
13+
. "github.com/onsi/ginkgo/v2"
14+
. "github.com/onsi/gomega"
15+
)
16+
17+
var _ = When("an ephemeral cluster is installed", Label("e2e"), func() {
18+
var virtualCluster *VirtualCluster
19+
20+
BeforeEach(func() {
21+
virtualCluster = NewVirtualCluster()
22+
})
23+
24+
AfterEach(func() {
25+
DeleteNamespaces(virtualCluster.Cluster.Namespace)
26+
})
27+
28+
It("can create a nginx pod", func() {
29+
_, _ = virtualCluster.NewNginxPod("")
30+
})
31+
32+
It("regenerates the bootstrap secret after a restart", func() {
33+
ctx := context.Background()
34+
35+
_, err := virtualCluster.Client.ServerVersion()
36+
Expect(err).To(Not(HaveOccurred()))
37+
38+
labelSelector := "cluster=" + virtualCluster.Cluster.Name + ",role=server"
39+
serverPods, err := k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
40+
Expect(err).To(Not(HaveOccurred()))
41+
42+
Expect(len(serverPods.Items)).To(Equal(1))
43+
serverPod := serverPods.Items[0]
44+
45+
GinkgoWriter.Printf("deleting pod %s/%s\n", serverPod.Namespace, serverPod.Name)
46+
47+
err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).Delete(ctx, serverPod.Name, v1.DeleteOptions{})
48+
Expect(err).To(Not(HaveOccurred()))
49+
50+
By("Deleting server pod")
51+
52+
// check that the server pods restarted
53+
Eventually(func() any {
54+
serverPods, err = k8s.CoreV1().Pods(virtualCluster.Cluster.Namespace).List(ctx, v1.ListOptions{LabelSelector: labelSelector})
55+
Expect(err).To(Not(HaveOccurred()))
56+
Expect(len(serverPods.Items)).To(Equal(1))
57+
return serverPods.Items[0].DeletionTimestamp
58+
}).
59+
WithTimeout(time.Minute).
60+
WithPolling(time.Second * 5).
61+
Should(BeNil())
62+
63+
By("Server pod up and running again")
64+
65+
By("Using old k8s client configuration should fail")
66+
67+
Eventually(func() bool {
68+
_, err = virtualCluster.Client.DiscoveryClient.ServerVersion()
69+
var unknownAuthorityErr x509.UnknownAuthorityError
70+
return errors.As(err, &unknownAuthorityErr)
71+
}).
72+
WithTimeout(time.Minute * 2).
73+
WithPolling(time.Second * 5).
74+
Should(BeTrue())
75+
76+
By("Recover new config should succeed")
77+
78+
Eventually(func() error {
79+
virtualCluster.Client, virtualCluster.RestConfig = NewVirtualK8sClientAndConfig(virtualCluster.Cluster)
80+
_, err = virtualCluster.Client.DiscoveryClient.ServerVersion()
81+
return err
82+
}).
83+
WithTimeout(time.Minute * 2).
84+
WithPolling(time.Second * 5).
85+
Should(BeNil())
86+
})
87+
})
88+
89+
var _ = When("a dynamic cluster is installed", Label("e2e"), func() {
90+
var virtualCluster *VirtualCluster
91+
92+
BeforeEach(func() {
93+
virtualCluster = NewVirtualClusterWithType(v1alpha1.DynamicPersistenceMode)
94+
})
95+
96+
AfterEach(func() {
97+
DeleteNamespaces(virtualCluster.Cluster.Namespace)
98+
})
99+
100+
It("can create a nginx pod", func() {
101+
_, _ = virtualCluster.NewNginxPod("")
102+
})
103+
104+
It("uses the same bootstrap secret after a restart", func() {
105+
ctx := context.Background()
106+
107+
_, err := virtualCluster.Client.ServerVersion()
108+
Expect(err).To(Not(HaveOccurred()))
109+
110+
restartServerPod(ctx, virtualCluster)
111+
112+
By("Server pod up and running again")
113+
114+
By("Using old k8s client configuration should succeed")
115+
116+
Eventually(func() error {
117+
_, err = virtualCluster.Client.DiscoveryClient.ServerVersion()
118+
return err
119+
}).
120+
WithTimeout(2 * time.Minute).
121+
WithPolling(time.Second * 5).
122+
Should(BeNil())
123+
})
124+
})

0 commit comments

Comments
 (0)