From b215c43c81e182a3589f404821e3ec6d6522ca55 Mon Sep 17 00:00:00 2001 From: Lance Bragstad Date: Mon, 8 Jul 2024 16:32:51 -0500 Subject: [PATCH 1/2] Only set node selectors for OpenShift deployments By default, the File Integrity Operator would set node selectors and taints that would only allow it to schedule on `master` nodes. While this is fine for OpenShift environments, some additional environments we test on, like Red Hat OpenShift on AWS (ROSA), don't provide `master` nodes at all. To accomodate this in our testing and tooling, this commit removes the node selector and taint from the default bundle, and moves it into an OpenShift-specific bundle where it will still get used for OpenShift deploy paths and bundles. This will be more useful in a future patch that implements support for running File Integrity Operator end-to-end testing on ROSA HCP, which only provides `worker` nodes and not `master` nodes. --- Makefile | 4 ++++ config/manager/deployment.yaml | 5 ----- config/openshift-bundle/kustomization.yaml | 13 +++++++++++++ config/openshift-bundle/manager_patch.yaml | 13 +++++++++++++ 4 files changed, 30 insertions(+), 5 deletions(-) create mode 100644 config/openshift-bundle/kustomization.yaml create mode 100644 config/openshift-bundle/manager_patch.yaml diff --git a/Makefile b/Makefile index be20e6043..c03a50d14 100644 --- a/Makefile +++ b/Makefile @@ -320,7 +320,11 @@ bundle: check-operator-version operator-sdk manifests update-skip-range kustomiz $(SDK_BIN) generate kustomize manifests --apis-dir=./pkg/apis -q @echo "kustomize using deployment image $(IMG)" cd config/manager && $(KUSTOMIZE) edit set image $(APP_NAME)=$(IMG) + if [ $(PLATFORM) = "openshift" ]; then \ + sed -i 's%../default-bundle%../openshift-bundle%' config/manifests/kustomization.yaml; \ + fi $(KUSTOMIZE) build config/manifests | $(SDK_BIN) generate bundle -q $(BUNDLE_SA_OPTS) --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) + git restore config/manifests/kustomization.yaml $(SDK_BIN) bundle validate ./bundle .PHONY: bundle-image diff --git a/config/manager/deployment.yaml b/config/manager/deployment.yaml index ddacb905f..fb6378c87 100644 --- a/config/manager/deployment.yaml +++ b/config/manager/deployment.yaml @@ -53,12 +53,7 @@ spec: secret: secretName: file-integrity-operator-serving-cert optional: true - nodeSelector: - node-role.kubernetes.io/master: "" tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Exists" - effect: "NoSchedule" - key: "node.kubernetes.io/unreachable" operator: "Exists" effect: "NoExecute" diff --git a/config/openshift-bundle/kustomization.yaml b/config/openshift-bundle/kustomization.yaml new file mode 100644 index 000000000..fe877e81a --- /dev/null +++ b/config/openshift-bundle/kustomization.yaml @@ -0,0 +1,13 @@ +namespace: openshift-compliance + +bases: +- ../crd +- ../rbac +- ../manager +- ../ns + +patches: +- path: manager_patch.yaml + target: + kind: Deployment + name: file-integrity-operator diff --git a/config/openshift-bundle/manager_patch.yaml b/config/openshift-bundle/manager_patch.yaml new file mode 100644 index 000000000..82cd33c18 --- /dev/null +++ b/config/openshift-bundle/manager_patch.yaml @@ -0,0 +1,13 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: file-integrity-operator +spec: + template: + spec: + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" From 1a98280b2a7030e087c57e929b46cb11f218037f Mon Sep 17 00:00:00 2001 From: Lance Bragstad Date: Mon, 8 Jul 2024 16:36:23 -0500 Subject: [PATCH 2/2] CMP-2624: Implement support for running e2e tests on ROSA This commit adds some test plumbing so that the e2e framework knows when it's running on ROSA. This is necessary so that the framework bypasses certain features that aren't supported on that platform, like Machine Configs and Machine Config Pools. By implementing this, we're able to integrate ROSA CI into the gating process to ensure the operator works on managed OpenShift offerings. --- Makefile | 4 ++++ tests/e2e/helpers.go | 13 ++++++++----- tests/framework/context.go | 6 ++++++ tests/framework/framework.go | 5 +++++ 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index c03a50d14..7fd3827bc 100644 --- a/Makefile +++ b/Makefile @@ -449,6 +449,10 @@ test-unit: fmt vet ## Run tests. e2e: e2e-set-image prep-e2e @$(GO) test ./tests/e2e $(E2E_GO_TEST_FLAGS) -args $(E2E_ARGS) +.PHONY: e2e-rosa +e2e-rosa: e2e-set-image prep-e2e + @$(GO) test ./tests/e2e $(E2E_GO_TEST_FLAGS) -args $(E2E_ARGS) --platform rosa + .PHONY: prep-e2e prep-e2e: kustomize rm -rf $(TEST_SETUP_DIR) diff --git a/tests/e2e/helpers.go b/tests/e2e/helpers.go index 9db7747b4..da38c32e2 100644 --- a/tests/e2e/helpers.go +++ b/tests/e2e/helpers.go @@ -416,16 +416,19 @@ func deleteStatusEvents(f *framework.Framework, namespace string) error { func setupTestRequirements(t *testing.T) *framework.Context { fileIntegrities := &v1alpha1.FileIntegrityList{} nodeStatus := &v1alpha1.FileIntegrityNodeStatusList{} + f := framework.NewContext(t) err := framework.AddToFrameworkScheme(v1alpha1.AddToScheme, fileIntegrities) if err != nil { t.Fatalf("TEST SETUP: failed to add custom resource scheme to framework: %v", err) } - mcList := &mcfgv1.MachineConfigList{} - err = framework.AddToFrameworkScheme(mcfgapi.Install, mcList) - if err != nil { - t.Fatalf("TEST SETUP: failed to add custom resource scheme to framework: %v", err) + if f.GetPlatform() != "rosa" { + mcList := &mcfgv1.MachineConfigList{} + err = framework.AddToFrameworkScheme(mcfgapi.Install, mcList) + if err != nil { + t.Fatalf("TEST SETUP: failed to add custom resource scheme to framework: %v", err) + } } err = framework.AddToFrameworkScheme(v1alpha1.AddToScheme, nodeStatus) @@ -439,7 +442,7 @@ func setupTestRequirements(t *testing.T) *framework.Context { if err != nil { t.Fatalf("TEST SETUP: failed to add custom resource scheme to framework: %v", err) } - return framework.NewContext(t) + return f } func replaceNamespaceFromManifest(t *testing.T, nsFrom, nsTo string, namespacedManPath *string) { diff --git a/tests/framework/context.go b/tests/framework/context.go index 60427c980..d1408e91e 100644 --- a/tests/framework/context.go +++ b/tests/framework/context.go @@ -27,6 +27,7 @@ type Context struct { kubeclient kubernetes.Interface restMapper *restmapper.DeferredDiscoveryRESTMapper skipCleanupOnError bool + platform string } // todo(camilamacedo86): Remove the following line just added for we are able to deprecated TestCtx @@ -71,6 +72,7 @@ func (f *Framework) newContext(t *testing.T) *Context { kubeclient: f.KubeClient, restMapper: f.restMapper, skipCleanupOnError: f.skipCleanupOnError, + platform: f.Platform, } } @@ -111,3 +113,7 @@ func (ctx *Context) Cleanup() { func (ctx *Context) AddCleanupFn(fn cleanupFn) { ctx.cleanupFns = append(ctx.cleanupFns, fn) } + +func (ctx *Context) GetPlatform() string { + return ctx.platform +} diff --git a/tests/framework/framework.go b/tests/framework/framework.go index b51045aed..bbb148a4e 100644 --- a/tests/framework/framework.go +++ b/tests/framework/framework.go @@ -68,6 +68,7 @@ type Framework struct { NamespacedManPath *string OperatorNamespace string WatchNamespace string + Platform string restMapper *restmapper.DeferredDiscoveryRESTMapper @@ -88,6 +89,7 @@ type frameworkOpts struct { localOperatorArgs string isLocalOperator bool skipCleanupOnError bool + platform string } const ( @@ -98,6 +100,7 @@ const ( LocalOperatorFlag = "localOperator" LocalOperatorArgs = "localOperatorArgs" SkipCleanupOnErrorFlag = "skipCleanupOnError" + PlatformFlag = "platform" TestOperatorNamespaceEnv = "TEST_OPERATOR_NAMESPACE" TestWatchNamespaceEnv = "TEST_WATCH_NAMESPACE" @@ -115,6 +118,7 @@ func (opts *frameworkOpts) addToFlagSet(flagset *flag.FlagSet) { flagset.BoolVar(&opts.skipCleanupOnError, SkipCleanupOnErrorFlag, false, "If set as true, the cleanup function responsible to remove all artifacts "+ "will be skipped if an error is faced.") + flagset.StringVar(&opts.platform, PlatformFlag, "openshift", "The type of deployment hosting the tests. The only supported option besides OpenShift is \"rosa\".") } func newFramework(opts *frameworkOpts) (*Framework, error) { @@ -168,6 +172,7 @@ func newFramework(opts *frameworkOpts) (*Framework, error) { kubeconfigPath: opts.kubeconfigPath, restMapper: restMapper, skipCleanupOnError: opts.skipCleanupOnError, + Platform: opts.platform, } // This is required because controller-runtime expects its consumers to