diff --git a/assets/overlays/openstack-cinder/generated/hypershift/controller.yaml b/assets/overlays/openstack-cinder/generated/hypershift/controller.yaml index 2c9338b2d..1d1e630fc 100644 --- a/assets/overlays/openstack-cinder/generated/hypershift/controller.yaml +++ b/assets/overlays/openstack-cinder/generated/hypershift/controller.yaml @@ -46,7 +46,7 @@ spec: template: metadata: annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: cacert,config-cinderplugin,secret-cinderplugin,socket-dir + cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: config-cinderplugin,cloud-credentials,legacy-cacert,socket-dir openshift.io/required-scc: restricted-v2 target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}' labels: @@ -132,16 +132,17 @@ spec: memory: 50Mi terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - - mountPath: /etc/kubernetes/static-pod-resources/configmaps/cloud-config - name: cacert + - mountPath: /csi + name: socket-dir - mountPath: /etc/kubernetes/config name: config-cinderplugin readOnly: true - - mountPath: /etc/kubernetes/secret - name: secret-cinderplugin + - mountPath: /etc/openstack + name: cloud-credentials + readOnly: true + - mountPath: /etc/kubernetes/static-pod-resources/configmaps/cloud-config + name: legacy-cacert readOnly: true - - mountPath: /csi - name: socket-dir - args: - --secure-listen-address=0.0.0.0:9202 - --upstream=http://127.0.0.1:8202/ @@ -404,12 +405,20 @@ spec: - name: metrics-serving-cert secret: secretName: openstack-cinder-csi-driver-controller-metrics-serving-cert - - name: secret-cinderplugin - secret: - items: - - key: clouds.yaml - path: clouds.yaml - secretName: openstack-cloud-credentials + - name: cloud-credentials + projected: + sources: + - secret: + items: + - key: cacert + path: ca.crt + name: openstack-cloud-credentials + optional: true + - secret: + items: + - key: clouds.yaml + path: clouds.yaml + name: openstack-cloud-credentials - configMap: items: - key: cloud.conf @@ -422,7 +431,7 @@ spec: path: ca-bundle.pem name: cloud-conf optional: true - name: cacert + name: legacy-cacert - name: hosted-kubeconfig secret: defaultMode: 420 diff --git a/assets/overlays/openstack-cinder/generated/hypershift/node.yaml b/assets/overlays/openstack-cinder/generated/hypershift/node.yaml index 6a56f8646..c5fe6058a 100644 --- a/assets/overlays/openstack-cinder/generated/hypershift/node.yaml +++ b/assets/overlays/openstack-cinder/generated/hypershift/node.yaml @@ -90,13 +90,14 @@ spec: name: etc-selinux - mountPath: /sys/fs name: sys-fs - - mountPath: /etc/kubernetes/static-pod-resources/configmaps/cloud-config - name: cacert - mountPath: /etc/kubernetes/config name: config-cinderplugin readOnly: true - - mountPath: /etc/kubernetes/secret - name: secret-cinderplugin + - mountPath: /etc/openstack + name: cloud-credentials + readOnly: true + - mountPath: /etc/kubernetes/static-pod-resources/configmaps/cloud-config + name: legacy-cacert readOnly: true - args: - --csi-address=/csi/csi.sock @@ -191,25 +192,33 @@ spec: - name: metrics-serving-cert secret: secretName: openstack-cinder-csi-driver-node-metrics-serving-cert - - configMap: - items: - - key: ca-bundle.pem - path: ca-bundle.pem - name: cloud-conf - optional: true - name: cacert + - name: cloud-credentials + projected: + sources: + - secret: + items: + - key: cacert + path: ca.crt + name: openstack-cloud-credentials + optional: true + - secret: + items: + - key: clouds.yaml + path: clouds.yaml + name: openstack-cloud-credentials - configMap: items: - key: cloud.conf path: cloud.conf name: cloud-conf name: config-cinderplugin - - name: secret-cinderplugin - secret: + - configMap: items: - - key: clouds.yaml - path: clouds.yaml - secretName: openstack-cloud-credentials + - key: ca-bundle.pem + path: ca-bundle.pem + name: cloud-conf + optional: true + name: legacy-cacert updateStrategy: rollingUpdate: maxUnavailable: 10% diff --git a/assets/overlays/openstack-cinder/generated/standalone/controller.yaml b/assets/overlays/openstack-cinder/generated/standalone/controller.yaml index 19c739816..2c1144bf1 100644 --- a/assets/overlays/openstack-cinder/generated/standalone/controller.yaml +++ b/assets/overlays/openstack-cinder/generated/standalone/controller.yaml @@ -40,7 +40,7 @@ spec: template: metadata: annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: cacert,config-cinderplugin,secret-cinderplugin,socket-dir + cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: config-cinderplugin,cloud-credentials,legacy-cacert,socket-dir openshift.io/required-scc: restricted-v2 target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}' labels: @@ -101,16 +101,17 @@ spec: memory: 50Mi terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - - mountPath: /etc/kubernetes/static-pod-resources/configmaps/cloud-config - name: cacert + - mountPath: /csi + name: socket-dir - mountPath: /etc/kubernetes/config name: config-cinderplugin readOnly: true - - mountPath: /etc/kubernetes/secret - name: secret-cinderplugin + - mountPath: /etc/openstack + name: cloud-credentials + readOnly: true + - mountPath: /etc/kubernetes/static-pod-resources/configmaps/cloud-config + name: legacy-cacert readOnly: true - - mountPath: /csi - name: socket-dir - args: - --secure-listen-address=0.0.0.0:9202 - --upstream=http://127.0.0.1:8202/ @@ -346,12 +347,20 @@ spec: - name: metrics-serving-cert secret: secretName: openstack-cinder-csi-driver-controller-metrics-serving-cert - - name: secret-cinderplugin - secret: - items: - - key: clouds.yaml - path: clouds.yaml - secretName: openstack-cloud-credentials + - name: cloud-credentials + projected: + sources: + - secret: + items: + - key: cacert + path: ca.crt + name: openstack-cloud-credentials + optional: true + - secret: + items: + - key: clouds.yaml + path: clouds.yaml + name: openstack-cloud-credentials - configMap: items: - key: cloud.conf @@ -364,4 +373,4 @@ spec: path: ca-bundle.pem name: cloud-conf optional: true - name: cacert + name: legacy-cacert diff --git a/assets/overlays/openstack-cinder/generated/standalone/node.yaml b/assets/overlays/openstack-cinder/generated/standalone/node.yaml index 6a56f8646..c5fe6058a 100644 --- a/assets/overlays/openstack-cinder/generated/standalone/node.yaml +++ b/assets/overlays/openstack-cinder/generated/standalone/node.yaml @@ -90,13 +90,14 @@ spec: name: etc-selinux - mountPath: /sys/fs name: sys-fs - - mountPath: /etc/kubernetes/static-pod-resources/configmaps/cloud-config - name: cacert - mountPath: /etc/kubernetes/config name: config-cinderplugin readOnly: true - - mountPath: /etc/kubernetes/secret - name: secret-cinderplugin + - mountPath: /etc/openstack + name: cloud-credentials + readOnly: true + - mountPath: /etc/kubernetes/static-pod-resources/configmaps/cloud-config + name: legacy-cacert readOnly: true - args: - --csi-address=/csi/csi.sock @@ -191,25 +192,33 @@ spec: - name: metrics-serving-cert secret: secretName: openstack-cinder-csi-driver-node-metrics-serving-cert - - configMap: - items: - - key: ca-bundle.pem - path: ca-bundle.pem - name: cloud-conf - optional: true - name: cacert + - name: cloud-credentials + projected: + sources: + - secret: + items: + - key: cacert + path: ca.crt + name: openstack-cloud-credentials + optional: true + - secret: + items: + - key: clouds.yaml + path: clouds.yaml + name: openstack-cloud-credentials - configMap: items: - key: cloud.conf path: cloud.conf name: cloud-conf name: config-cinderplugin - - name: secret-cinderplugin - secret: + - configMap: items: - - key: clouds.yaml - path: clouds.yaml - secretName: openstack-cloud-credentials + - key: ca-bundle.pem + path: ca-bundle.pem + name: cloud-conf + optional: true + name: legacy-cacert updateStrategy: rollingUpdate: maxUnavailable: 10% diff --git a/assets/overlays/openstack-cinder/patches/controller_add_driver.yaml b/assets/overlays/openstack-cinder/patches/controller_add_driver.yaml index a0bdcaa90..eee418eef 100644 --- a/assets/overlays/openstack-cinder/patches/controller_add_driver.yaml +++ b/assets/overlays/openstack-cinder/patches/controller_add_driver.yaml @@ -8,7 +8,7 @@ spec: template: metadata: annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: "cacert,config-cinderplugin,secret-cinderplugin,socket-dir" + cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: "config-cinderplugin,cloud-credentials,legacy-cacert,socket-dir" openshift.io/required-scc: restricted-v2 labels: openshift.storage.network-policy.all-egress: allow @@ -71,43 +71,50 @@ spec: periodSeconds: 30 failureThreshold: 5 volumeMounts: - - name: cacert - mountPath: /etc/kubernetes/static-pod-resources/configmaps/cloud-config + - name: socket-dir + mountPath: /csi + # credentials and configuration - name: config-cinderplugin mountPath: /etc/kubernetes/config readOnly: true - - name: secret-cinderplugin - mountPath: /etc/kubernetes/secret + - name: cloud-credentials + mountPath: /etc/openstack + readOnly: true + # TODO(stephenfin): Remove in 4.22 + - name: legacy-cacert + mountPath: /etc/kubernetes/static-pod-resources/configmaps/cloud-config readOnly: true - - name: socket-dir - mountPath: /csi resources: requests: memory: 50Mi cpu: 10m terminationMessagePolicy: FallbackToLogsOnError volumes: - - name: secret-cinderplugin - secret: - secretName: openstack-cloud-credentials - items: - - key: clouds.yaml - path: clouds.yaml + - name: cloud-credentials + projected: + sources: + - secret: + name: openstack-cloud-credentials + items: + - key: cacert + path: ca.crt + optional: true + - secret: + name: openstack-cloud-credentials + items: + - key: clouds.yaml + path: clouds.yaml - name: config-cinderplugin configMap: name: cloud-conf items: - key: cloud.conf path: cloud.conf - - name: cacert - # If present, extract ca-bundle.pem to - # /etc/kubernetes/static-pod-resources/configmaps/cloud-config - # Let the pod start when the ConfigMap does not exist or the certificate - # is not preset there. The certificate file will be created once the - # ConfigMap is created / the certificate is added to it. + # TODO(stephenfin): Remove in 4.22 + - name: legacy-cacert configMap: name: cloud-conf items: - - key: ca-bundle.pem - path: ca-bundle.pem + - key: ca-bundle.pem + path: ca-bundle.pem optional: true diff --git a/assets/overlays/openstack-cinder/patches/node_add_driver.yaml b/assets/overlays/openstack-cinder/patches/node_add_driver.yaml index 87ccadfc1..a69546223 100644 --- a/assets/overlays/openstack-cinder/patches/node_add_driver.yaml +++ b/assets/overlays/openstack-cinder/patches/node_add_driver.yaml @@ -50,13 +50,16 @@ spec: mountPath: /etc/selinux - name: sys-fs mountPath: /sys/fs - - name: cacert - mountPath: /etc/kubernetes/static-pod-resources/configmaps/cloud-config + # credentials and configuration - name: config-cinderplugin mountPath: /etc/kubernetes/config readOnly: true - - name: secret-cinderplugin - mountPath: /etc/kubernetes/secret + - name: cloud-credentials + mountPath: /etc/openstack + readOnly: true + # TODO(stephenfin): Remove in 4.22 + - name: legacy-cacert + mountPath: /etc/kubernetes/static-pod-resources/configmaps/cloud-config readOnly: true ports: - name: healthz @@ -77,26 +80,31 @@ spec: cpu: 10m terminationMessagePolicy: FallbackToLogsOnError volumes: - - name: cacert - # Extract ca-bundle.pem to /etc/kubernetes/static-pod-resources/configmaps/cloud-config if present. - # Let the pod start when the ConfigMap does not exist or the certificate - # is not preset there. The certificate file will be created once the - # ConfigMap is created / the certificate is added to it. - configMap: - name: cloud-conf - items: - - key: ca-bundle.pem - path: ca-bundle.pem - optional: true + - name: cloud-credentials + projected: + sources: + - secret: + name: openstack-cloud-credentials + items: + - key: cacert + path: ca.crt + optional: true + - secret: + name: openstack-cloud-credentials + items: + - key: clouds.yaml + path: clouds.yaml - name: config-cinderplugin configMap: name: cloud-conf items: - key: cloud.conf path: cloud.conf - - name: secret-cinderplugin - secret: - secretName: openstack-cloud-credentials + # TODO(stephenfin): Remove in 4.22 + - name: legacy-cacert + configMap: + name: cloud-conf items: - - key: clouds.yaml - path: clouds.yaml + - key: ca-bundle.pem + path: ca-bundle.pem + optional: true diff --git a/assets/overlays/openstack-manila/generated/hypershift/controller.yaml b/assets/overlays/openstack-manila/generated/hypershift/controller.yaml index febff678b..84060e949 100644 --- a/assets/overlays/openstack-manila/generated/hypershift/controller.yaml +++ b/assets/overlays/openstack-manila/generated/hypershift/controller.yaml @@ -138,8 +138,10 @@ spec: volumeMounts: - mountPath: /plugin name: socket-dir + - mountPath: /etc/openstack + name: cloud-credentials - mountPath: /etc/kubernetes/static-pod-resources/configmaps/cloud-config - name: cacert + name: legacy-cacert - args: - --nodeid=$(NODE_ID) - --endpoint=unix://plugin/csi-nfs.sock @@ -350,13 +352,19 @@ spec: - name: metrics-serving-cert secret: secretName: manila-csi-driver-controller-metrics-serving-cert + - name: cloud-credentials + secret: + items: + - key: cacert + path: ca.crt + secretName: manila-cloud-credentials - configMap: items: - key: ca-bundle.pem path: ca-bundle.pem name: openstack-cloud-config optional: true - name: cacert + name: legacy-cacert - name: hosted-kubeconfig secret: defaultMode: 420 diff --git a/assets/overlays/openstack-manila/generated/hypershift/node.yaml b/assets/overlays/openstack-manila/generated/hypershift/node.yaml index dc7fdbf09..c193ce5d1 100644 --- a/assets/overlays/openstack-manila/generated/hypershift/node.yaml +++ b/assets/overlays/openstack-manila/generated/hypershift/node.yaml @@ -84,12 +84,14 @@ spec: name: plugin-dir - mountPath: /var/lib/kubelet/plugins/csi-nfsplugin name: fwd-plugin-dir - - mountPath: /etc/kubernetes/static-pod-resources/configmaps/cloud-config - name: cacert - mountPath: /etc/selinux name: etc-selinux - mountPath: /sys/fs name: sys-fs + - mountPath: /etc/openstack + name: cloud-credentials + - mountPath: /etc/kubernetes/static-pod-resources/configmaps/cloud-config + name: legacy-cacert - args: - --csi-address=/csi/csi.sock - --http-endpoint=127.0.0.1:10305 @@ -192,13 +194,19 @@ spec: path: /var/lib/kubelet/plugins/csi-nfsplugin type: DirectoryOrCreate name: fwd-plugin-dir + - name: cloud-credentials + secret: + items: + - key: cacert + path: ca.crt + secretName: manila-cloud-credentials - configMap: items: - key: ca-bundle.pem path: ca-bundle.pem name: cloud-provider-config optional: true - name: cacert + name: legacy-cacert updateStrategy: rollingUpdate: maxUnavailable: 10% diff --git a/assets/overlays/openstack-manila/generated/standalone/controller.yaml b/assets/overlays/openstack-manila/generated/standalone/controller.yaml index a070905a5..65511ac54 100644 --- a/assets/overlays/openstack-manila/generated/standalone/controller.yaml +++ b/assets/overlays/openstack-manila/generated/standalone/controller.yaml @@ -107,8 +107,10 @@ spec: volumeMounts: - mountPath: /plugin name: socket-dir + - mountPath: /etc/openstack + name: cloud-credentials - mountPath: /etc/kubernetes/static-pod-resources/configmaps/cloud-config - name: cacert + name: legacy-cacert - args: - --nodeid=$(NODE_ID) - --endpoint=unix://plugin/csi-nfs.sock @@ -298,10 +300,16 @@ spec: - name: metrics-serving-cert secret: secretName: manila-csi-driver-controller-metrics-serving-cert + - name: cloud-credentials + secret: + items: + - key: cacert + path: ca.crt + secretName: manila-cloud-credentials - configMap: items: - key: ca-bundle.pem path: ca-bundle.pem name: cloud-provider-config optional: true - name: cacert + name: legacy-cacert diff --git a/assets/overlays/openstack-manila/generated/standalone/node.yaml b/assets/overlays/openstack-manila/generated/standalone/node.yaml index dc7fdbf09..c193ce5d1 100644 --- a/assets/overlays/openstack-manila/generated/standalone/node.yaml +++ b/assets/overlays/openstack-manila/generated/standalone/node.yaml @@ -84,12 +84,14 @@ spec: name: plugin-dir - mountPath: /var/lib/kubelet/plugins/csi-nfsplugin name: fwd-plugin-dir - - mountPath: /etc/kubernetes/static-pod-resources/configmaps/cloud-config - name: cacert - mountPath: /etc/selinux name: etc-selinux - mountPath: /sys/fs name: sys-fs + - mountPath: /etc/openstack + name: cloud-credentials + - mountPath: /etc/kubernetes/static-pod-resources/configmaps/cloud-config + name: legacy-cacert - args: - --csi-address=/csi/csi.sock - --http-endpoint=127.0.0.1:10305 @@ -192,13 +194,19 @@ spec: path: /var/lib/kubelet/plugins/csi-nfsplugin type: DirectoryOrCreate name: fwd-plugin-dir + - name: cloud-credentials + secret: + items: + - key: cacert + path: ca.crt + secretName: manila-cloud-credentials - configMap: items: - key: ca-bundle.pem path: ca-bundle.pem name: cloud-provider-config optional: true - name: cacert + name: legacy-cacert updateStrategy: rollingUpdate: maxUnavailable: 10% diff --git a/assets/overlays/openstack-manila/patches/controller_add_driver.yaml b/assets/overlays/openstack-manila/patches/controller_add_driver.yaml index 0b30487ab..c8746cc16 100644 --- a/assets/overlays/openstack-manila/patches/controller_add_driver.yaml +++ b/assets/overlays/openstack-manila/patches/controller_add_driver.yaml @@ -80,7 +80,9 @@ spec: volumeMounts: - name: socket-dir mountPath: /plugin - - name: cacert + - name: cloud-credentials + mountPath: /etc/openstack + - name: legacy-cacert mountPath: /etc/kubernetes/static-pod-resources/configmaps/cloud-config resources: requests: @@ -111,12 +113,17 @@ spec: volumes: - name: socket-dir emptyDir: {} - - name: cacert - # If present, extract ca-bundle.pem to - # /etc/kubernetes/static-pod-resources/configmaps/cloud-config - # Let the pod start when the ConfigMap does not exist or the certificate - # is not preset there. The certificate file will be created once the - # ConfigMap is created / the cerificate is added to it. + - name: cloud-credentials + secret: + secretName: manila-cloud-credentials + # TODO(stephenfin): We should also mount the clouds.yaml file + # here and start consuming that rather than converting it into + # the cloud.conf format + items: + - key: cacert + path: ca.crt + - name: legacy-cacert + # TODO(stephenfin): Remove in 4.22 configMap: name: cloud-provider-config items: diff --git a/assets/overlays/openstack-manila/patches/controller_rename_config_map.yaml b/assets/overlays/openstack-manila/patches/controller_rename_config_map.yaml index aaaca5043..08d655bd7 100644 --- a/assets/overlays/openstack-manila/patches/controller_rename_config_map.yaml +++ b/assets/overlays/openstack-manila/patches/controller_rename_config_map.yaml @@ -2,10 +2,11 @@ spec: template: spec: volumes: - - configMap: - items: - - key: ca-bundle.pem - path: ca-bundle.pem - name: openstack-cloud-config - optional: true - name: cacert + # TODO(stephenfin): Remove in 4.22 + - name: legacy-cacert + configMap: + name: openstack-cloud-config + items: + - key: ca-bundle.pem + path: ca-bundle.pem + optional: true diff --git a/assets/overlays/openstack-manila/patches/node_add_driver.yaml b/assets/overlays/openstack-manila/patches/node_add_driver.yaml index d131b6c9c..a81c97fee 100644 --- a/assets/overlays/openstack-manila/patches/node_add_driver.yaml +++ b/assets/overlays/openstack-manila/patches/node_add_driver.yaml @@ -63,12 +63,15 @@ spec: mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org - name: fwd-plugin-dir mountPath: /var/lib/kubelet/plugins/csi-nfsplugin - - name: cacert - mountPath: /etc/kubernetes/static-pod-resources/configmaps/cloud-config - name: etc-selinux mountPath: /etc/selinux - name: sys-fs mountPath: /sys/fs + # credentials and configuration + - name: cloud-credentials + mountPath: /etc/openstack + - name: legacy-cacert + mountPath: /etc/kubernetes/static-pod-resources/configmaps/cloud-config ports: - name: healthz containerPort: 10305 @@ -100,17 +103,6 @@ spec: hostPath: path: /var/lib/kubelet/plugins/csi-nfsplugin type: DirectoryOrCreate - - name: cacert - # Extract ca-bundle.pem to /etc/kubernetes/static-pod-resources/configmaps/cloud-config if present. - # Let the pod start when the ConfigMap does not exist or the certificate - # is not preset there. The certificate file will be created once the - # ConfigMap is created / the cerificate is added to it. - configMap: - name: cloud-provider-config - items: - - key: ca-bundle.pem - path: ca-bundle.pem - optional: true - name: etc-selinux hostPath: path: /etc/selinux @@ -119,3 +111,20 @@ spec: hostPath: path: /sys/fs type: Directory + - name: cloud-credentials + secret: + secretName: manila-cloud-credentials + # TODO(stephenfin): We should also mount the clouds.yaml file + # here and start consuming that rather than converting it into + # the cloud.conf format + items: + - key: cacert + path: ca.crt + # TODO(stephenfin): Remove in 4.22 + - name: legacy-cacert + configMap: + name: cloud-provider-config + items: + - key: ca-bundle.pem + path: ca-bundle.pem + optional: true diff --git a/pkg/openstack-cinder/config/cloudinfo.go b/pkg/openstack-cinder/config/cloudinfo.go index 2e41047cd..70c47e4c3 100644 --- a/pkg/openstack-cinder/config/cloudinfo.go +++ b/pkg/openstack-cinder/config/cloudinfo.go @@ -13,8 +13,6 @@ import ( "github.com/openshift/csi-operator/pkg/version" ) -const caFile = "/etc/kubernetes/static-pod-resources/configmaps/cloud-config/ca-bundle.pem" - // CloudInfo caches data fetched from the user's openstack cloud type CloudInfo struct { ComputeZones []string @@ -87,10 +85,12 @@ func getCloudInfo() (*CloudInfo, error) { // our assets mount the CA file at a known path and we don't want to rely on other things // setting the 'cafile' value in our clouds.yaml file to the same path, so we explicitly // override this if a CA file is present + // NOTE(stephenfin): gophercloud (or rather, the clientconfig package) doesn't currently + // provide the way to override configuration other than via environment variables if _, err := os.Stat(caFile); err == nil { - // NOTE(stephenfin): gophercloud (or rather, the clientconfig package) doesn't currently - // provide the way to override configuration other than via environment variables os.Setenv("OS_CACERT", caFile) + } else if _, err := os.Stat(legacyCAFile); err == nil { // legacy path + os.Setenv("OS_CACERT", legacyCAFile) } ci.clients.computeClient, err = clientconfig.NewServiceClient(context.TODO(), "compute", opts) diff --git a/pkg/openstack-cinder/config/configsync.go b/pkg/openstack-cinder/config/configsync.go index 50f2b4e74..576cbc87c 100644 --- a/pkg/openstack-cinder/config/configsync.go +++ b/pkg/openstack-cinder/config/configsync.go @@ -37,6 +37,13 @@ type configDestination struct { kubeClient kubernetes.Interface } +type caCertSource string + +const ( + configCACertSource caCertSource = "config" + secretCACertSource caCertSource = "secret" +) + // This ConfigSyncController generates the configuration file needed for the Cinder CSI controller // and driver, including optional configuration from the user, and save it to a config map in a // well-known location that both services can use @@ -159,27 +166,39 @@ func (c *ConfigSyncController) sync(ctx context.Context, syncCtx factory.SyncCon enableTopology = strconv.FormatBool(enableTopologyFeature) } - // ...and we watch the clusters-${NAME} namespace (on the control plane cluster) - configMapLister = c.controlPlaneInformers.InformersFor(c.controlPlaneNamespace).Core().V1().ConfigMaps().Lister() - - // FIXME(stephenfin): We extract the CA cert from the cloud-provider-config because we know - // it will exist here in both a standalone (IPI) deployment and in both clusters in a - // Hypershift deployment. However, this is less than ideal: cloud providers and CSI drivers are - // now separate things and we should be able to get this information from the credentials - // secret, along with our clouds.yaml. Fixing this will requires changes to - // cloud-credential-operator and hypershift (because the latter doesn't currently use the - // former). Once we have that feature, we should rejig this. - cloudProviderConfig := "cloud-provider-config" - if c.isHypershift { - // unfortunately hypershift uses a different name - cloudProviderConfig = "openstack-cloud-config" - } - caCert, err := getCACert(configMapLister, c.controlPlaneNamespace, cloudProviderConfig) + secretLister := c.controlPlaneInformers.InformersFor(c.controlPlaneNamespace).Core().V1().Secrets().Lister() + + var caCertSource caCertSource + + hasCACert, err := hasCACert(secretLister, c.controlPlaneNamespace, "openstack-cloud-credentials") if err != nil { return err } - generatedConfig, err := generateConfig(sourceConfig, caCert) + if hasCACert { + caCertSource = secretCACertSource + } else { + // TODO(stephenfin): Remove this fallback in 4.22. At that point, the + // cloud-credentials-operator will have ensured the root credential has + // been updated. + configMapLister = c.controlPlaneInformers.InformersFor(c.controlPlaneNamespace).Core().V1().ConfigMaps().Lister() + cloudProviderConfig := "cloud-provider-config" + if c.isHypershift { + // unfortunately hypershift uses a different name + cloudProviderConfig = "openstack-cloud-config" + } + + hasCACert, err = getCACertLegacy(configMapLister, c.controlPlaneNamespace, cloudProviderConfig) + if err != nil { + return err + } + + if hasCACert { + caCertSource = configCACertSource + } + } + + generatedConfig, err := generateConfig(sourceConfig, caCertSource) if err != nil { return err } @@ -195,9 +214,6 @@ func (c *ConfigSyncController) sync(ctx context.Context, syncCtx factory.SyncCon "enable_topology": enableTopology, }, } - if caCert != nil { - targetConfig.Data["ca-bundle.pem"] = *caCert - } _, _, err = resourceapply.ApplyConfigMap(ctx, configDestination.kubeClient.CoreV1(), c.eventRecorder, targetConfig) if err != nil { return err @@ -206,18 +222,35 @@ func (c *ConfigSyncController) sync(ctx context.Context, syncCtx factory.SyncCon return nil } -// getCACert gets the (optional) embedded CA Cert provided in a config map by either the Installer -// or the hypershift-operator so that we can inject it into our config files -func getCACert(configMapLister corelisters.ConfigMapLister, ns, name string) (*string, error) { +// hasCACert determines whether there is a CA Cert provided in the credentials +// secret by either the cloud-credential-operator or hypershift-operator so +// that we can inject it into our configuration +func hasCACert(secretLister corelisters.SecretLister, ns, name string) (bool, error) { + cm, err := secretLister.Secrets(ns).Get(name) + if err != nil { + return false, err + } + caCert, ok := cm.Data["cacert"] + if !ok || len(caCert) == 0 { + return false, nil + } + return true, nil +} + +// getCACertLegacy determines whether there is a CA Cert provided in the cloud +// provider config map by either the Installer or the hypershift-operator so +// that we can inject it into our config files. This is the legacy path for the +// transition period as the CA cert is now provided in the credentials secret +func getCACertLegacy(configMapLister corelisters.ConfigMapLister, ns, name string) (bool, error) { cm, err := configMapLister.ConfigMaps(ns).Get(name) if err != nil { - return nil, err + return false, err } caCert, ok := cm.Data["ca-bundle.pem"] if !ok || len(caCert) == 0 { - return nil, nil + return false, nil } - return &caCert, nil + return true, nil } // getSourceConfig extracts any source configuration present in any of the provided for the legacy, @@ -258,7 +291,7 @@ func getSourceConfig( // '[BlockStorage]' section (and only that section) will be used in generation. func generateConfig( sourceContent []byte, - caCert *string, + caCertSource caCertSource, ) (string, error) { var cfg *ini.File @@ -295,7 +328,7 @@ func generateConfig( } for _, o := range []struct{ k, v string }{ {"use-clouds", "true"}, - {"clouds-file", "/etc/kubernetes/secret/clouds.yaml"}, + {"clouds-file", "/etc/openstack/clouds.yaml"}, {"cloud", "openstack"}, } { _, err = global.NewKey(o.k, o.v) @@ -304,12 +337,18 @@ func generateConfig( } } - if caCert != nil { + if caCertSource != "" { // We don't have a (non-deprecated) way to inject the CA cert itself into the config // file, so instead we pass a file path. The path itself may look like a magic value but // its where we have configured both the deployment (for controller) and daemonset (for // driver) assets to mount the cert, if present. - _, err = global.NewKey("ca-file", "/etc/kubernetes/static-pod-resources/configmaps/cloud-config/ca-bundle.pem") + var path string + if caCertSource == secretCACertSource { + path = caFile + } else { // legacy path + path = legacyCAFile + } + _, err = global.NewKey("ca-file", path) if err != nil { return "", err } diff --git a/pkg/openstack-cinder/config/configsync_test.go b/pkg/openstack-cinder/config/configsync_test.go index 54dea154f..d476587e4 100644 --- a/pkg/openstack-cinder/config/configsync_test.go +++ b/pkg/openstack-cinder/config/configsync_test.go @@ -6,7 +6,6 @@ import ( . "github.com/onsi/gomega" "github.com/onsi/gomega/format" - "k8s.io/utils/ptr" ) func TestGenerateConfigMap(t *testing.T) { @@ -14,25 +13,25 @@ func TestGenerateConfigMap(t *testing.T) { format.TruncatedDiff = false tc := []struct { - name string - source []byte - caCert *string - expected string - errMsg string + name string + source []byte + caCertSource caCertSource + expected string + errMsg string }{ { name: "Unset config", source: nil, expected: `[Global] use-clouds = true -clouds-file = /etc/kubernetes/secret/clouds.yaml +clouds-file = /etc/openstack/clouds.yaml cloud = openstack`, }, { name: "Empty config", source: []byte(""), expected: `[Global] use-clouds = true -clouds-file = /etc/kubernetes/secret/clouds.yaml +clouds-file = /etc/openstack/clouds.yaml cloud = openstack`, }, { name: "Minimal config", @@ -43,15 +42,24 @@ ignore-volume-az = True [Global] use-clouds = true -clouds-file = /etc/kubernetes/secret/clouds.yaml +clouds-file = /etc/openstack/clouds.yaml cloud = openstack`, }, { - name: "With CA cert", - source: nil, - caCert: ptr.To("not-so-secret CA data goes here"), + name: "With CA cert", + source: nil, + caCertSource: secretCACertSource, + expected: `[Global] +use-clouds = true +clouds-file = /etc/openstack/clouds.yaml +cloud = openstack +ca-file = /etc/openstack/ca.crt`, + }, { + name: "With CA cert (legacy)", + source: nil, + caCertSource: configCACertSource, expected: `[Global] use-clouds = true -clouds-file = /etc/kubernetes/secret/clouds.yaml +clouds-file = /etc/openstack/clouds.yaml cloud = openstack ca-file = /etc/kubernetes/static-pod-resources/configmaps/cloud-config/ca-bundle.pem`, }, { @@ -61,7 +69,7 @@ ca-file = /etc/kubernetes/static-pod-resources/configmaps/cloud-config/ca-bu trust-device-path = /dev/sdb1`), expected: `[Global] use-clouds = true -clouds-file = /etc/kubernetes/secret/clouds.yaml +clouds-file = /etc/openstack/clouds.yaml cloud = openstack`, }, } @@ -71,7 +79,7 @@ cloud = openstack`, g := NewWithT(t) actual, err := generateConfig( []byte(tc.source), - tc.caCert, + tc.caCertSource, ) if tc.errMsg != "" { g.Expect(err).Should(MatchError(tc.errMsg)) diff --git a/pkg/openstack-cinder/config/const.go b/pkg/openstack-cinder/config/const.go new file mode 100644 index 000000000..26b09b852 --- /dev/null +++ b/pkg/openstack-cinder/config/const.go @@ -0,0 +1,5 @@ +package config + +// caFile/legacyCAFile is based on the path defined in the assets +const caFile = "/etc/openstack/ca.crt" +const legacyCAFile = "/etc/kubernetes/static-pod-resources/configmaps/cloud-config/ca-bundle.pem" diff --git a/pkg/openstack-manila/client/openstack.go b/pkg/openstack-manila/client/openstack.go index d40932d55..fa19905ac 100644 --- a/pkg/openstack-manila/client/openstack.go +++ b/pkg/openstack-manila/client/openstack.go @@ -4,8 +4,8 @@ import ( "context" "crypto/tls" "crypto/x509" + "errors" "fmt" - "io/ioutil" "net/http" "os" "time" @@ -59,7 +59,7 @@ func (o *openStackClient) GetShareTypes() ([]sharetypes.ShareType, error) { provider.UserAgent = ua cert, err := getCloudProviderCert() - if err != nil && !os.IsNotExist(err) { + if err != nil && !errors.Is(err, os.ErrNotExist) { return nil, fmt.Errorf("failed to get cloud provider CA certificate: %w", err) } @@ -103,7 +103,7 @@ func (o *openStackClient) GetShareTypes() ([]sharetypes.ShareType, error) { } func getCloudFromFile(filename string) (*clientconfig.Cloud, error) { - cloudConfig, err := ioutil.ReadFile(filename) + cloudConfig, err := os.ReadFile(filename) if err != nil { return nil, err } @@ -121,5 +121,11 @@ func getCloudFromFile(filename string) (*clientconfig.Cloud, error) { } func getCloudProviderCert() ([]byte, error) { - return ioutil.ReadFile(util.CertFile) + data, err := os.ReadFile(util.CertFile) + if err == nil || !errors.Is(err, os.ErrNotExist) { + return data, err + } + + // legacy path; remove in 4.22 + return os.ReadFile(util.LegacyCertFile) } diff --git a/pkg/openstack-manila/secret/secretsync.go b/pkg/openstack-manila/secret/secretsync.go index 20c1a907b..450ef1e6f 100644 --- a/pkg/openstack-manila/secret/secretsync.go +++ b/pkg/openstack-manila/secret/secretsync.go @@ -35,9 +35,10 @@ type SecretSyncController struct { const ( // Name of key with clouds.yaml in Secret provided by cloud-credentials-operator. cloudSecretKey = "clouds.yaml" - // Name of OpenStack in clouds.yaml - // Canonical path for custom ca certificates - cacertPath = "/etc/kubernetes/static-pod-resources/configmaps/cloud-config/ca-bundle.pem" + // Path for custom CA certificates when provided by cloud-credentials-operator. + defaultCACertPath = "/etc/openstack/ca.crt" + // Path for custom CA certificates when provided by Installer (legacy path). + legacyCACertPath = "/etc/kubernetes/static-pod-resources/configmaps/cloud-config/ca-bundle.pem" ) func NewSecretSyncController( @@ -115,11 +116,29 @@ func (c *SecretSyncController) translateSecret(cloudSecret *v1.Secret) (*v1.Secr data := cloudToConf(cloud) - // In the hypershift secret, the clouds.yaml field might not have the cacert defined. The content of the certificate - // is defined in the ca-bundle.pem field instead. - _, ok = cloudSecret.Data["ca-bundle.pem"] - if ok { - data["os-certAuthorityPath"] = []byte(cacertPath) + // Determine where our CA cert is stored. + // TODO(stephenfin): Remove most of this in 4.22 + var caCertPath string + if _, ok := cloudSecret.Data["cacert"]; ok { + // Option A: We have the CA cert in our credentials under the 'cacert' + // key which indicates a recent (>= 4.19) version of + // cluster-credential-operator (CCO) or hypershift + caCertPath = defaultCACertPath + } else if _, ok = cloudSecret.Data["ca-bundle.pem"]; ok { + // Option B: We have the CA cert in our credentials but under the + // 'ca-bundle.pem' key, which indicates an older (< 4.19) version of + // hypershift + caCertPath = legacyCACertPath + } else if cloud.CACertFile != "" { + // Option C: We have a non-empty 'cafile' field in our clouds.yaml. + // This means our root credential secret has this defined yet + // cloud-credential-operator (CCO) didn't populate the 'cacert' key of + // the secret. This indicates an older (< 4.19) version of CCO. + caCertPath = legacyCACertPath + } + + if caCertPath != "" { + data["os-certAuthorityPath"] = []byte(caCertPath) } secret := v1.Secret{ @@ -182,10 +201,8 @@ func cloudToConf(cloud clientconfig.Cloud) map[string][]byte { data["os-userDomainName"] = []byte(cloud.AuthInfo.UserDomainName) data["os-domainName"] = []byte(cloud.AuthInfo.UserDomainName) } - if cloud.CACertFile != "" { - // Replace the original cert authority path from clouds.yaml with the canonical one - data["os-certAuthorityPath"] = []byte(cacertPath) - } + + // We don't set os-certAuthorityPath here as it's handled separately. return data } diff --git a/pkg/openstack-manila/util/const.go b/pkg/openstack-manila/util/const.go index bdd82f42d..335664b5e 100644 --- a/pkg/openstack-manila/util/const.go +++ b/pkg/openstack-manila/util/const.go @@ -11,7 +11,8 @@ const ( // OpenStack config file name (as present in the operator Deployment) CloudConfigFilename = "/etc/openstack/clouds.yaml" - CertFile = "/etc/openstack-ca/ca-bundle.pem" + CertFile = "/etc/openstack/ca.crt" + LegacyCertFile = "/etc/openstack-ca/ca-bundle.pem" // Name of cloud in secret provided by cloud-credentials-operator CloudName = "openstack"