diff --git a/config/core/configmaps/deployment.yaml b/config/core/configmaps/deployment.yaml index 026760ef1104..9a4197090c3a 100644 --- a/config/core/configmaps/deployment.yaml +++ b/config/core/configmaps/deployment.yaml @@ -22,7 +22,7 @@ metadata: app.kubernetes.io/component: controller app.kubernetes.io/version: devel annotations: - knative.dev/example-checksum: "b99000ec" + knative.dev/example-checksum: "555b4826" data: # This is the Go import path for the binary that is containerized # and substituted here. @@ -92,6 +92,25 @@ data: # If omitted, or empty, no rootCA is added to the golang rootCAs queue-sidecar-rootca: "" + # Sets the minimum TLS version for the queue proxy sidecar's TLS server. + # Accepted values: "1.2", "1.3". Default is "1.3" if not specified. + queue-sidecar-tls-min-version: "" + + # Sets the maximum TLS version for the queue proxy sidecar's TLS server. + # Accepted values: "1.2", "1.3". If omitted, the Go default is used. + queue-sidecar-tls-max-version: "" + + # Sets the cipher suites for the queue proxy sidecar's TLS server. + # Comma-separated list of cipher suite names (e.g. "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"). + # If omitted, the Go default cipher suites are used. + # Note: cipher suites are not configurable in TLS 1.3. + queue-sidecar-tls-cipher-suites: "" + + # Sets the elliptic curve preferences for the queue proxy sidecar's TLS server. + # Comma-separated list of curve names (e.g. "X25519,CurveP256"). + # If omitted, the Go default curves are used. + queue-sidecar-tls-curve-preferences: "" + # If set, it automatically configures pod anti-affinity requirements for all Knative services. # It employs the `preferredDuringSchedulingIgnoredDuringExecution` weighted pod affinity term, # aligning with the Knative revision label. It yields the configuration below in all workloads' deployments: diff --git a/pkg/deployment/config.go b/pkg/deployment/config.go index 990fcdeff00c..6ebcf3771d23 100644 --- a/pkg/deployment/config.go +++ b/pkg/deployment/config.go @@ -74,6 +74,12 @@ const ( queueSidecarTokenAudiencesKey = "queue-sidecar-token-audiences" queueSidecarRooCAKey = "queue-sidecar-rootca" + // queueSidecar TLS configuration keys. + queueSidecarTLSMinVersionKey = "queue-sidecar-tls-min-version" + queueSidecarTLSMaxVersionKey = "queue-sidecar-tls-max-version" + queueSidecarTLSCipherSuitesKey = "queue-sidecar-tls-cipher-suites" + queueSidecarTLSCurvePreferencesKey = "queue-sidecar-tls-curve-preferences" + defaultAffinityTypeKey = "default-affinity-type" defaultAffinityTypeValue = PreferSpreadRevisionOverNodes @@ -202,6 +208,11 @@ func NewConfigFromMap(configMap map[string]string) (*Config, error) { cm.AsStringSet(queueSidecarTokenAudiencesKey, &nc.QueueSidecarTokenAudiences), cm.AsString(queueSidecarRooCAKey, &nc.QueueSidecarRootCA), + cm.AsString(queueSidecarTLSMinVersionKey, &nc.QueueSidecarTLSMinVersion), + cm.AsString(queueSidecarTLSMaxVersionKey, &nc.QueueSidecarTLSMaxVersion), + cm.AsString(queueSidecarTLSCipherSuitesKey, &nc.QueueSidecarTLSCipherSuites), + cm.AsString(queueSidecarTLSCurvePreferencesKey, &nc.QueueSidecarTLSCurvePreferences), + cm.AsString(RuntimeClassNameKey, &runtimeClassNames), cm.AsBool(podIsAlwaysSchedulableKey, &nc.PodIsAlwaysSchedulable), @@ -308,6 +319,18 @@ type Config struct { // QueueSidecarRootCA is a root certificate to be trusted by the queue proxy sidecar qpoptions. QueueSidecarRootCA string + // QueueSidecarTLSMinVersion is the minimum TLS version for the queue proxy sidecar (e.g. "1.2", "1.3"). + QueueSidecarTLSMinVersion string + + // QueueSidecarTLSMaxVersion is the maximum TLS version for the queue proxy sidecar (e.g. "1.2", "1.3"). + QueueSidecarTLSMaxVersion string + + // QueueSidecarTLSCipherSuites is a comma-separated list of cipher suites for the queue proxy sidecar. + QueueSidecarTLSCipherSuites string + + // QueueSidecarTLSCurvePreferences is a comma-separated list of elliptic curves for the queue proxy sidecar. + QueueSidecarTLSCurvePreferences string + // DefaultAffinityType is a string that controls what affinity rules will be automatically // applied to the PodSpec of all Knative services. DefaultAffinityType AffinityType diff --git a/pkg/deployment/config_test.go b/pkg/deployment/config_test.go index c44a8782290e..7d783ea2829e 100644 --- a/pkg/deployment/config_test.go +++ b/pkg/deployment/config_test.go @@ -472,6 +472,28 @@ kata: podIsAlwaysSchedulableKey: "true", QueueSidecarImageKey: defaultSidecarImage, }, + }, { + name: "controller configuration with queue sidecar TLS settings", + wantConfig: &Config{ + RegistriesSkippingTagResolving: sets.New("kind.local", "ko.local", "dev.local"), + DigestResolutionTimeout: digestResolutionTimeoutDefault, + QueueSidecarImage: defaultSidecarImage, + QueueSidecarCPURequest: &QueueSidecarCPURequestDefault, + QueueSidecarTokenAudiences: sets.New(""), + ProgressDeadline: ProgressDeadlineDefault, + DefaultAffinityType: defaultAffinityTypeValue, + QueueSidecarTLSMinVersion: "1.2", + QueueSidecarTLSMaxVersion: "1.3", + QueueSidecarTLSCipherSuites: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + QueueSidecarTLSCurvePreferences: "X25519,CurveP256", + }, + data: map[string]string{ + QueueSidecarImageKey: defaultSidecarImage, + queueSidecarTLSMinVersionKey: "1.2", + queueSidecarTLSMaxVersionKey: "1.3", + queueSidecarTLSCipherSuitesKey: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + queueSidecarTLSCurvePreferencesKey: "X25519,CurveP256", + }, }} for _, tt := range configTests { diff --git a/pkg/queue/sharedmain/main.go b/pkg/queue/sharedmain/main.go index 5b4ad8990a2b..ec819973d573 100644 --- a/pkg/queue/sharedmain/main.go +++ b/pkg/queue/sharedmain/main.go @@ -18,7 +18,6 @@ package sharedmain import ( "context" - "crypto/tls" "encoding/json" "errors" "fmt" @@ -31,6 +30,7 @@ import ( netproxy "knative.dev/networking/pkg/http/proxy" pkghandler "knative.dev/pkg/network/handlers" + knativetls "knative.dev/pkg/tls" "knative.dev/serving/pkg/activator" "github.com/kelseyhightower/envconfig" @@ -265,23 +265,10 @@ func Main(opts ...Option) error { httpServers["profile"] = pprof.Server } - tlsServers := make(map[string]*http.Server) + var tlsServer *http.Server var certWatcher *certificate.CertWatcher var err error - if tlsEnabled { - tlsServers["main"] = mainServer(":"+env.QueueServingTLSPort, mainHandler) - // Keep admin server on HTTP even with TLS enabled since it's only accessed locally by kubelet - - certWatcher, err = certificate.NewCertWatcher(certPath, keyPath, 1*time.Minute, logger) - if err != nil { - logger.Fatal("failed to create certWatcher", zap.Error(err)) - } - defer certWatcher.Stop() - } - - logger.Info("Starting queue-proxy") - errCh := make(chan error) for name, server := range httpServers { go func(name string, s *http.Server) { @@ -292,20 +279,34 @@ func Main(opts ...Option) error { } }(name, server) } - for name, server := range tlsServers { - go func(name string, s *http.Server) { - logger.Info("Starting tls server ", name, s.Addr) - s.TLSConfig = &tls.Config{ - GetCertificate: certWatcher.GetCertificate, - MinVersion: tls.VersionTLS13, - } + + if tlsEnabled { + tlsServer = mainServer(":"+env.QueueServingTLSPort, mainHandler) + // Keep admin server on HTTP even with TLS enabled since it's only accessed locally by kubelet + + certWatcher, err = certificate.NewCertWatcher(certPath, keyPath, 1*time.Minute, logger) + if err != nil { + logger.Fatal("failed to create certWatcher", zap.Error(err)) + } + defer certWatcher.Stop() + + tlsCfg, err := knativetls.DefaultConfigFromEnv("QUEUE_PROXY_") + if err != nil { + logger.Fatalw("Failed to read TLS configuration from environment", zap.Error(err)) + } + go func() { + logger.Info("Starting tls server main ", tlsServer.Addr) + tlsServer.TLSConfig = tlsCfg + tlsServer.TLSConfig.GetCertificate = certWatcher.GetCertificate // Don't forward ErrServerClosed as that indicates we're already shutting down. - if err := s.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) { - errCh <- fmt.Errorf("%s server failed to serve: %w", name, err) + if err := tlsServer.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) { + errCh <- fmt.Errorf("main tls server failed to serve: %w", err) } - }(name, server) + }() } + logger.Info("Starting queue-proxy") + // Blocks until we actually receive a TERM signal or one of the servers // exits unexpectedly. We fold both signals together because we only want // to act on the first of those to reach here. @@ -326,10 +327,10 @@ func Main(opts ...Option) error { logger.Errorw("Failed to shutdown server", zap.String("server", name), zap.Error(err)) } } - for name, srv := range tlsServers { - logger.Info("Shutting down server: ", name) - if err := srv.Shutdown(ctx); err != nil { - logger.Errorw("Failed to shutdown server", zap.String("server", name), zap.Error(err)) + if tlsServer != nil { + logger.Info("Shutting down server: main tls") + if err := tlsServer.Shutdown(ctx); err != nil { + logger.Errorw("Failed to shutdown server", zap.String("server", "main tls"), zap.Error(err)) } } diff --git a/pkg/reconciler/revision/resources/deploy_test.go b/pkg/reconciler/revision/resources/deploy_test.go index 873c7e4d8e52..44abafe49a42 100644 --- a/pkg/reconciler/revision/resources/deploy_test.go +++ b/pkg/reconciler/revision/resources/deploy_test.go @@ -158,6 +158,18 @@ var ( }, { Name: "ROOT_CA", Value: "", + }, { + Name: "QUEUE_PROXY_TLS_MIN_VERSION", + Value: "", + }, { + Name: "QUEUE_PROXY_TLS_MAX_VERSION", + Value: "", + }, { + Name: "QUEUE_PROXY_TLS_CIPHER_SUITES", + Value: "", + }, { + Name: "QUEUE_PROXY_TLS_CURVE_PREFERENCES", + Value: "", }, { Name: "ENABLE_MULTI_CONTAINER_PROBES", Value: "false", diff --git a/pkg/reconciler/revision/resources/queue.go b/pkg/reconciler/revision/resources/queue.go index 23348cf67389..60a2292df044 100644 --- a/pkg/reconciler/revision/resources/queue.go +++ b/pkg/reconciler/revision/resources/queue.go @@ -428,6 +428,18 @@ func makeQueueContainer(rev *v1.Revision, cfg *config.Config) (*corev1.Container }, { Name: "ROOT_CA", Value: cfg.Deployment.QueueSidecarRootCA, + }, { + Name: "QUEUE_PROXY_TLS_MIN_VERSION", + Value: cfg.Deployment.QueueSidecarTLSMinVersion, + }, { + Name: "QUEUE_PROXY_TLS_MAX_VERSION", + Value: cfg.Deployment.QueueSidecarTLSMaxVersion, + }, { + Name: "QUEUE_PROXY_TLS_CIPHER_SUITES", + Value: cfg.Deployment.QueueSidecarTLSCipherSuites, + }, { + Name: "QUEUE_PROXY_TLS_CURVE_PREFERENCES", + Value: cfg.Deployment.QueueSidecarTLSCurvePreferences, }, { Name: "ENABLE_MULTI_CONTAINER_PROBES", Value: strconv.FormatBool(multiContainerProbingEnabled), diff --git a/pkg/reconciler/revision/resources/queue_test.go b/pkg/reconciler/revision/resources/queue_test.go index 1f7f05a4bc61..56ee18165d15 100644 --- a/pkg/reconciler/revision/resources/queue_test.go +++ b/pkg/reconciler/revision/resources/queue_test.go @@ -1122,6 +1122,10 @@ var defaultEnv = map[string]string{ "SYSTEM_NAMESPACE": system.Namespace(), "USER_PORT": strconv.Itoa(v1.DefaultUserPort), "ROOT_CA": "", + "QUEUE_PROXY_TLS_MIN_VERSION": "", + "QUEUE_PROXY_TLS_MAX_VERSION": "", + "QUEUE_PROXY_TLS_CIPHER_SUITES": "", + "QUEUE_PROXY_TLS_CURVE_PREFERENCES": "", "ENABLE_MULTI_CONTAINER_PROBES": "false", "OBSERVABILITY_CONFIG": `{"tracing":{},"metrics":{},"runtime":{},"requestMetrics":{}}`, }