feat: use knative.dev/pkg/tls for queue-proxy TLS configuration

Replace the hardcoded tls.VersionTLS13 in queue-proxy's TLS
server with the shared knative.dev/pkg/tls package, allowing
TLS settings to be configured via QUEUE_PROXY_TLS_MIN_VERSION,
QUEUE_PROXY_TLS_MAX_VERSION, QUEUE_PROXY_TLS_CIPHER_SUITES,
and QUEUE_PROXY_TLS_CURVE_PREFERENCES environment variables.
The default remains TLS 1.3 when no env var is set.

Add four new keys to the config-deployment ConfigMap
(queue-sidecar-tls-min-version, queue-sidecar-tls-max-version,
queue-sidecar-tls-cipher-suites, queue-sidecar-tls-curve-preferences)
and forward them as QUEUE_PROXY_TLS_* environment variables in
makeQueueContainer. This allows cluster admins to configure the
queue-proxy's TLS server via the same ConfigMap used for other
queue-proxy settings (like queue-sidecar-rootca), since the operator
cannot inject env vars into the dynamically created sidecar via
manifestival.

Signed-off-by: Mikhail Fedosin <mfedosin@redhat.com>

Author: Fedosin

config/core/configmaps/deployment.yaml

@@ -22,7 +22,7 @@ metadata:
     app.kubernetes.io/component: controller
     app.kubernetes.io/version: devel
   annotations:
-    knative.dev/example-checksum: "b99000ec"
+    knative.dev/example-checksum: "555b4826"
 data:
   # This is the Go import path for the binary that is containerized
   # and substituted here.
@@ -92,6 +92,25 @@ data:
     # If omitted, or empty, no rootCA is added to the golang rootCAs
     queue-sidecar-rootca: ""
 
+    # Sets the minimum TLS version for the queue proxy sidecar's TLS server.
+    # Accepted values: "1.2", "1.3". Default is "1.3" if not specified.
+    queue-sidecar-tls-min-version: ""
+
+    # Sets the maximum TLS version for the queue proxy sidecar's TLS server.
+    # Accepted values: "1.2", "1.3". If omitted, the Go default is used.
+    queue-sidecar-tls-max-version: ""
+
+    # Sets the cipher suites for the queue proxy sidecar's TLS server.
+    # Comma-separated list of cipher suite names (e.g. "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256").
+    # If omitted, the Go default cipher suites are used.
+    # Note: cipher suites are not configurable in TLS 1.3.
+    queue-sidecar-tls-cipher-suites: ""
+
+    # Sets the elliptic curve preferences for the queue proxy sidecar's TLS server.
+    # Comma-separated list of curve names (e.g. "X25519,CurveP256").
+    # If omitted, the Go default curves are used.
+    queue-sidecar-tls-curve-preferences: ""
+
     # If set, it automatically configures pod anti-affinity requirements for all Knative services.
     # It employs the `preferredDuringSchedulingIgnoredDuringExecution` weighted pod affinity term,
     # aligning with the Knative revision label. It yields the configuration below in all workloads' deployments:

pkg/deployment/config.go

@@ -74,6 +74,12 @@ const (
 	queueSidecarTokenAudiencesKey = "queue-sidecar-token-audiences"
 	queueSidecarRooCAKey          = "queue-sidecar-rootca"
 
+	// queueSidecar TLS configuration keys.
+	queueSidecarTLSMinVersionKey       = "queue-sidecar-tls-min-version"
+	queueSidecarTLSMaxVersionKey       = "queue-sidecar-tls-max-version"
+	queueSidecarTLSCipherSuitesKey     = "queue-sidecar-tls-cipher-suites"
+	queueSidecarTLSCurvePreferencesKey = "queue-sidecar-tls-curve-preferences"
+
 	defaultAffinityTypeKey   = "default-affinity-type"
 	defaultAffinityTypeValue = PreferSpreadRevisionOverNodes
 
@@ -202,6 +208,11 @@ func NewConfigFromMap(configMap map[string]string) (*Config, error) {
 		cm.AsStringSet(queueSidecarTokenAudiencesKey, &nc.QueueSidecarTokenAudiences),
 		cm.AsString(queueSidecarRooCAKey, &nc.QueueSidecarRootCA),
 
+		cm.AsString(queueSidecarTLSMinVersionKey, &nc.QueueSidecarTLSMinVersion),
+		cm.AsString(queueSidecarTLSMaxVersionKey, &nc.QueueSidecarTLSMaxVersion),
+		cm.AsString(queueSidecarTLSCipherSuitesKey, &nc.QueueSidecarTLSCipherSuites),
+		cm.AsString(queueSidecarTLSCurvePreferencesKey, &nc.QueueSidecarTLSCurvePreferences),
+
 		cm.AsString(RuntimeClassNameKey, &runtimeClassNames),
 
 		cm.AsBool(podIsAlwaysSchedulableKey, &nc.PodIsAlwaysSchedulable),
@@ -308,6 +319,18 @@ type Config struct {
 	// QueueSidecarRootCA is a root certificate to be trusted by the queue proxy sidecar  qpoptions.
 	QueueSidecarRootCA string
 
+	// QueueSidecarTLSMinVersion is the minimum TLS version for the queue proxy sidecar (e.g. "1.2", "1.3").
+	QueueSidecarTLSMinVersion string
+
+	// QueueSidecarTLSMaxVersion is the maximum TLS version for the queue proxy sidecar (e.g. "1.2", "1.3").
+	QueueSidecarTLSMaxVersion string
+
+	// QueueSidecarTLSCipherSuites is a comma-separated list of cipher suites for the queue proxy sidecar.
+	QueueSidecarTLSCipherSuites string
+
+	// QueueSidecarTLSCurvePreferences is a comma-separated list of elliptic curves for the queue proxy sidecar.
+	QueueSidecarTLSCurvePreferences string
+
 	// DefaultAffinityType is a string that controls what affinity rules will be automatically
 	// applied to the PodSpec of all Knative services.
 	DefaultAffinityType AffinityType

pkg/deployment/config_test.go

@@ -472,6 +472,28 @@ kata:
 			podIsAlwaysSchedulableKey: "true",
 			QueueSidecarImageKey:      defaultSidecarImage,
 		},
+	}, {
+		name: "controller configuration with queue sidecar TLS settings",
+		wantConfig: &Config{
+			RegistriesSkippingTagResolving:  sets.New("kind.local", "ko.local", "dev.local"),
+			DigestResolutionTimeout:         digestResolutionTimeoutDefault,
+			QueueSidecarImage:               defaultSidecarImage,
+			QueueSidecarCPURequest:          &QueueSidecarCPURequestDefault,
+			QueueSidecarTokenAudiences:      sets.New(""),
+			ProgressDeadline:                ProgressDeadlineDefault,
+			DefaultAffinityType:             defaultAffinityTypeValue,
+			QueueSidecarTLSMinVersion:       "1.2",
+			QueueSidecarTLSMaxVersion:       "1.3",
+			QueueSidecarTLSCipherSuites:     "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+			QueueSidecarTLSCurvePreferences: "X25519,CurveP256",
+		},
+		data: map[string]string{
+			QueueSidecarImageKey:               defaultSidecarImage,
+			queueSidecarTLSMinVersionKey:       "1.2",
+			queueSidecarTLSMaxVersionKey:       "1.3",
+			queueSidecarTLSCipherSuitesKey:     "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+			queueSidecarTLSCurvePreferencesKey: "X25519,CurveP256",
+		},
 	}}
 
 	for _, tt := range configTests {

pkg/queue/sharedmain/main.go

@@ -18,7 +18,6 @@ package sharedmain
 
 import (
 	"context"
-	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -31,6 +30,7 @@ import (
 
 	netproxy "knative.dev/networking/pkg/http/proxy"
 	pkghandler "knative.dev/pkg/network/handlers"
+	knativetls "knative.dev/pkg/tls"
 	"knative.dev/serving/pkg/activator"
 
 	"github.com/kelseyhightower/envconfig"
@@ -265,23 +265,10 @@ func Main(opts ...Option) error {
 		httpServers["profile"] = pprof.Server
 	}
 
-	tlsServers := make(map[string]*http.Server)
+	var tlsServer *http.Server
 	var certWatcher *certificate.CertWatcher
 	var err error
 
-	if tlsEnabled {
-		tlsServers["main"] = mainServer(":"+env.QueueServingTLSPort, mainHandler)
-		// Keep admin server on HTTP even with TLS enabled since it's only accessed locally by kubelet
-
-		certWatcher, err = certificate.NewCertWatcher(certPath, keyPath, 1*time.Minute, logger)
-		if err != nil {
-			logger.Fatal("failed to create certWatcher", zap.Error(err))
-		}
-		defer certWatcher.Stop()
-	}
-
-	logger.Info("Starting queue-proxy")
-
 	errCh := make(chan error)
 	for name, server := range httpServers {
 		go func(name string, s *http.Server) {
@@ -292,20 +279,34 @@ func Main(opts ...Option) error {
 			}
 		}(name, server)
 	}
-	for name, server := range tlsServers {
-		go func(name string, s *http.Server) {
-			logger.Info("Starting tls server ", name, s.Addr)
-			s.TLSConfig = &tls.Config{
-				GetCertificate: certWatcher.GetCertificate,
-				MinVersion:     tls.VersionTLS13,
-			}
+
+	if tlsEnabled {
+		tlsServer = mainServer(":"+env.QueueServingTLSPort, mainHandler)
+		// Keep admin server on HTTP even with TLS enabled since it's only accessed locally by kubelet
+
+		certWatcher, err = certificate.NewCertWatcher(certPath, keyPath, 1*time.Minute, logger)
+		if err != nil {
+			logger.Fatal("failed to create certWatcher", zap.Error(err))
+		}
+		defer certWatcher.Stop()
+
+		tlsCfg, err := knativetls.DefaultConfigFromEnv("QUEUE_PROXY_")
+		if err != nil {
+			logger.Fatalw("Failed to read TLS configuration from environment", zap.Error(err))
+		}
+		go func() {
+			logger.Info("Starting tls server main ", tlsServer.Addr)
+			tlsServer.TLSConfig = tlsCfg
+			tlsServer.TLSConfig.GetCertificate = certWatcher.GetCertificate
 			// Don't forward ErrServerClosed as that indicates we're already shutting down.
-			if err := s.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) {
-				errCh <- fmt.Errorf("%s server failed to serve: %w", name, err)
+			if err := tlsServer.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) {
+				errCh <- fmt.Errorf("main tls server failed to serve: %w", err)
 			}
-		}(name, server)
+		}()
 	}
 
+	logger.Info("Starting queue-proxy")
+
 	// Blocks until we actually receive a TERM signal or one of the servers
 	// exits unexpectedly. We fold both signals together because we only want
 	// to act on the first of those to reach here.
@@ -326,10 +327,10 @@ func Main(opts ...Option) error {
 				logger.Errorw("Failed to shutdown server", zap.String("server", name), zap.Error(err))
 			}
 		}
-		for name, srv := range tlsServers {
-			logger.Info("Shutting down server: ", name)
-			if err := srv.Shutdown(ctx); err != nil {
-				logger.Errorw("Failed to shutdown server", zap.String("server", name), zap.Error(err))
+		if tlsServer != nil {
+			logger.Info("Shutting down server: main tls")
+			if err := tlsServer.Shutdown(ctx); err != nil {
+				logger.Errorw("Failed to shutdown server", zap.String("server", "main tls"), zap.Error(err))
 			}
 		}
 

pkg/reconciler/revision/resources/deploy_test.go

@@ -158,6 +158,18 @@ var (
 		}, {
 			Name:  "ROOT_CA",
 			Value: "",
+		}, {
+			Name:  "QUEUE_PROXY_TLS_MIN_VERSION",
+			Value: "",
+		}, {
+			Name:  "QUEUE_PROXY_TLS_MAX_VERSION",
+			Value: "",
+		}, {
+			Name:  "QUEUE_PROXY_TLS_CIPHER_SUITES",
+			Value: "",
+		}, {
+			Name:  "QUEUE_PROXY_TLS_CURVE_PREFERENCES",
+			Value: "",
 		}, {
 			Name:  "ENABLE_MULTI_CONTAINER_PROBES",
 			Value: "false",

pkg/reconciler/revision/resources/queue.go

@@ -428,6 +428,18 @@ func makeQueueContainer(rev *v1.Revision, cfg *config.Config) (*corev1.Container
 		}, {
 			Name:  "ROOT_CA",
 			Value: cfg.Deployment.QueueSidecarRootCA,
+		}, {
+			Name:  "QUEUE_PROXY_TLS_MIN_VERSION",
+			Value: cfg.Deployment.QueueSidecarTLSMinVersion,
+		}, {
+			Name:  "QUEUE_PROXY_TLS_MAX_VERSION",
+			Value: cfg.Deployment.QueueSidecarTLSMaxVersion,
+		}, {
+			Name:  "QUEUE_PROXY_TLS_CIPHER_SUITES",
+			Value: cfg.Deployment.QueueSidecarTLSCipherSuites,
+		}, {
+			Name:  "QUEUE_PROXY_TLS_CURVE_PREFERENCES",
+			Value: cfg.Deployment.QueueSidecarTLSCurvePreferences,
 		}, {
 			Name:  "ENABLE_MULTI_CONTAINER_PROBES",
 			Value: strconv.FormatBool(multiContainerProbingEnabled),

pkg/reconciler/revision/resources/queue_test.go

@@ -1122,6 +1122,10 @@ var defaultEnv = map[string]string{
 	"SYSTEM_NAMESPACE":                        system.Namespace(),
 	"USER_PORT":                               strconv.Itoa(v1.DefaultUserPort),
 	"ROOT_CA":                                 "",
+	"QUEUE_PROXY_TLS_MIN_VERSION":             "",
+	"QUEUE_PROXY_TLS_MAX_VERSION":             "",
+	"QUEUE_PROXY_TLS_CIPHER_SUITES":           "",
+	"QUEUE_PROXY_TLS_CURVE_PREFERENCES":       "",
 	"ENABLE_MULTI_CONTAINER_PROBES":           "false",
 	"OBSERVABILITY_CONFIG":                    `{"tracing":{},"metrics":{},"runtime":{},"requestMetrics":{}}`,
 }