[Cherry-pick v1.11] fix security related issues

cmd/agent/app/app.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/controller-manager/pkg/clientbuilder"
 	"k8s.io/klog/v2"
 
+	"volcano.sh/apis/pkg/apis/helpers"
 	"volcano.sh/volcano/cmd/agent/app/options"
 	"volcano.sh/volcano/pkg/agent/healthcheck"
 	"volcano.sh/volcano/pkg/agent/utils"
@@ -81,8 +82,11 @@ func RunServer(checker healthcheck.HealthChecker, address string, port int) {
 		mux.HandleFunc("/healthz", checker.HealthCheck)
 		mux.Handle("/metrics", promhttp.Handler())
 		s := &http.Server{
-			Addr:    net.JoinHostPort(address, strconv.Itoa(port)),
-			Handler: mux,
+			Addr:              net.JoinHostPort(address, strconv.Itoa(port)),
+			Handler:           mux,
+			ReadHeaderTimeout: helpers.DefaultReadHeaderTimeout,
+			ReadTimeout:       helpers.DefaultReadTimeout,
+			WriteTimeout:      helpers.DefaultWriteTimeout,
 		}
 		if err := s.ListenAndServe(); err != nil {
 			klog.Fatalf("failed to start health check server: %v", err)

cmd/controller-manager/app/server.go

@@ -58,8 +58,17 @@ func Run(opt *options.ServerOption) error {
 
 	if opt.EnableMetrics {
 		go func() {
-			http.Handle("/metrics", commonutil.PromHandler())
-			klog.Fatalf("Prometheus Http Server failed %s", http.ListenAndServe(opt.ListenAddress, nil))
+			mux := http.NewServeMux()
+			mux.Handle("/metrics", commonutil.PromHandler())
+
+			server := &http.Server{
+				Addr:              opt.ListenAddress,
+				Handler:           mux,
+				ReadHeaderTimeout: helpers.DefaultReadHeaderTimeout,
+				ReadTimeout:       helpers.DefaultReadTimeout,
+				WriteTimeout:      helpers.DefaultWriteTimeout,
+			}
+			klog.Fatalf("Prometheus Http Server failed: %s", server.ListenAndServe())
 		}()
 	}
 

cmd/scheduler/app/options/options.go

@@ -67,6 +67,7 @@ type ServerOption struct {
 	DefaultQueue        string
 	PrintVersion        bool
 	EnableMetrics       bool
+	EnablePprof         bool
 	ListenAddress       string
 	EnablePriorityClass bool
 	EnableCSIStorage    bool
@@ -141,6 +142,7 @@ func (s *ServerOption) AddFlags(fs *pflag.FlagSet) {
 		"Enable tracking of available storage capacity that CSI drivers provide; it is false by default")
 	fs.BoolVar(&s.EnableHealthz, "enable-healthz", false, "Enable the health check; it is false by default")
 	fs.BoolVar(&s.EnableMetrics, "enable-metrics", false, "Enable the metrics function; it is false by default")
+	fs.BoolVar(&s.EnablePprof, "enable-pprof", false, "Enable the pprof endpoint; it is false by default")
 	fs.StringSliceVar(&s.NodeSelector, "node-selector", nil, "volcano only work with the labeled node, like: --node-selector=volcano.sh/role:train --node-selector=volcano.sh/role:serving")
 	fs.BoolVar(&s.EnableCacheDumper, "cache-dumper", true, "Enable the cache dumper, it's true by default")
 	fs.StringVar(&s.CacheDumpFileDir, "cache-dump-dir", "/tmp", "The target dir where the json file put at when dump cache info to json file")

cmd/scheduler/app/server.go

@@ -20,10 +20,10 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"net/http/pprof"
 	"os"
 
 	"volcano.sh/apis/pkg/apis/helpers"
-
 	"volcano.sh/volcano/cmd/scheduler/app/options"
 	"volcano.sh/volcano/pkg/kube"
 	"volcano.sh/volcano/pkg/scheduler"
@@ -69,11 +69,8 @@ func Run(opt *options.ServerOption) error {
 		panic(err)
 	}
 
-	if opt.EnableMetrics {
-		go func() {
-			http.Handle("/metrics", commonutil.PromHandler())
-			klog.Fatalf("Prometheus Http Server failed %s", http.ListenAndServe(opt.ListenAddress, nil))
-		}()
+	if opt.EnableMetrics || opt.EnablePprof {
+		go startMetricsServer(opt)
 	}
 
 	if opt.EnableHealthz {
@@ -142,3 +139,31 @@ func Run(opt *options.ServerOption) error {
 	})
 	return fmt.Errorf("lost lease")
 }
+
+func startMetricsServer(opt *options.ServerOption) {
+	mux := http.NewServeMux()
+
+	if opt.EnableMetrics {
+		mux.Handle("/metrics", commonutil.PromHandler())
+	}
+
+	if opt.EnablePprof {
+		mux.HandleFunc("/debug/pprof/", pprof.Index)
+		mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+		mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+	}
+
+	server := &http.Server{
+		Addr:              opt.ListenAddress,
+		Handler:           mux,
+		ReadHeaderTimeout: helpers.DefaultReadHeaderTimeout,
+		ReadTimeout:       helpers.DefaultReadTimeout,
+		WriteTimeout:      helpers.DefaultWriteTimeout,
+	}
+
+	if err := server.ListenAndServe(); err != nil {
+		klog.Errorf("start metrics/pprof http server failed: %v", err)
+	}
+}

cmd/scheduler/main.go

@@ -29,9 +29,6 @@ import (
 	componentbaseoptions "k8s.io/component-base/config/options"
 	"k8s.io/klog/v2"
 
-	// init pprof server
-	_ "net/http/pprof"
-
 	"volcano.sh/volcano/cmd/scheduler/app"
 	"volcano.sh/volcano/cmd/scheduler/app/options"
 	commonutil "volcano.sh/volcano/pkg/util"

cmd/webhook-manager/app/server.go

@@ -108,8 +108,11 @@ func Run(config *options.Config) error {
 	}
 
 	server := &http.Server{
-		Addr:      config.ListenAddress + ":" + strconv.Itoa(config.Port),
-		TLSConfig: configTLS(config, restConfig),
+		Addr:              config.ListenAddress + ":" + strconv.Itoa(config.Port),
+		TLSConfig:         configTLS(config, restConfig),
+		ReadHeaderTimeout: helpers.DefaultReadHeaderTimeout,
+		ReadTimeout:       helpers.DefaultReadTimeout,
+		WriteTimeout:      helpers.DefaultWriteTimeout,
 	}
 	go func() {
 		err = server.ListenAndServeTLS("", "")

go.mod

@@ -46,7 +46,7 @@ require (
 	sigs.k8s.io/controller-runtime v0.13.0
 	sigs.k8s.io/yaml v1.4.0
 	stathat.com/c/consistent v1.0.0
-	volcano.sh/apis v1.11.1
+	volcano.sh/apis v1.11.2
 )
 
 require (

go.sum

@@ -510,5 +510,5 @@ sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
 sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
 stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c=
 stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0=
-volcano.sh/apis v1.11.1 h1:BuewlHccLIJVJmVcBF32KewXJmtwpCjx4d7fxVxG900=
-volcano.sh/apis v1.11.1/go.mod h1:FOdmG++9+8lgENJ9XXDh+O3Jcb9YVRnlMSpgIh3NSVI=
+volcano.sh/apis v1.11.2 h1:Vz8NzP0af8vyxRccrEUt6/FikD5eeEOnCZRolVzZvK8=
+volcano.sh/apis v1.11.2/go.mod h1:FOdmG++9+8lgENJ9XXDh+O3Jcb9YVRnlMSpgIh3NSVI=

installer/helm/chart/volcano/templates/scheduler.yaml

@@ -185,6 +185,9 @@ spec:
             {{- if .Values.custom.scheduler_metrics_enable }}
             - --enable-metrics=true
             {{- end }}
+            {{- if .Values.custom.scheduler_pprof_enable }}
+            - --enable-pprof=true
+            {{- end }}
             - --leader-elect={{ .Values.custom.leader_elect_enable }}
             {{- if .Values.custom.leader_elect_enable }}
             - --leader-elect-resource-namespace={{ .Release.Namespace }}

installer/helm/chart/volcano/values.yaml

@@ -21,6 +21,7 @@ custom:
   scheduler_enable: true
   scheduler_replicas: 1
   scheduler_metrics_enable: true
+  scheduler_pprof_enable: false
   scheduler_name: ~
   leader_elect_enable: false
   controller_kube_api_qps: 50

pkg/scheduler/metrics/source/metrics_client_elasticsearch.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/elastic/go-elasticsearch/v7"
+	"k8s.io/klog/v2"
 )
 
 const (
@@ -65,6 +66,10 @@ func NewElasticsearchMetricsClient(conf map[string]string) (*ElasticsearchMetric
 	}
 	var err error
 	insecureSkipVerify := conf["tls.insecureSkipVerify"] == "true"
+	if insecureSkipVerify {
+		klog.Warningf("WARNING: TLS certificate verification is disabled which is insecure. This should not be used in production environments")
+	}
+
 	e.es, err = elasticsearch.NewClient(elasticsearch.Config{
 		Addresses: []string{address},
 		Username:  conf["elasticsearch.username"],

pkg/scheduler/metrics/source/metrics_client_prometheus.go

@@ -61,6 +61,10 @@ func (p *PrometheusMetricsClient) NodeMetricsAvg(ctx context.Context, nodeName s
 	var client api.Client
 	var err error
 	insecureSkipVerify := p.conf["tls.insecureSkipVerify"] == "true"
+	if insecureSkipVerify {
+		klog.Warningf("WARNING: TLS certificate verification is disabled which is insecure. This should not be used in production environments")
+	}
+
 	tr := &http.Transport{
 		TLSClientConfig: &tls.Config{
 			InsecureSkipVerify: insecureSkipVerify,

pkg/util/socket.go

@@ -31,6 +31,8 @@ import (
 	"golang.org/x/sys/unix"
 	"k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/klog/v2"
+
+	"volcano.sh/apis/pkg/apis/helpers"
 )
 
 const (
@@ -201,7 +203,10 @@ func listenUnix(componentName string, socketDir string) (net.Listener, error) {
 // serveOnListener starts the server using given listener, loops forever.
 func serveOnListener(l net.Listener, m *http.ServeMux) error {
 	server := http.Server{
-		Handler: m,
+		Handler:           m,
+		ReadHeaderTimeout: helpers.DefaultReadHeaderTimeout,
+		ReadTimeout:       helpers.DefaultReadTimeout,
+		WriteTimeout:      helpers.DefaultWriteTimeout,
 	}
 	return server.Serve(l)
 }