[Cherry-pick v1.9] fix security related issues

cmd/scheduler/app/options/options.go

@@ -62,6 +62,7 @@ type ServerOption struct {
 	DefaultQueue         string
 	PrintVersion         bool
 	EnableMetrics        bool
+	EnablePprof          bool
 	ListenAddress        string
 	EnablePriorityClass  bool
 	EnableCSIStorage     bool
@@ -138,6 +139,7 @@ func (s *ServerOption) AddFlags(fs *pflag.FlagSet) {
 		"Enable tracking of available storage capacity that CSI drivers provide; it is false by default")
 	fs.BoolVar(&s.EnableHealthz, "enable-healthz", false, "Enable the health check; it is false by default")
 	fs.BoolVar(&s.EnableMetrics, "enable-metrics", false, "Enable the metrics function; it is false by default")
+	fs.BoolVar(&s.EnablePprof, "enable-pprof", false, "Enable the pprof endpoint; it is false by default")
 	fs.StringSliceVar(&s.NodeSelector, "node-selector", nil, "volcano only work with the labeled node, like: --node-selector=volcano.sh/role:train --node-selector=volcano.sh/role:serving")
 	fs.BoolVar(&s.EnableCacheDumper, "cache-dumper", true, "Enable the cache dumper, it's true by default")
 	fs.StringVar(&s.CacheDumpFileDir, "cache-dump-dir", "/tmp", "The target dir where the json file put at when dump cache info to json file")

cmd/scheduler/app/server.go

@@ -20,11 +20,11 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"net/http/pprof"
 	"os"
 	"time"
 
 	"volcano.sh/apis/pkg/apis/helpers"
-
 	"volcano.sh/volcano/cmd/scheduler/app/options"
 	"volcano.sh/volcano/pkg/kube"
 	"volcano.sh/volcano/pkg/scheduler"
@@ -85,11 +85,8 @@ func Run(opt *options.ServerOption) error {
 		panic(err)
 	}
 
-	if opt.EnableMetrics {
-		go func() {
-			http.Handle("/metrics", promHandler())
-			klog.Fatalf("Prometheus Http Server failed %s", http.ListenAndServe(opt.ListenAddress, nil))
-		}()
+	if opt.EnableMetrics || opt.EnablePprof {
+		go startMetricsServer(opt)
 	}
 
 	if opt.EnableHealthz {
@@ -160,3 +157,31 @@ func promHandler() http.Handler {
 	prometheus.DefaultRegisterer.Unregister(collectors.NewGoCollector())
 	return promhttp.InstrumentMetricHandler(prometheus.DefaultRegisterer, promhttp.HandlerFor(prometheus.Gatherers{prometheus.DefaultGatherer, legacyregistry.DefaultGatherer}, promhttp.HandlerOpts{}))
 }
+
+func startMetricsServer(opt *options.ServerOption) {
+	mux := http.NewServeMux()
+
+	if opt.EnableMetrics {
+		mux.Handle("/metrics", promHandler())
+	}
+
+	if opt.EnablePprof {
+		mux.HandleFunc("/debug/pprof/", pprof.Index)
+		mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+		mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+	}
+
+	server := &http.Server{
+		Addr:              opt.ListenAddress,
+		Handler:           mux,
+		ReadHeaderTimeout: helpers.DefaultReadHeaderTimeout,
+		ReadTimeout:       helpers.DefaultReadTimeout,
+		WriteTimeout:      helpers.DefaultWriteTimeout,
+	}
+
+	if err := server.ListenAndServe(); err != nil {
+		klog.Errorf("start metrics/pprof http server failed: %v", err)
+	}
+}

cmd/scheduler/main.go

@@ -21,9 +21,6 @@ import (
 	"runtime"
 	"time"
 
-	// init pprof server
-	_ "net/http/pprof"
-
 	"github.com/spf13/pflag"
 	_ "go.uber.org/automaxprocs"
 

cmd/webhook-manager/app/server.go

@@ -102,8 +102,11 @@ func Run(config *options.Config) error {
 	signal.Notify(stopChannel, syscall.SIGTERM, syscall.SIGINT)
 
 	server := &http.Server{
-		Addr:      config.ListenAddress + ":" + strconv.Itoa(config.Port),
-		TLSConfig: configTLS(config, restConfig),
+		Addr:              config.ListenAddress + ":" + strconv.Itoa(config.Port),
+		TLSConfig:         configTLS(config, restConfig),
+		ReadHeaderTimeout: helpers.DefaultReadHeaderTimeout,
+		ReadTimeout:       helpers.DefaultReadTimeout,
+		WriteTimeout:      helpers.DefaultWriteTimeout,
 	}
 	go func() {
 		err = server.ListenAndServeTLS("", "")

installer/helm/chart/volcano/templates/scheduler.yaml

@@ -155,6 +155,9 @@ spec:
             - --scheduler-conf=/volcano.scheduler/{{base .Values.basic.scheduler_config_file}}
             - --enable-healthz=true
             - --enable-metrics=true
+            {{- if .Values.custom.scheduler_pprof_enable }}
+            - --enable-pprof=true
+            {{- end }}
             - --leader-elect={{ .Values.custom.leader_elect_enable }}
             {{- if .Values.custom.leader_elect_enable }}
             - --lock-object-namespace={{ .Release.Namespace }}

installer/helm/chart/volcano/values.yaml

@@ -17,6 +17,7 @@ custom:
   controller_replicas: 1
   scheduler_enable: true
   scheduler_replicas: 1
+  scheduler_pprof_enable: false
   leader_elect_enable: false
   enabled_admissions: "/jobs/mutate,/jobs/validate,/podgroups/mutate,/pods/validate,/pods/mutate,/queues/mutate,/queues/validate"
 

pkg/scheduler/metrics/source/metrics_client_elasticsearch.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/elastic/go-elasticsearch/v7"
+	"k8s.io/klog/v2"
 )
 
 const (
@@ -65,6 +66,10 @@ func NewElasticsearchMetricsClient(conf map[string]string) (*ElasticsearchMetric
 	}
 	var err error
 	insecureSkipVerify := conf["tls.insecureSkipVerify"] == "true"
+	if insecureSkipVerify {
+		klog.Warningf("WARNING: TLS certificate verification is disabled which is insecure. This should not be used in production environments")
+	}
+
 	e.es, err = elasticsearch.NewClient(elasticsearch.Config{
 		Addresses: []string{address},
 		Username:  conf["elasticsearch.username"],

pkg/scheduler/metrics/source/metrics_client_prometheus.go

@@ -61,6 +61,10 @@ func (p *PrometheusMetricsClient) NodeMetricsAvg(ctx context.Context, nodeName s
 	var client api.Client
 	var err error
 	insecureSkipVerify := p.conf["tls.insecureSkipVerify"] == "true"
+	if insecureSkipVerify {
+		klog.Warningf("WARNING: TLS certificate verification is disabled which is insecure. This should not be used in production environments")
+	}
+
 	tr := &http.Transport{
 		TLSClientConfig: &tls.Config{
 			InsecureSkipVerify: insecureSkipVerify,

pkg/util/socket.go

@@ -31,6 +31,8 @@ import (
 	"golang.org/x/sys/unix"
 	"k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/klog/v2"
+
+	"volcano.sh/apis/pkg/apis/helpers"
 )
 
 const (
@@ -201,7 +203,10 @@ func listenUnix(componentName string, socketDir string) (net.Listener, error) {
 // serveOnListener starts the server using given listener, loops forever.
 func serveOnListener(l net.Listener, m *http.ServeMux) error {
 	server := http.Server{
-		Handler: m,
+		Handler:           m,
+		ReadHeaderTimeout: helpers.DefaultReadHeaderTimeout,
+		ReadTimeout:       helpers.DefaultReadTimeout,
+		WriteTimeout:      helpers.DefaultWriteTimeout,
 	}
 	return server.Serve(l)
 }