Merge pull request #4250 from JesseStutler/network-topology-cherry-pick

[Cherry-pick network-topology] Add security context configuration

Author: volcano-sh-bot

installer/build/volcano-agent/install.sh

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -e
+
 VOLCANO_AGENT_LOG_DIR="/var/log/volcano/agent"
 VOLCANO_AGENT_LOG_PATH="${VOLCANO_AGENT_LOG_DIR}/volcano-agent.log"
 NETWORK_QOS_LOG_PATH="${VOLCANO_AGENT_LOG_DIR}/network-qos.log"
@@ -53,5 +55,9 @@ touch ${VOLCANO_AGENT_LOG_PATH}
 touch ${NETWORK_QOS_LOG_PATH}
 touch ${NETWORK_QOS_TOOLS_LOG_PATH}
 
+chmod 750 ${VOLCANO_AGENT_LOG_DIR}
+chown -R 1000:1000 ${VOLCANO_AGENT_LOG_DIR}
+chmod 640 ${VOLCANO_AGENT_LOG_DIR}/*.log
+
 set_memory_qos_enabled
 set_sched_prio_load_balance_enabled

installer/dockerfile/agent/Dockerfile

@@ -29,10 +29,14 @@ RUN yum install -y cpio && \
     rpm2cpio $(ls | grep oncn-bwm) | cpio -div
 
 FROM alpine:latest
-RUN apk add sudo
+RUN apk add sudo libcap
 COPY --from=builder /go/src/volcano.sh/volcano/_output/bin/vc-agent /vc-agent
 COPY --from=builder /go/src/volcano.sh/volcano/_output/bin/network-qos \
                     /go/src/volcano.sh/volcano/installer/build/volcano-agent/install.sh /usr/local/bin/
 COPY --from=repo /usr/share/bwmcli/bwm_tc.o /usr/local/bin/
-RUN chmod +x /usr/local/bin/install.sh
+RUN adduser -u 1000 -D appuser
+RUN chmod +x /usr/local/bin/install.sh \
+    && setcap "cap_dac_override=eip" /vc-agent \
+    && setcap "cap_dac_override=eip" /usr/local/bin/network-qos \
+    && echo -e '%appuser ALL=(root) NOPASSWD: /bin/cp -f /usr/local/bin/network-qos /opt/cni/bin\n%appuser ALL=(root) NOPASSWD: /bin/cp -f /usr/local/bin/bwm_tc.o /usr/share/bwmcli' >> /etc/sudoers
 

installer/helm/chart/volcano/templates/admission-init.yaml

@@ -2,6 +2,7 @@
 {{ $admission_affinity := or .Values.custom.admission_affinity .Values.custom.default_affinity }}
 {{ $admission_tolerations := or .Values.custom.admission_tolerations .Values.custom.default_tolerations }}
 {{ $admission_sc := or .Values.custom.admission_sc .Values.custom.default_sc }}
+{{ $admission_init_csc := or .Values.custom.admission_init_csc .Values.custom.default_csc }}
 {{ $admission_ns := or .Values.custom.admission_ns .Values.custom.default_ns }}
 
 apiVersion: v1
@@ -110,8 +111,8 @@ spec:
           imagePullPolicy: {{ .Values.basic.image_pull_policy }}
           command: ["./gen-admission-secret.sh", "--service", "{{ .Release.Name }}-admission-service", "--namespace",
                     "{{ .Release.Namespace }}", "--secret", "{{.Values.basic.admission_secret_name}}"]
-          {{- if .Values.custom.admission_default_csc }}
+          {{- if $admission_init_csc }}
           securityContext:
-            {{- toYaml .Values.custom.admission_default_csc | nindent 12 }}
+            {{- toYaml $admission_init_csc | nindent 12 }}
           {{- end }}
 {{- end }}

installer/helm/chart/volcano/templates/admission.yaml

@@ -2,6 +2,7 @@
 {{ $admission_affinity := or .Values.custom.admission_affinity .Values.custom.default_affinity }}
 {{ $admission_tolerations := or .Values.custom.admission_tolerations .Values.custom.default_tolerations }}
 {{ $admission_sc := or .Values.custom.admission_sc .Values.custom.default_sc }}
+{{ $admission_main_csc := or .Values.custom.admission_main_csc .Values.custom.default_csc }}
 {{ $admission_ns := or .Values.custom.admission_ns .Values.custom.default_ns }}
 {{ $scheduler_name := .Values.custom.scheduler_name }}
 apiVersion: v1
@@ -155,9 +156,9 @@ spec:
               readOnly: true
             - mountPath: /admission.local.config/configmap
               name: admission-config
-          {{- if .Values.custom.admission_default_csc }}
+          {{- if $admission_main_csc }}
           securityContext:
-            {{- toYaml .Values.custom.admission_default_csc | nindent 12 }}
+            {{- toYaml $admission_main_csc | nindent 12 }}
           {{- end }}
       volumes:
         - name: admission-certs

installer/helm/chart/volcano/templates/agent.yaml

@@ -2,6 +2,8 @@
 {{ $agent_affinity := or .Values.custom.agent_affinity .Values.custom.default_affinity }}
 {{ $agent_tolerations := or .Values.custom.agent_tolerations .Values.custom.default_tolerations }}
 {{ $agent_sc := or .Values.custom.agent_sc .Values.custom.default_sc }}
+{{ $agent_main_csc := or .Values.custom.agent_main_csc .Values.custom.default_csc }}
+{{ $agent_init_csc := or .Values.custom.agent_init_csc .Values.custom.default_csc }}
 {{ $agent_ns := or .Values.custom.agent_ns .Values.custom.default_ns }}
 apiVersion: apps/v1
 kind: DaemonSet
@@ -86,6 +88,10 @@ spec:
       initContainers:
         - name: volcano-agent-init
           image: {{ .Values.basic.image_registry }}/{{.Values.basic.agent_image_name}}:{{.Values.basic.image_tag_version}}
+          {{- if $agent_init_csc }}
+          securityContext:
+            {{- toYaml $agent_init_csc | nindent 12 }}
+          {{- end }}
           command:
             - /bin/sh
             - '-c'
@@ -111,6 +117,10 @@ spec:
       containers:
         - name: volcano-agent
           image: {{ .Values.basic.image_registry }}/{{.Values.basic.agent_image_name}}:{{.Values.basic.image_tag_version}}
+          {{- if $agent_main_csc }}
+          securityContext:
+            {{- toYaml $agent_main_csc | nindent 12 }}
+          {{- end }}
           command:
             - /bin/sh
             - '-c'

installer/helm/chart/volcano/templates/controllers.yaml

@@ -2,6 +2,7 @@
 {{ $controller_affinity := or .Values.custom.controller_affinity .Values.custom.default_affinity }}
 {{ $controller_tolerations := or .Values.custom.controller_tolerations .Values.custom.default_tolerations }}
 {{ $controller_sc := or .Values.custom.controller_sc .Values.custom.default_sc }}
+{{ $controller_main_csc := or .Values.custom.controller_main_csc .Values.custom.default_csc }}
 {{ $controller_ns := or .Values.custom.controller_ns .Values.custom.default_ns }}
 {{ $scheduler_name := .Values.custom.scheduler_name }}
 apiVersion: v1
@@ -184,9 +185,9 @@ spec:
             - -v={{.Values.custom.controller_log_level}}
             - 2>&1
           imagePullPolicy: {{ .Values.basic.image_pull_policy }}
-            {{- if .Values.custom.controller_default_csc }}
+            {{- if $controller_main_csc }}
           securityContext:
-              {{- toYaml .Values.custom.controller_default_csc | nindent 14 }}
+              {{- toYaml $controller_main_csc | nindent 14 }}
             {{- end }}
 ---
 apiVersion: v1

installer/helm/chart/volcano/templates/scheduler.yaml

@@ -2,6 +2,7 @@
 {{ $scheduler_affinity := or .Values.custom.scheduler_affinity .Values.custom.default_affinity }}
 {{ $scheduler_tolerations := or .Values.custom.scheduler_tolerations .Values.custom.default_tolerations }}
 {{ $scheduler_sc := or .Values.custom.scheduler_sc .Values.custom.default_sc }}
+{{ $scheduler_main_csc := or .Values.custom.scheduler_main_csc .Values.custom.default_csc }}
 {{ $scheduler_ns := or .Values.custom.scheduler_ns .Values.custom.default_ns }}
 {{ $scheduler_name := .Values.custom.scheduler_name }}
 apiVersion: v1
@@ -218,9 +219,9 @@ spec:
               mountPath: /volcano.scheduler
             - name: klog-sock
               mountPath: /tmp/klog-socks
-          {{- if .Values.custom.scheduler_default_csc }}
+          {{- if $scheduler_main_csc }}
           securityContext:
-            {{- toYaml .Values.custom.scheduler_default_csc | nindent 12 }}
+            {{- toYaml $scheduler_main_csc | nindent 12 }}
           {{- end }}
       volumes:
         - name: scheduler-config

installer/helm/chart/volcano/values.yaml

@@ -98,7 +98,11 @@ custom:
 #  default_sc:
 #    runAsUser: 3000
 #    runAsGroup: 3000
-  default_sc: ~
+  default_sc:
+    seccompProfile:
+      type: RuntimeDefault
+    seLinuxOptions:
+      level: "s0:c123,c456"
   scheduler_sc: ~
   admission_sc: ~
   controller_sc: ~
@@ -174,12 +178,33 @@ custom:
 # Specify container security context for admission
 # For example:
 #
-# admission_default_csc:
+# default_csc:
 #   allowPrivilegeEscalation: false
 #   runAsUser: 2000
-  admission_default_csc: ~
-  scheduler_default_csc: ~
-  controller_default_csc: ~
+  default_csc:
+    runAsNonRoot: true
+    runAsUser: 1000
+    # Disable all capabilities by default, components can add capabilities as needed
+    capabilities:
+      add: ["DAC_OVERRIDE"]
+      drop: [ "ALL" ]
+    allowPrivilegeEscalation: false
+  admission_main_csc: ~
+  admission_init_csc: ~
+  scheduler_main_csc: ~
+  controller_main_csc: ~
+  agent_main_csc:
+    runAsNonRoot: true
+    runAsUser: 1000
+    capabilities:
+      add: ["DAC_OVERRIDE", "SETUID", "SETGID", "SETFCAP", "BPF"]
+      drop: [ "ALL" ]
+  agent_init_csc:
+    runAsUser: 0
+    capabilities:
+      add: ["CHOWN", "DAC_OVERRIDE", "FOWNER"]
+      drop: [ "ALL" ]
+    allowPrivilegeEscalation: false
 
   # Specify agent cni config path.
   agent_cni_config_path: /etc/cni/net.d/cni.conflist

installer/volcano-agent-development.yaml

@@ -82,6 +82,11 @@ spec:
         - effect: NoSchedule
           key: volcano.sh/offline-job-evicting
           operator: Exists
+      securityContext:
+        seLinuxOptions:
+          level: s0:c123,c456
+        seccompProfile:
+          type: RuntimeDefault
       serviceAccountName: volcano-agent
       hostNetwork: true
       priorityClassName: system-node-critical
@@ -127,6 +132,16 @@ spec:
       initContainers:
         - name: volcano-agent-init
           image: docker.io/volcanosh/vc-agent:v1.11.0-network-topology-preview.3
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              add:
+              - CHOWN
+              - DAC_OVERRIDE
+              - FOWNER
+              drop:
+              - ALL
+            runAsUser: 0
           command:
             - /bin/sh
             - '-c'
@@ -148,6 +163,18 @@ spec:
       containers:
         - name: volcano-agent
           image: docker.io/volcanosh/vc-agent:v1.11.0-network-topology-preview.3
+          securityContext:
+            capabilities:
+              add:
+              - DAC_OVERRIDE
+              - SETUID
+              - SETGID
+              - SETFCAP
+              - BPF
+              drop:
+              - ALL
+            runAsNonRoot: true
+            runAsUser: 1000
           command:
             - /bin/sh
             - '-c'

installer/volcano-development.yaml

@@ -131,6 +131,11 @@ spec:
       labels:
         app: volcano-admission
     spec:
+      securityContext:
+        seLinuxOptions:
+          level: s0:c123,c456
+        seccompProfile:
+          type: RuntimeDefault
       serviceAccount: volcano-admission
       priorityClassName: system-cluster-critical
       containers:
@@ -156,6 +161,15 @@ spec:
               readOnly: true
             - mountPath: /admission.local.config/configmap
               name: admission-config
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              add:
+              - DAC_OVERRIDE
+              drop:
+              - ALL
+            runAsNonRoot: true
+            runAsUser: 1000
       volumes:
         - name: admission-certs
           secret:
@@ -226,6 +240,11 @@ spec:
   backoffLimit: 3
   template:
     spec:
+      securityContext:
+        seLinuxOptions:
+          level: s0:c123,c456
+        seccompProfile:
+          type: RuntimeDefault
       serviceAccountName: volcano-admission-init
       priorityClassName: system-cluster-critical
       restartPolicy: Never
@@ -235,6 +254,15 @@ spec:
           imagePullPolicy: Always
           command: ["./gen-admission-secret.sh", "--service", "volcano-admission-service", "--namespace",
                     "volcano-system", "--secret", "volcano-admission-secret"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              add:
+              - DAC_OVERRIDE
+              drop:
+              - ALL
+            runAsNonRoot: true
+            runAsUser: 1000
 ---
 # Source: volcano/templates/batch_v1alpha1_job.yaml
 apiVersion: apiextensions.k8s.io/v1
@@ -4465,6 +4493,11 @@ spec:
       labels:
         app: volcano-controller
     spec:
+      securityContext:
+        seLinuxOptions:
+          level: s0:c123,c456
+        seccompProfile:
+          type: RuntimeDefault
       serviceAccount: volcano-controllers
       priorityClassName: system-cluster-critical
       containers:
@@ -4483,6 +4516,15 @@ spec:
             - -v=4
             - 2>&1
           imagePullPolicy: Always
+          securityContext:
+              allowPrivilegeEscalation: false
+              capabilities:
+                add:
+                - DAC_OVERRIDE
+                drop:
+                - ALL
+              runAsNonRoot: true
+              runAsUser: 1000
 ---
 # Source: volcano/templates/scheduler.yaml
 apiVersion: v1
@@ -4645,6 +4687,11 @@ spec:
       labels:
         app: volcano-scheduler
     spec:
+      securityContext:
+        seLinuxOptions:
+          level: s0:c123,c456
+        seccompProfile:
+          type: RuntimeDefault
       serviceAccount: volcano-scheduler
       priorityClassName: system-cluster-critical
       containers:
@@ -4671,6 +4718,15 @@ spec:
               mountPath: /volcano.scheduler
             - name: klog-sock
               mountPath: /tmp/klog-socks
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              add:
+              - DAC_OVERRIDE
+              drop:
+              - ALL
+            runAsNonRoot: true
+            runAsUser: 1000
       volumes:
         - name: scheduler-config
           configMap: