Cherry-pick some bug fix into release-1.9 and bump image to v1.9.1

.github/workflows/code_verify.yaml

@@ -26,7 +26,7 @@ jobs:
           fetch-depth: 0
           path: ./src/github.com/${{ github.repository }}
 
-      - uses: actions/cache@v2
+      - uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -35,7 +35,7 @@ jobs:
         run: |
           make lint
           make verify
-          make TAG=v1.9.0 generate-yaml
+          make TAG=v1.9.1 generate-yaml
           make verify-generated-yaml
           make unit-test
         working-directory: ./src/github.com/${{ github.repository }}

.github/workflows/e2e_parallel_jobs.yaml

@@ -24,7 +24,7 @@ jobs:
           tar -xf musl-1.2.1.tar.gz && cd musl-1.2.1
           ./configure
           make && sudo make install
-      - uses: actions/cache@v2
+      - uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}

.github/workflows/e2e_scheduling_actions.yaml

@@ -24,7 +24,7 @@ jobs:
           tar -xf musl-1.2.1.tar.gz && cd musl-1.2.1
           ./configure
           make && sudo make install
-      - uses: actions/cache@v2
+      - uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}

.github/workflows/e2e_scheduling_basic.yaml

@@ -24,7 +24,7 @@ jobs:
           tar -xf musl-1.2.1.tar.gz && cd musl-1.2.1
           ./configure
           make && sudo make install
-      - uses: actions/cache@v2
+      - uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}

.github/workflows/e2e_sequence.yaml

@@ -24,7 +24,7 @@ jobs:
           tar -xf musl-1.2.1.tar.gz && cd musl-1.2.1
           ./configure
           make && sudo make install
-      - uses: actions/cache@v2
+      - uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}

.github/workflows/e2e_spark.yaml

@@ -25,7 +25,7 @@ jobs:
         ref: branch-3.4
         path: ${{ github.workspace }}/spark
     - name: Cache Scala, SBT and Maven
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: |
           build/apache-maven-*
@@ -36,7 +36,7 @@ jobs:
         restore-keys: |
           build-
     - name: Cache Coursier local repository
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ~/.cache/coursier
         key: k8s-integration-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
@@ -61,8 +61,8 @@ jobs:
     - name: Build lastest volcano images
       run: |
         eval $(minikube docker-env)
-        make TAG=v1.9.0 update-development-yaml
-        make TAG=v1.9.0 images
+        make TAG=v1.9.1 update-development-yaml
+        make TAG=v1.9.1 images
         docker images | grep volcano
         cat ./installer/volcano-development.yaml  | grep "image: "
         sed 's/imagePullPolicy: Always/imagePullPolicy: IfNotPresent/' ./installer/volcano-development.yaml | minikube kubectl -- replace --force -f -

.github/workflows/e2e_vcctl.yaml

@@ -24,7 +24,7 @@ jobs:
           tar -xf musl-1.2.1.tar.gz && cd musl-1.2.1
           ./configure
           make && sudo make install
-      - uses: actions/cache@v2
+      - uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}

.github/workflows/release.yaml

@@ -44,7 +44,7 @@ jobs:
           echo "TAG=latest" >> $GITHUB_ENV
           echo "RELEASE_VER=latest" >> $GITHUB_ENV
 
-      - uses: actions/cache@v2
+      - uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}

Makefile

@@ -203,8 +203,8 @@ CONTROLLER_GEN=$(shell which controller-gen)
 endif
 
 update-development-yaml:
-	make generate-yaml TAG=v1.9.0 RELEASE_DIR=installer
-	mv installer/volcano-v1.9.0.yaml installer/volcano-development.yaml
+	make generate-yaml TAG=v1.9.1 RELEASE_DIR=installer
+	mv installer/volcano-v1.9.1.yaml installer/volcano-development.yaml
 
 mod-download-go:
 	@-GOFLAGS="-mod=readonly" find -name go.mod -execdir go mod download \;

Makefile.def

@@ -1,9 +1,9 @@
 
 # If tag not explicitly set in users default to the git sha.
-TAG ?= v1.9.0
+TAG ?= v1.9.1
 GitSHA=`git rev-parse HEAD`
 Date=`date "+%Y-%m-%d %H:%M:%S"`
-RELEASE_VER=v1.9.0
+RELEASE_VER=v1.9.1
 LD_FLAGS=" \
     -X '${REPO_PATH}/pkg/version.GitSHA=${GitSHA}' \
     -X '${REPO_PATH}/pkg/version.Built=${Date}'   \

README.md

@@ -140,8 +140,8 @@ This way is only available for x86_64 temporarily.
 If you want to get prometheus and grafana volcano dashboard after volcano installed, try following commands:
 
 ```bash
-make TAG=v1.9.0 generate-yaml
-kubectl create -f _output/release/volcano-monitoring-v1.9.0.yaml
+make TAG=v1.9.1 generate-yaml
+kubectl create -f _output/release/volcano-monitoring-v1.9.1.yaml
 ```
 
 ## Kubernetes compatibility

go.mod

@@ -40,7 +40,7 @@ require (
 	sigs.k8s.io/controller-runtime v0.13.0
 	sigs.k8s.io/yaml v1.3.0
 	stathat.com/c/consistent v1.0.0
-	volcano.sh/apis v1.9.0
+	volcano.sh/apis v1.9.1
 )
 
 require (

go.sum

@@ -434,5 +434,5 @@ sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
 sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
 stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c=
 stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0=
-volcano.sh/apis v1.9.0 h1:e+9yEbQOi6HvgaayAxYULT6n+59mkYvmqjKhp9Z06sY=
-volcano.sh/apis v1.9.0/go.mod h1:yXNfsZRzAOq6EUyPJYFrlMorh1XsYQGonGWyr4IiznM=
+volcano.sh/apis v1.9.1 h1:EdRTk8KF1ARsnjg7RAlaW6EfktXJhDyfn4GzOMOWQnE=
+volcano.sh/apis v1.9.1/go.mod h1:yXNfsZRzAOq6EUyPJYFrlMorh1XsYQGonGWyr4IiznM=

hack/check-generated-yaml.sh

@@ -21,12 +21,12 @@ set -o pipefail
 VK_ROOT=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/..
 export RELEASE_FOLDER=${VK_ROOT}/${RELEASE_DIR}
 
-if ! diff ${VK_ROOT}/installer/volcano-development.yaml ${RELEASE_FOLDER}/volcano-v1.9.0.yaml ; then
+if ! diff ${VK_ROOT}/installer/volcano-development.yaml ${RELEASE_FOLDER}/volcano-v1.9.1.yaml ; then
 	{
 		echo
 		echo "The Generated yaml is different from the one in installer/volcano-development.yaml"
-		echo "please run 'make generate-yaml TAG=v1.9.0 RELEASE_DIR=installer \
-		&& mv ${VK_ROOT}/installer/volcano-v1.9.0.yaml ${VK_ROOT}/installer/volcano-development.yaml' to update"
+		echo "please run 'make generate-yaml TAG=v1.9.1 RELEASE_DIR=installer \
+		&& mv ${VK_ROOT}/installer/volcano-v1.9.1.yaml ${VK_ROOT}/installer/volcano-development.yaml' to update"
 		echo
 	} >&2
 	false

installer/helm/chart/volcano/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 description: A Helm chart for Volcano
 name: volcano
-version: "1.9.0"
+version: "1.9.1"
 appVersion: "0.1"
 icon: https://raw.githubusercontent.com/volcano-sh/charts/master/docs/images/volcano-logo.png
 home: https://volcano.sh

installer/helm/chart/volcano/values.yaml

@@ -7,7 +7,7 @@ basic:
   scheduler_config_file: "config/volcano-scheduler.conf"
   image_pull_secret: ""
   image_pull_policy: "Always"
-  image_tag_version: "v1.9.0"
+  image_tag_version: "v1.9.1"
   admission_port: 8443
 custom:
   metrics_enable: false

installer/volcano-development.yaml

@@ -150,7 +150,7 @@ spec:
             - --port=8443
             - -v=4
             - 2>&1
-          image: volcanosh/vc-webhook-manager:v1.9.0
+          image: volcanosh/vc-webhook-manager:v1.9.1
           imagePullPolicy: Always
           name: admission
           volumeMounts:
@@ -185,7 +185,7 @@ spec:
       restartPolicy: Never
       containers:
         - name: main
-          image: volcanosh/vc-webhook-manager:v1.9.0
+          image: volcanosh/vc-webhook-manager:v1.9.1
           imagePullPolicy: Always
           command: ["./gen-admission-secret.sh", "--service", "volcano-admission-service", "--namespace",
                     "volcano-system", "--secret", "volcano-admission-secret"]
@@ -4033,7 +4033,7 @@ spec:
       priorityClassName: system-cluster-critical
       containers:
           - name: volcano-controllers
-            image: volcanosh/vc-controller-manager:v1.9.0
+            image: volcanosh/vc-controller-manager:v1.9.1
             args:
               - --logtostderr
               - --enable-healthz=true
@@ -4201,7 +4201,7 @@ spec:
       priorityClassName: system-cluster-critical
       containers:
         - name: volcano-scheduler
-          image: volcanosh/vc-scheduler:v1.9.0
+          image: volcanosh/vc-scheduler:v1.9.1
           args:
             - --logtostderr
             - --scheduler-conf=/volcano.scheduler/volcano-scheduler.conf

pkg/controllers/jobflow/jobflow_controller_action_test.go

@@ -19,6 +19,7 @@ package jobflow
 import (
 	"context"
 	"reflect"
+	"sort"
 	"testing"
 	"time"
 
@@ -405,6 +406,9 @@ func TestGetAllJobStatusFunc(t *testing.T) {
 			if got != nil {
 				got.JobStatusList[0].RunningHistories[0].StartTimestamp = metav1.Time{}
 				got.JobStatusList[1].RunningHistories[0].StartTimestamp = metav1.Time{}
+				sort.Slice(got.JobStatusList, func(i, j int) bool {
+					return got.JobStatusList[i].StartTimestamp.Before(&got.JobStatusList[j].StartTimestamp)
+				})
 			}
 			if !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("getAllJobStatus() got = %v, want %v", got, tt.want)

pkg/scheduler/api/job_info.go

@@ -492,12 +492,15 @@ func (ji *JobInfo) GetMinResources() *Resource {
 	return NewResource(*ji.PodGroup.Spec.MinResources)
 }
 
+// GetElasticResources returns those partly resources in allocated which are more than its minResource
 func (ji *JobInfo) GetElasticResources() *Resource {
-	minResource := ji.GetMinResources()
-	if ji.Allocated.LessEqualPartly(minResource, Zero) {
+	if ji.Allocated == nil {
 		return EmptyResource()
 	}
-	return ji.Allocated.Clone().Sub(minResource)
+	minResource := ji.GetMinResources()
+	elastic := ExceededPart(ji.Allocated, minResource)
+
+	return elastic
 }
 
 func (ji *JobInfo) addTaskIndex(ti *TaskInfo) {

pkg/scheduler/api/job_info_test.go

@@ -383,3 +383,58 @@ func TestJobInfo(t *testing.T) {
 		}
 	}
 }
+
+func TestGetElasticResources(t *testing.T) {
+	resNoGPU := BuildResourceList("1", "1G")
+	resWithGPU := BuildResourceListWithGPU("1", "1G", "1")
+	wantNoGPU := BuildResourceList("1", "1G", []ScalarResource{{Name: "pods", Value: "1"}}...)
+	wantWithGPU := BuildResourceListWithGPU("1", "1G", "1", []ScalarResource{{Name: "pods", Value: "1"}}...)
+	tests := []struct {
+		pods     []*v1.Pod
+		podgroup scheduling.PodGroup
+		want     *Resource
+	}{
+		{
+			pods: []*v1.Pod{
+				buildPod("ns1", "task-1", "node1", v1.PodRunning, resWithGPU, nil, make(map[string]string)),
+			},
+			podgroup: BuildPodgroup("pg1", "ns1", 1, wantWithGPU),
+			want:     EmptyResource(),
+		},
+		{
+			pods: []*v1.Pod{
+				buildPod("ns1", "task-1", "node1", v1.PodRunning, resWithGPU, nil, make(map[string]string)),
+				buildPod("ns1", "task-2", "node2", v1.PodRunning, resNoGPU, nil, make(map[string]string)),
+			},
+			podgroup: BuildPodgroup("pg1", "ns1", 1, wantWithGPU),
+			want:     NewResource(wantNoGPU),
+		},
+		{
+			pods: []*v1.Pod{
+				buildPod("ns1", "task-1", "node1", v1.PodRunning, resNoGPU, nil, make(map[string]string)),
+				buildPod("ns1", "task-2", "node2", v1.PodRunning, resNoGPU, nil, make(map[string]string)),
+			},
+			podgroup: BuildPodgroup("pg1", "ns1", 1, wantWithGPU),
+			want:     NewResource(wantNoGPU),
+		},
+		{
+			pods: []*v1.Pod{
+				buildPod("ns1", "task-1", "node1", v1.PodRunning, resWithGPU, nil, make(map[string]string)),
+				buildPod("ns1", "task-2", "node2", v1.PodRunning, resWithGPU, nil, make(map[string]string)),
+			},
+			podgroup: BuildPodgroup("pg1", "ns1", 1, wantWithGPU),
+			want:     NewResource(wantWithGPU),
+		},
+	}
+
+	for i, test := range tests {
+		job := NewJobInfo("job")
+		for _, pod := range test.pods {
+			job.AddTaskInfo(NewTaskInfo(pod))
+		}
+		job.SetPodGroup(&PodGroup{PodGroup: test.podgroup})
+		if elastic := job.GetElasticResources(); !elastic.Equal(test.want, Zero) {
+			t.Fatalf("case %d:expected %+v, got %+v", i, test.want, elastic)
+		}
+	}
+}

pkg/scheduler/api/resource_info.go

@@ -732,3 +732,17 @@ func (r ResourceNameList) Contains(rr ResourceNameList) bool {
 func IsCountQuota(name v1.ResourceName) bool {
 	return strings.HasPrefix(string(name), "count/")
 }
+
+// ExceededPart returns the partly resource in left which exceed right
+func ExceededPart(left, right *Resource) *Resource {
+	if right == nil {
+		return left
+	}
+
+	if left == nil {
+		return EmptyResource()
+	}
+
+	diff, _ := left.Diff(right, Zero)
+	return diff
+}

pkg/scheduler/api/test_utils.go

@@ -23,6 +23,8 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+
+	"volcano.sh/apis/pkg/apis/scheduling"
 )
 
 func buildNode(name string, alloc v1.ResourceList) *v1.Node {
@@ -90,7 +92,7 @@ type ScalarResource struct {
 	Value string
 }
 
-// BuildResourceList builts resource list object
+// BuildResourceList builds resource list object
 func BuildResourceList(cpu string, memory string, scalarResources ...ScalarResource) v1.ResourceList {
 	resourceList := v1.ResourceList{
 		v1.ResourceCPU:    resource.MustParse(cpu),
@@ -103,7 +105,7 @@ func BuildResourceList(cpu string, memory string, scalarResources ...ScalarResou
 	return resourceList
 }
 
-// BuildResourceListWithGPU builts resource list with GPU
+// BuildResourceListWithGPU builds resource list with GPU
 func BuildResourceListWithGPU(cpu string, memory string, GPU string, scalarResources ...ScalarResource) v1.ResourceList {
 	resourceList := v1.ResourceList{
 		v1.ResourceCPU:    resource.MustParse(cpu),
@@ -116,3 +118,17 @@ func BuildResourceListWithGPU(cpu string, memory string, GPU string, scalarResou
 
 	return resourceList
 }
+
+// BuildPodgroup builds podgroup
+func BuildPodgroup(name, ns string, minMember int32, minResource v1.ResourceList) scheduling.PodGroup {
+	return scheduling.PodGroup{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: ns,
+			Name:      name,
+		},
+		Spec: scheduling.PodGroupSpec{
+			MinMember:    minMember,
+			MinResources: &minResource,
+		},
+	}
+}

pkg/scheduler/plugins/capacity/capacity.go

@@ -116,7 +116,7 @@ func (cp *capacityPlugin) OnSessionOpen(ssn *framework.Session) {
 			if len(queue.Queue.Spec.Guarantee.Resource) != 0 {
 				attr.guarantee = api.NewResource(queue.Queue.Spec.Guarantee.Resource)
 			}
-			realCapability := cp.totalResource.Clone().Sub(cp.totalGuarantee).Add(attr.guarantee)
+			realCapability := api.ExceededPart(cp.totalResource, cp.totalGuarantee).Add(attr.guarantee)
 			if attr.capability == nil {
 				attr.realCapability = realCapability
 			} else {

pkg/scheduler/plugins/proportion/proportion.go

@@ -117,7 +117,7 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) {
 			if len(queue.Queue.Spec.Guarantee.Resource) != 0 {
 				attr.guarantee = api.NewResource(queue.Queue.Spec.Guarantee.Resource)
 			}
-			realCapability := pp.totalResource.Clone().Sub(pp.totalGuarantee).Add(attr.guarantee)
+			realCapability := api.ExceededPart(pp.totalResource, pp.totalGuarantee).Add(attr.guarantee)
 			if attr.capability == nil {
 				attr.realCapability = realCapability
 			} else {