diff --git a/pkg/scheduler/core/estimation.go b/pkg/scheduler/core/estimation.go new file mode 100644 index 000000000000..b5629437ccd4 --- /dev/null +++ b/pkg/scheduler/core/estimation.go @@ -0,0 +1,103 @@ +/* +Copyright 2025 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "context" + + "k8s.io/klog/v2" + + clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1" + policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" + workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" + estimatorclient "github.com/karmada-io/karmada/pkg/estimator/client" + "github.com/karmada-io/karmada/pkg/util/names" +) + +// isMultiTemplateSchedulingApplicable checks if the given ResourceBindingSpec +// meets the criteria for multi-template scheduling: +// 1. The referenced resource holds multiple pod templates (multiple components). +// 2. The placement configuration schedules the resource to exactly one cluster. +// This is currently determined by checking if spread constraints is set and requires exactly one cluster. +// +// Returns true if both conditions are satisfied, false otherwise. +// Note: We do not infer required cluster number from placement.clusterAffinity and +// placement.clusterAffinities because it's impossible to determine without cluster metadata +// whether the affinity rule matches exactly one cluster in the current environment, and the +// only reliable way is spread constraints. +func isMultiTemplateSchedulingApplicable(spec *workv1alpha2.ResourceBindingSpec) bool { + if spec == nil { + return false + } + + if len(spec.Components) < 2 { + return false + } + + // Check if placement targets exactly one cluster + if spec.Placement == nil { + return false + } + for i := range spec.Placement.SpreadConstraints { + if spec.Placement.SpreadConstraints[i].SpreadByField == policyv1alpha1.SpreadByFieldCluster && + spec.Placement.SpreadConstraints[i].MinGroups == 1 && + spec.Placement.SpreadConstraints[i].MaxGroups == 1 { + return true + } + } + + return false +} + +// calculateMultiTemplateAvailableSets calculates available sets for multi-template scheduling. +// It uses MaxAvailableComponentSets to estimate capacity for workloads with multiple pod templates. +func calculateMultiTemplateAvailableSets(ctx context.Context, estimator estimatorclient.ReplicaEstimator, name string, clusters []*clusterv1alpha1.Cluster, spec *workv1alpha2.ResourceBindingSpec, availableTargetClusters []workv1alpha2.TargetCluster) ([]workv1alpha2.TargetCluster, error) { + req := estimatorclient.ComponentSetEstimationRequest{ + Clusters: clusters, + Components: spec.Components, + Namespace: spec.Resource.Namespace, + } + + namespacedKey := names.NamespacedKey(spec.Resource.Namespace, spec.Resource.Name) + resp, err := estimator.MaxAvailableComponentSets(ctx, req) + if err != nil { + klog.Errorf("Failed to calculate available component set with estimator(%s) for workload(%s, kind=%s, %s): %v", + name, spec.Resource.APIVersion, spec.Resource.Kind, namespacedKey, err) + return availableTargetClusters, err + } + + // Use a map to safely update replicas regardless of order. + resMap := make(map[string]int32, len(resp)) + for i := range resp { + if resp[i].Sets == estimatorclient.UnauthenticReplica { + continue + } + resMap[resp[i].Name] = resp[i].Sets + } + for i := range availableTargetClusters { + if newReplicas, ok := resMap[availableTargetClusters[i].Name]; ok { + if availableTargetClusters[i].Replicas > newReplicas { + availableTargetClusters[i].Replicas = newReplicas + } + } else { + klog.Warningf("The estimator(%s) missed estimation from cluster(%s) when estimating for workload(%s, kind=%s, %s).", + name, availableTargetClusters[i].Name, spec.Resource.APIVersion, spec.Resource.Kind, namespacedKey) + } + } + + return availableTargetClusters, nil +} diff --git a/pkg/scheduler/core/estimation_test.go b/pkg/scheduler/core/estimation_test.go new file mode 100644 index 000000000000..b6c42411ef97 --- /dev/null +++ b/pkg/scheduler/core/estimation_test.go @@ -0,0 +1,398 @@ +/* +Copyright 2025 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + + clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1" + policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" + workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" + estimatorclient "github.com/karmada-io/karmada/pkg/estimator/client" + "github.com/karmada-io/karmada/test/helper" +) + +// mockReplicaEstimator is a mock implementation of ReplicaEstimator for testing +type mockReplicaEstimator struct { + maxAvailableComponentSetsResponse []estimatorclient.ComponentSetEstimationResponse + maxAvailableComponentSetsError error +} + +func (m *mockReplicaEstimator) MaxAvailableReplicas(_ context.Context, _ []*clusterv1alpha1.Cluster, _ *workv1alpha2.ReplicaRequirements) ([]workv1alpha2.TargetCluster, error) { + return nil, nil +} + +func (m *mockReplicaEstimator) MaxAvailableComponentSets(_ context.Context, _ estimatorclient.ComponentSetEstimationRequest) ([]estimatorclient.ComponentSetEstimationResponse, error) { + return m.maxAvailableComponentSetsResponse, m.maxAvailableComponentSetsError +} + +func Test_isMultiTemplateSchedulingApplicable(t *testing.T) { + tests := []struct { + name string + spec *workv1alpha2.ResourceBindingSpec + want bool + }{ + { + name: "nil spec should not be applicable", + spec: nil, + want: false, + }, + { + name: "spec with multiple components but without placement should not be applicable", + spec: &workv1alpha2.ResourceBindingSpec{ + Components: []workv1alpha2.Component{ + {Name: "component1"}, + {Name: "component2"}, + }, + }, + want: false, + }, + { + name: "spec with nil placement should not be applicable", + spec: &workv1alpha2.ResourceBindingSpec{ + Components: []workv1alpha2.Component{ + {Name: "component1"}, + {Name: "component2"}, + }, + Placement: nil, + }, + want: false, + }, + { + name: "spec with empty spread constraints should not be applicable", + spec: &workv1alpha2.ResourceBindingSpec{ + Components: []workv1alpha2.Component{ + {Name: "component1"}, + {Name: "component2"}, + }, + Placement: &policyv1alpha1.Placement{ + SpreadConstraints: []policyv1alpha1.SpreadConstraint{}, + }, + }, + want: false, + }, + { + name: "spec with non-cluster spread constraint should not be applicable", + spec: &workv1alpha2.ResourceBindingSpec{ + Components: []workv1alpha2.Component{ + {Name: "component1"}, + {Name: "component2"}, + }, + Placement: &policyv1alpha1.Placement{ + SpreadConstraints: []policyv1alpha1.SpreadConstraint{ + { + SpreadByField: policyv1alpha1.SpreadByFieldRegion, + MinGroups: 1, + MaxGroups: 1, + }, + }, + }, + }, + want: false, + }, + { + name: "spec with cluster spread constraint but wrong min/max groups should not be applicable", + spec: &workv1alpha2.ResourceBindingSpec{ + Components: []workv1alpha2.Component{ + {Name: "component1"}, + {Name: "component2"}, + }, + Placement: &policyv1alpha1.Placement{ + SpreadConstraints: []policyv1alpha1.SpreadConstraint{ + { + SpreadByField: policyv1alpha1.SpreadByFieldCluster, + MinGroups: 2, + MaxGroups: 2, + }, + }, + }, + }, + want: false, + }, + { + name: "spec with single component should not be applicable", + spec: &workv1alpha2.ResourceBindingSpec{ + Components: []workv1alpha2.Component{ + {Name: "component1"}, + }, + Placement: &policyv1alpha1.Placement{ + SpreadConstraints: []policyv1alpha1.SpreadConstraint{ + { + SpreadByField: policyv1alpha1.SpreadByFieldCluster, + MinGroups: 1, + MaxGroups: 1, + }, + }, + }, + }, + want: false, + }, + { + name: "spec with valid cluster spread constraint should be applicable", + spec: &workv1alpha2.ResourceBindingSpec{ + Components: []workv1alpha2.Component{ + {Name: "component1"}, + {Name: "component2"}, + }, + Placement: &policyv1alpha1.Placement{ + SpreadConstraints: []policyv1alpha1.SpreadConstraint{ + { + SpreadByField: policyv1alpha1.SpreadByFieldCluster, + MinGroups: 1, + MaxGroups: 1, + }, + }, + }, + }, + want: true, + }, + { + name: "spec with multiple spread constraints, one valid should be applicable", + spec: &workv1alpha2.ResourceBindingSpec{ + Components: []workv1alpha2.Component{ + {Name: "component1"}, + {Name: "component2"}, + }, + Placement: &policyv1alpha1.Placement{ + SpreadConstraints: []policyv1alpha1.SpreadConstraint{ + { + SpreadByField: policyv1alpha1.SpreadByFieldRegion, + MinGroups: 1, + MaxGroups: 1, + }, + { + SpreadByField: policyv1alpha1.SpreadByFieldCluster, + MinGroups: 1, + MaxGroups: 1, + }, + }, + }, + }, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isMultiTemplateSchedulingApplicable(tt.spec) + assert.Equal(t, tt.want, got) + }) + } +} + +func Test_calculateMultiTemplateAvailableSets(t *testing.T) { + ctx := context.Background() + estimatorName := "test-estimator" + clusters := []*clusterv1alpha1.Cluster{ + helper.NewCluster("cluster1"), + helper.NewCluster("cluster2"), + helper.NewCluster("cluster3"), + } + + spec := &workv1alpha2.ResourceBindingSpec{ + Resource: workv1alpha2.ObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Namespace: "default", + Name: "test-deployment", + }, + Components: []workv1alpha2.Component{ + {Name: "component1"}, + {Name: "component2"}, + }, + Placement: &policyv1alpha1.Placement{ + SpreadConstraints: []policyv1alpha1.SpreadConstraint{ + { + SpreadByField: policyv1alpha1.SpreadByFieldCluster, + MinGroups: 1, + MaxGroups: 1, + }, + }, + }, + } + + tests := []struct { + name string + availableTargetClusters []workv1alpha2.TargetCluster + mockResponse []estimatorclient.ComponentSetEstimationResponse + mockError error + expectedResult []workv1alpha2.TargetCluster + expectedError bool + }{ + { + name: "successful calculation with reduced replicas", + availableTargetClusters: []workv1alpha2.TargetCluster{ + {Name: "cluster1", Replicas: 100}, + {Name: "cluster2", Replicas: 200}, + {Name: "cluster3", Replicas: 300}, + }, + mockResponse: []estimatorclient.ComponentSetEstimationResponse{ + {Name: "cluster1", Sets: 50}, + {Name: "cluster2", Sets: 150}, + {Name: "cluster3", Sets: 250}, + }, + mockError: nil, + expectedResult: []workv1alpha2.TargetCluster{ + {Name: "cluster1", Replicas: 50}, + {Name: "cluster2", Replicas: 150}, + {Name: "cluster3", Replicas: 250}, + }, + expectedError: false, + }, + { + name: "successful calculation with some clusters having higher replicas", + availableTargetClusters: []workv1alpha2.TargetCluster{ + {Name: "cluster1", Replicas: 100}, + {Name: "cluster2", Replicas: 200}, + {Name: "cluster3", Replicas: 300}, + }, + mockResponse: []estimatorclient.ComponentSetEstimationResponse{ + {Name: "cluster1", Sets: 150}, // Higher than current, should not change + {Name: "cluster2", Sets: 100}, // Lower than current, should change + {Name: "cluster3", Sets: 250}, // Lower than current, should change + }, + mockError: nil, + expectedResult: []workv1alpha2.TargetCluster{ + {Name: "cluster1", Replicas: 100}, // Unchanged + {Name: "cluster2", Replicas: 100}, // Changed + {Name: "cluster3", Replicas: 250}, // Changed + }, + expectedError: false, + }, + { + name: "successful calculation with unauthentic replica", + availableTargetClusters: []workv1alpha2.TargetCluster{ + {Name: "cluster1", Replicas: 100}, + {Name: "cluster2", Replicas: 200}, + {Name: "cluster3", Replicas: 300}, + }, + mockResponse: []estimatorclient.ComponentSetEstimationResponse{ + {Name: "cluster1", Sets: estimatorclient.UnauthenticReplica}, // Should be skipped + {Name: "cluster2", Sets: 150}, + {Name: "cluster3", Sets: 250}, + }, + mockError: nil, + expectedResult: []workv1alpha2.TargetCluster{ + {Name: "cluster1", Replicas: 100}, // Unchanged due to unauthentic replica + {Name: "cluster2", Replicas: 150}, // Changed + {Name: "cluster3", Replicas: 250}, // Changed + }, + expectedError: false, + }, + { + name: "successful calculation with different cluster order", + availableTargetClusters: []workv1alpha2.TargetCluster{ + {Name: "cluster1", Replicas: 100}, + {Name: "cluster2", Replicas: 200}, + {Name: "cluster3", Replicas: 300}, + }, + mockResponse: []estimatorclient.ComponentSetEstimationResponse{ + {Name: "cluster3", Sets: 250}, // Different order + {Name: "cluster1", Sets: 50}, + {Name: "cluster2", Sets: 150}, + }, + mockError: nil, + expectedResult: []workv1alpha2.TargetCluster{ + {Name: "cluster1", Replicas: 50}, + {Name: "cluster2", Replicas: 150}, + {Name: "cluster3", Replicas: 250}, + }, + expectedError: false, + }, + { + name: "successful calculation with missing cluster in response", + availableTargetClusters: []workv1alpha2.TargetCluster{ + {Name: "cluster1", Replicas: 100}, + {Name: "cluster2", Replicas: 200}, + {Name: "cluster3", Replicas: 300}, + }, + mockResponse: []estimatorclient.ComponentSetEstimationResponse{ + {Name: "cluster1", Sets: 50}, + // cluster2 missing from response + {Name: "cluster3", Sets: 250}, + }, + mockError: nil, + expectedResult: []workv1alpha2.TargetCluster{ + {Name: "cluster1", Replicas: 50}, + {Name: "cluster2", Replicas: 200}, // Unchanged due to missing response + {Name: "cluster3", Replicas: 250}, + }, + expectedError: false, + }, + { + name: "estimator error", + availableTargetClusters: []workv1alpha2.TargetCluster{ + {Name: "cluster1", Replicas: 100}, + {Name: "cluster2", Replicas: 200}, + }, + mockResponse: nil, + mockError: errors.New("estimator error"), + expectedResult: []workv1alpha2.TargetCluster{ + {Name: "cluster1", Replicas: 100}, // Unchanged due to error + {Name: "cluster2", Replicas: 200}, // Unchanged due to error + }, + expectedError: true, + }, + { + name: "empty available target clusters", + availableTargetClusters: []workv1alpha2.TargetCluster{}, + mockResponse: []estimatorclient.ComponentSetEstimationResponse{ + {Name: "cluster1", Sets: 50}, + }, + mockError: nil, + expectedResult: []workv1alpha2.TargetCluster{}, // Empty result + expectedError: false, + }, + { + name: "empty estimator response", + availableTargetClusters: []workv1alpha2.TargetCluster{ + {Name: "cluster1", Replicas: 100}, + {Name: "cluster2", Replicas: 200}, + }, + mockResponse: []estimatorclient.ComponentSetEstimationResponse{}, // Empty response + mockError: nil, + expectedResult: []workv1alpha2.TargetCluster{ + {Name: "cluster1", Replicas: 100}, // Unchanged + {Name: "cluster2", Replicas: 200}, // Unchanged + }, + expectedError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockEstimator := &mockReplicaEstimator{ + maxAvailableComponentSetsResponse: tt.mockResponse, + maxAvailableComponentSetsError: tt.mockError, + } + + result, err := calculateMultiTemplateAvailableSets(ctx, mockEstimator, estimatorName, clusters, spec, tt.availableTargetClusters) + + if tt.expectedError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + + assert.Equal(t, tt.expectedResult, result) + }) + } +} diff --git a/pkg/scheduler/core/util.go b/pkg/scheduler/core/util.go index 795d7a3267b7..d1033eee26d5 100644 --- a/pkg/scheduler/core/util.go +++ b/pkg/scheduler/core/util.go @@ -28,6 +28,7 @@ import ( policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" estimatorclient "github.com/karmada-io/karmada/pkg/estimator/client" + "github.com/karmada-io/karmada/pkg/features" "github.com/karmada-io/karmada/pkg/scheduler/core/spreadconstraint" "github.com/karmada-io/karmada/pkg/util" "github.com/karmada-io/karmada/pkg/util/names" @@ -64,7 +65,7 @@ func calAvailableReplicas(clusters []*clusterv1alpha1.Cluster, spec *workv1alpha // For non-workload, like ServiceAccount, ConfigMap, Secret and etc, it's unnecessary to calculate available replicas in member clusters. // See issue: https://github.com/karmada-io/karmada/issues/3743. namespacedKey := names.NamespacedKey(spec.Resource.Namespace, spec.Resource.Name) - if spec.Replicas == 0 { + if spec.Replicas == 0 && len(spec.Components) == 0 { klog.V(4).Infof("Do not calculate available replicas for non-workload(%s, kind=%s, %s).", spec.Resource.APIVersion, spec.Resource.Kind, namespacedKey) return availableTargetClusters @@ -75,19 +76,27 @@ func calAvailableReplicas(clusters []*clusterv1alpha1.Cluster, spec *workv1alpha ctx := context.WithValue(context.TODO(), util.ContextKeyObject, fmt.Sprintf("kind=%s, name=%s/%s", spec.Resource.Kind, spec.Resource.Namespace, spec.Resource.Name)) for name, estimator := range estimators { - res, err := estimator.MaxAvailableReplicas(ctx, clusters, spec.ReplicaRequirements) - if err != nil { - klog.Errorf("Max cluster available replicas error: %v", err) - continue - } - klog.V(4).Infof("Invoked MaxAvailableReplicas of estimator %s for workload(%s, kind=%s, %s): %v", name, - spec.Resource.APIVersion, spec.Resource.Kind, namespacedKey, res) - for i := range res { - if res[i].Replicas == estimatorclient.UnauthenticReplica { + if features.FeatureGate.Enabled(features.MultiplePodTemplatesScheduling) && isMultiTemplateSchedulingApplicable(spec) { + var err error + availableTargetClusters, err = calculateMultiTemplateAvailableSets(ctx, estimator, name, clusters, spec, availableTargetClusters) + if err != nil { + continue + } + } else { + res, err := estimator.MaxAvailableReplicas(ctx, clusters, spec.ReplicaRequirements) + if err != nil { + klog.Errorf("Max cluster available replicas error: %v", err) continue } - if availableTargetClusters[i].Name == res[i].Name && availableTargetClusters[i].Replicas > res[i].Replicas { - availableTargetClusters[i].Replicas = res[i].Replicas + klog.V(4).Infof("Invoked MaxAvailableReplicas of estimator %s for workload(%s, kind=%s, %s): %v", name, + spec.Resource.APIVersion, spec.Resource.Kind, namespacedKey, res) + for i := range res { + if res[i].Replicas == estimatorclient.UnauthenticReplica { + continue + } + if availableTargetClusters[i].Name == res[i].Name && availableTargetClusters[i].Replicas > res[i].Replicas { + availableTargetClusters[i].Replicas = res[i].Replicas + } } } }