From 367086634dcdf47a55413541656a54871adbc8de Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Tue, 7 Oct 2025 20:43:48 +0200 Subject: [PATCH 1/2] Fix race conditions ScaleDownOldMS OnDelete --- .../machinedeployment_controller.go | 2 +- .../machinedeployment_rollout_ondelete.go | 220 +++--- ...machinedeployment_rollout_ondelete_test.go | 719 ++++++++++++++++++ ...achinedeployment_rollout_rollingupdate.go} | 30 +- ...edeployment_rollout_rollingupdate_test.go} | 446 ++++++++++- ...machinedeployment_rollout_sequence_test.go | 464 ++--------- ...scale down to 6, random(0).test.log.golden | 287 +++++++ ...ilable 1, scale down to 6.test.log.golden | 199 +++++ ...erunavailable 1, random(0).test.log.golden | 217 ++++++ ...icas, maxuserunavailable 1.test.log.golden | 165 ++++ ... 1, change spec, random(0).test.log.golden | 656 ++++++++++++++++ ...unavailable 1, change spec.test.log.golden | 413 ++++++++++ ..., scale up to 9, random(0).test.log.golden | 490 ++++++++++++ ...available 1, scale up to 9.test.log.golden | 229 ++++++ ...erunavailable 2, random(0).test.log.golden | 327 ++++++++ ...icas, maxuserunavailable 2.test.log.golden | 215 ++++++ ...scale down to 6, random(0).test.log.golden | 0 ...ailable 1, scale down to 6.test.log.golden | 0 ...axunavailable 1, random(0).test.log.golden | 0 ...xsurge 0, maxunavailable 1.test.log.golden | 0 ...axunavailable 0, random(0).test.log.golden | 0 ...xsurge 1, maxunavailable 0.test.log.golden | 0 ...xunavailable 10, random(0).test.log.golden | 0 ...surge 0, maxunavailable 10.test.log.golden | 0 ...axunavailable 3, random(0).test.log.golden | 0 ...xsurge 1, maxunavailable 3.test.log.golden | 0 ...axunavailable 0, random(0).test.log.golden | 0 ...surge 10, maxunavailable 0.test.log.golden | 0 ... 1, change spec, random(0).test.log.golden | 0 ...unavailable 1, change spec.test.log.golden | 0 ...axunavailable 1, random(0).test.log.golden | 0 ... scale up to 12, random(0).test.log.golden | 0 ...vailable 1, scale up to 12.test.log.golden | 0 ...xsurge 3, maxunavailable 1.test.log.golden | 0 34 files changed, 4517 insertions(+), 562 deletions(-) create mode 100644 internal/controllers/machinedeployment/machinedeployment_rollout_ondelete_test.go rename internal/controllers/machinedeployment/{machinedeployment_rolling.go => machinedeployment_rollout_rollingupdate.go} (90%) rename internal/controllers/machinedeployment/{machinedeployment_rolling_test.go => machinedeployment_rollout_rollingupdate_test.go} (62%) create mode 100644 internal/controllers/machinedeployment/testdata/ondelete/12 replicas, maxuserunavailable 1, scale down to 6, random(0).test.log.golden create mode 100644 internal/controllers/machinedeployment/testdata/ondelete/12 replicas, maxuserunavailable 1, scale down to 6.test.log.golden create mode 100644 internal/controllers/machinedeployment/testdata/ondelete/3 replicas, maxuserunavailable 1, random(0).test.log.golden create mode 100644 internal/controllers/machinedeployment/testdata/ondelete/3 replicas, maxuserunavailable 1.test.log.golden create mode 100644 internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 1, change spec, random(0).test.log.golden create mode 100644 internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 1, change spec.test.log.golden create mode 100644 internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 1, scale up to 9, random(0).test.log.golden create mode 100644 internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 1, scale up to 9.test.log.golden create mode 100644 internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 2, random(0).test.log.golden create mode 100644 internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 2.test.log.golden rename internal/controllers/machinedeployment/testdata/{ => rollingupdate}/regular rollout, 12 replicas, maxsurge 3, maxunavailable 1, scale down to 6, random(0).test.log.golden (100%) rename internal/controllers/machinedeployment/testdata/{ => rollingupdate}/regular rollout, 12 replicas, maxsurge 3, maxunavailable 1, scale down to 6.test.log.golden (100%) rename internal/controllers/machinedeployment/testdata/{ => rollingupdate}/regular rollout, 3 replicas, maxsurge 0, maxunavailable 1, random(0).test.log.golden (100%) rename internal/controllers/machinedeployment/testdata/{ => rollingupdate}/regular rollout, 3 replicas, maxsurge 0, maxunavailable 1.test.log.golden (100%) rename internal/controllers/machinedeployment/testdata/{ => rollingupdate}/regular rollout, 3 replicas, maxsurge 1, maxunavailable 0, random(0).test.log.golden (100%) rename internal/controllers/machinedeployment/testdata/{ => rollingupdate}/regular rollout, 3 replicas, maxsurge 1, maxunavailable 0.test.log.golden (100%) rename internal/controllers/machinedeployment/testdata/{ => rollingupdate}/regular rollout, 6 replicas, maxsurge 0, maxunavailable 10, random(0).test.log.golden (100%) rename internal/controllers/machinedeployment/testdata/{ => rollingupdate}/regular rollout, 6 replicas, maxsurge 0, maxunavailable 10.test.log.golden (100%) rename internal/controllers/machinedeployment/testdata/{ => rollingupdate}/regular rollout, 6 replicas, maxsurge 1, maxunavailable 3, random(0).test.log.golden (100%) rename internal/controllers/machinedeployment/testdata/{ => rollingupdate}/regular rollout, 6 replicas, maxsurge 1, maxunavailable 3.test.log.golden (100%) rename internal/controllers/machinedeployment/testdata/{ => rollingupdate}/regular rollout, 6 replicas, maxsurge 10, maxunavailable 0, random(0).test.log.golden (100%) rename internal/controllers/machinedeployment/testdata/{ => rollingupdate}/regular rollout, 6 replicas, maxsurge 10, maxunavailable 0.test.log.golden (100%) rename internal/controllers/machinedeployment/testdata/{ => rollingupdate}/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, change spec, random(0).test.log.golden (100%) rename internal/controllers/machinedeployment/testdata/{ => rollingupdate}/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, change spec.test.log.golden (100%) rename internal/controllers/machinedeployment/testdata/{ => rollingupdate}/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, random(0).test.log.golden (100%) rename internal/controllers/machinedeployment/testdata/{ => rollingupdate}/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, scale up to 12, random(0).test.log.golden (100%) rename internal/controllers/machinedeployment/testdata/{ => rollingupdate}/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, scale up to 12.test.log.golden (100%) rename internal/controllers/machinedeployment/testdata/{ => rollingupdate}/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1.test.log.golden (100%) diff --git a/internal/controllers/machinedeployment/machinedeployment_controller.go b/internal/controllers/machinedeployment/machinedeployment_controller.go index 679294153355..aeb80bfa4640 100644 --- a/internal/controllers/machinedeployment/machinedeployment_controller.go +++ b/internal/controllers/machinedeployment/machinedeployment_controller.go @@ -293,7 +293,7 @@ func (r *Reconciler) reconcile(ctx context.Context, s *scope) error { } if md.Spec.Rollout.Strategy.Type == clusterv1.RollingUpdateMachineDeploymentStrategyType { - return r.rolloutRolling(ctx, md, s.machineSets, templateExists) + return r.rolloutRollingUpdate(ctx, md, s.machineSets, templateExists) } if md.Spec.Rollout.Strategy.Type == clusterv1.OnDeleteMachineDeploymentStrategyType { diff --git a/internal/controllers/machinedeployment/machinedeployment_rollout_ondelete.go b/internal/controllers/machinedeployment/machinedeployment_rollout_ondelete.go index bfb8a249e461..766257613cfe 100644 --- a/internal/controllers/machinedeployment/machinedeployment_rollout_ondelete.go +++ b/internal/controllers/machinedeployment/machinedeployment_rollout_ondelete.go @@ -19,47 +19,53 @@ package machinedeployment import ( "context" "fmt" + "sort" - "github.com/pkg/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" "sigs.k8s.io/cluster-api/internal/controllers/machinedeployment/mdutil" "sigs.k8s.io/cluster-api/util/patch" ) -// rolloutOnDelete implements the logic for the OnDelete rollout strategy. +// rolloutOnDelete reconcile machine sets controlled by a MachineDeployment that is using the OnDelete strategy. func (r *Reconciler) rolloutOnDelete(ctx context.Context, md *clusterv1.MachineDeployment, msList []*clusterv1.MachineSet, templateExists bool) error { + // TODO(in-place): move create newMS into rolloutPlanner newMS, oldMSs, err := r.getAllMachineSetsAndSyncRevision(ctx, md, msList, true, templateExists) if err != nil { return err } - // newMS can be nil in case there is already a MachineSet associated with this deployment, - // but there are only either changes in annotations or MinReadySeconds. Or in other words, - // this can be nil if there are changes, but no replacement of existing machines is needed. - if newMS == nil { - return nil + planner := newRolloutPlanner() + planner.md = md + planner.newMS = newMS + planner.oldMSs = oldMSs + + if err := planner.planOnDelete(ctx); err != nil { + return err } allMSs := append(oldMSs, newMS) - // Scale up, if we can. - if err := r.reconcileNewMachineSetOnDelete(ctx, md, oldMSs, newMS); err != nil { + // TODO(in-place): also apply/remove labels to MS should go into rolloutPlanner + if err := r.cleanupDisableMachineCreateAnnotation(ctx, newMS); err != nil { return err } - - if err := r.syncDeploymentStatus(allMSs, newMS, md); err != nil { + if err := r.addDisableMachineCreateAnnotation(ctx, oldMSs); err != nil { return err } - // Scale down, if we can. - if err := r.reconcileOldMachineSetsOnDelete(ctx, oldMSs, allMSs, md); err != nil { - return err + // TODO(in-place): this should be changed as soon as rolloutPlanner support MS creation and adding/removing labels from MS + for _, ms := range allMSs { + scaleIntent := ptr.Deref(ms.Spec.Replicas, 0) + if v, ok := planner.scaleIntents[ms.Name]; ok { + scaleIntent = v + } + if err := r.scaleMachineSet(ctx, ms, scaleIntent, md); err != nil { + return err + } } if err := r.syncDeploymentStatus(allMSs, newMS, md); err != nil { @@ -75,115 +81,109 @@ func (r *Reconciler) rolloutOnDelete(ctx context.Context, md *clusterv1.MachineD return nil } +// planOnDelete determine how to proceed with the rollout when using the OnDelete strategy if we are not yet at the desired state. +func (p *rolloutPlanner) planOnDelete(ctx context.Context) error { + // Scale up, if we can. + if err := p.reconcileNewMachineSet(ctx); err != nil { + return err + } + + // Scale down, if we can. + p.reconcileOldMachineSetsOnDelete(ctx) + return nil +} + // reconcileOldMachineSetsOnDelete handles reconciliation of Old MachineSets associated with the MachineDeployment in the OnDelete rollout strategy. -func (r *Reconciler) reconcileOldMachineSetsOnDelete(ctx context.Context, oldMSs []*clusterv1.MachineSet, allMSs []*clusterv1.MachineSet, deployment *clusterv1.MachineDeployment) error { +func (p *rolloutPlanner) reconcileOldMachineSetsOnDelete(ctx context.Context) { log := ctrl.LoggerFrom(ctx) - if deployment.Spec.Replicas == nil { - return errors.Errorf("spec replicas for MachineDeployment %q/%q is nil, this is unexpected", - deployment.Namespace, deployment.Name) + oldMachinesCount := mdutil.GetReplicaCountForMachineSets(p.oldMSs) + if oldMachinesCount == 0 { + // Can't scale down further + return } - log.V(4).Info("Checking to see if machines have been deleted or are in the process of deleting for old machine sets") - totalReplicas := mdutil.GetReplicaCountForMachineSets(allMSs) - scaleDownAmount := totalReplicas - *deployment.Spec.Replicas - for _, oldMS := range oldMSs { - log := log.WithValues("MachineSet", klog.KObj(oldMS)) - if oldMS.Spec.Replicas == nil || *oldMS.Spec.Replicas <= 0 { - log.V(4).Info("fully scaled down") + + // Determine if there are replicas to be scaled down due to a scale down in MD. + newMSReplicas := ptr.Deref(p.newMS.Spec.Replicas, 0) + if v, ok := p.scaleIntents[p.newMS.Name]; ok { + newMSReplicas = v + } + totReplicas := oldMachinesCount + newMSReplicas + totalScaleDownCount := max(totReplicas-ptr.Deref(p.md.Spec.Replicas, 0), 0) + + // Sort oldMSs so the system will start deleting from the oldest MS first. + sort.Sort(mdutil.MachineSetsByCreationTimestamp(p.oldMSs)) + + // Start scaling down old machine sets to acknowledge spec.replicas without corresponding status.replicas. + // Note: spec.replicas without corresponding status.replicas exists + // - after a user manually deletes a replica + // - when a newMS non yet fully provisioned suddenly becomes an oldMS. + // In both cases spec.replicas without corresponding status.replicas should be dropped, no matter + // if there are replicas to be scaled down due to a scale down in MD or not. + // However, if there are replicas to be scaled down due to a scale down in MD, deleted replicas should + // be deducted from the totalScaleDownCount. + for _, oldMS := range p.oldMSs { + // No op if this MS has been already scaled down to zero. + if ptr.Deref(oldMS.Spec.Replicas, 0) <= 0 { continue } - if oldMS.Annotations == nil { - oldMS.Annotations = map[string]string{} + + scaleDownCount := max(ptr.Deref(oldMS.Spec.Replicas, 0)-ptr.Deref(oldMS.Status.Replicas, 0), 0) + if scaleDownCount > 0 { + newScaleIntent := max(ptr.Deref(oldMS.Spec.Replicas, 0)-scaleDownCount, 0) + log.V(5).Info(fmt.Sprintf("Setting scale down intent for MachineSet %s to %d replicas (-%d)", oldMS.Name, newScaleIntent, scaleDownCount), "MachineSet", klog.KObj(oldMS)) + p.scaleIntents[oldMS.Name] = newScaleIntent + + totalScaleDownCount -= scaleDownCount + } + } + + // Scale down additional replicas if replicas removed in the for loop above were not enough to align to MD replicas. + for _, oldMS := range p.oldMSs { + // No op if there is no scaling down left. + if totalScaleDownCount <= 0 { + break + } + + // No op if this MS has been already scaled down to zero. + scaleIntent := ptr.Deref(oldMS.Spec.Replicas, 0) + if v, ok := p.scaleIntents[oldMS.Name]; ok { + scaleIntent = min(scaleIntent, v) } + + if scaleIntent <= 0 { + continue + } + + scaleDownCount := min(scaleIntent, totalScaleDownCount) + if scaleDownCount > 0 { + newScaleIntent := max(ptr.Deref(oldMS.Spec.Replicas, 0)-scaleDownCount, 0) + log.V(5).Info(fmt.Sprintf("Setting scale down intent for MachineSet %s to %d replicas (-%d)", oldMS.Name, newScaleIntent, scaleDownCount), "MachineSet", klog.KObj(oldMS)) + p.scaleIntents[oldMS.Name] = newScaleIntent + + totalScaleDownCount -= scaleDownCount + } + } +} + +// addDisableMachineCreateAnnotation will add the disable machine create annotation to old MachineSets. +func (r *Reconciler) addDisableMachineCreateAnnotation(ctx context.Context, oldMSs []*clusterv1.MachineSet) error { + for _, oldMS := range oldMSs { + log := ctrl.LoggerFrom(ctx, "MachineSet", klog.KObj(oldMS)) if _, ok := oldMS.Annotations[clusterv1.DisableMachineCreateAnnotation]; !ok { - log.V(4).Info("setting annotation on old MachineSet to disable machine creation") + log.V(4).Info("adding annotation on old MachineSet to disable machine creation") patchHelper, err := patch.NewHelper(oldMS, r.Client) if err != nil { return err } + if oldMS.Annotations == nil { + oldMS.Annotations = map[string]string{} + } oldMS.Annotations[clusterv1.DisableMachineCreateAnnotation] = "true" - if err := patchHelper.Patch(ctx, oldMS); err != nil { + err = patchHelper.Patch(ctx, oldMS) + if err != nil { return err } } - selectorMap, err := metav1.LabelSelectorAsMap(&oldMS.Spec.Selector) - if err != nil { - log.V(4).Info("Failed to convert MachineSet label selector to a map", "err", err) - continue - } - log.V(4).Info("Fetching Machines associated with MachineSet") - // Get all Machines linked to this MachineSet. - allMachinesInOldMS := &clusterv1.MachineList{} - if err := r.Client.List(ctx, - allMachinesInOldMS, - client.InNamespace(oldMS.Namespace), - client.MatchingLabels(selectorMap), - ); err != nil { - return errors.Wrap(err, "failed to list machines") - } - totalMachineCount := int32(len(allMachinesInOldMS.Items)) - log.V(4).Info("Retrieved machines", "totalMachineCount", totalMachineCount) - updatedReplicaCount := totalMachineCount - mdutil.GetDeletingMachineCount(allMachinesInOldMS) - if updatedReplicaCount < 0 { - return errors.Errorf("negative updated replica count %d for MachineSet %q, this is unexpected", updatedReplicaCount, oldMS.Name) - } - machineSetScaleDownAmountDueToMachineDeletion := *oldMS.Spec.Replicas - updatedReplicaCount - if machineSetScaleDownAmountDueToMachineDeletion < 0 { - log.V(4).Info(fmt.Sprintf("Error reconciling MachineSet %s", oldMS.Name), "err", errors.Errorf("Unexpected negative scale down amount: %d", machineSetScaleDownAmountDueToMachineDeletion)) - } - scaleDownAmount -= machineSetScaleDownAmountDueToMachineDeletion - log.V(4).Info("Adjusting replica count for deleted machines", "oldReplicas", oldMS.Spec.Replicas, "newReplicas", updatedReplicaCount) - log.V(4).Info("Scaling down", "replicas", updatedReplicaCount) - if err := r.scaleMachineSet(ctx, oldMS, updatedReplicaCount, deployment); err != nil { - return err - } } - log.V(4).Info("Finished reconcile of Old MachineSets to account for deleted machines. Now analyzing if there's more potential to scale down") - for _, oldMS := range oldMSs { - log := log.WithValues("MachineSet", klog.KObj(oldMS)) - if scaleDownAmount <= 0 { - break - } - if oldMS.Spec.Replicas == nil || *oldMS.Spec.Replicas <= 0 { - log.V(4).Info("Fully scaled down") - continue - } - updatedReplicaCount := *oldMS.Spec.Replicas - if updatedReplicaCount >= scaleDownAmount { - updatedReplicaCount -= scaleDownAmount - scaleDownAmount = 0 - } else { - scaleDownAmount -= updatedReplicaCount - updatedReplicaCount = 0 - } - log.V(4).Info("Scaling down", "replicas", updatedReplicaCount) - if err := r.scaleMachineSet(ctx, oldMS, updatedReplicaCount, deployment); err != nil { - return err - } - } - log.V(4).Info("Finished reconcile of all old MachineSets") return nil } - -// reconcileNewMachineSetOnDelete handles reconciliation of the latest MachineSet associated with the MachineDeployment in the OnDelete rollout strategy. -func (r *Reconciler) reconcileNewMachineSetOnDelete(ctx context.Context, md *clusterv1.MachineDeployment, oldMSs []*clusterv1.MachineSet, newMS *clusterv1.MachineSet) error { - // TODO(in-place): also apply/remove labels should go into rolloutPlanner - if err := r.cleanupDisableMachineCreateAnnotation(ctx, newMS); err != nil { - return err - } - - planner := newRolloutPlanner() - planner.md = md - planner.newMS = newMS - planner.oldMSs = oldMSs - - if err := planner.reconcileNewMachineSet(ctx); err != nil { - return err - } - - // TODO(in-place): this should be changed as soon as rolloutPlanner support MS creation and adding/removing labels from MS - scaleIntent := ptr.Deref(newMS.Spec.Replicas, 0) - if v, ok := planner.scaleIntents[newMS.Name]; ok { - scaleIntent = v - } - return r.scaleMachineSet(ctx, newMS, scaleIntent, md) -} diff --git a/internal/controllers/machinedeployment/machinedeployment_rollout_ondelete_test.go b/internal/controllers/machinedeployment/machinedeployment_rollout_ondelete_test.go new file mode 100644 index 000000000000..a0149c7d4900 --- /dev/null +++ b/internal/controllers/machinedeployment/machinedeployment_rollout_ondelete_test.go @@ -0,0 +1,719 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package machinedeployment + +import ( + "context" + "fmt" + "math/rand" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/cluster-api/internal/controllers/machinedeployment/mdutil" +) + +func TestReconcileOldMachineSetsOnDelete(t *testing.T) { + testCases := []struct { + name string + machineDeployment *clusterv1.MachineDeployment + scaleIntent map[string]int32 + newMachineSet *clusterv1.MachineSet + oldMachineSets []*clusterv1.MachineSet + expectScaleIntent map[string]int32 + error error + }{ + { + name: "OnDelete strategy: do not scale oldMSs when no machines have been deleted", + machineDeployment: &clusterv1.MachineDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "md", + }, + Spec: clusterv1.MachineDeploymentSpec{ + Rollout: clusterv1.MachineDeploymentRolloutSpec{ + Strategy: clusterv1.MachineDeploymentRolloutStrategy{ + Type: clusterv1.OnDeleteMachineDeploymentStrategyType, + }, + }, + Replicas: ptr.To[int32](4), + }, + }, + scaleIntent: map[string]int32{}, + newMachineSet: &clusterv1.MachineSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ms3", + }, + Spec: clusterv1.MachineSetSpec{ + Replicas: ptr.To[int32](0), + }, + }, + oldMachineSets: []*clusterv1.MachineSet{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "ms1", + }, + Spec: clusterv1.MachineSetSpec{ + Replicas: ptr.To[int32](3), + }, + Status: clusterv1.MachineSetStatus{ + Replicas: ptr.To[int32](3), + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "ms2", + }, + Spec: clusterv1.MachineSetSpec{ + Replicas: ptr.To[int32](1), + }, + Status: clusterv1.MachineSetStatus{ + Replicas: ptr.To[int32](1), + }, + }, + }, + expectScaleIntent: map[string]int32{ + // no new intent for old MS + }, + }, + { + name: "OnDelete strategy: scale down oldMSs when machines have been deleted", + machineDeployment: &clusterv1.MachineDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "md", + }, + Spec: clusterv1.MachineDeploymentSpec{ + Rollout: clusterv1.MachineDeploymentRolloutSpec{ + Strategy: clusterv1.MachineDeploymentRolloutStrategy{ + Type: clusterv1.OnDeleteMachineDeploymentStrategyType, + }, + }, + Replicas: ptr.To[int32](3), + }, + }, + scaleIntent: map[string]int32{}, + newMachineSet: &clusterv1.MachineSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ms3", + }, + Spec: clusterv1.MachineSetSpec{ + Replicas: ptr.To[int32](0), + }, + }, + oldMachineSets: []*clusterv1.MachineSet{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "ms1", + }, + Spec: clusterv1.MachineSetSpec{ + Replicas: ptr.To[int32](3), + }, + Status: clusterv1.MachineSetStatus{ + Replicas: ptr.To[int32](2), // one machines has been deleted + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "ms2", + }, + Spec: clusterv1.MachineSetSpec{ + Replicas: ptr.To[int32](1), + }, + Status: clusterv1.MachineSetStatus{ + Replicas: ptr.To[int32](0), // one machines has been deleted + }, + }, + }, + expectScaleIntent: map[string]int32{ + // new intent for old MS + "ms1": 2, // scale down by one, one machines has been deleted + "ms2": 0, // scale down by one, one machines has been deleted + }, + }, + { + name: "OnDelete strategy: scale down oldMSs when md has been scaled down", + machineDeployment: &clusterv1.MachineDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "md", + }, + Spec: clusterv1.MachineDeploymentSpec{ + Rollout: clusterv1.MachineDeploymentRolloutSpec{ + Strategy: clusterv1.MachineDeploymentRolloutStrategy{ + Type: clusterv1.OnDeleteMachineDeploymentStrategyType, + }, + }, + Replicas: ptr.To[int32](3), // scaled down to 3 replicas, there are exceeding machines in the cluster + }, + }, + scaleIntent: map[string]int32{}, + newMachineSet: &clusterv1.MachineSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ms3", + }, + Spec: clusterv1.MachineSetSpec{ + Replicas: ptr.To[int32](2), + }, + }, + oldMachineSets: []*clusterv1.MachineSet{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "ms1", + }, + Spec: clusterv1.MachineSetSpec{ + Replicas: ptr.To[int32](1), + }, + Status: clusterv1.MachineSetStatus{ + Replicas: ptr.To[int32](1), + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "ms2", + }, + Spec: clusterv1.MachineSetSpec{ + Replicas: ptr.To[int32](2), + }, + Status: clusterv1.MachineSetStatus{ + Replicas: ptr.To[int32](2), + }, + }, + }, + expectScaleIntent: map[string]int32{ + // new intent for old MS + "ms1": 0, // scale down by one,1 exceeding machine removed from ms1 + "ms2": 1, // scale down by one,1 exceeding machine removed from ms1 + }, + }, + { + name: "OnDelete strategy: scale down oldMSs when md has been scaled down, keeps into account newMS scale intent", + machineDeployment: &clusterv1.MachineDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "md", + }, + Spec: clusterv1.MachineDeploymentSpec{ + Rollout: clusterv1.MachineDeploymentRolloutSpec{ + Strategy: clusterv1.MachineDeploymentRolloutStrategy{ + Type: clusterv1.OnDeleteMachineDeploymentStrategyType, + }, + }, + Replicas: ptr.To[int32](3), // scaled down to 3 replicas, there are exceeding machines in the cluster + }, + }, + scaleIntent: map[string]int32{ + "ms3": 2, // scaling intent for new MS (+1) + }, + newMachineSet: &clusterv1.MachineSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ms3", + }, + Spec: clusterv1.MachineSetSpec{ + Replicas: ptr.To[int32](1), + }, + }, + oldMachineSets: []*clusterv1.MachineSet{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "ms1", + }, + Spec: clusterv1.MachineSetSpec{ + Replicas: ptr.To[int32](1), + }, + Status: clusterv1.MachineSetStatus{ + Replicas: ptr.To[int32](1), + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "ms2", + }, + Spec: clusterv1.MachineSetSpec{ + Replicas: ptr.To[int32](2), + }, + Status: clusterv1.MachineSetStatus{ + Replicas: ptr.To[int32](2), + }, + }, + }, + expectScaleIntent: map[string]int32{ + "ms3": 2, + // new intent for old MS + "ms1": 0, // scale down by one, 1 exceeding machine removed from ms1 + "ms2": 1, // scale down by one,1 exceeding machine removed from ms1 + }, + }, + { + name: "OnDelete strategy: scale down oldMSs when md has been scaled down keeps into account deleted machines", + machineDeployment: &clusterv1.MachineDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "md", + }, + Spec: clusterv1.MachineDeploymentSpec{ + Rollout: clusterv1.MachineDeploymentRolloutSpec{ + Strategy: clusterv1.MachineDeploymentRolloutStrategy{ + Type: clusterv1.OnDeleteMachineDeploymentStrategyType, + }, + }, + Replicas: ptr.To[int32](3), // scaled down to 3 replicas, there are exceeding machines in the cluster + }, + }, + scaleIntent: map[string]int32{}, + newMachineSet: &clusterv1.MachineSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ms3", + }, + Spec: clusterv1.MachineSetSpec{ + Replicas: ptr.To[int32](2), + }, + }, + oldMachineSets: []*clusterv1.MachineSet{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "ms1", + }, + Spec: clusterv1.MachineSetSpec{ + Replicas: ptr.To[int32](1), + }, + Status: clusterv1.MachineSetStatus{ + Replicas: ptr.To[int32](0), // one machines has been deleted, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "ms2", + }, + Spec: clusterv1.MachineSetSpec{ + Replicas: ptr.To[int32](2), + }, + Status: clusterv1.MachineSetStatus{ + Replicas: ptr.To[int32](2), + }, + }, + }, + expectScaleIntent: map[string]int32{ + // new intent for old MS + "ms1": 0, // scale down by one, one machines has been deleted + "ms2": 1, // scale down by one, 1 exceeding machine removed from ms1 + }, + }, + } + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + planner := newRolloutPlanner() + planner.scaleIntents = tt.scaleIntent + planner.md = tt.machineDeployment + planner.newMS = tt.newMachineSet + planner.oldMSs = tt.oldMachineSets + + planner.reconcileOldMachineSetsOnDelete(ctx) + g.Expect(planner.scaleIntents).To(Equal(tt.expectScaleIntent), "unexpected scaleIntents") + }) + } +} + +type onDeleteSequenceTestCase struct { + name string + + // currentMachineNames is the list of machines before the rollout, and provides a simplified alternative to currentScope. + // all the machines in this list are initialized as upToDate and owned by the new MS before the rollout (which is different from the new MS after the rollout). + // Please name machines as "mX" where X is a progressive number starting from 1 (do not skip numbers), + // e.g. "m1","m2","m3" + currentMachineNames []string + + // currentScope defines the current state at the beginning of the test case. + // When the test case start from a stable state (there are no previous rollout in progress), use currentMachineNames instead. + // Please name machines as "mX" where X is a progressive number starting from 1 (do not skip numbers), + // e.g. "m1","m2","m3" + // machineUID must be set to the last used number. + currentScope *rolloutScope + + // desiredMachineNames is the list of machines at the end of the rollout. + // all the machines in this list are expected to be upToDate and owned by the new MS after the rollout (which is different from the new MS before the rollout). + // if this list contains old machines names (machine names already in currentMachineNames), it implies those machine have been upgraded in places. + // if this list contains new machines names (machine names not in currentMachineNames), it implies those machines have been created during a rollout; + // please name new machines names as "mX" where X is a progressive number starting after the max number in currentMachineNames (do not skip numbers), + // e.g. desiredMachineNames "m4","m5","m6" (desired machine names after a regular rollout of a MD with currentMachineNames "m1","m2","m3") + // e.g. desiredMachineNames "m1","m2","m3" (desired machine names after rollout performed using in-place upgrade for an MD with currentMachineNames "m1","m2","m3") + desiredMachineNames []string + + // maxUserUnavailable define the maximum numbers of unavailable machines a user want to have in the system. + // Unavailable machines includes both machines being deleted or machines already deleted. + maxUserUnavailable int32 + + // skipLogToFileAndGoldenFileCheck allows to skip storing the log to file and golden file Check. + // NOTE: this field is controlled by the test itself. + skipLogToFileAndGoldenFileCheck bool + + // name of the log to file and the golden file. + // NOTE: this field is controlled by the test itself. + logAndGoldenFileName string + + // randomControllerOrder force the tests to run controllers in random order, mimicking what happens in production. + // NOTE. We are using a pseudo randomizer, so the random order remains consistent across runs of the same groups of tests. + // NOTE: this field is controlled by the test itself. + randomControllerOrder bool + + // maxIterations defines the max number of iterations the system must attempt before assuming the logic has an issue + // in reaching the desired state. + // When the test is using default controller order, an iteration implies reconcile MD + reconcile all MS in a predictable order; + // while using randomControllerOrder the concept of iteration is less defined, but it can still be used to prevent + // the test from running indefinitely. + // NOTE: this field is controlled by the test itself. + maxIterations int + + // seed value to initialize the generator. + // NOTE: this field is controlled by the test itself. + seed int64 +} + +func Test_OnDeleteSequences(t *testing.T) { + ctx := context.Background() + ctx = ctrl.LoggerInto(ctx, klog.Background()) + + tests := []onDeleteSequenceTestCase{ + { // delete 1 + name: "3 replicas, maxUserUnavailable 1", + currentMachineNames: []string{"m1", "m2", "m3"}, + maxUserUnavailable: 1, + desiredMachineNames: []string{"m4", "m5", "m6"}, + }, + { // delete 2 + name: "6 replicas, maxUserUnavailable 2", + currentMachineNames: []string{"m1", "m2", "m3", "m4", "m5", "m6"}, + maxUserUnavailable: 2, + desiredMachineNames: []string{"m7", "m8", "m9", "m10", "m11", "m12"}, + }, + { // delete 1 + scale up machine deployment in the middle + name: "6 Replicas, maxUserUnavailable 1, scale up to 9", + currentScope: &rolloutScope{ // Manually providing a scope simulating a MD originally with 6 replica in the middle of a rollout, with 3 machines already created in the newMS and 3 still on the oldMS, and then MD scaled up to 9. + machineDeployment: createMD("v2", 9, withOnDeleteStrategy()), + machineSets: []*clusterv1.MachineSet{ + createMS("ms1", "v1", 3), + createMS("ms2", "v2", 3), + }, + machineSetMachines: map[string][]*clusterv1.Machine{ + "ms1": { + // "m1", "m2", "m3" already deleted + createM("m4", "ms1", "v1"), + createM("m5", "ms1", "v1"), + createM("m6", "ms1", "v1"), + }, + "ms2": { + createM("m7", "ms2", "v2"), + createM("m8", "ms2", "v2"), + createM("m9", "ms2", "v2"), + }, + }, + machineUID: 9, + }, + maxUserUnavailable: 1, + desiredMachineNames: []string{"m7", "m8", "m9", "m10", "m11", "m12", "m13", "m14", "m15"}, + }, + { // delete 1 + scale down machine deployment in the middle + name: "12 Replicas, maxUserUnavailable 1, scale down to 6", + currentScope: &rolloutScope{ // Manually providing a scope simulating a MD originally with 12 replica in the middle of a rollout, with 3 machines already created in the newMS and 9 still on the oldMS, and then MD scaled down to 6. + machineDeployment: createMD("v2", 6, withOnDeleteStrategy()), + machineSets: []*clusterv1.MachineSet{ + createMS("ms1", "v1", 9), + createMS("ms2", "v2", 3), + }, + machineSetMachines: map[string][]*clusterv1.Machine{ + "ms1": { + // "m1", "m2", "m3" already deleted + createM("m4", "ms1", "v1"), + createM("m5", "ms1", "v1"), + createM("m6", "ms1", "v1"), + createM("m7", "ms1", "v1"), + createM("m8", "ms1", "v1"), + createM("m9", "ms1", "v1"), + createM("m10", "ms1", "v1"), + createM("m11", "ms1", "v1"), + createM("m12", "ms1", "v1"), + }, + "ms2": { + createM("m13", "ms2", "v2"), + createM("m14", "ms2", "v2"), + createM("m15", "ms2", "v2"), + }, + }, + machineUID: 15, + }, + maxUserUnavailable: 1, + desiredMachineNames: []string{"m13", "m14", "m15", "m16", "m17", "m18"}, + }, + { // delete 1 + change spec in the middle + name: "6 Replicas, maxUserUnavailable 1, change spec", + currentScope: &rolloutScope{ // Manually providing a scope simulating a MD with 6 replica in the middle of a rollout, with 3 machines already created in the newMS and 3 still on the oldMS, and then MD spec is changed. + machineDeployment: createMD("v3", 6, withOnDeleteStrategy()), + machineSets: []*clusterv1.MachineSet{ + createMS("ms1", "v1", 3), + createMS("ms2", "v2", 3), + createMS("ms3", "v3", 0), + }, + machineSetMachines: map[string][]*clusterv1.Machine{ + "ms1": { + // "m1", "m2", "m3" already deleted + createM("m4", "ms1", "v1"), + createM("m5", "ms1", "v1"), + createM("m6", "ms1", "v1"), + }, + "ms2": { + createM("m7", "ms2", "v2"), + createM("m8", "ms2", "v2"), + createM("m9", "ms2", "v2"), + }, + }, + machineUID: 9, + }, + maxUserUnavailable: 1, + desiredMachineNames: []string{"m10", "m11", "m12", "m13", "m14", "m15"}, // NOTE: Machines created before the spec change are deleted + }, + } + + testWithPredictableReconcileOrder := true + testWithRandomReconcileOrderFromConstantSeed := true + testWithRandomReconcileOrderFromRandomSeed := true + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + name := tt.name + + if testWithPredictableReconcileOrder { + tt.maxIterations = 50 + tt.randomControllerOrder = false + tt.logAndGoldenFileName = strings.ToLower(tt.name) + t.Run("default", func(t *testing.T) { + runOnDeleteTestCase(ctx, t, tt) + }) + } + + if testWithRandomReconcileOrderFromConstantSeed { + tt.maxIterations = 70 + tt.name = fmt.Sprintf("%s, random(0)", name) + tt.randomControllerOrder = true + tt.seed = 0 + tt.logAndGoldenFileName = strings.ToLower(tt.name) + t.Run("random(0)", func(t *testing.T) { + runOnDeleteTestCase(ctx, t, tt) + }) + } + + if testWithRandomReconcileOrderFromRandomSeed { + for range 100 { + tt.maxIterations = 150 + tt.seed = time.Now().UnixNano() + tt.name = fmt.Sprintf("%s, random(%d)", name, tt.seed) + tt.randomControllerOrder = true + tt.skipLogToFileAndGoldenFileCheck = true + t.Run(fmt.Sprintf("random(%d)", tt.seed), func(t *testing.T) { + runOnDeleteTestCase(ctx, t, tt) + }) + } + } + }) + } +} + +func runOnDeleteTestCase(ctx context.Context, t *testing.T, tt onDeleteSequenceTestCase) { + t.Helper() + g := NewWithT(t) + + rng := rand.New(rand.NewSource(tt.seed)) //nolint:gosec // it is ok to use a weak randomizer here + fLogger := newFileLogger(t, tt.name, fmt.Sprintf("testdata/ondelete/%s", tt.logAndGoldenFileName)) + // uncomment this line to automatically generate/update golden files: fLogger.writeGoldenFile = true + + // Init current and desired state from test case + current := tt.currentScope.Clone() + if current == nil { + current = initCurrentRolloutScope(tt.currentMachineNames, withOnDeleteStrategy()) + } + desired := computeDesiredRolloutScope(current, tt.desiredMachineNames) + + // Log initial state + fLogger.Logf("[Test] Initial state\n%s", current) + random := "" + if tt.randomControllerOrder { + random = fmt.Sprintf(", random(%d)", tt.seed) + } + fLogger.Logf("[Test] Rollout %d replicas, onDeleteStrategy%s\n", len(current.machines()), random) + i := 1 + maxIterations := tt.maxIterations + + // Prevent deletion to start till the rollout planner did create the newMS. + // Note: this prevent machines being deleted and re-created on oldMSs in case of random sequences. + canDelete := false + + for { + taskList := getTaskListOnDelete(current) + taskCount := len(taskList) + taskOrder := defaultTaskOrder(taskCount) + if tt.randomControllerOrder { + taskOrder = randomTaskOrder(taskCount, rng) + } + for _, taskID := range taskOrder { + task := taskList[taskID] + if task == "md" { + fLogger.Logf("[MD controller] Iteration %d, Reconcile md", i) + fLogger.Logf("[MD controller] - Input to rollout planner\n%s", current) + + // Running a small subset of MD reconcile (the rollout logic and a bit of setReplicas) + + // create rollout planner and override the func that computeDesiredMS so it uses a predictable MS name when creating newMS. + p := newRolloutPlanner() + p.md = current.machineDeployment + p.newMS = current.newMS() + p.oldMSs = current.oldMSs() + + err := p.planOnDelete(ctx) + g.Expect(err).ToNot(HaveOccurred()) + + // Apply changes. + delete(p.newMS.Annotations, clusterv1.DisableMachineCreateAnnotation) + for _, oldMS := range current.oldMSs() { + if oldMS.Annotations == nil { + oldMS.Annotations = map[string]string{} + } + oldMS.Annotations[clusterv1.DisableMachineCreateAnnotation] = "true" + } + + for _, ms := range current.machineSets { + if scaleIntent, ok := p.scaleIntents[ms.Name]; ok { + ms.Spec.Replicas = ptr.To(scaleIntent) + } + } + + // As soon as the newMS exists, unblock machine deletion (at this stage we are sure new machines will be created on the newMS only). + if p.newMS != nil { + canDelete = true + } + + // Running a small subset of setReplicas (we don't want to run the full func to avoid unnecessary noise on the test) + current.machineDeployment.Status.Replicas = mdutil.GetActualReplicaCountForMachineSets(current.machineSets) + current.machineDeployment.Status.AvailableReplicas = mdutil.GetAvailableReplicaCountForMachineSets(current.machineSets) + + // Log state after this reconcile + fLogger.Logf("[MD controller] - Result of rollout planner\n%s", current) + } + + // Simulate the user deleting machines not upToDate; in order to make this realistic deletion will be performed + // if the operation respects maxUserUnavailable e.g. + // - with maxUserUnavailable=1, perform deletion only when previous deletion has been completed and a replacement machines, + // which we assume it is the most common behaviour. + // - maxUserUnavailable > 1 can be used to test scenarios where the users delete machines in a more aggressive + // Also deletion cannot happen before the newMS has been created (thus preventing machines being deleted and + // re-created on oldMSs in case of random sequences, which will lead to result different than what expected) + if task == "delete-machine" { + deletionsInFlight := int32(0) + for _, m := range current.machines() { + if !m.DeletionTimestamp.IsZero() { + deletionsInFlight++ + } + } + + totAvailable := max(int32(len(current.machines()))-deletionsInFlight, 0) + minAvailable := max(ptr.Deref(current.machineDeployment.Spec.Replicas, 0)-tt.maxUserUnavailable, 0) + if totAvailable > minAvailable && canDelete { + // Determine the list of machines that should be deleted manually, which are the machines not yet upd to date. + machinesToDelete := []*clusterv1.Machine{} + for _, m := range current.machines() { + if upToDate, _ := mdutil.MachineTemplateUpToDate(&clusterv1.MachineTemplateSpec{Spec: m.Spec}, &desired.machineDeployment.Spec.Template); !upToDate { + machinesToDelete = append(machinesToDelete, m) + } + } + + if len(machinesToDelete) > 0 { + // When running with default controller order, also delete machines in order; + // conversely, when running with random controller order, also machine deletion happens in rando order. + n := 0 + if tt.randomControllerOrder { + n = rng.Intn(len(machinesToDelete)) + } + + if machinesToDelete[n].DeletionTimestamp.IsZero() { + fLogger.Logf("[User] Iteration %d, Deleting machine %s", i, machinesToDelete[n].Name) + machinesToDelete[n].DeletionTimestamp = ptr.To(metav1.Now()) + } + } + } + } + + // Run mutators faking MS controllers + for _, ms := range current.machineSets { + if ms.Name == task { + fLogger.Logf("[MS controller] Iteration %d, Reconcile %s, %s", i, ms.Name, msLog(ms, current.machineSetMachines[ms.Name])) + machineSetControllerMutator(fLogger, ms, current) + break + } + } + + // Run mutators faking M controllers + for _, ms := range current.machineSets { + for _, m := range current.machineSetMachines[ms.Name] { + if m.Name == task { + fLogger.Logf("[M controller] Iteration %d, Reconcile %s", i, m.Name) + machineControllerMutator(fLogger, m, current) + } + } + } + } + + // Check if we are at the desired state + if current.Equal(desired) { + fLogger.Logf("[Test] Final state\n%s", current) + break + } + + // Safeguard for infinite reconcile + i++ + if i > maxIterations { + // NOTE: the following can be used to set a breakpoint for debugging why the system is not reaching desired state after maxIterations (to check what is not yet equal) + current.Equal(desired) + // Log desired state we never reached + fLogger.Logf("[Test] Desired state\n%s", desired) + g.Fail(fmt.Sprintf("Failed to reach desired state in %d iterations", maxIterations)) + } + } + + if !tt.skipLogToFileAndGoldenFileCheck { + currentLog, goldenLog, err := fLogger.WriteLogAndCompareWithGoldenFile() + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(currentLog).To(Equal(goldenLog), "current test case log and golden test case log are different\n%s", cmp.Diff(currentLog, goldenLog)) + } +} + +func getTaskListOnDelete(current *rolloutScope) []string { + taskList := make([]string, 0) + taskList = append(taskList, "md") + for _, ms := range current.machineSets { + taskList = append(taskList, ms.Name) + } + taskList = append(taskList, + fmt.Sprintf("ms%d", len(current.machineSets)+1), // r the MachineSet that might be created when reconciling md + "delete-machine", + ) + for _, m := range current.machines() { + taskList = append(taskList, m.Name) + } + return taskList +} diff --git a/internal/controllers/machinedeployment/machinedeployment_rolling.go b/internal/controllers/machinedeployment/machinedeployment_rollout_rollingupdate.go similarity index 90% rename from internal/controllers/machinedeployment/machinedeployment_rolling.go rename to internal/controllers/machinedeployment/machinedeployment_rollout_rollingupdate.go index d2e4a8556527..83be656bcbb8 100644 --- a/internal/controllers/machinedeployment/machinedeployment_rolling.go +++ b/internal/controllers/machinedeployment/machinedeployment_rollout_rollingupdate.go @@ -32,8 +32,8 @@ import ( "sigs.k8s.io/cluster-api/util/patch" ) -// rolloutRolling implements the logic for rolling a new MachineSet. -func (r *Reconciler) rolloutRolling(ctx context.Context, md *clusterv1.MachineDeployment, msList []*clusterv1.MachineSet, templateExists bool) error { +// rolloutRollingUpdate reconcile machine sets controlled by a MachineDeployment that is using the RolloutUpdate strategy. +func (r *Reconciler) rolloutRollingUpdate(ctx context.Context, md *clusterv1.MachineDeployment, msList []*clusterv1.MachineSet, templateExists bool) error { // TODO(in-place): move create newMS into rolloutPlanner newMS, oldMSs, err := r.getAllMachineSetsAndSyncRevision(ctx, md, msList, true, templateExists) if err != nil { @@ -50,6 +50,14 @@ func (r *Reconciler) rolloutRolling(ctx context.Context, md *clusterv1.MachineDe allMSs := append(oldMSs, newMS) // TODO(in-place): also apply/remove labels to MS should go into rolloutPlanner + // Note: looks like the current implementation is missing proper management of the DisableMachineCreateAnnotation on oldMS: + // - Currently the DisableMachineCreateAnnotation is not removed from oldMS, so the annotation might be there or not depending it the users + // transitioned back and forth from RolloutUpdate strategy and OnDelete strategy. + // - TBD is we want to implement a proper cleanup thus ensuring create machines on oldMS can always happen when using the + // RolloutUpdate strategy, or if we want to always prevent machine creation on oldMS also in this case. + // Note: When rollout is paused the code assumes it can always scale up oldMS (see scale in machinedeployment_sync.go). + // however current implement it is of missing removal of the DisableMachineCreateAnnotation that might exist or not + // on oldMS (depending on strategy / proper cleanup etc.). if err := r.cleanupDisableMachineCreateAnnotation(ctx, newMS); err != nil { return err } @@ -59,7 +67,7 @@ func (r *Reconciler) rolloutRolling(ctx context.Context, md *clusterv1.MachineDe planner.newMS = newMS planner.oldMSs = oldMSs - if err := planner.planRolloutRolling(ctx); err != nil { + if err := planner.planRollingUpdate(ctx); err != nil { return err } @@ -100,8 +108,8 @@ func newRolloutPlanner() *rolloutPlanner { } } -// planRolloutRolling determine how to proceed with the rollout when using the RolloutRolling strategy if the system is not yet at the desired state. -func (p *rolloutPlanner) planRolloutRolling(ctx context.Context) error { +// planRollingUpdate determine how to proceed with the rollout when using the RollingUpdate strategy if the system is not yet at the desired state. +func (p *rolloutPlanner) planRollingUpdate(ctx context.Context) error { if p.md.Spec.Replicas == nil { return errors.Errorf("spec.replicas for MachineDeployment %v is nil, this is unexpected", client.ObjectKeyFromObject(p.md)) } @@ -122,22 +130,22 @@ func (p *rolloutPlanner) planRolloutRolling(ctx context.Context) error { } // Scale down, if we can. - if err := p.reconcileOldMachineSetsRolloutRolling(ctx); err != nil { + if err := p.reconcileOldMachineSetsRollingUpdate(ctx); err != nil { return err } // This func tries to detect and address the case when a rollout is not making progress because both scaling down and scaling up are blocked. // Note: This func must be called after computing scale up/down intent for all the MachineSets. // Note: This func only addresses deadlocks due to unavailable replicas not getting deleted on oldMSs, which can happen - // because reconcileOldMachineSetsRolloutRolling called above always assumes the worst case when deleting replicas e.g. + // because reconcileOldMachineSetsRollingUpdate called above always assumes the worst case when deleting replicas e.g. // - MD with spec.replicas 3, MaxSurge 1, MaxUnavailable 0 // - OldMS with 3 replicas, 2 available replica (and thus 1 unavailable replica) // - NewMS with 1 replica, 1 available replica // - In theory it is possible to scale down oldMS from 3->2 replicas by deleting the unavailable replica. - // - However, reconcileOldMachineSetsRolloutRolling cannot assume that the MachineSet controller is going to delete + // - However, reconcileOldMachineSetsRollingUpdate cannot assume that the MachineSet controller is going to delete // the unavailable replica when scaling down from 3->2, because it might happen that one of the available replicas // is deleted instead. - // - As a consequence reconcileOldMachineSetsRolloutRolling, which assumes the worst case when deleting replicas, did not scaled down oldMS. + // - As a consequence reconcileOldMachineSetsRollingUpdate, which assumes the worst case when deleting replicas, did not scaled down oldMS. // This situation, rollout not proceeding due to unavailable replicas, is considered a deadlock to be addressed by reconcileDeadlockBreaker. // Note: Unblocking deadlocks when unavailable replicas exist only on oldMSs, is required also because replicas on oldMSs are not remediated by MHC. p.reconcileDeadlockBreaker(ctx) @@ -178,7 +186,7 @@ func (p *rolloutPlanner) reconcileNewMachineSet(ctx context.Context) error { return nil } -func (p *rolloutPlanner) reconcileOldMachineSetsRolloutRolling(ctx context.Context) error { +func (p *rolloutPlanner) reconcileOldMachineSetsRollingUpdate(ctx context.Context) error { allMSs := append(p.oldMSs, p.newMS) // no op if there are no replicas on old machinesets @@ -387,7 +395,7 @@ func (p *rolloutPlanner) reconcileDeadlockBreaker(ctx context.Context) { // // Note: In most cases this is only a formal violation of maxUnavailability, because there is a good chance // that the machine that will be deleted is one of the unavailable machines. - // Note: This for loop relies on the same ordering of oldMSs that has been applied by reconcileOldMachineSetsRolloutRolling. + // Note: This for loop relies on the same ordering of oldMSs that has been applied by reconcileOldMachineSetsRollingUpdate. for _, oldMS := range p.oldMSs { if ptr.Deref(oldMS.Status.AvailableReplicas, 0) == ptr.Deref(oldMS.Status.Replicas, 0) || ptr.Deref(oldMS.Spec.Replicas, 0) == 0 { continue diff --git a/internal/controllers/machinedeployment/machinedeployment_rolling_test.go b/internal/controllers/machinedeployment/machinedeployment_rollout_rollingupdate_test.go similarity index 62% rename from internal/controllers/machinedeployment/machinedeployment_rolling_test.go rename to internal/controllers/machinedeployment/machinedeployment_rollout_rollingupdate_test.go index ab8bfff996ab..51350ad658fd 100644 --- a/internal/controllers/machinedeployment/machinedeployment_rolling_test.go +++ b/internal/controllers/machinedeployment/machinedeployment_rollout_rollingupdate_test.go @@ -18,11 +18,19 @@ package machinedeployment import ( "context" + "fmt" + "math/rand" + "strings" "testing" + "time" + "github.com/google/go-cmp/cmp" + "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" "sigs.k8s.io/cluster-api/internal/controllers/machinedeployment/mdutil" @@ -269,7 +277,7 @@ func TestReconcileNewMachineSet(t *testing.T) { } } -func Test_reconcileOldMachineSetsRolloutRolling(t *testing.T) { +func Test_reconcileOldMachineSetsRollingUpdate(t *testing.T) { var ctx = context.Background() tests := []struct { @@ -284,7 +292,7 @@ func Test_reconcileOldMachineSetsRolloutRolling(t *testing.T) { { name: "no op if there are no replicas on old machinesets", scaleIntent: map[string]int32{}, - md: createMD("v2", 10, withRolloutStrategy(1, 0)), + md: createMD("v2", 10, withRollingUpdateStrategy(1, 0)), newMS: createMS("ms2", "v2", 10, withStatusReplicas(10), withStatusAvailableReplicas(10)), oldMSs: []*clusterv1.MachineSet{ createMS("ms1", "v1", 0, withStatusReplicas(0), withStatusAvailableReplicas(0)), @@ -296,7 +304,7 @@ func Test_reconcileOldMachineSetsRolloutRolling(t *testing.T) { { name: "do not scale down if replicas is equal to minAvailable replicas", scaleIntent: map[string]int32{}, - md: createMD("v2", 3, withRolloutStrategy(1, 0)), + md: createMD("v2", 3, withRollingUpdateStrategy(1, 0)), newMS: createMS("ms2", "v2", 1, withStatusReplicas(1), withStatusAvailableReplicas(1)), oldMSs: []*clusterv1.MachineSet{ createMS("ms1", "v1", 2, withStatusReplicas(2), withStatusAvailableReplicas(2)), @@ -309,7 +317,7 @@ func Test_reconcileOldMachineSetsRolloutRolling(t *testing.T) { { name: "do not scale down if replicas is less then minAvailable replicas", scaleIntent: map[string]int32{}, - md: createMD("v2", 3, withRolloutStrategy(1, 0)), + md: createMD("v2", 3, withRollingUpdateStrategy(1, 0)), newMS: createMS("ms2", "v2", 1, withStatusReplicas(1), withStatusAvailableReplicas(1)), oldMSs: []*clusterv1.MachineSet{ createMS("ms1", "v1", 2, withStatusReplicas(2), withStatusAvailableReplicas(1)), @@ -323,7 +331,7 @@ func Test_reconcileOldMachineSetsRolloutRolling(t *testing.T) { { name: "do not scale down if there are more replicas than minAvailable replicas, but scale down from a previous reconcile already takes the availability buffer", scaleIntent: map[string]int32{}, - md: createMD("v2", 3, withRolloutStrategy(1, 0)), + md: createMD("v2", 3, withRollingUpdateStrategy(1, 0)), newMS: createMS("ms2", "v2", 1, withStatusReplicas(1), withStatusAvailableReplicas(1)), oldMSs: []*clusterv1.MachineSet{ createMS("ms1", "v1", 2, withStatusReplicas(3), withStatusAvailableReplicas(3)), // OldMS is scaling down from a previous reconcile @@ -336,7 +344,7 @@ func Test_reconcileOldMachineSetsRolloutRolling(t *testing.T) { { name: "do not scale down if there are more replicas than minAvailable replicas, but scale down from a previous reconcile already takes the availability buffer, scale down from a previous reconcile on another MS", scaleIntent: map[string]int32{}, - md: createMD("v2", 6, withRolloutStrategy(3, 1)), + md: createMD("v2", 6, withRollingUpdateStrategy(3, 1)), newMS: createMS("ms3", "v2", 3, withStatusReplicas(0), withStatusAvailableReplicas(0)), // NewMS is scaling up from previous reconcile, but replicas do not exist yet oldMSs: []*clusterv1.MachineSet{ createMS("ms1", "v1", 2, withStatusReplicas(3), withStatusAvailableReplicas(3)), // OldMS is scaling down from a previous reconcile @@ -352,7 +360,7 @@ func Test_reconcileOldMachineSetsRolloutRolling(t *testing.T) { scaleIntent: map[string]int32{ "ms2": 1, // newMS (ms2) has a scaling down intent from current reconcile }, - md: createMD("v2", 3, withRolloutStrategy(1, 0)), + md: createMD("v2", 3, withRollingUpdateStrategy(1, 0)), newMS: createMS("ms2", "v2", 2, withStatusReplicas(2), withStatusAvailableReplicas(2)), oldMSs: []*clusterv1.MachineSet{ createMS("ms1", "v1", 2, withStatusReplicas(2), withStatusAvailableReplicas(2)), @@ -368,7 +376,7 @@ func Test_reconcileOldMachineSetsRolloutRolling(t *testing.T) { scaleIntent: map[string]int32{ "ms1": 1, // oldMS (ms1) has a scaling down intent from current reconcile }, - md: createMD("v2", 3, withRolloutStrategy(1, 0)), + md: createMD("v2", 3, withRollingUpdateStrategy(1, 0)), newMS: createMS("ms2", "v2", 2, withStatusReplicas(2), withStatusAvailableReplicas(2)), oldMSs: []*clusterv1.MachineSet{ createMS("ms1", "v1", 2, withStatusReplicas(2), withStatusAvailableReplicas(2)), @@ -382,7 +390,7 @@ func Test_reconcileOldMachineSetsRolloutRolling(t *testing.T) { { name: "do not scale down replicas when there are more replicas than minAvailable replicas, but not all the replicas are available (unavailability on newMS)", scaleIntent: map[string]int32{}, - md: createMD("v2", 3, withRolloutStrategy(1, 0)), + md: createMD("v2", 3, withRollingUpdateStrategy(1, 0)), newMS: createMS("ms2", "v2", 1, withStatusReplicas(1), withStatusAvailableReplicas(0)), // no replicas are available oldMSs: []*clusterv1.MachineSet{ createMS("ms1", "v1", 3, withStatusReplicas(3), withStatusAvailableReplicas(3)), @@ -395,7 +403,7 @@ func Test_reconcileOldMachineSetsRolloutRolling(t *testing.T) { { name: "do not scale down replicas when there are more replicas than minAvailable replicas, but not all the replicas are available (unavailability on oldMS)", scaleIntent: map[string]int32{}, - md: createMD("v2", 3, withRolloutStrategy(1, 0)), + md: createMD("v2", 3, withRollingUpdateStrategy(1, 0)), newMS: createMS("ms2", "v2", 1, withStatusReplicas(1), withStatusAvailableReplicas(1)), oldMSs: []*clusterv1.MachineSet{ createMS("ms1", "v1", 3, withStatusReplicas(3), withStatusAvailableReplicas(2)), // only 2 replicas are available @@ -408,7 +416,7 @@ func Test_reconcileOldMachineSetsRolloutRolling(t *testing.T) { { name: "scale down replicas when there are more replicas than minAvailable replicas, all replicas are available", scaleIntent: map[string]int32{}, - md: createMD("v2", 3, withRolloutStrategy(1, 0)), + md: createMD("v2", 3, withRollingUpdateStrategy(1, 0)), newMS: createMS("ms2", "v2", 1, withStatusReplicas(1), withStatusAvailableReplicas(1)), oldMSs: []*clusterv1.MachineSet{ createMS("ms1", "v1", 3, withStatusReplicas(3), withStatusAvailableReplicas(3)), @@ -421,7 +429,7 @@ func Test_reconcileOldMachineSetsRolloutRolling(t *testing.T) { { name: "scale down replicas when there are more replicas than minAvailable replicas, unavailable replicas are scaled down first", scaleIntent: map[string]int32{}, - md: createMD("v2", 3, withRolloutStrategy(1, 0)), + md: createMD("v2", 3, withRollingUpdateStrategy(1, 0)), newMS: createMS("ms4", "v3", 1, withStatusReplicas(1), withStatusAvailableReplicas(1)), oldMSs: []*clusterv1.MachineSet{ createMS("ms1", "v0", 2, withStatusReplicas(2), withStatusAvailableReplicas(2)), @@ -439,7 +447,7 @@ func Test_reconcileOldMachineSetsRolloutRolling(t *testing.T) { { name: "scale down replicas when there are more replicas than minAvailable replicas, unavailable replicas are scaled down first, available replicas are scaled down when unavailable replicas are gone", scaleIntent: map[string]int32{}, - md: createMD("v2", 3, withRolloutStrategy(1, 0)), + md: createMD("v2", 3, withRollingUpdateStrategy(1, 0)), newMS: createMS("ms4", "v3", 1, withStatusReplicas(1), withStatusAvailableReplicas(1)), oldMSs: []*clusterv1.MachineSet{ createMS("ms1", "v0", 3, withStatusReplicas(3), withStatusAvailableReplicas(3)), @@ -457,7 +465,7 @@ func Test_reconcileOldMachineSetsRolloutRolling(t *testing.T) { { name: "scale down replicas when there are more replicas than minAvailable replicas, unavailable replicas are scaled down first, available replicas are scaled down when unavailable replicas are gone is not affected by replicas without machines", scaleIntent: map[string]int32{}, - md: createMD("v2", 3, withRolloutStrategy(1, 0)), + md: createMD("v2", 3, withRollingUpdateStrategy(1, 0)), newMS: createMS("ms4", "v3", 1, withStatusReplicas(1), withStatusAvailableReplicas(1)), oldMSs: []*clusterv1.MachineSet{ createMS("ms1", "v0", 4, withStatusReplicas(3), withStatusAvailableReplicas(3)), // 1 replica without machine @@ -475,7 +483,7 @@ func Test_reconcileOldMachineSetsRolloutRolling(t *testing.T) { { name: "scale down replicas when there are more replicas than minAvailable replicas, unavailable replicas are scaled down first, scale down stops before breaching minAvailable replicas", scaleIntent: map[string]int32{}, - md: createMD("v2", 3, withRolloutStrategy(1, 0)), + md: createMD("v2", 3, withRollingUpdateStrategy(1, 0)), newMS: createMS("ms4", "v3", 1, withStatusReplicas(1), withStatusAvailableReplicas(1)), oldMSs: []*clusterv1.MachineSet{ createMS("ms1", "v0", 1, withStatusReplicas(1), withStatusAvailableReplicas(1)), @@ -492,7 +500,7 @@ func Test_reconcileOldMachineSetsRolloutRolling(t *testing.T) { { name: "scale down replicas when there are more replicas than minAvailable replicas, unavailable replicas are scaled down first, scale down stops before breaching minAvailable replicas is not affected by replicas without machines", scaleIntent: map[string]int32{}, - md: createMD("v2", 3, withRolloutStrategy(1, 0)), + md: createMD("v2", 3, withRollingUpdateStrategy(1, 0)), newMS: createMS("ms4", "v3", 1, withStatusReplicas(1), withStatusAvailableReplicas(1)), oldMSs: []*clusterv1.MachineSet{ createMS("ms1", "v0", 2, withStatusReplicas(1), withStatusAvailableReplicas(1)), // 1 replica without machine @@ -510,7 +518,7 @@ func Test_reconcileOldMachineSetsRolloutRolling(t *testing.T) { { name: "scale down replicas when there are more replicas than minAvailable replicas, scale down keeps into account scale downs from a previous reconcile", scaleIntent: map[string]int32{}, - md: createMD("v2", 3, withRolloutStrategy(1, 0)), + md: createMD("v2", 3, withRollingUpdateStrategy(1, 0)), newMS: createMS("ms2", "v3", 2, withStatusReplicas(2), withStatusAvailableReplicas(2)), oldMSs: []*clusterv1.MachineSet{ createMS("ms1", "v0", 3, withStatusReplicas(4), withStatusAvailableReplicas(4)), // OldMS is scaling down from a previous reconcile @@ -525,7 +533,7 @@ func Test_reconcileOldMachineSetsRolloutRolling(t *testing.T) { scaleIntent: map[string]int32{ "ms2": 1, // newMS (ms2) has a scaling down intent from current reconcile }, - md: createMD("v2", 3, withRolloutStrategy(1, 0)), + md: createMD("v2", 3, withRollingUpdateStrategy(1, 0)), newMS: createMS("ms2", "v3", 2, withStatusReplicas(2), withStatusAvailableReplicas(2)), oldMSs: []*clusterv1.MachineSet{ createMS("ms1", "v0", 3, withStatusReplicas(3), withStatusAvailableReplicas(3)), @@ -541,7 +549,7 @@ func Test_reconcileOldMachineSetsRolloutRolling(t *testing.T) { scaleIntent: map[string]int32{ "ms1": 2, // oldMS (ms1) has a scaling down intent from current reconcile }, - md: createMD("v2", 3, withRolloutStrategy(1, 0)), + md: createMD("v2", 3, withRollingUpdateStrategy(1, 0)), newMS: createMS("ms2", "v3", 2, withStatusReplicas(2), withStatusAvailableReplicas(2)), oldMSs: []*clusterv1.MachineSet{ createMS("ms1", "v0", 3, withStatusReplicas(3), withStatusAvailableReplicas(3)), @@ -562,7 +570,7 @@ func Test_reconcileOldMachineSetsRolloutRolling(t *testing.T) { oldMSs: tt.oldMSs, scaleIntents: tt.scaleIntent, } - err := p.reconcileOldMachineSetsRolloutRolling(ctx) + err := p.reconcileOldMachineSetsRollingUpdate(ctx) g.Expect(err).ToNot(HaveOccurred()) g.Expect(p.scaleIntents).To(Equal(tt.expectScaleIntent), "unexpected scaleIntents") @@ -709,3 +717,401 @@ func Test_reconcileDeadlockBreaker(t *testing.T) { }) } } + +type rollingUpdateSequenceTestCase struct { + name string + maxSurge int32 + maxUnavailable int32 + + // currentMachineNames is the list of machines before the rollout, and provides a simplified alternative to currentScope. + // all the machines in this list are initialized as upToDate and owned by the new MS before the rollout (which is different from the new MS after the rollout). + // Please name machines as "mX" where X is a progressive number starting from 1 (do not skip numbers), + // e.g. "m1","m2","m3" + currentMachineNames []string + + // currentScope defines the current state at the beginning of the test case. + // When the test case start from a stable state (there are no previous rollout in progress), use currentMachineNames instead. + // Please name machines as "mX" where X is a progressive number starting from 1 (do not skip numbers), + // e.g. "m1","m2","m3" + // machineUID must be set to the last used number. + currentScope *rolloutScope + + // maxUnavailableBreachToleration can be used to temporarily silence MaxUnavailable breaches + // + // maxUnavailableBreachToleration: func(log *logger, i int, scope *rolloutScope, minAvailableReplicas, totAvailableReplicas int32) bool { + // if i == 5 { + // t.Log("[Toleration] tolerate minAvailable breach after scale up") + // return true + // } + // return false + // }, + maxUnavailableBreachToleration func(log *fileLogger, i int, scope *rolloutScope, minAvailableReplicas, totAvailableReplicas int32) bool + + // maxSurgeBreachToleration can be used to temporarily silence MaxSurge breaches + // (see maxUnavailableBreachToleration example) + maxSurgeBreachToleration func(log *fileLogger, i int, scope *rolloutScope, maxAllowedReplicas, totReplicas int32) bool + + // desiredMachineNames is the list of machines at the end of the rollout. + // all the machines in this list are expected to be upToDate and owned by the new MS after the rollout (which is different from the new MS before the rollout). + // if this list contains old machines names (machine names already in currentMachineNames), it implies those machine have been upgraded in places. + // if this list contains new machines names (machine names not in currentMachineNames), it implies those machines have been created during a rollout; + // please name new machines names as "mX" where X is a progressive number starting after the max number in currentMachineNames (do not skip numbers), + // e.g. desiredMachineNames "m4","m5","m6" (desired machine names after a regular rollout of a MD with currentMachineNames "m1","m2","m3") + // e.g. desiredMachineNames "m1","m2","m3" (desired machine names after rollout performed using in-place upgrade for an MD with currentMachineNames "m1","m2","m3") + desiredMachineNames []string + + // skipLogToFileAndGoldenFileCheck allows to skip storing the log to file and golden file Check. + // NOTE: this field is controlled by the test itself. + skipLogToFileAndGoldenFileCheck bool + + // name of the log to file and the golden file. + // NOTE: this field is controlled by the test itself. + logAndGoldenFileName string + + // randomControllerOrder force the tests to run controllers in random order, mimicking what happens in production. + // NOTE. We are using a pseudo randomizer, so the random order remains consistent across runs of the same groups of tests. + // NOTE: this field is controlled by the test itself. + randomControllerOrder bool + + // maxIterations defines the max number of iterations the system must attempt before assuming the logic has an issue + // in reaching the desired state. + // When the test is using default controller order, an iteration implies reconcile MD + reconcile all MS in a predictable order; + // while using randomControllerOrder the concept of iteration is less defined, but it can still be used to prevent + // the test from running indefinitely. + // NOTE: this field is controlled by the test itself. + maxIterations int + + // seed value to initialize the generator. + // NOTE: this field is controlled by the test itself. + seed int64 +} + +func Test_RollingUpdateSequences(t *testing.T) { + ctx := context.Background() + ctx = ctrl.LoggerInto(ctx, klog.Background()) + klog.SetOutput(ginkgo.GinkgoWriter) + + tests := []rollingUpdateSequenceTestCase{ + // Regular rollout (no in-place) + + { // scale out by 1 + name: "Regular rollout, 3 Replicas, maxSurge 1, maxUnavailable 0", + maxSurge: 1, + maxUnavailable: 0, + currentMachineNames: []string{"m1", "m2", "m3"}, + desiredMachineNames: []string{"m4", "m5", "m6"}, + }, + { // scale in by 1 + name: "Regular rollout, 3 Replicas, maxSurge 0, maxUnavailable 1", + maxSurge: 0, + maxUnavailable: 1, + currentMachineNames: []string{"m1", "m2", "m3"}, + desiredMachineNames: []string{"m4", "m5", "m6"}, + }, + { // scale out by 3, scale in by 1 (maxSurge > maxUnavailable) + name: "Regular rollout, 6 Replicas, maxSurge 3, maxUnavailable 1", + maxSurge: 3, + maxUnavailable: 1, + currentMachineNames: []string{"m1", "m2", "m3", "m4", "m5", "m6"}, + desiredMachineNames: []string{"m7", "m8", "m9", "m10", "m11", "m12"}, + }, + { // scale out by 1, scale in by 3 (maxSurge < maxUnavailable) + name: "Regular rollout, 6 Replicas, maxSurge 1, maxUnavailable 3", + maxSurge: 1, + maxUnavailable: 3, + currentMachineNames: []string{"m1", "m2", "m3", "m4", "m5", "m6"}, + desiredMachineNames: []string{"m7", "m8", "m9", "m10", "m11", "m12"}, + }, + { // scale out by 10 (maxSurge >= replicas) + name: "Regular rollout, 6 Replicas, maxSurge 10, maxUnavailable 0", + maxSurge: 10, + maxUnavailable: 0, + currentMachineNames: []string{"m1", "m2", "m3", "m4", "m5", "m6"}, + desiredMachineNames: []string{"m7", "m8", "m9", "m10", "m11", "m12"}, + }, + { // scale in by 10 (maxUnavailable >= replicas) + name: "Regular rollout, 6 Replicas, maxSurge 0, maxUnavailable 10", + maxSurge: 0, + maxUnavailable: 10, + currentMachineNames: []string{"m1", "m2", "m3", "m4", "m5", "m6"}, + desiredMachineNames: []string{"m7", "m8", "m9", "m10", "m11", "m12"}, + }, + { // scale out by 3, scale in by 1 (maxSurge > maxUnavailable) + scale up machine deployment in the middle + name: "Regular rollout, 6 Replicas, maxSurge 3, maxUnavailable 1, scale up to 12", + maxSurge: 3, + maxUnavailable: 1, + currentScope: &rolloutScope{ // Manually providing a scope simulating a MD originally with 6 replica in the middle of a rollout, with 3 machines already created in the newMS and 3 still on the oldMS, and then MD scaled up to 12. + machineDeployment: createMD("v2", 12, withRollingUpdateStrategy(3, 1)), + machineSets: []*clusterv1.MachineSet{ + createMS("ms1", "v1", 3), + createMS("ms2", "v2", 3), + }, + machineSetMachines: map[string][]*clusterv1.Machine{ + "ms1": { + // "m1", "m2", "m3" already deleted + createM("m4", "ms1", "v1"), + createM("m5", "ms1", "v1"), + createM("m6", "ms1", "v1"), + }, + "ms2": { + createM("m7", "ms2", "v2"), + createM("m8", "ms2", "v2"), + createM("m9", "ms2", "v2"), + }, + }, + machineUID: 9, + }, + desiredMachineNames: []string{"m7", "m8", "m9", "m10", "m11", "m12", "m13", "m14", "m15", "m16", "m17", "m18"}, + maxUnavailableBreachToleration: maxUnavailableBreachToleration(), + }, + { // scale out by 3, scale in by 1 (maxSurge > maxUnavailable) + scale down machine deployment in the middle + name: "Regular rollout, 12 Replicas, maxSurge 3, maxUnavailable 1, scale down to 6", + maxSurge: 3, + maxUnavailable: 1, + currentScope: &rolloutScope{ // Manually providing a scope simulating a MD originally with 12 replica in the middle of a rollout, with 3 machines already created in the newMS and 9 still on the oldMS, and then MD scaled down to 6. + machineDeployment: createMD("v2", 6, withRollingUpdateStrategy(3, 1)), + machineSets: []*clusterv1.MachineSet{ + createMS("ms1", "v1", 9), + createMS("ms2", "v2", 3), + }, + machineSetMachines: map[string][]*clusterv1.Machine{ + "ms1": { + // "m1", "m2", "m3" already deleted + createM("m4", "ms1", "v1"), + createM("m5", "ms1", "v1"), + createM("m6", "ms1", "v1"), + createM("m7", "ms1", "v1"), + createM("m8", "ms1", "v1"), + createM("m9", "ms1", "v1"), + createM("m10", "ms1", "v1"), + createM("m11", "ms1", "v1"), + createM("m12", "ms1", "v1"), + }, + "ms2": { + createM("m13", "ms2", "v2"), + createM("m14", "ms2", "v2"), + createM("m15", "ms2", "v2"), + }, + }, + machineUID: 15, + }, + desiredMachineNames: []string{"m13", "m14", "m15", "m16", "m17", "m18"}, + maxSurgeBreachToleration: maxSurgeToleration(), + }, + { // scale out by 3, scale in by 1 (maxSurge > maxUnavailable) + change spec in the middle + name: "Regular rollout, 6 Replicas, maxSurge 3, maxUnavailable 1, change spec", + maxSurge: 3, + maxUnavailable: 1, + currentScope: &rolloutScope{ // Manually providing a scope simulating a MD with 6 replica in the middle of a rollout, with 3 machines already created in the newMS and 3 still on the oldMS, and then MD spec is changed. + machineDeployment: createMD("v3", 6, withRollingUpdateStrategy(3, 1)), + machineSets: []*clusterv1.MachineSet{ + createMS("ms1", "v1", 3), + createMS("ms2", "v2", 3), + createMS("ms3", "v3", 0), + }, + machineSetMachines: map[string][]*clusterv1.Machine{ + "ms1": { + // "m1", "m2", "m3" already deleted + createM("m4", "ms1", "v1"), + createM("m5", "ms1", "v1"), + createM("m6", "ms1", "v1"), + }, + "ms2": { + createM("m7", "ms2", "v2"), + createM("m8", "ms2", "v2"), + createM("m9", "ms2", "v2"), + }, + }, + machineUID: 9, + }, + desiredMachineNames: []string{"m10", "m11", "m12", "m13", "m14", "m15"}, // NOTE: Machines created before the spec change are deleted + }, + } + + testWithPredictableReconcileOrder := true + testWithRandomReconcileOrderFromConstantSeed := true + testWithRandomReconcileOrderFromRandomSeed := true + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + name := tt.name + + if testWithPredictableReconcileOrder { + tt.maxIterations = 50 + tt.randomControllerOrder = false + tt.logAndGoldenFileName = strings.ToLower(tt.name) + t.Run("default", func(t *testing.T) { + runRollingUpdateTestCase(ctx, t, tt) + }) + } + + if testWithRandomReconcileOrderFromConstantSeed { + tt.maxIterations = 70 + tt.name = fmt.Sprintf("%s, random(0)", name) + tt.randomControllerOrder = true + tt.seed = 0 + tt.logAndGoldenFileName = strings.ToLower(tt.name) + t.Run("random(0)", func(t *testing.T) { + runRollingUpdateTestCase(ctx, t, tt) + }) + } + + if testWithRandomReconcileOrderFromRandomSeed { + for range 100 { + tt.maxIterations = 150 + tt.seed = time.Now().UnixNano() + tt.name = fmt.Sprintf("%s, random(%d)", name, tt.seed) + tt.randomControllerOrder = true + tt.skipLogToFileAndGoldenFileCheck = true + t.Run(fmt.Sprintf("random(%d)", tt.seed), func(t *testing.T) { + runRollingUpdateTestCase(ctx, t, tt) + }) + } + } + }) + } +} + +func runRollingUpdateTestCase(ctx context.Context, t *testing.T, tt rollingUpdateSequenceTestCase) { + t.Helper() + g := NewWithT(t) + + rng := rand.New(rand.NewSource(tt.seed)) //nolint:gosec // it is ok to use a weak randomizer here + fLogger := newFileLogger(t, tt.name, fmt.Sprintf("testdata/rollingupdate/%s", tt.logAndGoldenFileName)) + // uncomment this line to automatically generate/update golden files: fLogger.writeGoldenFile = true + + // Init current and desired state from test case + current := tt.currentScope.Clone() + if current == nil { + current = initCurrentRolloutScope(tt.currentMachineNames, withRollingUpdateStrategy(tt.maxSurge, tt.maxUnavailable)) + } + desired := computeDesiredRolloutScope(current, tt.desiredMachineNames) + + // Log initial state + fLogger.Logf("[Test] Initial state\n%s", current) + random := "" + if tt.randomControllerOrder { + random = fmt.Sprintf(", random(%d)", tt.seed) + } + fLogger.Logf("[Test] Rollout %d replicas, MaxSurge=%d, MaxUnavailable=%d%s\n", len(current.machines()), tt.maxSurge, tt.maxUnavailable, random) + i := 1 + maxIterations := tt.maxIterations + for { + taskList := getTaskListRollingUpdate(current) + taskCount := len(taskList) + taskOrder := defaultTaskOrder(taskCount) + if tt.randomControllerOrder { + taskOrder = randomTaskOrder(taskCount, rng) + } + for _, taskID := range taskOrder { + task := taskList[taskID] + if task == "md" { + fLogger.Logf("[MD controller] Iteration %d, Reconcile md", i) + fLogger.Logf("[MD controller] - Input to rollout planner\n%s", current) + + // Running a small subset of MD reconcile (the rollout logic and a bit of setReplicas) + p := newRolloutPlanner() + p.md = current.machineDeployment + p.newMS = current.newMS() + p.oldMSs = current.oldMSs() + + err := p.planRollingUpdate(ctx) + g.Expect(err).ToNot(HaveOccurred()) + + // Apply changes. + for _, ms := range current.machineSets { + if scaleIntent, ok := p.scaleIntents[ms.Name]; ok { + ms.Spec.Replicas = ptr.To(scaleIntent) + } + } + + // Running a small subset of setReplicas (we don't want to run the full func to avoid unnecessary noise on the test) + current.machineDeployment.Status.Replicas = mdutil.GetActualReplicaCountForMachineSets(current.machineSets) + current.machineDeployment.Status.AvailableReplicas = mdutil.GetAvailableReplicaCountForMachineSets(current.machineSets) + + // Log state after this reconcile + fLogger.Logf("[MD controller] - Result of rollout planner\n%s", current) + + // Check we are not breaching rollout constraints + minAvailableReplicas := ptr.Deref(current.machineDeployment.Spec.Replicas, 0) - mdutil.MaxUnavailable(*current.machineDeployment) + totAvailableReplicas := ptr.Deref(current.machineDeployment.Status.AvailableReplicas, 0) + if totAvailableReplicas < minAvailableReplicas { + tolerateBreach := false + if tt.maxUnavailableBreachToleration != nil { + tolerateBreach = tt.maxUnavailableBreachToleration(fLogger, i, current, minAvailableReplicas, totAvailableReplicas) + } + if !tolerateBreach { + g.Expect(totAvailableReplicas).To(BeNumerically(">=", minAvailableReplicas), "totAvailable machines is less than md.spec.replicas - maxUnavailable") + } + } + + maxAllowedReplicas := ptr.Deref(current.machineDeployment.Spec.Replicas, 0) + mdutil.MaxSurge(*current.machineDeployment) + totReplicas := mdutil.TotalMachineSetsReplicaSum(current.machineSets) + if totReplicas > maxAllowedReplicas { + tolerateBreach := false + if tt.maxSurgeBreachToleration != nil { + tolerateBreach = tt.maxSurgeBreachToleration(fLogger, i, current, maxAllowedReplicas, totReplicas) + } + if !tolerateBreach { + g.Expect(totReplicas).To(BeNumerically("<=", maxAllowedReplicas), "totReplicas machines is greater than md.spec.replicas + maxSurge") + } + } + } + + // Run mutators faking other controllers + for _, ms := range current.machineSets { + if ms.Name == task { + fLogger.Logf("[MS controller] Iteration %d, Reconcile %s, %s", i, ms.Name, msLog(ms, current.machineSetMachines[ms.Name])) + machineSetControllerMutator(fLogger, ms, current) + break + } + } + } + + // Check if we are at the desired state + if current.Equal(desired) { + fLogger.Logf("[Test] Final state\n%s", current) + break + } + + // Safeguard for infinite reconcile + i++ + if i > maxIterations { + // NOTE: the following can be used to set a breakpoint for debugging why the system is not reaching desired state after maxIterations (to check what is not yet equal) + current.Equal(desired) + // Log desired state we never reached + fLogger.Logf("[Test] Desired state\n%s", desired) + g.Fail(fmt.Sprintf("Failed to reach desired state in %d iterations", maxIterations)) + } + } + + if !tt.skipLogToFileAndGoldenFileCheck { + currentLog, goldenLog, err := fLogger.WriteLogAndCompareWithGoldenFile() + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(currentLog).To(Equal(goldenLog), "current test case log and golden test case log are different\n%s", cmp.Diff(currentLog, goldenLog)) + } +} + +func maxUnavailableBreachToleration() func(log *fileLogger, _ int, _ *rolloutScope, _, _ int32) bool { + return func(log *fileLogger, _ int, _ *rolloutScope, _, _ int32) bool { + log.Logf("[Toleration] tolerate maxUnavailable breach") + return true + } +} + +func maxSurgeToleration() func(log *fileLogger, _ int, _ *rolloutScope, _, _ int32) bool { + return func(log *fileLogger, _ int, _ *rolloutScope, _, _ int32) bool { + log.Logf("[Toleration] tolerate maxSurge breach") + return true + } +} + +func getTaskListRollingUpdate(current *rolloutScope) []string { + taskList := make([]string, 0) + taskList = append(taskList, "md") + for _, ms := range current.machineSets { + taskList = append(taskList, ms.Name) + } + taskList = append(taskList, fmt.Sprintf("ms%d", len(current.machineSets)+1)) // r the MachineSet that might be created when reconciling md + return taskList +} diff --git a/internal/controllers/machinedeployment/machinedeployment_rollout_sequence_test.go b/internal/controllers/machinedeployment/machinedeployment_rollout_sequence_test.go index cec7073bdb69..b08ef97e4a81 100644 --- a/internal/controllers/machinedeployment/machinedeployment_rollout_sequence_test.go +++ b/internal/controllers/machinedeployment/machinedeployment_rollout_sequence_test.go @@ -17,7 +17,6 @@ limitations under the License. package machinedeployment import ( - "context" "fmt" "math/rand" "os" @@ -25,393 +24,31 @@ import ( "strconv" "strings" "testing" - "time" - "github.com/google/go-cmp/cmp" - "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/klog/v2" "k8s.io/utils/ptr" - ctrl "sigs.k8s.io/controller-runtime" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" "sigs.k8s.io/cluster-api/internal/controllers/machinedeployment/mdutil" ) -type rolloutRollingSequenceTestCase struct { - name string - maxSurge int32 - maxUnavailable int32 - - // currentMachineNames is the list of machines before the rollout, and provides a simplified alternative to currentScope. - // all the machines in this list are initialized as upToDate and owned by the new MS before the rollout (which is different from the new MS after the rollout). - // Please name machines as "mX" where X is a progressive number starting from 1 (do not skip numbers), - // e.g. "m1","m2","m3" - currentMachineNames []string - - // currentScope defines the current state at the beginning of the test case. - // When the test case start from a stable state (there are no previous rollout in progress), use currentMachineNames instead. - // Please name machines as "mX" where X is a progressive number starting from 1 (do not skip numbers), - // e.g. "m1","m2","m3" - // machineUID must be set to the last used number. - currentScope *rolloutScope - - // maxUnavailableBreachToleration can be used to temporarily silence MaxUnavailable breaches - // - // maxUnavailableBreachToleration: func(log *logger, i int, scope *rolloutScope, minAvailableReplicas, totAvailableReplicas int32) bool { - // if i == 5 { - // t.Log("[Toleration] tolerate minAvailable breach after scale up") - // return true - // } - // return false - // }, - maxUnavailableBreachToleration func(log *fileLogger, i int, scope *rolloutScope, minAvailableReplicas, totAvailableReplicas int32) bool - - // maxSurgeBreachToleration can be used to temporarily silence MaxSurge breaches - // (see maxUnavailableBreachToleration example) - maxSurgeBreachToleration func(log *fileLogger, i int, scope *rolloutScope, maxAllowedReplicas, totReplicas int32) bool - - // desiredMachineNames is the list of machines at the end of the rollout. - // all the machines in this list are expected to be upToDate and owned by the new MS after the rollout (which is different from the new MS before the rollout). - // if this list contains old machines names (machine names already in currentMachineNames), it implies those machine have been upgraded in places. - // if this list contains new machines names (machine names not in currentMachineNames), it implies those machines have been created during a rollout; - // please name new machines names as "mX" where X is a progressive number starting after the max number in currentMachineNames (do not skip numbers), - // e.g. desiredMachineNames "m4","m5","m6" (desired machine names after a regular rollout of a MD with currentMachineNames "m1","m2","m3") - // e.g. desiredMachineNames "m1","m2","m3" (desired machine names after rollout performed using in-place upgrade for an MD with currentMachineNames "m1","m2","m3") - desiredMachineNames []string - - // skipLogToFileAndGoldenFileCheck allows to skip storing the log to file and golden file Check. - // NOTE: this field is controlled by the test itself. - skipLogToFileAndGoldenFileCheck bool - - // name of the log to file and the golden file. - // NOTE: this field is controlled by the test itself. - logAndGoldenFileName string - - // randomControllerOrder force the tests to run controllers in random order, mimicking what happens in production. - // NOTE. We are using a pseudo randomizer, so the random order remains consistent across runs of the same groups of tests. - // NOTE: this field is controlled by the test itself. - randomControllerOrder bool - - // maxIterations defines the max number of iterations the system must attempt before assuming the logic has an issue - // in reaching the desired state. - // When the test is using default controller order, an iteration implies reconcile MD + reconcile all MS in a predictable order; - // while using randomControllerOrder the concept of iteration is less defined, but it can still be used to prevent - // the test from running indefinitely. - // NOTE: this field is controlled by the test itself. - maxIterations int - - // seed value to initialize the generator. - // NOTE: this field is controlled by the test itself. - seed int64 -} - -func Test_rolloutRollingSequences(t *testing.T) { - ctx := context.Background() - ctx = ctrl.LoggerInto(ctx, klog.Background()) - klog.SetOutput(ginkgo.GinkgoWriter) - - tests := []rolloutRollingSequenceTestCase{ - // Regular rollout (no in-place) - - { // scale out by 1 - name: "Regular rollout, 3 Replicas, maxSurge 1, maxUnavailable 0", - maxSurge: 1, - maxUnavailable: 0, - currentMachineNames: []string{"m1", "m2", "m3"}, - desiredMachineNames: []string{"m4", "m5", "m6"}, - }, - { // scale in by 1 - name: "Regular rollout, 3 Replicas, maxSurge 0, maxUnavailable 1", - maxSurge: 0, - maxUnavailable: 1, - currentMachineNames: []string{"m1", "m2", "m3"}, - desiredMachineNames: []string{"m4", "m5", "m6"}, - }, - { // scale out by 3, scale in by 1 (maxSurge > maxUnavailable) - name: "Regular rollout, 6 Replicas, maxSurge 3, maxUnavailable 1", - maxSurge: 3, - maxUnavailable: 1, - currentMachineNames: []string{"m1", "m2", "m3", "m4", "m5", "m6"}, - desiredMachineNames: []string{"m7", "m8", "m9", "m10", "m11", "m12"}, - }, - { // scale out by 1, scale in by 3 (maxSurge < maxUnavailable) - name: "Regular rollout, 6 Replicas, maxSurge 1, maxUnavailable 3", - maxSurge: 1, - maxUnavailable: 3, - currentMachineNames: []string{"m1", "m2", "m3", "m4", "m5", "m6"}, - desiredMachineNames: []string{"m7", "m8", "m9", "m10", "m11", "m12"}, - }, - { // scale out by 10 (maxSurge >= replicas) - name: "Regular rollout, 6 Replicas, maxSurge 10, maxUnavailable 0", - maxSurge: 10, - maxUnavailable: 0, - currentMachineNames: []string{"m1", "m2", "m3", "m4", "m5", "m6"}, - desiredMachineNames: []string{"m7", "m8", "m9", "m10", "m11", "m12"}, - }, - { // scale in by 10 (maxUnavailable >= replicas) - name: "Regular rollout, 6 Replicas, maxSurge 0, maxUnavailable 10", - maxSurge: 0, - maxUnavailable: 10, - currentMachineNames: []string{"m1", "m2", "m3", "m4", "m5", "m6"}, - desiredMachineNames: []string{"m7", "m8", "m9", "m10", "m11", "m12"}, - }, - { // scale out by 3, scale in by 1 (maxSurge > maxUnavailable) + scale up machine deployment in the middle - name: "Regular rollout, 6 Replicas, maxSurge 3, maxUnavailable 1, scale up to 12", - maxSurge: 3, - maxUnavailable: 1, - currentScope: &rolloutScope{ // Manually providing a scope simulating a MD originally with 6 replica in the middle of a rollout, with 3 machines already created in the newMS and 3 still on the oldMS, and then MD scaled up to 12. - machineDeployment: createMD("v2", 12, withRolloutStrategy(3, 1)), - machineSets: []*clusterv1.MachineSet{ - createMS("ms1", "v1", 3), - createMS("ms2", "v2", 3), - }, - machineSetMachines: map[string][]*clusterv1.Machine{ - "ms1": { - // "m1", "m2", "m3" already deleted - createM("m4", "ms1", "v1"), - createM("m5", "ms1", "v1"), - createM("m6", "ms1", "v1"), - }, - "ms2": { - createM("m7", "ms2", "v2"), - createM("m8", "ms2", "v2"), - createM("m9", "ms2", "v2"), - }, - }, - machineUID: 9, - }, - desiredMachineNames: []string{"m7", "m8", "m9", "m10", "m11", "m12", "m13", "m14", "m15", "m16", "m17", "m18"}, - maxUnavailableBreachToleration: maxUnavailableBreachToleration(), - }, - { // scale out by 3, scale in by 1 (maxSurge > maxUnavailable) + scale down machine deployment in the middle - name: "Regular rollout, 12 Replicas, maxSurge 3, maxUnavailable 1, scale down to 6", - maxSurge: 3, - maxUnavailable: 1, - currentScope: &rolloutScope{ // Manually providing a scope simulating a MD originally with 12 replica in the middle of a rollout, with 3 machines already created in the newMS and 9 still on the oldMS, and then MD scaled down to 6. - machineDeployment: createMD("v2", 6, withRolloutStrategy(3, 1)), - machineSets: []*clusterv1.MachineSet{ - createMS("ms1", "v1", 9), - createMS("ms2", "v2", 3), - }, - machineSetMachines: map[string][]*clusterv1.Machine{ - "ms1": { - // "m1", "m2", "m3" already deleted - createM("m4", "ms1", "v1"), - createM("m5", "ms1", "v1"), - createM("m6", "ms1", "v1"), - createM("m7", "ms1", "v1"), - createM("m8", "ms1", "v1"), - createM("m9", "ms1", "v1"), - createM("m10", "ms1", "v1"), - createM("m11", "ms1", "v1"), - createM("m12", "ms1", "v1"), - }, - "ms2": { - createM("m13", "ms2", "v2"), - createM("m14", "ms2", "v2"), - createM("m15", "ms2", "v2"), - }, - }, - machineUID: 15, - }, - desiredMachineNames: []string{"m13", "m14", "m15", "m16", "m17", "m18"}, - maxSurgeBreachToleration: maxSurgeToleration(), - }, - { // scale out by 3, scale in by 1 (maxSurge > maxUnavailable) + change spec in the middle - name: "Regular rollout, 6 Replicas, maxSurge 3, maxUnavailable 1, change spec", - maxSurge: 3, - maxUnavailable: 1, - currentScope: &rolloutScope{ // Manually providing a scope simulating a MD with 6 replica in the middle of a rollout, with 3 machines already created in the newMS and 3 still on the oldMS, and then MD spec is changed. - machineDeployment: createMD("v3", 6, withRolloutStrategy(3, 1)), - machineSets: []*clusterv1.MachineSet{ - createMS("ms1", "v1", 3), - createMS("ms2", "v2", 3), - createMS("ms3", "v3", 0), - }, - machineSetMachines: map[string][]*clusterv1.Machine{ - "ms1": { - // "m1", "m2", "m3" already deleted - createM("m4", "ms1", "v1"), - createM("m5", "ms1", "v1"), - createM("m6", "ms1", "v1"), - }, - "ms2": { - createM("m7", "ms2", "v2"), - createM("m8", "ms2", "v2"), - createM("m9", "ms2", "v2"), - }, - }, - machineUID: 9, - }, - desiredMachineNames: []string{"m10", "m11", "m12", "m13", "m14", "m15"}, // NOTE: Machines created before the spec change are deleted - }, - } - - testWithPredictableReconcileOrder := true - testWithRandomReconcileOrderFromConstantSeed := true - testWithRandomReconcileOrderFromRandomSeed := true - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - name := tt.name - - if testWithPredictableReconcileOrder { - tt.maxIterations = 50 - tt.randomControllerOrder = false - tt.logAndGoldenFileName = strings.ToLower(tt.name) - t.Run("default", func(t *testing.T) { - runRolloutRollingTestCase(ctx, t, tt) - }) - } - - if testWithRandomReconcileOrderFromConstantSeed { - tt.maxIterations = 70 - tt.name = fmt.Sprintf("%s, random(0)", name) - tt.randomControllerOrder = true - tt.seed = 0 - tt.logAndGoldenFileName = strings.ToLower(tt.name) - t.Run("random(0)", func(t *testing.T) { - runRolloutRollingTestCase(ctx, t, tt) - }) - } - - if testWithRandomReconcileOrderFromRandomSeed { - for range 100 { - tt.maxIterations = 150 - tt.seed = time.Now().UnixNano() - tt.name = fmt.Sprintf("%s, random(%d)", name, tt.seed) - tt.randomControllerOrder = true - tt.skipLogToFileAndGoldenFileCheck = true - t.Run(fmt.Sprintf("random(%d)", tt.seed), func(t *testing.T) { - runRolloutRollingTestCase(ctx, t, tt) - }) - } - } - }) +// machineControllerMutator fakes a small part of the Machine controller, just what is required for the rollout to progress. +func machineControllerMutator(log *fileLogger, m *clusterv1.Machine, scope *rolloutScope) { + if m.DeletionTimestamp.IsZero() { + return } -} - -func runRolloutRollingTestCase(ctx context.Context, t *testing.T, tt rolloutRollingSequenceTestCase) { - t.Helper() - g := NewWithT(t) - rng := rand.New(rand.NewSource(tt.seed)) //nolint:gosec // it is ok to use a weak randomizer here - fLogger := newFileLogger(t, tt.name, fmt.Sprintf("testdata/%s", tt.logAndGoldenFileName)) - // uncomment this line to automatically generate/update golden files: fLogger.writeGoldenFile = true - - // Init current and desired state from test case - current := tt.currentScope.Clone() - if current == nil { - current = initCurrentRolloutScope(tt.currentMachineNames, withRolloutStrategy(tt.maxSurge, tt.maxUnavailable)) - } - desired := computeDesiredRolloutScope(current, tt.desiredMachineNames) - - // Log initial state - fLogger.Logf("[Test] Initial state\n%s", current) - random := "" - if tt.randomControllerOrder { - random = fmt.Sprintf(", random(%d)", tt.seed) - } - fLogger.Logf("[Test] Rollout %d replicas, MaxSurge=%d, MaxUnavailable=%d%s\n", len(current.machines()), tt.maxSurge, tt.maxUnavailable, random) - i := 1 - maxIterations := tt.maxIterations - for { - taskList := getTaskListRolloutRolling(current) - taskCount := len(taskList) - taskOrder := defaultTaskOrder(taskCount) - if tt.randomControllerOrder { - taskOrder = randomTaskOrder(taskCount, rng) - } - for _, taskID := range taskOrder { - task := taskList[taskID] - if task == "md" { - fLogger.Logf("[MD controller] Iteration %d, Reconcile md", i) - fLogger.Logf("[MD controller] - Input to rollout planner\n%s", current) - - // Running a small subset of MD reconcile (the rollout logic and a bit of setReplicas) - p := newRolloutPlanner() - p.md = current.machineDeployment - p.newMS = current.newMS() - p.oldMSs = current.oldMSs() - - err := p.planRolloutRolling(ctx) - g.Expect(err).ToNot(HaveOccurred()) - - // Apply changes. - for _, ms := range current.machineSets { - if scaleIntent, ok := p.scaleIntents[ms.Name]; ok { - ms.Spec.Replicas = ptr.To(scaleIntent) - } - } - - // Running a small subset of setReplicas (we don't want to run the full func to avoid unnecessary noise on the test) - current.machineDeployment.Status.Replicas = mdutil.GetActualReplicaCountForMachineSets(current.machineSets) - current.machineDeployment.Status.AvailableReplicas = mdutil.GetAvailableReplicaCountForMachineSets(current.machineSets) - - // Log state after this reconcile - fLogger.Logf("[MD controller] - Result of rollout planner\n%s", current) - - // Check we are not breaching rollout constraints - minAvailableReplicas := ptr.Deref(current.machineDeployment.Spec.Replicas, 0) - mdutil.MaxUnavailable(*current.machineDeployment) - totAvailableReplicas := ptr.Deref(current.machineDeployment.Status.AvailableReplicas, 0) - if totAvailableReplicas < minAvailableReplicas { - tolerateBreach := false - if tt.maxUnavailableBreachToleration != nil { - tolerateBreach = tt.maxUnavailableBreachToleration(fLogger, i, current, minAvailableReplicas, totAvailableReplicas) - } - if !tolerateBreach { - g.Expect(totAvailableReplicas).To(BeNumerically(">=", minAvailableReplicas), "totAvailable machines is less than md.spec.replicas - maxUnavailable") - } - } - - maxAllowedReplicas := ptr.Deref(current.machineDeployment.Spec.Replicas, 0) + mdutil.MaxSurge(*current.machineDeployment) - totReplicas := mdutil.TotalMachineSetsReplicaSum(current.machineSets) - if totReplicas > maxAllowedReplicas { - tolerateBreach := false - if tt.maxSurgeBreachToleration != nil { - tolerateBreach = tt.maxSurgeBreachToleration(fLogger, i, current, maxAllowedReplicas, totReplicas) - } - if !tolerateBreach { - g.Expect(totReplicas).To(BeNumerically("<=", maxAllowedReplicas), "totReplicas machines is greater than md.spec.replicas + maxSurge") - } - } - } - - // Run mutators faking other controllers - for _, ms := range current.machineSets { - if ms.Name == task { - fLogger.Logf("[MS controller] Iteration %d, Reconcile %s, %s", i, ms.Name, msLog(ms, current.machineSetMachines[ms.Name])) - machineSetControllerMutator(fLogger, ms, current) - break - } - } - } - - // Check if we are at the desired state - if current.Equal(desired) { - fLogger.Logf("[Test] Final state\n%s", current) - break - } - - // Safeguard for infinite reconcile - i++ - if i > maxIterations { - // NOTE: the following can be used to set a breakpoint for debugging why the system is not reaching desired state after maxIterations (to check what is not yet equal) - current.Equal(desired) - // Log desired state we never reached - fLogger.Logf("[Test] Desired state\n%s", desired) - g.Fail(fmt.Sprintf("Failed to reach desired state in %d iterations", maxIterations)) + log.Logf("[M controller] - %s finalizer removed", m.Name) + ms := m.OwnerReferences[0].Name + machinesSetMachines := []*clusterv1.Machine{} + for _, mx := range scope.machineSetMachines[ms] { + if mx.Name == m.Name { + continue } + machinesSetMachines = append(machinesSetMachines, mx) } - - if !tt.skipLogToFileAndGoldenFileCheck { - currentLog, goldenLog, err := fLogger.WriteLogAndCompareWithGoldenFile() - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(currentLog).To(Equal(goldenLog), "current test case log and golden test case log are different\n%s", cmp.Diff(currentLog, goldenLog)) - } + scope.machineSetMachines[ms] = machinesSetMachines } // machineSetControllerMutator fakes a small part of the MachineSet controller, just what is required for the rollout to progress. @@ -427,30 +64,33 @@ func machineSetControllerMutator(log *fileLogger, ms *clusterv1.MachineSet, scop // if too few machines, create missing machine. // new machines are created with a predictable name, so it is easier to write test case and validate rollout sequences. // e.g. if the cluster is initialized with m1, m2, m3, new machines will be m4, m5, m6 - machinesToAdd := ptr.Deref(ms.Spec.Replicas, 0) - ptr.Deref(ms.Status.Replicas, 0) - if machinesToAdd > 0 { - machinesAdded := []string{} - for range machinesToAdd { - machineName := fmt.Sprintf("m%d", scope.GetNextMachineUID()) - scope.machineSetMachines[ms.Name] = append(scope.machineSetMachines[ms.Name], - &clusterv1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Name: machineName, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: clusterv1.GroupVersion.String(), - Kind: "MachineSet", - Name: ms.Name, - Controller: ptr.To(true), + if _, ok := ms.Annotations[clusterv1.DisableMachineCreateAnnotation]; !ok { + machinesToAdd := ptr.Deref(ms.Spec.Replicas, 0) - ptr.Deref(ms.Status.Replicas, 0) + if machinesToAdd > 0 { + machinesAdded := []string{} + for range machinesToAdd { + machineName := fmt.Sprintf("m%d", scope.GetNextMachineUID()) + scope.machineSetMachines[ms.Name] = append(scope.machineSetMachines[ms.Name], + &clusterv1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: machineName, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: clusterv1.GroupVersion.String(), + Kind: "MachineSet", + Name: ms.Name, + Controller: ptr.To(true), + }, }, }, + Spec: *ms.Spec.Template.Spec.DeepCopy(), }, - }, - ) - machinesAdded = append(machinesAdded, machineName) - } + ) + machinesAdded = append(machinesAdded, machineName) + } - log.Logf("[MS controller] - %s scale up to %d/%[2]d replicas (%s created)", ms.Name, ptr.Deref(ms.Spec.Replicas, 0), strings.Join(machinesAdded, ",")) + log.Logf("[MS controller] - %s scale up to %d/%[2]d replicas (%s created)", ms.Name, ptr.Deref(ms.Spec.Replicas, 0), strings.Join(machinesAdded, ",")) + } } // if too many replicas, delete exceeding machines. @@ -818,30 +458,6 @@ func sortMachineSetMachines(machines []*clusterv1.Machine) { }) } -func maxUnavailableBreachToleration() func(log *fileLogger, _ int, _ *rolloutScope, _, _ int32) bool { - return func(log *fileLogger, _ int, _ *rolloutScope, _, _ int32) bool { - log.Logf("[Toleration] tolerate maxUnavailable breach") - return true - } -} - -func maxSurgeToleration() func(log *fileLogger, _ int, _ *rolloutScope, _, _ int32) bool { - return func(log *fileLogger, _ int, _ *rolloutScope, _, _ int32) bool { - log.Logf("[Toleration] tolerate maxSurge breach") - return true - } -} - -func getTaskListRolloutRolling(current *rolloutScope) []string { - taskList := make([]string, 0) - taskList = append(taskList, "md") - for _, ms := range current.machineSets { - taskList = append(taskList, ms.Name) - } - taskList = append(taskList, fmt.Sprintf("ms%d", len(current.machineSets)+1)) // r the MachineSet that might be created when reconciling md - return taskList -} - // default task order ensure the controllers are run in a consistent and predictable way: md, ms1, ms2 and so on. func defaultTaskOrder(taskCount int) []int { taskOrder := []int{} @@ -873,7 +489,7 @@ func randomTaskOrder(taskCount int, rng *rand.Rand) []int { type machineDeploymentOption func(md *clusterv1.MachineDeployment) -func withRolloutStrategy(maxSurge, maxUnavailable int32) func(md *clusterv1.MachineDeployment) { +func withRollingUpdateStrategy(maxSurge, maxUnavailable int32) func(md *clusterv1.MachineDeployment) { return func(md *clusterv1.MachineDeployment) { md.Spec.Rollout.Strategy = clusterv1.MachineDeploymentRolloutStrategy{ Type: clusterv1.RollingUpdateMachineDeploymentStrategyType, @@ -885,6 +501,14 @@ func withRolloutStrategy(maxSurge, maxUnavailable int32) func(md *clusterv1.Mach } } +func withOnDeleteStrategy() func(md *clusterv1.MachineDeployment) { + return func(md *clusterv1.MachineDeployment) { + md.Spec.Rollout.Strategy = clusterv1.MachineDeploymentRolloutStrategy{ + Type: clusterv1.OnDeleteMachineDeploymentStrategyType, + } + } +} + func createMD(failureDomain string, replicas int32, options ...machineDeploymentOption) *clusterv1.MachineDeployment { md := &clusterv1.MachineDeployment{ ObjectMeta: metav1.ObjectMeta{Name: "md"}, diff --git a/internal/controllers/machinedeployment/testdata/ondelete/12 replicas, maxuserunavailable 1, scale down to 6, random(0).test.log.golden b/internal/controllers/machinedeployment/testdata/ondelete/12 replicas, maxuserunavailable 1, scale down to 6, random(0).test.log.golden new file mode 100644 index 000000000000..909841973d2d --- /dev/null +++ b/internal/controllers/machinedeployment/testdata/ondelete/12 replicas, maxuserunavailable 1, scale down to 6, random(0).test.log.golden @@ -0,0 +1,287 @@ +## 12 Replicas, maxUserUnavailable 1, scale down to 6, random(0) + +[Test] Initial state + md, 6/6 replicas + - ms1, 9/9 replicas (m4,m5,m6,m7,m8,m9,m10,m11,m12) + - ms2, 3/3 replicas (m13,m14,m15) +[Test] Rollout 12 replicas, onDeleteStrategy, random(0) +[M controller] Iteration 1, Reconcile m5 +[M controller] Iteration 1, Reconcile m4 +[M controller] Iteration 1, Reconcile m13 +[MD controller] Iteration 1, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 9/9 replicas (m4,m5,m6,m7,m8,m9,m10,m11,m12) + - ms2, 3/3 replicas (m13,m14,m15) +[MD controller] - Result of rollout planner + md, 12/6 replicas + - ms1, 9/3 replicas (m4,m5,m6,m7,m8,m9,m10,m11,m12) + - ms2, 3/3 replicas (m13,m14,m15) +[MS controller] Iteration 1, Reconcile ms1, 9/3 replicas (m4,m5,m6,m7,m8,m9,m10,m11,m12) +[MS controller] - ms1 scale down to 3/3 replicas (m4,m5,m6,m7,m8,m9 deleted) +[M controller] Iteration 1, Reconcile m12 +[M controller] Iteration 1, Reconcile m10 +[MS controller] Iteration 1, Reconcile ms2, 3/3 replicas (m13,m14,m15) +[M controller] Iteration 2, Reconcile m13 +[M controller] Iteration 2, Reconcile m11 +[M controller] Iteration 2, Reconcile m10 +[M controller] Iteration 2, Reconcile m13 +[M controller] Iteration 2, Reconcile m10 +[M controller] Iteration 2, Reconcile m12 +[MS controller] Iteration 2, Reconcile ms2, 3/3 replicas (m13,m14,m15) +[MS controller] Iteration 3, Reconcile ms1, 3/3 replicas (m10,m11,m12) +[M controller] Iteration 3, Reconcile m14 +[M controller] Iteration 3, Reconcile m14 +[M controller] Iteration 3, Reconcile m13 +[M controller] Iteration 3, Reconcile m10 +[M controller] Iteration 3, Reconcile m10 +[MS controller] Iteration 3, Reconcile ms2, 3/3 replicas (m13,m14,m15) +[M controller] Iteration 4, Reconcile m14 +[MS controller] Iteration 4, Reconcile ms2, 3/3 replicas (m13,m14,m15) +[M controller] Iteration 4, Reconcile m13 +[M controller] Iteration 4, Reconcile m11 +[M controller] Iteration 4, Reconcile m10 +[M controller] Iteration 4, Reconcile m12 +[MS controller] Iteration 4, Reconcile ms1, 3/3 replicas (m10,m11,m12) +[M controller] Iteration 4, Reconcile m15 +[MD controller] Iteration 4, Reconcile md +[MD controller] - Input to rollout planner + md, 12/6 replicas + - ms1, 3/3 replicas (m10,m11,m12) + - ms2, 3/3 replicas (m13,m14,m15) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m10,m11,m12) + - ms2, 3/3 replicas (m13,m14,m15) +[MD controller] Iteration 5, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m10,m11,m12) + - ms2, 3/3 replicas (m13,m14,m15) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m10,m11,m12) + - ms2, 3/3 replicas (m13,m14,m15) +[M controller] Iteration 5, Reconcile m15 +[M controller] Iteration 5, Reconcile m11 +[M controller] Iteration 5, Reconcile m10 +[M controller] Iteration 5, Reconcile m15 +[User] Iteration 5, Deleting machine m12 +[M controller] Iteration 5, Reconcile m13 +[MS controller] Iteration 5, Reconcile ms1, 3/3 replicas (m10,m11,m12) +[M controller] Iteration 5, Reconcile m11 +[M controller] Iteration 6, Reconcile m11 +[MD controller] Iteration 6, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m10,m11,m12) + - ms2, 3/3 replicas (m13,m14,m15) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m10,m11,m12) + - ms2, 3/3 replicas (m13,m14,m15) +[MS controller] Iteration 6, Reconcile ms1, 3/3 replicas (m10,m11,m12) +[MS controller] Iteration 6, Reconcile ms2, 3/3 replicas (m13,m14,m15) +[M controller] Iteration 6, Reconcile m15 +[M controller] Iteration 6, Reconcile m14 +[M controller] Iteration 6, Reconcile m10 +[MS controller] Iteration 6, Reconcile ms1, 3/3 replicas (m10,m11,m12) +[M controller] Iteration 6, Reconcile m12 +[M controller] - m12 finalizer removed +[MD controller] Iteration 7, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m10,m11) + - ms2, 3/3 replicas (m13,m14,m15) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m10,m11) + - ms2, 3/3 replicas (m13,m14,m15) +[MS controller] Iteration 7, Reconcile ms1, 3/3 replicas (m10,m11) +[M controller] Iteration 7, Reconcile m14 +[M controller] Iteration 7, Reconcile m15 +[M controller] Iteration 7, Reconcile m11 +[M controller] Iteration 7, Reconcile m13 +[MD controller] Iteration 8, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 2/3 replicas (m10,m11) + - ms2, 3/3 replicas (m13,m14,m15) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m10,m11) + - ms2, 3/3 replicas (m13,m14,m15) +[M controller] Iteration 8, Reconcile m10 +[MS controller] Iteration 8, Reconcile ms2, 3/3 replicas (m13,m14,m15) +[M controller] Iteration 8, Reconcile m15 +[MD controller] Iteration 9, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m10,m11) + - ms2, 3/3 replicas (m13,m14,m15) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m10,m11) + - ms2, 3/4 replicas (m13,m14,m15) +[M controller] Iteration 9, Reconcile m15 +[MS controller] Iteration 9, Reconcile ms1, 2/2 replicas (m10,m11) +[M controller] Iteration 9, Reconcile m13 +[MS controller] Iteration 9, Reconcile ms2, 3/4 replicas (m13,m14,m15) +[MS controller] - ms2 scale up to 4/4 replicas (m16 created) +[M controller] Iteration 9, Reconcile m11 +[MS controller] Iteration 10, Reconcile ms1, 2/2 replicas (m10,m11) +[M controller] Iteration 10, Reconcile m10 +[MS controller] Iteration 10, Reconcile ms1, 2/2 replicas (m10,m11) +[M controller] Iteration 10, Reconcile m14 +[User] Iteration 10, Deleting machine m11 +[MD controller] Iteration 10, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m10,m11) + - ms2, 4/4 replicas (m13,m14,m15,m16) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m10,m11) + - ms2, 4/4 replicas (m13,m14,m15,m16) +[MS controller] Iteration 11, Reconcile ms1, 2/2 replicas (m10,m11) +[M controller] Iteration 11, Reconcile m14 +[M controller] Iteration 11, Reconcile m16 +[M controller] Iteration 11, Reconcile m13 +[MS controller] Iteration 11, Reconcile ms2, 4/4 replicas (m13,m14,m15,m16) +[MS controller] Iteration 11, Reconcile ms1, 2/2 replicas (m10,m11) +[M controller] Iteration 11, Reconcile m10 +[M controller] Iteration 11, Reconcile m11 +[M controller] - m11 finalizer removed +[M controller] Iteration 11, Reconcile m15 +[MD controller] Iteration 11, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m10) + - ms2, 4/4 replicas (m13,m14,m15,m16) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m10) + - ms2, 4/4 replicas (m13,m14,m15,m16) +[M controller] Iteration 11, Reconcile m10 +[M controller] Iteration 12, Reconcile m15 +[MD controller] Iteration 12, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m10) + - ms2, 4/4 replicas (m13,m14,m15,m16) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m10) + - ms2, 4/4 replicas (m13,m14,m15,m16) +[MS controller] Iteration 12, Reconcile ms1, 2/2 replicas (m10) +[M controller] Iteration 12, Reconcile m13 +[MS controller] Iteration 12, Reconcile ms2, 4/4 replicas (m13,m14,m15,m16) +[M controller] Iteration 12, Reconcile m15 +[M controller] Iteration 12, Reconcile m14 +[M controller] Iteration 13, Reconcile m14 +[M controller] Iteration 13, Reconcile m15 +[MS controller] Iteration 13, Reconcile ms2, 4/4 replicas (m13,m14,m15,m16) +[M controller] Iteration 13, Reconcile m13 +[M controller] Iteration 13, Reconcile m16 +[M controller] Iteration 13, Reconcile m10 +[M controller] Iteration 14, Reconcile m16 +[M controller] Iteration 14, Reconcile m13 +[MD controller] Iteration 14, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 1/2 replicas (m10) + - ms2, 4/4 replicas (m13,m14,m15,m16) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 1/1 replicas (m10) + - ms2, 4/4 replicas (m13,m14,m15,m16) +[M controller] Iteration 14, Reconcile m10 +[M controller] Iteration 14, Reconcile m15 +[MS controller] Iteration 14, Reconcile ms1, 1/1 replicas (m10) +[MS controller] Iteration 14, Reconcile ms2, 4/4 replicas (m13,m14,m15,m16) +[M controller] Iteration 15, Reconcile m15 +[M controller] Iteration 15, Reconcile m13 +[MD controller] Iteration 15, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 1/1 replicas (m10) + - ms2, 4/4 replicas (m13,m14,m15,m16) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 1/1 replicas (m10) + - ms2, 4/5 replicas (m13,m14,m15,m16) +[M controller] Iteration 15, Reconcile m14 +[M controller] Iteration 15, Reconcile m13 +[MS controller] Iteration 15, Reconcile ms1, 1/1 replicas (m10) +[MS controller] Iteration 15, Reconcile ms2, 4/5 replicas (m13,m14,m15,m16) +[MS controller] - ms2 scale up to 5/5 replicas (m17 created) +[M controller] Iteration 16, Reconcile m15 +[M controller] Iteration 16, Reconcile m14 +[M controller] Iteration 16, Reconcile m10 +[M controller] Iteration 16, Reconcile m17 +[M controller] Iteration 16, Reconcile m17 +[M controller] Iteration 16, Reconcile m10 +[MD controller] Iteration 16, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 1/1 replicas (m10) + - ms2, 5/5 replicas (m13,m14,m15,m16,m17) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 1/1 replicas (m10) + - ms2, 5/5 replicas (m13,m14,m15,m16,m17) +[M controller] Iteration 16, Reconcile m13 +[M controller] Iteration 16, Reconcile m16 +[User] Iteration 17, Deleting machine m10 +[M controller] Iteration 17, Reconcile m16 +[M controller] Iteration 17, Reconcile m14 +[M controller] Iteration 17, Reconcile m10 +[M controller] - m10 finalizer removed +[MS controller] Iteration 17, Reconcile ms1, 1/1 replicas () +[M controller] Iteration 17, Reconcile m16 +[MD controller] Iteration 17, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 0/1 replicas () + - ms2, 5/5 replicas (m13,m14,m15,m16,m17) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 5/5 replicas (m13,m14,m15,m16,m17) +[M controller] Iteration 17, Reconcile m17 +[M controller] Iteration 18, Reconcile m17 +[MD controller] Iteration 18, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 5/5 replicas (m13,m14,m15,m16,m17) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 5/6 replicas (m13,m14,m15,m16,m17) +[M controller] Iteration 18, Reconcile m13 +[M controller] Iteration 18, Reconcile m15 +[MS controller] Iteration 18, Reconcile ms1, 0/0 replicas () +[M controller] Iteration 18, Reconcile m14 +[MS controller] Iteration 19, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 19, Reconcile ms2, 5/6 replicas (m13,m14,m15,m16,m17) +[MS controller] - ms2 scale up to 6/6 replicas (m18 created) +[M controller] Iteration 19, Reconcile m17 +[M controller] Iteration 19, Reconcile m13 +[M controller] Iteration 19, Reconcile m15 +[M controller] Iteration 19, Reconcile m16 +[M controller] Iteration 19, Reconcile m14 +[M controller] Iteration 19, Reconcile m14 +[MD controller] Iteration 19, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m13,m14,m15,m16,m17,m18) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m13,m14,m15,m16,m17,m18) +[Test] Final state + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m13,m14,m15,m16,m17,m18) diff --git a/internal/controllers/machinedeployment/testdata/ondelete/12 replicas, maxuserunavailable 1, scale down to 6.test.log.golden b/internal/controllers/machinedeployment/testdata/ondelete/12 replicas, maxuserunavailable 1, scale down to 6.test.log.golden new file mode 100644 index 000000000000..fcdfba907874 --- /dev/null +++ b/internal/controllers/machinedeployment/testdata/ondelete/12 replicas, maxuserunavailable 1, scale down to 6.test.log.golden @@ -0,0 +1,199 @@ +## 12 Replicas, maxUserUnavailable 1, scale down to 6 + +[Test] Initial state + md, 6/6 replicas + - ms1, 9/9 replicas (m4,m5,m6,m7,m8,m9,m10,m11,m12) + - ms2, 3/3 replicas (m13,m14,m15) +[Test] Rollout 12 replicas, onDeleteStrategy +[MD controller] Iteration 1, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 9/9 replicas (m4,m5,m6,m7,m8,m9,m10,m11,m12) + - ms2, 3/3 replicas (m13,m14,m15) +[MD controller] - Result of rollout planner + md, 12/6 replicas + - ms1, 9/3 replicas (m4,m5,m6,m7,m8,m9,m10,m11,m12) + - ms2, 3/3 replicas (m13,m14,m15) +[MS controller] Iteration 1, Reconcile ms1, 9/3 replicas (m4,m5,m6,m7,m8,m9,m10,m11,m12) +[MS controller] - ms1 scale down to 3/3 replicas (m4,m5,m6,m7,m8,m9 deleted) +[MS controller] Iteration 1, Reconcile ms2, 3/3 replicas (m13,m14,m15) +[User] Iteration 1, Deleting machine m10 +[M controller] Iteration 1, Reconcile m10 +[M controller] - m10 finalizer removed +[M controller] Iteration 1, Reconcile m11 +[M controller] Iteration 1, Reconcile m12 +[M controller] Iteration 1, Reconcile m13 +[M controller] Iteration 1, Reconcile m14 +[M controller] Iteration 1, Reconcile m15 +[MD controller] Iteration 2, Reconcile md +[MD controller] - Input to rollout planner + md, 12/6 replicas + - ms1, 3/3 replicas (m11,m12) + - ms2, 3/3 replicas (m13,m14,m15) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m11,m12) + - ms2, 3/3 replicas (m13,m14,m15) +[MS controller] Iteration 2, Reconcile ms1, 3/3 replicas (m11,m12) +[MS controller] Iteration 2, Reconcile ms2, 3/3 replicas (m13,m14,m15) +[M controller] Iteration 2, Reconcile m11 +[M controller] Iteration 2, Reconcile m12 +[M controller] Iteration 2, Reconcile m13 +[M controller] Iteration 2, Reconcile m14 +[M controller] Iteration 2, Reconcile m15 +[MD controller] Iteration 3, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 2/3 replicas (m11,m12) + - ms2, 3/3 replicas (m13,m14,m15) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m11,m12) + - ms2, 3/3 replicas (m13,m14,m15) +[MS controller] Iteration 3, Reconcile ms1, 2/2 replicas (m11,m12) +[MS controller] Iteration 3, Reconcile ms2, 3/3 replicas (m13,m14,m15) +[M controller] Iteration 3, Reconcile m11 +[M controller] Iteration 3, Reconcile m12 +[M controller] Iteration 3, Reconcile m13 +[M controller] Iteration 3, Reconcile m14 +[M controller] Iteration 3, Reconcile m15 +[MD controller] Iteration 4, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m11,m12) + - ms2, 3/3 replicas (m13,m14,m15) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m11,m12) + - ms2, 3/4 replicas (m13,m14,m15) +[MS controller] Iteration 4, Reconcile ms1, 2/2 replicas (m11,m12) +[MS controller] Iteration 4, Reconcile ms2, 3/4 replicas (m13,m14,m15) +[MS controller] - ms2 scale up to 4/4 replicas (m16 created) +[User] Iteration 4, Deleting machine m11 +[M controller] Iteration 4, Reconcile m11 +[M controller] - m11 finalizer removed +[M controller] Iteration 4, Reconcile m12 +[M controller] Iteration 4, Reconcile m13 +[M controller] Iteration 4, Reconcile m14 +[M controller] Iteration 4, Reconcile m15 +[MD controller] Iteration 5, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m12) + - ms2, 4/4 replicas (m13,m14,m15,m16) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m12) + - ms2, 4/4 replicas (m13,m14,m15,m16) +[MS controller] Iteration 5, Reconcile ms1, 2/2 replicas (m12) +[MS controller] Iteration 5, Reconcile ms2, 4/4 replicas (m13,m14,m15,m16) +[M controller] Iteration 5, Reconcile m12 +[M controller] Iteration 5, Reconcile m13 +[M controller] Iteration 5, Reconcile m14 +[M controller] Iteration 5, Reconcile m15 +[M controller] Iteration 5, Reconcile m16 +[MD controller] Iteration 6, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 1/2 replicas (m12) + - ms2, 4/4 replicas (m13,m14,m15,m16) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 1/1 replicas (m12) + - ms2, 4/4 replicas (m13,m14,m15,m16) +[MS controller] Iteration 6, Reconcile ms1, 1/1 replicas (m12) +[MS controller] Iteration 6, Reconcile ms2, 4/4 replicas (m13,m14,m15,m16) +[M controller] Iteration 6, Reconcile m12 +[M controller] Iteration 6, Reconcile m13 +[M controller] Iteration 6, Reconcile m14 +[M controller] Iteration 6, Reconcile m15 +[M controller] Iteration 6, Reconcile m16 +[MD controller] Iteration 7, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 1/1 replicas (m12) + - ms2, 4/4 replicas (m13,m14,m15,m16) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 1/1 replicas (m12) + - ms2, 4/5 replicas (m13,m14,m15,m16) +[MS controller] Iteration 7, Reconcile ms1, 1/1 replicas (m12) +[MS controller] Iteration 7, Reconcile ms2, 4/5 replicas (m13,m14,m15,m16) +[MS controller] - ms2 scale up to 5/5 replicas (m17 created) +[User] Iteration 7, Deleting machine m12 +[M controller] Iteration 7, Reconcile m12 +[M controller] - m12 finalizer removed +[M controller] Iteration 7, Reconcile m13 +[M controller] Iteration 7, Reconcile m14 +[M controller] Iteration 7, Reconcile m15 +[M controller] Iteration 7, Reconcile m16 +[MD controller] Iteration 8, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 1/1 replicas () + - ms2, 5/5 replicas (m13,m14,m15,m16,m17) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 1/1 replicas () + - ms2, 5/5 replicas (m13,m14,m15,m16,m17) +[MS controller] Iteration 8, Reconcile ms1, 1/1 replicas () +[MS controller] Iteration 8, Reconcile ms2, 5/5 replicas (m13,m14,m15,m16,m17) +[M controller] Iteration 8, Reconcile m13 +[M controller] Iteration 8, Reconcile m14 +[M controller] Iteration 8, Reconcile m15 +[M controller] Iteration 8, Reconcile m16 +[M controller] Iteration 8, Reconcile m17 +[MD controller] Iteration 9, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 0/1 replicas () + - ms2, 5/5 replicas (m13,m14,m15,m16,m17) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 5/5 replicas (m13,m14,m15,m16,m17) +[MS controller] Iteration 9, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 9, Reconcile ms2, 5/5 replicas (m13,m14,m15,m16,m17) +[M controller] Iteration 9, Reconcile m13 +[M controller] Iteration 9, Reconcile m14 +[M controller] Iteration 9, Reconcile m15 +[M controller] Iteration 9, Reconcile m16 +[M controller] Iteration 9, Reconcile m17 +[MD controller] Iteration 10, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 5/5 replicas (m13,m14,m15,m16,m17) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 5/6 replicas (m13,m14,m15,m16,m17) +[MS controller] Iteration 10, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 10, Reconcile ms2, 5/6 replicas (m13,m14,m15,m16,m17) +[MS controller] - ms2 scale up to 6/6 replicas (m18 created) +[M controller] Iteration 10, Reconcile m13 +[M controller] Iteration 10, Reconcile m14 +[M controller] Iteration 10, Reconcile m15 +[M controller] Iteration 10, Reconcile m16 +[M controller] Iteration 10, Reconcile m17 +[MD controller] Iteration 11, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m13,m14,m15,m16,m17,m18) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m13,m14,m15,m16,m17,m18) +[MS controller] Iteration 11, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 11, Reconcile ms2, 6/6 replicas (m13,m14,m15,m16,m17,m18) +[M controller] Iteration 11, Reconcile m13 +[M controller] Iteration 11, Reconcile m14 +[M controller] Iteration 11, Reconcile m15 +[M controller] Iteration 11, Reconcile m16 +[M controller] Iteration 11, Reconcile m17 +[M controller] Iteration 11, Reconcile m18 +[Test] Final state + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m13,m14,m15,m16,m17,m18) diff --git a/internal/controllers/machinedeployment/testdata/ondelete/3 replicas, maxuserunavailable 1, random(0).test.log.golden b/internal/controllers/machinedeployment/testdata/ondelete/3 replicas, maxuserunavailable 1, random(0).test.log.golden new file mode 100644 index 000000000000..f702a5ee06c6 --- /dev/null +++ b/internal/controllers/machinedeployment/testdata/ondelete/3 replicas, maxuserunavailable 1, random(0).test.log.golden @@ -0,0 +1,217 @@ +## 3 replicas, maxUserUnavailable 1, random(0) + +[Test] Initial state + md, 3/3 replicas + - ms1, 3/3 replicas (m1,m2,m3) + - ms2, 0/0 replicas () +[Test] Rollout 3 replicas, onDeleteStrategy, random(0) +[MS controller] Iteration 1, Reconcile ms2, 0/0 replicas () +[M controller] Iteration 1, Reconcile m1 +[MD controller] Iteration 1, Reconcile md +[MD controller] - Input to rollout planner + md, 3/3 replicas + - ms1, 3/3 replicas (m1,m2,m3) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 3/3 replicas + - ms1, 3/3 replicas (m1,m2,m3) + - ms2, 0/0 replicas () +[MS controller] Iteration 1, Reconcile ms1, 3/3 replicas (m1,m2,m3) +[MS controller] Iteration 1, Reconcile ms1, 3/3 replicas (m1,m2,m3) +[M controller] Iteration 2, Reconcile m2 +[M controller] Iteration 2, Reconcile m3 +[MS controller] Iteration 2, Reconcile ms2, 0/0 replicas () +[MD controller] Iteration 2, Reconcile md +[MD controller] - Input to rollout planner + md, 3/3 replicas + - ms1, 3/3 replicas (m1,m2,m3) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 3/3 replicas + - ms1, 3/3 replicas (m1,m2,m3) + - ms2, 0/0 replicas () +[M controller] Iteration 2, Reconcile m1 +[M controller] Iteration 3, Reconcile m3 +[MS controller] Iteration 3, Reconcile ms2, 0/0 replicas () +[User] Iteration 3, Deleting machine m2 +[M controller] Iteration 3, Reconcile m2 +[M controller] - m2 finalizer removed +[MD controller] Iteration 3, Reconcile md +[MD controller] - Input to rollout planner + md, 3/3 replicas + - ms1, 3/3 replicas (m1,m3) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 3/3 replicas + - ms1, 3/3 replicas (m1,m3) + - ms2, 0/0 replicas () +[M controller] Iteration 4, Reconcile m3 +[MS controller] Iteration 4, Reconcile ms2, 0/0 replicas () +[MD controller] Iteration 4, Reconcile md +[MD controller] - Input to rollout planner + md, 3/3 replicas + - ms1, 3/3 replicas (m1,m3) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 3/3 replicas + - ms1, 3/3 replicas (m1,m3) + - ms2, 0/0 replicas () +[MS controller] Iteration 4, Reconcile ms1, 3/3 replicas (m1,m3) +[MD controller] Iteration 5, Reconcile md +[MD controller] - Input to rollout planner + md, 3/3 replicas + - ms1, 2/3 replicas (m1,m3) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 2/3 replicas + - ms1, 2/2 replicas (m1,m3) + - ms2, 0/0 replicas () +[M controller] Iteration 5, Reconcile m3 +[MS controller] Iteration 5, Reconcile ms2, 0/0 replicas () +[MS controller] Iteration 5, Reconcile ms1, 2/2 replicas (m1,m3) +[MD controller] Iteration 5, Reconcile md +[MD controller] - Input to rollout planner + md, 2/3 replicas + - ms1, 2/2 replicas (m1,m3) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 2/3 replicas + - ms1, 2/2 replicas (m1,m3) + - ms2, 0/1 replicas () +[MS controller] Iteration 5, Reconcile ms1, 2/2 replicas (m1,m3) +[M controller] Iteration 6, Reconcile m3 +[M controller] Iteration 6, Reconcile m3 +[MS controller] Iteration 6, Reconcile ms2, 0/1 replicas () +[MS controller] - ms2 scale up to 1/1 replicas (m4 created) +[User] Iteration 6, Deleting machine m3 +[MS controller] Iteration 6, Reconcile ms1, 2/2 replicas (m1,m3) +[MS controller] Iteration 6, Reconcile ms2, 1/1 replicas (m4) +[MS controller] Iteration 7, Reconcile ms2, 1/1 replicas (m4) +[M controller] Iteration 7, Reconcile m4 +[MS controller] Iteration 7, Reconcile ms1, 2/2 replicas (m1,m3) +[M controller] Iteration 7, Reconcile m1 +[M controller] Iteration 7, Reconcile m3 +[M controller] - m3 finalizer removed +[MD controller] Iteration 8, Reconcile md +[MD controller] - Input to rollout planner + md, 2/3 replicas + - ms1, 2/2 replicas (m1) + - ms2, 1/1 replicas (m4) +[MD controller] - Result of rollout planner + md, 3/3 replicas + - ms1, 2/2 replicas (m1) + - ms2, 1/1 replicas (m4) +[MD controller] Iteration 8, Reconcile md +[MD controller] - Input to rollout planner + md, 3/3 replicas + - ms1, 2/2 replicas (m1) + - ms2, 1/1 replicas (m4) +[MD controller] - Result of rollout planner + md, 3/3 replicas + - ms1, 2/2 replicas (m1) + - ms2, 1/1 replicas (m4) +[MS controller] Iteration 8, Reconcile ms2, 1/1 replicas (m4) +[MS controller] Iteration 8, Reconcile ms1, 2/2 replicas (m1) +[MS controller] Iteration 8, Reconcile ms1, 1/2 replicas (m1) +[MS controller] Iteration 8, Reconcile ms1, 1/2 replicas (m1) +[MD controller] Iteration 9, Reconcile md +[MD controller] - Input to rollout planner + md, 3/3 replicas + - ms1, 1/2 replicas (m1) + - ms2, 1/1 replicas (m4) +[MD controller] - Result of rollout planner + md, 2/3 replicas + - ms1, 1/1 replicas (m1) + - ms2, 1/1 replicas (m4) +[MS controller] Iteration 9, Reconcile ms1, 1/1 replicas (m1) +[MD controller] Iteration 9, Reconcile md +[MD controller] - Input to rollout planner + md, 2/3 replicas + - ms1, 1/1 replicas (m1) + - ms2, 1/1 replicas (m4) +[MD controller] - Result of rollout planner + md, 2/3 replicas + - ms1, 1/1 replicas (m1) + - ms2, 1/2 replicas (m4) +[MS controller] Iteration 9, Reconcile ms1, 1/1 replicas (m1) +[MS controller] Iteration 9, Reconcile ms1, 1/1 replicas (m1) +[M controller] Iteration 9, Reconcile m4 +[MD controller] Iteration 9, Reconcile md +[MD controller] - Input to rollout planner + md, 2/3 replicas + - ms1, 1/1 replicas (m1) + - ms2, 1/2 replicas (m4) +[MD controller] - Result of rollout planner + md, 2/3 replicas + - ms1, 1/1 replicas (m1) + - ms2, 1/2 replicas (m4) +[MD controller] Iteration 9, Reconcile md +[MD controller] - Input to rollout planner + md, 2/3 replicas + - ms1, 1/1 replicas (m1) + - ms2, 1/2 replicas (m4) +[MD controller] - Result of rollout planner + md, 2/3 replicas + - ms1, 1/1 replicas (m1) + - ms2, 1/2 replicas (m4) +[M controller] Iteration 10, Reconcile m4 +[MS controller] Iteration 10, Reconcile ms2, 1/2 replicas (m4) +[MS controller] - ms2 scale up to 2/2 replicas (m5 created) +[User] Iteration 10, Deleting machine m1 +[M controller] Iteration 10, Reconcile m1 +[M controller] - m1 finalizer removed +[MS controller] Iteration 10, Reconcile ms1, 1/1 replicas () +[MD controller] Iteration 10, Reconcile md +[MD controller] - Input to rollout planner + md, 2/3 replicas + - ms1, 0/1 replicas () + - ms2, 2/2 replicas (m4,m5) +[MD controller] - Result of rollout planner + md, 2/3 replicas + - ms1, 0/0 replicas () + - ms2, 2/2 replicas (m4,m5) +[MD controller] Iteration 10, Reconcile md +[MD controller] - Input to rollout planner + md, 2/3 replicas + - ms1, 0/0 replicas () + - ms2, 2/2 replicas (m4,m5) +[MD controller] - Result of rollout planner + md, 2/3 replicas + - ms1, 0/0 replicas () + - ms2, 2/3 replicas (m4,m5) +[MD controller] Iteration 10, Reconcile md +[MD controller] - Input to rollout planner + md, 2/3 replicas + - ms1, 0/0 replicas () + - ms2, 2/3 replicas (m4,m5) +[MD controller] - Result of rollout planner + md, 2/3 replicas + - ms1, 0/0 replicas () + - ms2, 2/3 replicas (m4,m5) +[M controller] Iteration 11, Reconcile m5 +[M controller] Iteration 11, Reconcile m4 +[MS controller] Iteration 11, Reconcile ms2, 2/3 replicas (m4,m5) +[MS controller] - ms2 scale up to 3/3 replicas (m6 created) +[MS controller] Iteration 11, Reconcile ms1, 0/0 replicas () +[MD controller] Iteration 11, Reconcile md +[MD controller] - Input to rollout planner + md, 2/3 replicas + - ms1, 0/0 replicas () + - ms2, 3/3 replicas (m4,m5,m6) +[MD controller] - Result of rollout planner + md, 3/3 replicas + - ms1, 0/0 replicas () + - ms2, 3/3 replicas (m4,m5,m6) +[MD controller] Iteration 11, Reconcile md +[MD controller] - Input to rollout planner + md, 3/3 replicas + - ms1, 0/0 replicas () + - ms2, 3/3 replicas (m4,m5,m6) +[MD controller] - Result of rollout planner + md, 3/3 replicas + - ms1, 0/0 replicas () + - ms2, 3/3 replicas (m4,m5,m6) +[Test] Final state + md, 3/3 replicas + - ms1, 0/0 replicas () + - ms2, 3/3 replicas (m4,m5,m6) diff --git a/internal/controllers/machinedeployment/testdata/ondelete/3 replicas, maxuserunavailable 1.test.log.golden b/internal/controllers/machinedeployment/testdata/ondelete/3 replicas, maxuserunavailable 1.test.log.golden new file mode 100644 index 000000000000..079461a6318b --- /dev/null +++ b/internal/controllers/machinedeployment/testdata/ondelete/3 replicas, maxuserunavailable 1.test.log.golden @@ -0,0 +1,165 @@ +## 3 replicas, maxUserUnavailable 1 + +[Test] Initial state + md, 3/3 replicas + - ms1, 3/3 replicas (m1,m2,m3) + - ms2, 0/0 replicas () +[Test] Rollout 3 replicas, onDeleteStrategy +[MD controller] Iteration 1, Reconcile md +[MD controller] - Input to rollout planner + md, 3/3 replicas + - ms1, 3/3 replicas (m1,m2,m3) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 3/3 replicas + - ms1, 3/3 replicas (m1,m2,m3) + - ms2, 0/0 replicas () +[MS controller] Iteration 1, Reconcile ms1, 3/3 replicas (m1,m2,m3) +[MS controller] Iteration 1, Reconcile ms2, 0/0 replicas () +[User] Iteration 1, Deleting machine m1 +[M controller] Iteration 1, Reconcile m1 +[M controller] - m1 finalizer removed +[M controller] Iteration 1, Reconcile m2 +[M controller] Iteration 1, Reconcile m3 +[MD controller] Iteration 2, Reconcile md +[MD controller] - Input to rollout planner + md, 3/3 replicas + - ms1, 3/3 replicas (m2,m3) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 3/3 replicas + - ms1, 3/3 replicas (m2,m3) + - ms2, 0/0 replicas () +[MS controller] Iteration 2, Reconcile ms1, 3/3 replicas (m2,m3) +[MS controller] Iteration 2, Reconcile ms2, 0/0 replicas () +[M controller] Iteration 2, Reconcile m2 +[M controller] Iteration 2, Reconcile m3 +[MD controller] Iteration 3, Reconcile md +[MD controller] - Input to rollout planner + md, 3/3 replicas + - ms1, 2/3 replicas (m2,m3) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 2/3 replicas + - ms1, 2/2 replicas (m2,m3) + - ms2, 0/0 replicas () +[MS controller] Iteration 3, Reconcile ms1, 2/2 replicas (m2,m3) +[MS controller] Iteration 3, Reconcile ms2, 0/0 replicas () +[M controller] Iteration 3, Reconcile m2 +[M controller] Iteration 3, Reconcile m3 +[MD controller] Iteration 4, Reconcile md +[MD controller] - Input to rollout planner + md, 2/3 replicas + - ms1, 2/2 replicas (m2,m3) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 2/3 replicas + - ms1, 2/2 replicas (m2,m3) + - ms2, 0/1 replicas () +[MS controller] Iteration 4, Reconcile ms1, 2/2 replicas (m2,m3) +[MS controller] Iteration 4, Reconcile ms2, 0/1 replicas () +[MS controller] - ms2 scale up to 1/1 replicas (m4 created) +[User] Iteration 4, Deleting machine m2 +[M controller] Iteration 4, Reconcile m2 +[M controller] - m2 finalizer removed +[M controller] Iteration 4, Reconcile m3 +[MD controller] Iteration 5, Reconcile md +[MD controller] - Input to rollout planner + md, 2/3 replicas + - ms1, 2/2 replicas (m3) + - ms2, 1/1 replicas (m4) +[MD controller] - Result of rollout planner + md, 3/3 replicas + - ms1, 2/2 replicas (m3) + - ms2, 1/1 replicas (m4) +[MS controller] Iteration 5, Reconcile ms1, 2/2 replicas (m3) +[MS controller] Iteration 5, Reconcile ms2, 1/1 replicas (m4) +[M controller] Iteration 5, Reconcile m3 +[M controller] Iteration 5, Reconcile m4 +[MD controller] Iteration 6, Reconcile md +[MD controller] - Input to rollout planner + md, 3/3 replicas + - ms1, 1/2 replicas (m3) + - ms2, 1/1 replicas (m4) +[MD controller] - Result of rollout planner + md, 2/3 replicas + - ms1, 1/1 replicas (m3) + - ms2, 1/1 replicas (m4) +[MS controller] Iteration 6, Reconcile ms1, 1/1 replicas (m3) +[MS controller] Iteration 6, Reconcile ms2, 1/1 replicas (m4) +[M controller] Iteration 6, Reconcile m3 +[M controller] Iteration 6, Reconcile m4 +[MD controller] Iteration 7, Reconcile md +[MD controller] - Input to rollout planner + md, 2/3 replicas + - ms1, 1/1 replicas (m3) + - ms2, 1/1 replicas (m4) +[MD controller] - Result of rollout planner + md, 2/3 replicas + - ms1, 1/1 replicas (m3) + - ms2, 1/2 replicas (m4) +[MS controller] Iteration 7, Reconcile ms1, 1/1 replicas (m3) +[MS controller] Iteration 7, Reconcile ms2, 1/2 replicas (m4) +[MS controller] - ms2 scale up to 2/2 replicas (m5 created) +[User] Iteration 7, Deleting machine m3 +[M controller] Iteration 7, Reconcile m3 +[M controller] - m3 finalizer removed +[M controller] Iteration 7, Reconcile m4 +[MD controller] Iteration 8, Reconcile md +[MD controller] - Input to rollout planner + md, 2/3 replicas + - ms1, 1/1 replicas () + - ms2, 2/2 replicas (m4,m5) +[MD controller] - Result of rollout planner + md, 3/3 replicas + - ms1, 1/1 replicas () + - ms2, 2/2 replicas (m4,m5) +[MS controller] Iteration 8, Reconcile ms1, 1/1 replicas () +[MS controller] Iteration 8, Reconcile ms2, 2/2 replicas (m4,m5) +[M controller] Iteration 8, Reconcile m4 +[M controller] Iteration 8, Reconcile m5 +[MD controller] Iteration 9, Reconcile md +[MD controller] - Input to rollout planner + md, 3/3 replicas + - ms1, 0/1 replicas () + - ms2, 2/2 replicas (m4,m5) +[MD controller] - Result of rollout planner + md, 2/3 replicas + - ms1, 0/0 replicas () + - ms2, 2/2 replicas (m4,m5) +[MS controller] Iteration 9, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 9, Reconcile ms2, 2/2 replicas (m4,m5) +[M controller] Iteration 9, Reconcile m4 +[M controller] Iteration 9, Reconcile m5 +[MD controller] Iteration 10, Reconcile md +[MD controller] - Input to rollout planner + md, 2/3 replicas + - ms1, 0/0 replicas () + - ms2, 2/2 replicas (m4,m5) +[MD controller] - Result of rollout planner + md, 2/3 replicas + - ms1, 0/0 replicas () + - ms2, 2/3 replicas (m4,m5) +[MS controller] Iteration 10, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 10, Reconcile ms2, 2/3 replicas (m4,m5) +[MS controller] - ms2 scale up to 3/3 replicas (m6 created) +[M controller] Iteration 10, Reconcile m4 +[M controller] Iteration 10, Reconcile m5 +[MD controller] Iteration 11, Reconcile md +[MD controller] - Input to rollout planner + md, 2/3 replicas + - ms1, 0/0 replicas () + - ms2, 3/3 replicas (m4,m5,m6) +[MD controller] - Result of rollout planner + md, 3/3 replicas + - ms1, 0/0 replicas () + - ms2, 3/3 replicas (m4,m5,m6) +[MS controller] Iteration 11, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 11, Reconcile ms2, 3/3 replicas (m4,m5,m6) +[M controller] Iteration 11, Reconcile m4 +[M controller] Iteration 11, Reconcile m5 +[M controller] Iteration 11, Reconcile m6 +[Test] Final state + md, 3/3 replicas + - ms1, 0/0 replicas () + - ms2, 3/3 replicas (m4,m5,m6) diff --git a/internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 1, change spec, random(0).test.log.golden b/internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 1, change spec, random(0).test.log.golden new file mode 100644 index 000000000000..79dfff4ff0b5 --- /dev/null +++ b/internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 1, change spec, random(0).test.log.golden @@ -0,0 +1,656 @@ +## 6 Replicas, maxUserUnavailable 1, change spec, random(0) + +[Test] Initial state + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 0/0 replicas () +[Test] Rollout 6 replicas, onDeleteStrategy, random(0) +[M controller] Iteration 1, Reconcile m4 +[M controller] Iteration 1, Reconcile m8 +[M controller] Iteration 1, Reconcile m5 +[MD controller] Iteration 1, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 0/0 replicas () +[M controller] Iteration 1, Reconcile m9 +[User] Iteration 1, Deleting machine m8 +[M controller] Iteration 1, Reconcile m6 +[M controller] Iteration 1, Reconcile m6 +[MS controller] Iteration 1, Reconcile ms3, 0/0 replicas () +[M controller] Iteration 2, Reconcile m4 +[M controller] Iteration 2, Reconcile m6 +[M controller] Iteration 2, Reconcile m4 +[M controller] Iteration 2, Reconcile m5 +[MS controller] Iteration 2, Reconcile ms3, 0/0 replicas () +[MD controller] Iteration 2, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 0/0 replicas () +[MS controller] Iteration 3, Reconcile ms3, 0/0 replicas () +[MS controller] Iteration 3, Reconcile ms2, 3/3 replicas (m7,m8,m9) +[MS controller] Iteration 3, Reconcile ms1, 3/3 replicas (m4,m5,m6) +[M controller] Iteration 3, Reconcile m7 +[M controller] Iteration 3, Reconcile m5 +[M controller] Iteration 3, Reconcile m8 +[M controller] - m8 finalizer removed +[MS controller] Iteration 3, Reconcile ms1, 3/3 replicas (m4,m5,m6) +[M controller] Iteration 3, Reconcile m6 +[MD controller] Iteration 3, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/3 replicas (m7,m9) + - ms3, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/3 replicas (m7,m9) + - ms3, 0/0 replicas () +[M controller] Iteration 4, Reconcile m9 +[MS controller] Iteration 4, Reconcile ms3, 0/0 replicas () +[M controller] Iteration 4, Reconcile m6 +[MS controller] Iteration 4, Reconcile ms1, 3/3 replicas (m4,m5,m6) +[M controller] Iteration 4, Reconcile m7 +[MD controller] Iteration 4, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/3 replicas (m7,m9) + - ms3, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/3 replicas (m7,m9) + - ms3, 0/0 replicas () +[MS controller] Iteration 4, Reconcile ms1, 3/3 replicas (m4,m5,m6) +[M controller] Iteration 4, Reconcile m5 +[MS controller] Iteration 4, Reconcile ms3, 0/0 replicas () +[M controller] Iteration 4, Reconcile m4 +[M controller] Iteration 5, Reconcile m7 +[MS controller] Iteration 5, Reconcile ms2, 3/3 replicas (m7,m9) +[M controller] Iteration 5, Reconcile m6 +[M controller] Iteration 5, Reconcile m4 +[M controller] Iteration 5, Reconcile m5 +[MS controller] Iteration 5, Reconcile ms1, 3/3 replicas (m4,m5,m6) +[M controller] Iteration 5, Reconcile m9 +[MD controller] Iteration 5, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 2/3 replicas (m7,m9) + - ms3, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 2/2 replicas (m7,m9) + - ms3, 0/0 replicas () +[MS controller] Iteration 5, Reconcile ms3, 0/0 replicas () +[MD controller] Iteration 6, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 2/2 replicas (m7,m9) + - ms3, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 2/2 replicas (m7,m9) + - ms3, 0/1 replicas () +[M controller] Iteration 6, Reconcile m9 +[M controller] Iteration 6, Reconcile m4 +[M controller] Iteration 6, Reconcile m9 +[M controller] Iteration 6, Reconcile m6 +[MS controller] Iteration 6, Reconcile ms3, 0/1 replicas () +[MS controller] - ms3 scale up to 1/1 replicas (m10 created) +[MS controller] Iteration 6, Reconcile ms1, 3/3 replicas (m4,m5,m6) +[MS controller] Iteration 6, Reconcile ms3, 1/1 replicas (m10) +[M controller] Iteration 6, Reconcile m4 +[M controller] Iteration 7, Reconcile m6 +[M controller] Iteration 7, Reconcile m10 +[MS controller] Iteration 7, Reconcile ms2, 2/2 replicas (m7,m9) +[MS controller] Iteration 7, Reconcile ms1, 3/3 replicas (m4,m5,m6) +[M controller] Iteration 7, Reconcile m4 +[M controller] Iteration 7, Reconcile m7 +[M controller] Iteration 7, Reconcile m7 +[M controller] Iteration 7, Reconcile m5 +[M controller] Iteration 8, Reconcile m10 +[MS controller] Iteration 8, Reconcile ms2, 2/2 replicas (m7,m9) +[MS controller] Iteration 8, Reconcile ms2, 2/2 replicas (m7,m9) +[M controller] Iteration 8, Reconcile m6 +[User] Iteration 8, Deleting machine m7 +[M controller] Iteration 8, Reconcile m4 +[MS controller] Iteration 8, Reconcile ms1, 3/3 replicas (m4,m5,m6) +[M controller] Iteration 8, Reconcile m7 +[M controller] - m7 finalizer removed +[M controller] Iteration 8, Reconcile m9 +[M controller] Iteration 8, Reconcile m5 +[M controller] Iteration 8, Reconcile m9 +[MS controller] Iteration 8, Reconcile ms3, 1/1 replicas (m10) +[M controller] Iteration 9, Reconcile m9 +[M controller] Iteration 9, Reconcile m6 +[MD controller] Iteration 9, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 2/2 replicas (m9) + - ms3, 1/1 replicas (m10) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 2/2 replicas (m9) + - ms3, 1/1 replicas (m10) +[M controller] Iteration 9, Reconcile m4 +[M controller] Iteration 9, Reconcile m5 +[MS controller] Iteration 10, Reconcile ms3, 1/1 replicas (m10) +[MS controller] Iteration 10, Reconcile ms3, 1/1 replicas (m10) +[M controller] Iteration 10, Reconcile m6 +[MD controller] Iteration 10, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 2/2 replicas (m9) + - ms3, 1/1 replicas (m10) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 2/2 replicas (m9) + - ms3, 1/1 replicas (m10) +[M controller] Iteration 10, Reconcile m4 +[MS controller] Iteration 10, Reconcile ms1, 3/3 replicas (m4,m5,m6) +[M controller] Iteration 10, Reconcile m6 +[MS controller] Iteration 10, Reconcile ms3, 1/1 replicas (m10) +[MS controller] Iteration 10, Reconcile ms2, 2/2 replicas (m9) +[MS controller] Iteration 11, Reconcile ms2, 1/2 replicas (m9) +[MS controller] Iteration 11, Reconcile ms2, 1/2 replicas (m9) +[MS controller] Iteration 11, Reconcile ms3, 1/1 replicas (m10) +[MD controller] Iteration 11, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 1/2 replicas (m9) + - ms3, 1/1 replicas (m10) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 1/1 replicas (m9) + - ms3, 1/1 replicas (m10) +[MS controller] Iteration 11, Reconcile ms1, 3/3 replicas (m4,m5,m6) +[M controller] Iteration 11, Reconcile m10 +[M controller] Iteration 11, Reconcile m6 +[M controller] Iteration 11, Reconcile m10 +[M controller] Iteration 11, Reconcile m9 +[M controller] Iteration 12, Reconcile m5 +[MS controller] Iteration 12, Reconcile ms1, 3/3 replicas (m4,m5,m6) +[M controller] Iteration 12, Reconcile m6 +[M controller] Iteration 12, Reconcile m10 +[M controller] Iteration 12, Reconcile m5 +[MS controller] Iteration 12, Reconcile ms3, 1/1 replicas (m10) +[MS controller] Iteration 12, Reconcile ms2, 1/1 replicas (m9) +[MS controller] Iteration 12, Reconcile ms1, 3/3 replicas (m4,m5,m6) +[M controller] Iteration 12, Reconcile m4 +[M controller] Iteration 12, Reconcile m9 +[MD controller] Iteration 12, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 1/1 replicas (m9) + - ms3, 1/1 replicas (m10) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 1/1 replicas (m9) + - ms3, 1/2 replicas (m10) +[M controller] Iteration 13, Reconcile m6 +[MS controller] Iteration 13, Reconcile ms3, 1/2 replicas (m10) +[MS controller] - ms3 scale up to 2/2 replicas (m11 created) +[M controller] Iteration 13, Reconcile m4 +[M controller] Iteration 13, Reconcile m6 +[MS controller] Iteration 13, Reconcile ms3, 2/2 replicas (m10,m11) +[M controller] Iteration 13, Reconcile m6 +[MS controller] Iteration 13, Reconcile ms2, 1/1 replicas (m9) +[MD controller] Iteration 13, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 1/1 replicas (m9) + - ms3, 2/2 replicas (m10,m11) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 1/1 replicas (m9) + - ms3, 2/2 replicas (m10,m11) +[MD controller] Iteration 14, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 1/1 replicas (m9) + - ms3, 2/2 replicas (m10,m11) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 1/1 replicas (m9) + - ms3, 2/2 replicas (m10,m11) +[User] Iteration 14, Deleting machine m4 +[M controller] Iteration 14, Reconcile m11 +[MS controller] Iteration 14, Reconcile ms3, 2/2 replicas (m10,m11) +[MS controller] Iteration 14, Reconcile ms1, 3/3 replicas (m4,m5,m6) +[M controller] Iteration 14, Reconcile m4 +[M controller] - m4 finalizer removed +[M controller] Iteration 14, Reconcile m5 +[M controller] Iteration 14, Reconcile m6 +[MS controller] Iteration 14, Reconcile ms2, 1/1 replicas (m9) +[M controller] Iteration 15, Reconcile m5 +[MS controller] Iteration 15, Reconcile ms3, 2/2 replicas (m10,m11) +[MD controller] Iteration 15, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m5,m6) + - ms2, 1/1 replicas (m9) + - ms3, 2/2 replicas (m10,m11) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m5,m6) + - ms2, 1/1 replicas (m9) + - ms3, 2/2 replicas (m10,m11) +[MD controller] Iteration 15, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m5,m6) + - ms2, 1/1 replicas (m9) + - ms3, 2/2 replicas (m10,m11) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m5,m6) + - ms2, 1/1 replicas (m9) + - ms3, 2/2 replicas (m10,m11) +[M controller] Iteration 15, Reconcile m9 +[M controller] Iteration 15, Reconcile m9 +[MS controller] Iteration 15, Reconcile ms1, 3/3 replicas (m5,m6) +[MS controller] Iteration 15, Reconcile ms2, 1/1 replicas (m9) +[MD controller] Iteration 15, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 2/3 replicas (m5,m6) + - ms2, 1/1 replicas (m9) + - ms3, 2/2 replicas (m10,m11) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 1/1 replicas (m9) + - ms3, 2/2 replicas (m10,m11) +[M controller] Iteration 15, Reconcile m6 +[M controller] Iteration 16, Reconcile m11 +[MS controller] Iteration 16, Reconcile ms3, 2/2 replicas (m10,m11) +[M controller] Iteration 16, Reconcile m11 +[MD controller] Iteration 16, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 1/1 replicas (m9) + - ms3, 2/2 replicas (m10,m11) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 1/1 replicas (m9) + - ms3, 2/3 replicas (m10,m11) +[M controller] Iteration 16, Reconcile m9 +[M controller] Iteration 16, Reconcile m10 +[MS controller] Iteration 17, Reconcile ms3, 2/3 replicas (m10,m11) +[MS controller] - ms3 scale up to 3/3 replicas (m12 created) +[M controller] Iteration 17, Reconcile m10 +[M controller] Iteration 17, Reconcile m6 +[User] Iteration 17, Deleting machine m9 +[MS controller] Iteration 17, Reconcile ms1, 2/2 replicas (m5,m6) +[M controller] Iteration 17, Reconcile m10 +[MD controller] Iteration 17, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 1/1 replicas (m9) + - ms3, 3/3 replicas (m10,m11,m12) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 1/1 replicas (m9) + - ms3, 3/3 replicas (m10,m11,m12) +[M controller] Iteration 17, Reconcile m11 +[MS controller] Iteration 18, Reconcile ms1, 2/2 replicas (m5,m6) +[MS controller] Iteration 18, Reconcile ms2, 1/1 replicas (m9) +[MD controller] Iteration 18, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 1/1 replicas (m9) + - ms3, 3/3 replicas (m10,m11,m12) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 1/1 replicas (m9) + - ms3, 3/3 replicas (m10,m11,m12) +[M controller] Iteration 18, Reconcile m10 +[MS controller] Iteration 18, Reconcile ms3, 3/3 replicas (m10,m11,m12) +[M controller] Iteration 18, Reconcile m12 +[MS controller] Iteration 18, Reconcile ms2, 1/1 replicas (m9) +[MS controller] Iteration 18, Reconcile ms2, 1/1 replicas (m9) +[M controller] Iteration 19, Reconcile m9 +[M controller] - m9 finalizer removed +[MD controller] Iteration 19, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 1/1 replicas () + - ms3, 3/3 replicas (m10,m11,m12) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 1/1 replicas () + - ms3, 3/3 replicas (m10,m11,m12) +[M controller] Iteration 19, Reconcile m10 +[M controller] Iteration 19, Reconcile m10 +[MS controller] Iteration 19, Reconcile ms1, 2/2 replicas (m5,m6) +[M controller] Iteration 20, Reconcile m6 +[MD controller] Iteration 20, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 1/1 replicas () + - ms3, 3/3 replicas (m10,m11,m12) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 1/1 replicas () + - ms3, 3/3 replicas (m10,m11,m12) +[M controller] Iteration 20, Reconcile m11 +[MS controller] Iteration 20, Reconcile ms3, 3/3 replicas (m10,m11,m12) +[MS controller] Iteration 20, Reconcile ms2, 1/1 replicas () +[M controller] Iteration 20, Reconcile m11 +[M controller] Iteration 20, Reconcile m10 +[M controller] Iteration 20, Reconcile m12 +[MD controller] Iteration 20, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 0/1 replicas () + - ms3, 3/3 replicas (m10,m11,m12) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 0/0 replicas () + - ms3, 3/3 replicas (m10,m11,m12) +[MD controller] Iteration 20, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 0/0 replicas () + - ms3, 3/3 replicas (m10,m11,m12) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 0/0 replicas () + - ms3, 3/4 replicas (m10,m11,m12) +[MS controller] Iteration 20, Reconcile ms2, 0/0 replicas () +[MS controller] Iteration 20, Reconcile ms1, 2/2 replicas (m5,m6) +[M controller] Iteration 21, Reconcile m5 +[MD controller] Iteration 21, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 0/0 replicas () + - ms3, 3/4 replicas (m10,m11,m12) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 0/0 replicas () + - ms3, 3/4 replicas (m10,m11,m12) +[M controller] Iteration 21, Reconcile m12 +[M controller] Iteration 21, Reconcile m5 +[M controller] Iteration 21, Reconcile m5 +[M controller] Iteration 21, Reconcile m11 +[MS controller] Iteration 21, Reconcile ms2, 0/0 replicas () +[MS controller] Iteration 21, Reconcile ms1, 2/2 replicas (m5,m6) +[M controller] Iteration 21, Reconcile m6 +[M controller] Iteration 21, Reconcile m6 +[M controller] Iteration 22, Reconcile m11 +[M controller] Iteration 22, Reconcile m6 +[M controller] Iteration 22, Reconcile m5 +[MS controller] Iteration 22, Reconcile ms1, 2/2 replicas (m5,m6) +[M controller] Iteration 22, Reconcile m10 +[MS controller] Iteration 22, Reconcile ms3, 3/4 replicas (m10,m11,m12) +[MS controller] - ms3 scale up to 4/4 replicas (m13 created) +[M controller] Iteration 22, Reconcile m10 +[MS controller] Iteration 22, Reconcile ms2, 0/0 replicas () +[M controller] Iteration 22, Reconcile m12 +[M controller] Iteration 22, Reconcile m5 +[MS controller] Iteration 22, Reconcile ms1, 2/2 replicas (m5,m6) +[MS controller] Iteration 23, Reconcile ms1, 2/2 replicas (m5,m6) +[M controller] Iteration 23, Reconcile m10 +[User] Iteration 23, Deleting machine m6 +[M controller] Iteration 23, Reconcile m6 +[M controller] - m6 finalizer removed +[MD controller] Iteration 23, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m5) + - ms2, 0/0 replicas () + - ms3, 4/4 replicas (m10,m11,m12,m13) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m5) + - ms2, 0/0 replicas () + - ms3, 4/4 replicas (m10,m11,m12,m13) +[M controller] Iteration 23, Reconcile m13 +[MS controller] Iteration 23, Reconcile ms2, 0/0 replicas () +[MD controller] Iteration 24, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m5) + - ms2, 0/0 replicas () + - ms3, 4/4 replicas (m10,m11,m12,m13) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m5) + - ms2, 0/0 replicas () + - ms3, 4/4 replicas (m10,m11,m12,m13) +[MS controller] Iteration 24, Reconcile ms1, 2/2 replicas (m5) +[MD controller] Iteration 24, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 1/2 replicas (m5) + - ms2, 0/0 replicas () + - ms3, 4/4 replicas (m10,m11,m12,m13) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 1/1 replicas (m5) + - ms2, 0/0 replicas () + - ms3, 4/4 replicas (m10,m11,m12,m13) +[M controller] Iteration 24, Reconcile m5 +[M controller] Iteration 24, Reconcile m12 +[M controller] Iteration 24, Reconcile m13 +[M controller] Iteration 24, Reconcile m5 +[M controller] Iteration 25, Reconcile m12 +[MS controller] Iteration 25, Reconcile ms2, 0/0 replicas () +[M controller] Iteration 25, Reconcile m13 +[M controller] Iteration 25, Reconcile m5 +[M controller] Iteration 25, Reconcile m11 +[M controller] Iteration 25, Reconcile m12 +[MS controller] Iteration 25, Reconcile ms3, 4/4 replicas (m10,m11,m12,m13) +[MD controller] Iteration 26, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 1/1 replicas (m5) + - ms2, 0/0 replicas () + - ms3, 4/4 replicas (m10,m11,m12,m13) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 1/1 replicas (m5) + - ms2, 0/0 replicas () + - ms3, 4/5 replicas (m10,m11,m12,m13) +[M controller] Iteration 26, Reconcile m13 +[M controller] Iteration 26, Reconcile m5 +[MS controller] Iteration 26, Reconcile ms3, 4/5 replicas (m10,m11,m12,m13) +[MS controller] - ms3 scale up to 5/5 replicas (m14 created) +[MS controller] Iteration 26, Reconcile ms2, 0/0 replicas () +[M controller] Iteration 26, Reconcile m11 +[M controller] Iteration 27, Reconcile m12 +[MS controller] Iteration 27, Reconcile ms1, 1/1 replicas (m5) +[M controller] Iteration 27, Reconcile m14 +[M controller] Iteration 27, Reconcile m11 +[M controller] Iteration 27, Reconcile m10 +[MS controller] Iteration 27, Reconcile ms2, 0/0 replicas () +[MD controller] Iteration 28, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 1/1 replicas (m5) + - ms2, 0/0 replicas () + - ms3, 5/5 replicas (m10,m11,m12,m13,m14) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 1/1 replicas (m5) + - ms2, 0/0 replicas () + - ms3, 5/5 replicas (m10,m11,m12,m13,m14) +[User] Iteration 28, Deleting machine m5 +[MS controller] Iteration 28, Reconcile ms2, 0/0 replicas () +[M controller] Iteration 28, Reconcile m11 +[M controller] Iteration 28, Reconcile m5 +[M controller] - m5 finalizer removed +[M controller] Iteration 28, Reconcile m13 +[M controller] Iteration 28, Reconcile m13 +[M controller] Iteration 29, Reconcile m13 +[M controller] Iteration 29, Reconcile m12 +[MD controller] Iteration 29, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 1/1 replicas () + - ms2, 0/0 replicas () + - ms3, 5/5 replicas (m10,m11,m12,m13,m14) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 1/1 replicas () + - ms2, 0/0 replicas () + - ms3, 5/5 replicas (m10,m11,m12,m13,m14) +[M controller] Iteration 29, Reconcile m14 +[MS controller] Iteration 30, Reconcile ms2, 0/0 replicas () +[M controller] Iteration 30, Reconcile m11 +[MS controller] Iteration 30, Reconcile ms1, 1/1 replicas () +[M controller] Iteration 30, Reconcile m13 +[M controller] Iteration 30, Reconcile m14 +[M controller] Iteration 30, Reconcile m12 +[M controller] Iteration 30, Reconcile m13 +[M controller] Iteration 30, Reconcile m10 +[MS controller] Iteration 30, Reconcile ms3, 5/5 replicas (m10,m11,m12,m13,m14) +[M controller] Iteration 30, Reconcile m13 +[M controller] Iteration 30, Reconcile m13 +[M controller] Iteration 31, Reconcile m13 +[M controller] Iteration 31, Reconcile m10 +[M controller] Iteration 31, Reconcile m14 +[MS controller] Iteration 31, Reconcile ms3, 5/5 replicas (m10,m11,m12,m13,m14) +[M controller] Iteration 31, Reconcile m11 +[MS controller] Iteration 31, Reconcile ms1, 0/1 replicas () +[MS controller] Iteration 31, Reconcile ms2, 0/0 replicas () +[M controller] Iteration 32, Reconcile m12 +[MS controller] Iteration 32, Reconcile ms2, 0/0 replicas () +[M controller] Iteration 32, Reconcile m14 +[M controller] Iteration 32, Reconcile m10 +[M controller] Iteration 32, Reconcile m10 +[MS controller] Iteration 32, Reconcile ms1, 0/1 replicas () +[MS controller] Iteration 32, Reconcile ms3, 5/5 replicas (m10,m11,m12,m13,m14) +[MS controller] Iteration 33, Reconcile ms2, 0/0 replicas () +[MS controller] Iteration 33, Reconcile ms1, 0/1 replicas () +[M controller] Iteration 33, Reconcile m13 +[M controller] Iteration 33, Reconcile m14 +[MS controller] Iteration 34, Reconcile ms3, 5/5 replicas (m10,m11,m12,m13,m14) +[MD controller] Iteration 34, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 0/1 replicas () + - ms2, 0/0 replicas () + - ms3, 5/5 replicas (m10,m11,m12,m13,m14) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 0/0 replicas () + - ms3, 5/5 replicas (m10,m11,m12,m13,m14) +[M controller] Iteration 34, Reconcile m12 +[M controller] Iteration 34, Reconcile m11 +[M controller] Iteration 34, Reconcile m11 +[M controller] Iteration 34, Reconcile m14 +[M controller] Iteration 34, Reconcile m10 +[M controller] Iteration 34, Reconcile m13 +[M controller] Iteration 34, Reconcile m11 +[M controller] Iteration 34, Reconcile m13 +[M controller] Iteration 34, Reconcile m10 +[MS controller] Iteration 35, Reconcile ms3, 5/5 replicas (m10,m11,m12,m13,m14) +[MD controller] Iteration 35, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 0/0 replicas () + - ms3, 5/5 replicas (m10,m11,m12,m13,m14) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 0/0 replicas () + - ms3, 5/6 replicas (m10,m11,m12,m13,m14) +[MS controller] Iteration 35, Reconcile ms1, 0/0 replicas () +[M controller] Iteration 35, Reconcile m12 +[M controller] Iteration 35, Reconcile m12 +[M controller] Iteration 35, Reconcile m10 +[M controller] Iteration 35, Reconcile m11 +[M controller] Iteration 35, Reconcile m14 +[M controller] Iteration 35, Reconcile m11 +[MS controller] Iteration 36, Reconcile ms3, 5/6 replicas (m10,m11,m12,m13,m14) +[MS controller] - ms3 scale up to 6/6 replicas (m15 created) +[M controller] Iteration 36, Reconcile m14 +[MS controller] Iteration 36, Reconcile ms1, 0/0 replicas () +[M controller] Iteration 37, Reconcile m10 +[M controller] Iteration 37, Reconcile m14 +[M controller] Iteration 37, Reconcile m15 +[M controller] Iteration 37, Reconcile m11 +[MS controller] Iteration 37, Reconcile ms3, 6/6 replicas (m10,m11,m12,m13,m14,m15) +[MS controller] Iteration 37, Reconcile ms3, 6/6 replicas (m10,m11,m12,m13,m14,m15) +[M controller] Iteration 37, Reconcile m14 +[M controller] Iteration 37, Reconcile m13 +[MS controller] Iteration 37, Reconcile ms1, 0/0 replicas () +[M controller] Iteration 38, Reconcile m13 +[M controller] Iteration 38, Reconcile m14 +[M controller] Iteration 38, Reconcile m11 +[M controller] Iteration 38, Reconcile m10 +[M controller] Iteration 38, Reconcile m12 +[MS controller] Iteration 39, Reconcile ms2, 0/0 replicas () +[M controller] Iteration 39, Reconcile m12 +[M controller] Iteration 39, Reconcile m13 +[MS controller] Iteration 39, Reconcile ms3, 6/6 replicas (m10,m11,m12,m13,m14,m15) +[M controller] Iteration 39, Reconcile m10 +[M controller] Iteration 39, Reconcile m15 +[MS controller] Iteration 39, Reconcile ms2, 0/0 replicas () +[MD controller] Iteration 39, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 0/0 replicas () + - ms3, 6/6 replicas (m10,m11,m12,m13,m14,m15) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 0/0 replicas () + - ms3, 6/6 replicas (m10,m11,m12,m13,m14,m15) +[Test] Final state + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 0/0 replicas () + - ms3, 6/6 replicas (m10,m11,m12,m13,m14,m15) diff --git a/internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 1, change spec.test.log.golden b/internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 1, change spec.test.log.golden new file mode 100644 index 000000000000..503f376035e8 --- /dev/null +++ b/internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 1, change spec.test.log.golden @@ -0,0 +1,413 @@ +## 6 Replicas, maxUserUnavailable 1, change spec + +[Test] Initial state + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 0/0 replicas () +[Test] Rollout 6 replicas, onDeleteStrategy +[MD controller] Iteration 1, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 0/0 replicas () +[MS controller] Iteration 1, Reconcile ms1, 3/3 replicas (m4,m5,m6) +[MS controller] Iteration 1, Reconcile ms2, 3/3 replicas (m7,m8,m9) +[MS controller] Iteration 1, Reconcile ms3, 0/0 replicas () +[User] Iteration 1, Deleting machine m4 +[M controller] Iteration 1, Reconcile m4 +[M controller] - m4 finalizer removed +[M controller] Iteration 1, Reconcile m5 +[M controller] Iteration 1, Reconcile m6 +[M controller] Iteration 1, Reconcile m7 +[M controller] Iteration 1, Reconcile m8 +[M controller] Iteration 1, Reconcile m9 +[MD controller] Iteration 2, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 0/0 replicas () +[MS controller] Iteration 2, Reconcile ms1, 3/3 replicas (m5,m6) +[MS controller] Iteration 2, Reconcile ms2, 3/3 replicas (m7,m8,m9) +[MS controller] Iteration 2, Reconcile ms3, 0/0 replicas () +[M controller] Iteration 2, Reconcile m5 +[M controller] Iteration 2, Reconcile m6 +[M controller] Iteration 2, Reconcile m7 +[M controller] Iteration 2, Reconcile m8 +[M controller] Iteration 2, Reconcile m9 +[MD controller] Iteration 3, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 2/3 replicas (m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 0/0 replicas () +[MS controller] Iteration 3, Reconcile ms1, 2/2 replicas (m5,m6) +[MS controller] Iteration 3, Reconcile ms2, 3/3 replicas (m7,m8,m9) +[MS controller] Iteration 3, Reconcile ms3, 0/0 replicas () +[M controller] Iteration 3, Reconcile m5 +[M controller] Iteration 3, Reconcile m6 +[M controller] Iteration 3, Reconcile m7 +[M controller] Iteration 3, Reconcile m8 +[M controller] Iteration 3, Reconcile m9 +[MD controller] Iteration 4, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 0/1 replicas () +[MS controller] Iteration 4, Reconcile ms1, 2/2 replicas (m5,m6) +[MS controller] Iteration 4, Reconcile ms2, 3/3 replicas (m7,m8,m9) +[MS controller] Iteration 4, Reconcile ms3, 0/1 replicas () +[MS controller] - ms3 scale up to 1/1 replicas (m10 created) +[User] Iteration 4, Deleting machine m5 +[M controller] Iteration 4, Reconcile m5 +[M controller] - m5 finalizer removed +[M controller] Iteration 4, Reconcile m6 +[M controller] Iteration 4, Reconcile m7 +[M controller] Iteration 4, Reconcile m8 +[M controller] Iteration 4, Reconcile m9 +[MD controller] Iteration 5, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 1/1 replicas (m10) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 1/1 replicas (m10) +[MS controller] Iteration 5, Reconcile ms1, 2/2 replicas (m6) +[MS controller] Iteration 5, Reconcile ms2, 3/3 replicas (m7,m8,m9) +[MS controller] Iteration 5, Reconcile ms3, 1/1 replicas (m10) +[M controller] Iteration 5, Reconcile m6 +[M controller] Iteration 5, Reconcile m7 +[M controller] Iteration 5, Reconcile m8 +[M controller] Iteration 5, Reconcile m9 +[M controller] Iteration 5, Reconcile m10 +[MD controller] Iteration 6, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 1/2 replicas (m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 1/1 replicas (m10) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 1/1 replicas (m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 1/1 replicas (m10) +[MS controller] Iteration 6, Reconcile ms1, 1/1 replicas (m6) +[MS controller] Iteration 6, Reconcile ms2, 3/3 replicas (m7,m8,m9) +[MS controller] Iteration 6, Reconcile ms3, 1/1 replicas (m10) +[M controller] Iteration 6, Reconcile m6 +[M controller] Iteration 6, Reconcile m7 +[M controller] Iteration 6, Reconcile m8 +[M controller] Iteration 6, Reconcile m9 +[M controller] Iteration 6, Reconcile m10 +[MD controller] Iteration 7, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 1/1 replicas (m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 1/1 replicas (m10) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 1/1 replicas (m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 1/2 replicas (m10) +[MS controller] Iteration 7, Reconcile ms1, 1/1 replicas (m6) +[MS controller] Iteration 7, Reconcile ms2, 3/3 replicas (m7,m8,m9) +[MS controller] Iteration 7, Reconcile ms3, 1/2 replicas (m10) +[MS controller] - ms3 scale up to 2/2 replicas (m11 created) +[User] Iteration 7, Deleting machine m6 +[M controller] Iteration 7, Reconcile m6 +[M controller] - m6 finalizer removed +[M controller] Iteration 7, Reconcile m7 +[M controller] Iteration 7, Reconcile m8 +[M controller] Iteration 7, Reconcile m9 +[M controller] Iteration 7, Reconcile m10 +[MD controller] Iteration 8, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 1/1 replicas () + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 2/2 replicas (m10,m11) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 1/1 replicas () + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 2/2 replicas (m10,m11) +[MS controller] Iteration 8, Reconcile ms1, 1/1 replicas () +[MS controller] Iteration 8, Reconcile ms2, 3/3 replicas (m7,m8,m9) +[MS controller] Iteration 8, Reconcile ms3, 2/2 replicas (m10,m11) +[M controller] Iteration 8, Reconcile m7 +[M controller] Iteration 8, Reconcile m8 +[M controller] Iteration 8, Reconcile m9 +[M controller] Iteration 8, Reconcile m10 +[M controller] Iteration 8, Reconcile m11 +[MD controller] Iteration 9, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 0/1 replicas () + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 2/2 replicas (m10,m11) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 2/2 replicas (m10,m11) +[MS controller] Iteration 9, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 9, Reconcile ms2, 3/3 replicas (m7,m8,m9) +[MS controller] Iteration 9, Reconcile ms3, 2/2 replicas (m10,m11) +[M controller] Iteration 9, Reconcile m7 +[M controller] Iteration 9, Reconcile m8 +[M controller] Iteration 9, Reconcile m9 +[M controller] Iteration 9, Reconcile m10 +[M controller] Iteration 9, Reconcile m11 +[MD controller] Iteration 10, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 2/2 replicas (m10,m11) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 2/3 replicas (m10,m11) +[MS controller] Iteration 10, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 10, Reconcile ms2, 3/3 replicas (m7,m8,m9) +[MS controller] Iteration 10, Reconcile ms3, 2/3 replicas (m10,m11) +[MS controller] - ms3 scale up to 3/3 replicas (m12 created) +[User] Iteration 10, Deleting machine m7 +[M controller] Iteration 10, Reconcile m7 +[M controller] - m7 finalizer removed +[M controller] Iteration 10, Reconcile m8 +[M controller] Iteration 10, Reconcile m9 +[M controller] Iteration 10, Reconcile m10 +[M controller] Iteration 10, Reconcile m11 +[MD controller] Iteration 11, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 3/3 replicas (m8,m9) + - ms3, 3/3 replicas (m10,m11,m12) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 3/3 replicas (m8,m9) + - ms3, 3/3 replicas (m10,m11,m12) +[MS controller] Iteration 11, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 11, Reconcile ms2, 3/3 replicas (m8,m9) +[MS controller] Iteration 11, Reconcile ms3, 3/3 replicas (m10,m11,m12) +[M controller] Iteration 11, Reconcile m8 +[M controller] Iteration 11, Reconcile m9 +[M controller] Iteration 11, Reconcile m10 +[M controller] Iteration 11, Reconcile m11 +[M controller] Iteration 11, Reconcile m12 +[MD controller] Iteration 12, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 2/3 replicas (m8,m9) + - ms3, 3/3 replicas (m10,m11,m12) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 2/2 replicas (m8,m9) + - ms3, 3/3 replicas (m10,m11,m12) +[MS controller] Iteration 12, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 12, Reconcile ms2, 2/2 replicas (m8,m9) +[MS controller] Iteration 12, Reconcile ms3, 3/3 replicas (m10,m11,m12) +[M controller] Iteration 12, Reconcile m8 +[M controller] Iteration 12, Reconcile m9 +[M controller] Iteration 12, Reconcile m10 +[M controller] Iteration 12, Reconcile m11 +[M controller] Iteration 12, Reconcile m12 +[MD controller] Iteration 13, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 2/2 replicas (m8,m9) + - ms3, 3/3 replicas (m10,m11,m12) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 2/2 replicas (m8,m9) + - ms3, 3/4 replicas (m10,m11,m12) +[MS controller] Iteration 13, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 13, Reconcile ms2, 2/2 replicas (m8,m9) +[MS controller] Iteration 13, Reconcile ms3, 3/4 replicas (m10,m11,m12) +[MS controller] - ms3 scale up to 4/4 replicas (m13 created) +[User] Iteration 13, Deleting machine m8 +[M controller] Iteration 13, Reconcile m8 +[M controller] - m8 finalizer removed +[M controller] Iteration 13, Reconcile m9 +[M controller] Iteration 13, Reconcile m10 +[M controller] Iteration 13, Reconcile m11 +[M controller] Iteration 13, Reconcile m12 +[MD controller] Iteration 14, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 2/2 replicas (m9) + - ms3, 4/4 replicas (m10,m11,m12,m13) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 2/2 replicas (m9) + - ms3, 4/4 replicas (m10,m11,m12,m13) +[MS controller] Iteration 14, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 14, Reconcile ms2, 2/2 replicas (m9) +[MS controller] Iteration 14, Reconcile ms3, 4/4 replicas (m10,m11,m12,m13) +[M controller] Iteration 14, Reconcile m9 +[M controller] Iteration 14, Reconcile m10 +[M controller] Iteration 14, Reconcile m11 +[M controller] Iteration 14, Reconcile m12 +[M controller] Iteration 14, Reconcile m13 +[MD controller] Iteration 15, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 1/2 replicas (m9) + - ms3, 4/4 replicas (m10,m11,m12,m13) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 1/1 replicas (m9) + - ms3, 4/4 replicas (m10,m11,m12,m13) +[MS controller] Iteration 15, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 15, Reconcile ms2, 1/1 replicas (m9) +[MS controller] Iteration 15, Reconcile ms3, 4/4 replicas (m10,m11,m12,m13) +[M controller] Iteration 15, Reconcile m9 +[M controller] Iteration 15, Reconcile m10 +[M controller] Iteration 15, Reconcile m11 +[M controller] Iteration 15, Reconcile m12 +[M controller] Iteration 15, Reconcile m13 +[MD controller] Iteration 16, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 1/1 replicas (m9) + - ms3, 4/4 replicas (m10,m11,m12,m13) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 1/1 replicas (m9) + - ms3, 4/5 replicas (m10,m11,m12,m13) +[MS controller] Iteration 16, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 16, Reconcile ms2, 1/1 replicas (m9) +[MS controller] Iteration 16, Reconcile ms3, 4/5 replicas (m10,m11,m12,m13) +[MS controller] - ms3 scale up to 5/5 replicas (m14 created) +[User] Iteration 16, Deleting machine m9 +[M controller] Iteration 16, Reconcile m9 +[M controller] - m9 finalizer removed +[M controller] Iteration 16, Reconcile m10 +[M controller] Iteration 16, Reconcile m11 +[M controller] Iteration 16, Reconcile m12 +[M controller] Iteration 16, Reconcile m13 +[MD controller] Iteration 17, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 1/1 replicas () + - ms3, 5/5 replicas (m10,m11,m12,m13,m14) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 1/1 replicas () + - ms3, 5/5 replicas (m10,m11,m12,m13,m14) +[MS controller] Iteration 17, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 17, Reconcile ms2, 1/1 replicas () +[MS controller] Iteration 17, Reconcile ms3, 5/5 replicas (m10,m11,m12,m13,m14) +[M controller] Iteration 17, Reconcile m10 +[M controller] Iteration 17, Reconcile m11 +[M controller] Iteration 17, Reconcile m12 +[M controller] Iteration 17, Reconcile m13 +[M controller] Iteration 17, Reconcile m14 +[MD controller] Iteration 18, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 0/1 replicas () + - ms3, 5/5 replicas (m10,m11,m12,m13,m14) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 0/0 replicas () + - ms3, 5/5 replicas (m10,m11,m12,m13,m14) +[MS controller] Iteration 18, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 18, Reconcile ms2, 0/0 replicas () +[MS controller] Iteration 18, Reconcile ms3, 5/5 replicas (m10,m11,m12,m13,m14) +[M controller] Iteration 18, Reconcile m10 +[M controller] Iteration 18, Reconcile m11 +[M controller] Iteration 18, Reconcile m12 +[M controller] Iteration 18, Reconcile m13 +[M controller] Iteration 18, Reconcile m14 +[MD controller] Iteration 19, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 0/0 replicas () + - ms3, 5/5 replicas (m10,m11,m12,m13,m14) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 0/0 replicas () + - ms3, 5/6 replicas (m10,m11,m12,m13,m14) +[MS controller] Iteration 19, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 19, Reconcile ms2, 0/0 replicas () +[MS controller] Iteration 19, Reconcile ms3, 5/6 replicas (m10,m11,m12,m13,m14) +[MS controller] - ms3 scale up to 6/6 replicas (m15 created) +[M controller] Iteration 19, Reconcile m10 +[M controller] Iteration 19, Reconcile m11 +[M controller] Iteration 19, Reconcile m12 +[M controller] Iteration 19, Reconcile m13 +[M controller] Iteration 19, Reconcile m14 +[MD controller] Iteration 20, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 0/0 replicas () + - ms3, 6/6 replicas (m10,m11,m12,m13,m14,m15) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 0/0 replicas () + - ms3, 6/6 replicas (m10,m11,m12,m13,m14,m15) +[MS controller] Iteration 20, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 20, Reconcile ms2, 0/0 replicas () +[MS controller] Iteration 20, Reconcile ms3, 6/6 replicas (m10,m11,m12,m13,m14,m15) +[M controller] Iteration 20, Reconcile m10 +[M controller] Iteration 20, Reconcile m11 +[M controller] Iteration 20, Reconcile m12 +[M controller] Iteration 20, Reconcile m13 +[M controller] Iteration 20, Reconcile m14 +[M controller] Iteration 20, Reconcile m15 +[Test] Final state + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 0/0 replicas () + - ms3, 6/6 replicas (m10,m11,m12,m13,m14,m15) diff --git a/internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 1, scale up to 9, random(0).test.log.golden b/internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 1, scale up to 9, random(0).test.log.golden new file mode 100644 index 000000000000..fc37a3e55195 --- /dev/null +++ b/internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 1, scale up to 9, random(0).test.log.golden @@ -0,0 +1,490 @@ +## 6 Replicas, maxUserUnavailable 1, scale up to 9, random(0) + +[Test] Initial state + md, 9/9 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) +[Test] Rollout 6 replicas, onDeleteStrategy, random(0) +[M controller] Iteration 1, Reconcile m7 +[M controller] Iteration 1, Reconcile m8 +[M controller] Iteration 1, Reconcile m5 +[M controller] Iteration 1, Reconcile m9 +[M controller] Iteration 1, Reconcile m6 +[M controller] Iteration 2, Reconcile m7 +[M controller] Iteration 2, Reconcile m9 +[M controller] Iteration 2, Reconcile m5 +[MS controller] Iteration 2, Reconcile ms2, 3/3 replicas (m7,m8,m9) +[M controller] Iteration 2, Reconcile m6 +[M controller] Iteration 3, Reconcile m7 +[M controller] Iteration 3, Reconcile m8 +[M controller] Iteration 3, Reconcile m5 +[M controller] Iteration 3, Reconcile m4 +[MS controller] Iteration 3, Reconcile ms1, 3/3 replicas (m4,m5,m6) +[M controller] Iteration 3, Reconcile m6 +[MS controller] Iteration 3, Reconcile ms1, 3/3 replicas (m4,m5,m6) +[MS controller] Iteration 3, Reconcile ms2, 3/3 replicas (m7,m8,m9) +[M controller] Iteration 4, Reconcile m8 +[MS controller] Iteration 4, Reconcile ms2, 3/3 replicas (m7,m8,m9) +[M controller] Iteration 4, Reconcile m7 +[M controller] Iteration 4, Reconcile m5 +[M controller] Iteration 4, Reconcile m4 +[M controller] Iteration 4, Reconcile m6 +[MS controller] Iteration 4, Reconcile ms1, 3/3 replicas (m4,m5,m6) +[M controller] Iteration 4, Reconcile m9 +[MD controller] Iteration 4, Reconcile md +[MD controller] - Input to rollout planner + md, 9/9 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) +[MD controller] - Result of rollout planner + md, 6/9 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/6 replicas (m7,m8,m9) +[MD controller] Iteration 5, Reconcile md +[MD controller] - Input to rollout planner + md, 6/9 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/6 replicas (m7,m8,m9) +[MD controller] - Result of rollout planner + md, 6/9 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/6 replicas (m7,m8,m9) +[M controller] Iteration 5, Reconcile m9 +[M controller] Iteration 5, Reconcile m5 +[M controller] Iteration 5, Reconcile m4 +[M controller] Iteration 5, Reconcile m9 +[M controller] Iteration 5, Reconcile m7 +[MS controller] Iteration 5, Reconcile ms1, 3/3 replicas (m4,m5,m6) +[M controller] Iteration 5, Reconcile m5 +[M controller] Iteration 6, Reconcile m4 +[MD controller] Iteration 6, Reconcile md +[MD controller] - Input to rollout planner + md, 6/9 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/6 replicas (m7,m8,m9) +[MD controller] - Result of rollout planner + md, 6/9 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/6 replicas (m7,m8,m9) +[M controller] Iteration 6, Reconcile m5 +[MS controller] Iteration 6, Reconcile ms1, 3/3 replicas (m4,m5,m6) +[M controller] Iteration 6, Reconcile m9 +[M controller] Iteration 6, Reconcile m8 +[M controller] Iteration 6, Reconcile m6 +[M controller] Iteration 7, Reconcile m5 +[MS controller] Iteration 7, Reconcile ms2, 3/6 replicas (m7,m8,m9) +[MS controller] - ms2 scale up to 6/6 replicas (m10,m11,m12 created) +[M controller] Iteration 7, Reconcile m4 +[User] Iteration 7, Deleting machine m4 +[MD controller] Iteration 7, Reconcile md +[MD controller] - Input to rollout planner + md, 6/9 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MD controller] - Result of rollout planner + md, 9/9 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[M controller] Iteration 7, Reconcile m5 +[M controller] Iteration 7, Reconcile m7 +[M controller] Iteration 7, Reconcile m8 +[M controller] Iteration 7, Reconcile m7 +[M controller] Iteration 8, Reconcile m5 +[M controller] Iteration 8, Reconcile m4 +[M controller] - m4 finalizer removed +[MS controller] Iteration 8, Reconcile ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MD controller] Iteration 8, Reconcile md +[MD controller] - Input to rollout planner + md, 9/9 replicas + - ms1, 3/3 replicas (m5,m6) + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MD controller] - Result of rollout planner + md, 9/9 replicas + - ms1, 3/3 replicas (m5,m6) + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[M controller] Iteration 8, Reconcile m12 +[MS controller] Iteration 8, Reconcile ms1, 3/3 replicas (m5,m6) +[M controller] Iteration 8, Reconcile m6 +[M controller] Iteration 8, Reconcile m7 +[M controller] Iteration 8, Reconcile m10 +[M controller] Iteration 9, Reconcile m12 +[M controller] Iteration 9, Reconcile m12 +[M controller] Iteration 9, Reconcile m5 +[M controller] Iteration 9, Reconcile m10 +[M controller] Iteration 9, Reconcile m6 +[M controller] Iteration 9, Reconcile m10 +[M controller] Iteration 9, Reconcile m6 +[M controller] Iteration 9, Reconcile m11 +[M controller] Iteration 9, Reconcile m9 +[M controller] Iteration 9, Reconcile m7 +[M controller] Iteration 9, Reconcile m7 +[M controller] Iteration 9, Reconcile m11 +[M controller] Iteration 9, Reconcile m8 +[M controller] Iteration 10, Reconcile m11 +[MS controller] Iteration 10, Reconcile ms1, 2/3 replicas (m5,m6) +[M controller] Iteration 10, Reconcile m12 +[M controller] Iteration 10, Reconcile m9 +[M controller] Iteration 10, Reconcile m6 +[MS controller] Iteration 10, Reconcile ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[M controller] Iteration 10, Reconcile m7 +[MD controller] Iteration 10, Reconcile md +[MD controller] - Input to rollout planner + md, 9/9 replicas + - ms1, 2/3 replicas (m5,m6) + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MD controller] - Result of rollout planner + md, 8/9 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[M controller] Iteration 10, Reconcile m10 +[M controller] Iteration 10, Reconcile m7 +[M controller] Iteration 10, Reconcile m5 +[M controller] Iteration 11, Reconcile m5 +[M controller] Iteration 11, Reconcile m11 +[MS controller] Iteration 11, Reconcile ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[M controller] Iteration 11, Reconcile m9 +[MD controller] Iteration 11, Reconcile md +[MD controller] - Input to rollout planner + md, 8/9 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MD controller] - Result of rollout planner + md, 8/9 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 6/7 replicas (m7,m8,m9,m10,m11,m12) +[MD controller] Iteration 11, Reconcile md +[MD controller] - Input to rollout planner + md, 8/9 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 6/7 replicas (m7,m8,m9,m10,m11,m12) +[MD controller] - Result of rollout planner + md, 8/9 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 6/7 replicas (m7,m8,m9,m10,m11,m12) +[M controller] Iteration 11, Reconcile m8 +[M controller] Iteration 11, Reconcile m9 +[MS controller] Iteration 11, Reconcile ms1, 2/2 replicas (m5,m6) +[M controller] Iteration 12, Reconcile m8 +[MD controller] Iteration 12, Reconcile md +[MD controller] - Input to rollout planner + md, 8/9 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 6/7 replicas (m7,m8,m9,m10,m11,m12) +[MD controller] - Result of rollout planner + md, 8/9 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 6/7 replicas (m7,m8,m9,m10,m11,m12) +[M controller] Iteration 12, Reconcile m12 +[M controller] Iteration 12, Reconcile m9 +[M controller] Iteration 12, Reconcile m5 +[MS controller] Iteration 12, Reconcile ms2, 6/7 replicas (m7,m8,m9,m10,m11,m12) +[MS controller] - ms2 scale up to 7/7 replicas (m13 created) +[M controller] Iteration 12, Reconcile m7 +[M controller] Iteration 12, Reconcile m10 +[M controller] Iteration 12, Reconcile m6 +[MS controller] Iteration 12, Reconcile ms1, 2/2 replicas (m5,m6) +[MS controller] Iteration 12, Reconcile ms1, 2/2 replicas (m5,m6) +[M controller] Iteration 13, Reconcile m5 +[M controller] Iteration 13, Reconcile m7 +[MD controller] Iteration 13, Reconcile md +[MD controller] - Input to rollout planner + md, 8/9 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[MD controller] - Result of rollout planner + md, 9/9 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[MD controller] Iteration 13, Reconcile md +[MD controller] - Input to rollout planner + md, 9/9 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[MD controller] - Result of rollout planner + md, 9/9 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[M controller] Iteration 13, Reconcile m6 +[M controller] Iteration 13, Reconcile m12 +[M controller] Iteration 13, Reconcile m11 +[M controller] Iteration 13, Reconcile m13 +[MS controller] Iteration 13, Reconcile ms1, 2/2 replicas (m5,m6) +[User] Iteration 13, Deleting machine m6 +[M controller] Iteration 13, Reconcile m8 +[M controller] Iteration 13, Reconcile m11 +[M controller] Iteration 14, Reconcile m7 +[M controller] Iteration 14, Reconcile m8 +[M controller] Iteration 14, Reconcile m5 +[MD controller] Iteration 14, Reconcile md +[MD controller] - Input to rollout planner + md, 9/9 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[MD controller] - Result of rollout planner + md, 9/9 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[M controller] Iteration 14, Reconcile m12 +[MS controller] Iteration 14, Reconcile ms1, 2/2 replicas (m5,m6) +[M controller] Iteration 14, Reconcile m11 +[M controller] Iteration 14, Reconcile m13 +[M controller] Iteration 14, Reconcile m13 +[M controller] Iteration 14, Reconcile m10 +[MS controller] Iteration 14, Reconcile ms1, 2/2 replicas (m5,m6) +[M controller] Iteration 14, Reconcile m9 +[MS controller] Iteration 14, Reconcile ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[MS controller] Iteration 15, Reconcile ms1, 2/2 replicas (m5,m6) +[MS controller] Iteration 15, Reconcile ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[M controller] Iteration 15, Reconcile m6 +[M controller] - m6 finalizer removed +[M controller] Iteration 15, Reconcile m7 +[M controller] Iteration 15, Reconcile m12 +[MD controller] Iteration 15, Reconcile md +[MD controller] - Input to rollout planner + md, 9/9 replicas + - ms1, 2/2 replicas (m5) + - ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[MD controller] - Result of rollout planner + md, 9/9 replicas + - ms1, 2/2 replicas (m5) + - ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[M controller] Iteration 15, Reconcile m11 +[M controller] Iteration 15, Reconcile m11 +[M controller] Iteration 15, Reconcile m11 +[MD controller] Iteration 16, Reconcile md +[MD controller] - Input to rollout planner + md, 9/9 replicas + - ms1, 2/2 replicas (m5) + - ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[MD controller] - Result of rollout planner + md, 9/9 replicas + - ms1, 2/2 replicas (m5) + - ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[M controller] Iteration 16, Reconcile m11 +[M controller] Iteration 16, Reconcile m9 +[M controller] Iteration 16, Reconcile m5 +[M controller] Iteration 16, Reconcile m13 +[M controller] Iteration 16, Reconcile m5 +[M controller] Iteration 16, Reconcile m12 +[M controller] Iteration 16, Reconcile m5 +[M controller] Iteration 16, Reconcile m8 +[M controller] Iteration 16, Reconcile m13 +[MS controller] Iteration 16, Reconcile ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[M controller] Iteration 16, Reconcile m7 +[M controller] Iteration 16, Reconcile m13 +[M controller] Iteration 16, Reconcile m7 +[M controller] Iteration 16, Reconcile m5 +[M controller] Iteration 16, Reconcile m7 +[M controller] Iteration 17, Reconcile m9 +[M controller] Iteration 17, Reconcile m7 +[M controller] Iteration 17, Reconcile m12 +[MD controller] Iteration 17, Reconcile md +[MD controller] - Input to rollout planner + md, 9/9 replicas + - ms1, 2/2 replicas (m5) + - ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[MD controller] - Result of rollout planner + md, 9/9 replicas + - ms1, 2/2 replicas (m5) + - ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[M controller] Iteration 17, Reconcile m5 +[MS controller] Iteration 17, Reconcile ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[M controller] Iteration 17, Reconcile m5 +[M controller] Iteration 17, Reconcile m10 +[M controller] Iteration 17, Reconcile m11 +[M controller] Iteration 17, Reconcile m8 +[MS controller] Iteration 17, Reconcile ms1, 2/2 replicas (m5) +[MD controller] Iteration 18, Reconcile md +[MD controller] - Input to rollout planner + md, 9/9 replicas + - ms1, 1/2 replicas (m5) + - ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[MD controller] - Result of rollout planner + md, 8/9 replicas + - ms1, 1/1 replicas (m5) + - ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[M controller] Iteration 18, Reconcile m7 +[M controller] Iteration 18, Reconcile m13 +[MS controller] Iteration 18, Reconcile ms1, 1/1 replicas (m5) +[M controller] Iteration 18, Reconcile m11 +[MS controller] Iteration 18, Reconcile ms1, 1/1 replicas (m5) +[M controller] Iteration 18, Reconcile m9 +[M controller] Iteration 18, Reconcile m9 +[MS controller] Iteration 19, Reconcile ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[MD controller] Iteration 19, Reconcile md +[MD controller] - Input to rollout planner + md, 8/9 replicas + - ms1, 1/1 replicas (m5) + - ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[MD controller] - Result of rollout planner + md, 8/9 replicas + - ms1, 1/1 replicas (m5) + - ms2, 7/8 replicas (m7,m8,m9,m10,m11,m12,m13) +[M controller] Iteration 19, Reconcile m5 +[M controller] Iteration 19, Reconcile m8 +[M controller] Iteration 19, Reconcile m10 +[M controller] Iteration 19, Reconcile m12 +[M controller] Iteration 19, Reconcile m12 +[M controller] Iteration 20, Reconcile m12 +[M controller] Iteration 20, Reconcile m5 +[M controller] Iteration 20, Reconcile m9 +[MS controller] Iteration 20, Reconcile ms2, 7/8 replicas (m7,m8,m9,m10,m11,m12,m13) +[MS controller] - ms2 scale up to 8/8 replicas (m14 created) +[MD controller] Iteration 20, Reconcile md +[MD controller] - Input to rollout planner + md, 8/9 replicas + - ms1, 1/1 replicas (m5) + - ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[MD controller] - Result of rollout planner + md, 9/9 replicas + - ms1, 1/1 replicas (m5) + - ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[M controller] Iteration 20, Reconcile m9 +[M controller] Iteration 20, Reconcile m13 +[MS controller] Iteration 20, Reconcile ms1, 1/1 replicas (m5) +[M controller] Iteration 20, Reconcile m10 +[M controller] Iteration 20, Reconcile m11 +[M controller] Iteration 21, Reconcile m9 +[M controller] Iteration 21, Reconcile m5 +[MD controller] Iteration 21, Reconcile md +[MD controller] - Input to rollout planner + md, 9/9 replicas + - ms1, 1/1 replicas (m5) + - ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[MD controller] - Result of rollout planner + md, 9/9 replicas + - ms1, 1/1 replicas (m5) + - ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[M controller] Iteration 21, Reconcile m14 +[M controller] Iteration 21, Reconcile m8 +[M controller] Iteration 21, Reconcile m5 +[M controller] Iteration 21, Reconcile m13 +[M controller] Iteration 21, Reconcile m10 +[M controller] Iteration 21, Reconcile m11 +[M controller] Iteration 21, Reconcile m5 +[MS controller] Iteration 21, Reconcile ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[MD controller] Iteration 21, Reconcile md +[MD controller] - Input to rollout planner + md, 9/9 replicas + - ms1, 1/1 replicas (m5) + - ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[MD controller] - Result of rollout planner + md, 9/9 replicas + - ms1, 1/1 replicas (m5) + - ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[M controller] Iteration 21, Reconcile m9 +[M controller] Iteration 22, Reconcile m9 +[MD controller] Iteration 22, Reconcile md +[MD controller] - Input to rollout planner + md, 9/9 replicas + - ms1, 1/1 replicas (m5) + - ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[MD controller] - Result of rollout planner + md, 9/9 replicas + - ms1, 1/1 replicas (m5) + - ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[M controller] Iteration 22, Reconcile m11 +[M controller] Iteration 22, Reconcile m7 +[User] Iteration 22, Deleting machine m5 +[M controller] Iteration 22, Reconcile m8 +[MS controller] Iteration 22, Reconcile ms1, 1/1 replicas (m5) +[M controller] Iteration 22, Reconcile m7 +[MS controller] Iteration 22, Reconcile ms1, 1/1 replicas (m5) +[MS controller] Iteration 22, Reconcile ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[M controller] Iteration 22, Reconcile m14 +[M controller] Iteration 22, Reconcile m10 +[MS controller] Iteration 23, Reconcile ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[M controller] Iteration 23, Reconcile m7 +[M controller] Iteration 23, Reconcile m11 +[M controller] Iteration 23, Reconcile m13 +[M controller] Iteration 23, Reconcile m5 +[M controller] - m5 finalizer removed +[M controller] Iteration 23, Reconcile m9 +[M controller] Iteration 23, Reconcile m14 +[M controller] Iteration 23, Reconcile m10 +[M controller] Iteration 23, Reconcile m12 +[MS controller] Iteration 23, Reconcile ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[MS controller] Iteration 23, Reconcile ms1, 1/1 replicas () +[M controller] Iteration 23, Reconcile m14 +[M controller] Iteration 24, Reconcile m11 +[M controller] Iteration 24, Reconcile m10 +[MS controller] Iteration 24, Reconcile ms1, 0/1 replicas () +[M controller] Iteration 24, Reconcile m9 +[M controller] Iteration 24, Reconcile m12 +[MD controller] Iteration 24, Reconcile md +[MD controller] - Input to rollout planner + md, 9/9 replicas + - ms1, 0/1 replicas () + - ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[MD controller] - Result of rollout planner + md, 8/9 replicas + - ms1, 0/0 replicas () + - ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[MS controller] Iteration 24, Reconcile ms1, 0/0 replicas () +[M controller] Iteration 24, Reconcile m9 +[M controller] Iteration 25, Reconcile m8 +[M controller] Iteration 25, Reconcile m14 +[M controller] Iteration 25, Reconcile m7 +[M controller] Iteration 25, Reconcile m7 +[M controller] Iteration 25, Reconcile m13 +[M controller] Iteration 25, Reconcile m12 +[MD controller] Iteration 25, Reconcile md +[MD controller] - Input to rollout planner + md, 8/9 replicas + - ms1, 0/0 replicas () + - ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[MD controller] - Result of rollout planner + md, 8/9 replicas + - ms1, 0/0 replicas () + - ms2, 8/9 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[M controller] Iteration 25, Reconcile m9 +[M controller] Iteration 26, Reconcile m14 +[M controller] Iteration 26, Reconcile m7 +[MD controller] Iteration 26, Reconcile md +[MD controller] - Input to rollout planner + md, 8/9 replicas + - ms1, 0/0 replicas () + - ms2, 8/9 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[MD controller] - Result of rollout planner + md, 8/9 replicas + - ms1, 0/0 replicas () + - ms2, 8/9 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[MS controller] Iteration 26, Reconcile ms2, 8/9 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[MS controller] - ms2 scale up to 9/9 replicas (m15 created) +[M controller] Iteration 26, Reconcile m12 +[M controller] Iteration 26, Reconcile m11 +[M controller] Iteration 26, Reconcile m10 +[M controller] Iteration 26, Reconcile m7 +[M controller] Iteration 26, Reconcile m10 +[M controller] Iteration 27, Reconcile m15 +[M controller] Iteration 27, Reconcile m7 +[M controller] Iteration 27, Reconcile m13 +[MS controller] Iteration 27, Reconcile ms1, 0/0 replicas () +[M controller] Iteration 27, Reconcile m10 +[M controller] Iteration 27, Reconcile m12 +[MD controller] Iteration 27, Reconcile md +[MD controller] - Input to rollout planner + md, 8/9 replicas + - ms1, 0/0 replicas () + - ms2, 9/9 replicas (m7,m8,m9,m10,m11,m12,m13,m14,m15) +[MD controller] - Result of rollout planner + md, 9/9 replicas + - ms1, 0/0 replicas () + - ms2, 9/9 replicas (m7,m8,m9,m10,m11,m12,m13,m14,m15) +[M controller] Iteration 27, Reconcile m8 +[MD controller] Iteration 27, Reconcile md +[MD controller] - Input to rollout planner + md, 9/9 replicas + - ms1, 0/0 replicas () + - ms2, 9/9 replicas (m7,m8,m9,m10,m11,m12,m13,m14,m15) +[MD controller] - Result of rollout planner + md, 9/9 replicas + - ms1, 0/0 replicas () + - ms2, 9/9 replicas (m7,m8,m9,m10,m11,m12,m13,m14,m15) +[M controller] Iteration 27, Reconcile m9 +[MS controller] Iteration 27, Reconcile ms2, 9/9 replicas (m7,m8,m9,m10,m11,m12,m13,m14,m15) +[M controller] Iteration 27, Reconcile m13 +[Test] Final state + md, 9/9 replicas + - ms1, 0/0 replicas () + - ms2, 9/9 replicas (m7,m8,m9,m10,m11,m12,m13,m14,m15) diff --git a/internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 1, scale up to 9.test.log.golden b/internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 1, scale up to 9.test.log.golden new file mode 100644 index 000000000000..62fa44401988 --- /dev/null +++ b/internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 1, scale up to 9.test.log.golden @@ -0,0 +1,229 @@ +## 6 Replicas, maxUserUnavailable 1, scale up to 9 + +[Test] Initial state + md, 9/9 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) +[Test] Rollout 6 replicas, onDeleteStrategy +[MD controller] Iteration 1, Reconcile md +[MD controller] - Input to rollout planner + md, 9/9 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) +[MD controller] - Result of rollout planner + md, 6/9 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/6 replicas (m7,m8,m9) +[MS controller] Iteration 1, Reconcile ms1, 3/3 replicas (m4,m5,m6) +[MS controller] Iteration 1, Reconcile ms2, 3/6 replicas (m7,m8,m9) +[MS controller] - ms2 scale up to 6/6 replicas (m10,m11,m12 created) +[User] Iteration 1, Deleting machine m4 +[M controller] Iteration 1, Reconcile m4 +[M controller] - m4 finalizer removed +[M controller] Iteration 1, Reconcile m5 +[M controller] Iteration 1, Reconcile m6 +[M controller] Iteration 1, Reconcile m7 +[M controller] Iteration 1, Reconcile m8 +[M controller] Iteration 1, Reconcile m9 +[MD controller] Iteration 2, Reconcile md +[MD controller] - Input to rollout planner + md, 6/9 replicas + - ms1, 3/3 replicas (m5,m6) + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MD controller] - Result of rollout planner + md, 9/9 replicas + - ms1, 3/3 replicas (m5,m6) + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MS controller] Iteration 2, Reconcile ms1, 3/3 replicas (m5,m6) +[MS controller] Iteration 2, Reconcile ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[M controller] Iteration 2, Reconcile m5 +[M controller] Iteration 2, Reconcile m6 +[M controller] Iteration 2, Reconcile m7 +[M controller] Iteration 2, Reconcile m8 +[M controller] Iteration 2, Reconcile m9 +[M controller] Iteration 2, Reconcile m10 +[M controller] Iteration 2, Reconcile m11 +[M controller] Iteration 2, Reconcile m12 +[MD controller] Iteration 3, Reconcile md +[MD controller] - Input to rollout planner + md, 9/9 replicas + - ms1, 2/3 replicas (m5,m6) + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MD controller] - Result of rollout planner + md, 8/9 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MS controller] Iteration 3, Reconcile ms1, 2/2 replicas (m5,m6) +[MS controller] Iteration 3, Reconcile ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[M controller] Iteration 3, Reconcile m5 +[M controller] Iteration 3, Reconcile m6 +[M controller] Iteration 3, Reconcile m7 +[M controller] Iteration 3, Reconcile m8 +[M controller] Iteration 3, Reconcile m9 +[M controller] Iteration 3, Reconcile m10 +[M controller] Iteration 3, Reconcile m11 +[M controller] Iteration 3, Reconcile m12 +[MD controller] Iteration 4, Reconcile md +[MD controller] - Input to rollout planner + md, 8/9 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MD controller] - Result of rollout planner + md, 8/9 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 6/7 replicas (m7,m8,m9,m10,m11,m12) +[MS controller] Iteration 4, Reconcile ms1, 2/2 replicas (m5,m6) +[MS controller] Iteration 4, Reconcile ms2, 6/7 replicas (m7,m8,m9,m10,m11,m12) +[MS controller] - ms2 scale up to 7/7 replicas (m13 created) +[User] Iteration 4, Deleting machine m5 +[M controller] Iteration 4, Reconcile m5 +[M controller] - m5 finalizer removed +[M controller] Iteration 4, Reconcile m6 +[M controller] Iteration 4, Reconcile m7 +[M controller] Iteration 4, Reconcile m8 +[M controller] Iteration 4, Reconcile m9 +[M controller] Iteration 4, Reconcile m10 +[M controller] Iteration 4, Reconcile m11 +[M controller] Iteration 4, Reconcile m12 +[MD controller] Iteration 5, Reconcile md +[MD controller] - Input to rollout planner + md, 8/9 replicas + - ms1, 2/2 replicas (m6) + - ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[MD controller] - Result of rollout planner + md, 9/9 replicas + - ms1, 2/2 replicas (m6) + - ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[MS controller] Iteration 5, Reconcile ms1, 2/2 replicas (m6) +[MS controller] Iteration 5, Reconcile ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[M controller] Iteration 5, Reconcile m6 +[M controller] Iteration 5, Reconcile m7 +[M controller] Iteration 5, Reconcile m8 +[M controller] Iteration 5, Reconcile m9 +[M controller] Iteration 5, Reconcile m10 +[M controller] Iteration 5, Reconcile m11 +[M controller] Iteration 5, Reconcile m12 +[M controller] Iteration 5, Reconcile m13 +[MD controller] Iteration 6, Reconcile md +[MD controller] - Input to rollout planner + md, 9/9 replicas + - ms1, 1/2 replicas (m6) + - ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[MD controller] - Result of rollout planner + md, 8/9 replicas + - ms1, 1/1 replicas (m6) + - ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[MS controller] Iteration 6, Reconcile ms1, 1/1 replicas (m6) +[MS controller] Iteration 6, Reconcile ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[M controller] Iteration 6, Reconcile m6 +[M controller] Iteration 6, Reconcile m7 +[M controller] Iteration 6, Reconcile m8 +[M controller] Iteration 6, Reconcile m9 +[M controller] Iteration 6, Reconcile m10 +[M controller] Iteration 6, Reconcile m11 +[M controller] Iteration 6, Reconcile m12 +[M controller] Iteration 6, Reconcile m13 +[MD controller] Iteration 7, Reconcile md +[MD controller] - Input to rollout planner + md, 8/9 replicas + - ms1, 1/1 replicas (m6) + - ms2, 7/7 replicas (m7,m8,m9,m10,m11,m12,m13) +[MD controller] - Result of rollout planner + md, 8/9 replicas + - ms1, 1/1 replicas (m6) + - ms2, 7/8 replicas (m7,m8,m9,m10,m11,m12,m13) +[MS controller] Iteration 7, Reconcile ms1, 1/1 replicas (m6) +[MS controller] Iteration 7, Reconcile ms2, 7/8 replicas (m7,m8,m9,m10,m11,m12,m13) +[MS controller] - ms2 scale up to 8/8 replicas (m14 created) +[User] Iteration 7, Deleting machine m6 +[M controller] Iteration 7, Reconcile m6 +[M controller] - m6 finalizer removed +[M controller] Iteration 7, Reconcile m7 +[M controller] Iteration 7, Reconcile m8 +[M controller] Iteration 7, Reconcile m9 +[M controller] Iteration 7, Reconcile m10 +[M controller] Iteration 7, Reconcile m11 +[M controller] Iteration 7, Reconcile m12 +[M controller] Iteration 7, Reconcile m13 +[MD controller] Iteration 8, Reconcile md +[MD controller] - Input to rollout planner + md, 8/9 replicas + - ms1, 1/1 replicas () + - ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[MD controller] - Result of rollout planner + md, 9/9 replicas + - ms1, 1/1 replicas () + - ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[MS controller] Iteration 8, Reconcile ms1, 1/1 replicas () +[MS controller] Iteration 8, Reconcile ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[M controller] Iteration 8, Reconcile m7 +[M controller] Iteration 8, Reconcile m8 +[M controller] Iteration 8, Reconcile m9 +[M controller] Iteration 8, Reconcile m10 +[M controller] Iteration 8, Reconcile m11 +[M controller] Iteration 8, Reconcile m12 +[M controller] Iteration 8, Reconcile m13 +[M controller] Iteration 8, Reconcile m14 +[MD controller] Iteration 9, Reconcile md +[MD controller] - Input to rollout planner + md, 9/9 replicas + - ms1, 0/1 replicas () + - ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[MD controller] - Result of rollout planner + md, 8/9 replicas + - ms1, 0/0 replicas () + - ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[MS controller] Iteration 9, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 9, Reconcile ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[M controller] Iteration 9, Reconcile m7 +[M controller] Iteration 9, Reconcile m8 +[M controller] Iteration 9, Reconcile m9 +[M controller] Iteration 9, Reconcile m10 +[M controller] Iteration 9, Reconcile m11 +[M controller] Iteration 9, Reconcile m12 +[M controller] Iteration 9, Reconcile m13 +[M controller] Iteration 9, Reconcile m14 +[MD controller] Iteration 10, Reconcile md +[MD controller] - Input to rollout planner + md, 8/9 replicas + - ms1, 0/0 replicas () + - ms2, 8/8 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[MD controller] - Result of rollout planner + md, 8/9 replicas + - ms1, 0/0 replicas () + - ms2, 8/9 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[MS controller] Iteration 10, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 10, Reconcile ms2, 8/9 replicas (m7,m8,m9,m10,m11,m12,m13,m14) +[MS controller] - ms2 scale up to 9/9 replicas (m15 created) +[M controller] Iteration 10, Reconcile m7 +[M controller] Iteration 10, Reconcile m8 +[M controller] Iteration 10, Reconcile m9 +[M controller] Iteration 10, Reconcile m10 +[M controller] Iteration 10, Reconcile m11 +[M controller] Iteration 10, Reconcile m12 +[M controller] Iteration 10, Reconcile m13 +[M controller] Iteration 10, Reconcile m14 +[MD controller] Iteration 11, Reconcile md +[MD controller] - Input to rollout planner + md, 8/9 replicas + - ms1, 0/0 replicas () + - ms2, 9/9 replicas (m7,m8,m9,m10,m11,m12,m13,m14,m15) +[MD controller] - Result of rollout planner + md, 9/9 replicas + - ms1, 0/0 replicas () + - ms2, 9/9 replicas (m7,m8,m9,m10,m11,m12,m13,m14,m15) +[MS controller] Iteration 11, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 11, Reconcile ms2, 9/9 replicas (m7,m8,m9,m10,m11,m12,m13,m14,m15) +[M controller] Iteration 11, Reconcile m7 +[M controller] Iteration 11, Reconcile m8 +[M controller] Iteration 11, Reconcile m9 +[M controller] Iteration 11, Reconcile m10 +[M controller] Iteration 11, Reconcile m11 +[M controller] Iteration 11, Reconcile m12 +[M controller] Iteration 11, Reconcile m13 +[M controller] Iteration 11, Reconcile m14 +[M controller] Iteration 11, Reconcile m15 +[Test] Final state + md, 9/9 replicas + - ms1, 0/0 replicas () + - ms2, 9/9 replicas (m7,m8,m9,m10,m11,m12,m13,m14,m15) diff --git a/internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 2, random(0).test.log.golden b/internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 2, random(0).test.log.golden new file mode 100644 index 000000000000..6b72ae6d3ccf --- /dev/null +++ b/internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 2, random(0).test.log.golden @@ -0,0 +1,327 @@ +## 6 replicas, maxUserUnavailable 2, random(0) + +[Test] Initial state + md, 6/6 replicas + - ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 0/0 replicas () +[Test] Rollout 6 replicas, onDeleteStrategy, random(0) +[M controller] Iteration 1, Reconcile m4 +[M controller] Iteration 1, Reconcile m5 +[M controller] Iteration 1, Reconcile m2 +[M controller] Iteration 1, Reconcile m6 +[M controller] Iteration 1, Reconcile m3 +[M controller] Iteration 2, Reconcile m4 +[M controller] Iteration 2, Reconcile m6 +[M controller] Iteration 2, Reconcile m2 +[MS controller] Iteration 2, Reconcile ms2, 0/0 replicas () +[M controller] Iteration 2, Reconcile m3 +[M controller] Iteration 3, Reconcile m4 +[M controller] Iteration 3, Reconcile m5 +[M controller] Iteration 3, Reconcile m2 +[M controller] Iteration 3, Reconcile m1 +[MS controller] Iteration 3, Reconcile ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) +[M controller] Iteration 3, Reconcile m3 +[MS controller] Iteration 3, Reconcile ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) +[MS controller] Iteration 3, Reconcile ms2, 0/0 replicas () +[M controller] Iteration 4, Reconcile m5 +[MS controller] Iteration 4, Reconcile ms2, 0/0 replicas () +[M controller] Iteration 4, Reconcile m4 +[M controller] Iteration 4, Reconcile m2 +[M controller] Iteration 4, Reconcile m1 +[M controller] Iteration 4, Reconcile m3 +[MS controller] Iteration 4, Reconcile ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) +[M controller] Iteration 4, Reconcile m6 +[MD controller] Iteration 4, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 0/0 replicas () +[MD controller] Iteration 5, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 0/0 replicas () +[M controller] Iteration 5, Reconcile m6 +[M controller] Iteration 5, Reconcile m2 +[M controller] Iteration 5, Reconcile m1 +[M controller] Iteration 5, Reconcile m6 +[User] Iteration 5, Deleting machine m3 +[M controller] Iteration 5, Reconcile m4 +[MS controller] Iteration 5, Reconcile ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) +[User] Iteration 5, Deleting machine m2 +[M controller] Iteration 5, Reconcile m2 +[M controller] - m2 finalizer removed +[M controller] Iteration 6, Reconcile m6 +[M controller] Iteration 6, Reconcile m3 +[M controller] - m3 finalizer removed +[MS controller] Iteration 6, Reconcile ms1, 6/6 replicas (m1,m4,m5,m6) +[M controller] Iteration 6, Reconcile m4 +[M controller] Iteration 6, Reconcile m4 +[MS controller] Iteration 6, Reconcile ms2, 0/0 replicas () +[M controller] Iteration 6, Reconcile m4 +[M controller] Iteration 6, Reconcile m1 +[M controller] Iteration 6, Reconcile m5 +[MS controller] Iteration 7, Reconcile ms2, 0/0 replicas () +[M controller] Iteration 7, Reconcile m4 +[M controller] Iteration 7, Reconcile m6 +[M controller] Iteration 7, Reconcile m1 +[MD controller] Iteration 8, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 4/6 replicas (m1,m4,m5,m6) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 4/6 replicas + - ms1, 4/4 replicas (m1,m4,m5,m6) + - ms2, 0/0 replicas () +[M controller] Iteration 8, Reconcile m1 +[MS controller] Iteration 8, Reconcile ms2, 0/0 replicas () +[M controller] Iteration 8, Reconcile m5 +[M controller] Iteration 8, Reconcile m6 +[MS controller] Iteration 8, Reconcile ms1, 4/4 replicas (m1,m4,m5,m6) +[M controller] Iteration 8, Reconcile m4 +[M controller] Iteration 9, Reconcile m6 +[MS controller] Iteration 9, Reconcile ms1, 4/4 replicas (m1,m4,m5,m6) +[M controller] Iteration 9, Reconcile m5 +[M controller] Iteration 9, Reconcile m1 +[M controller] Iteration 9, Reconcile m4 +[M controller] Iteration 9, Reconcile m4 +[MS controller] Iteration 10, Reconcile ms1, 4/4 replicas (m1,m4,m5,m6) +[M controller] Iteration 10, Reconcile m5 +[M controller] Iteration 10, Reconcile m1 +[M controller] Iteration 10, Reconcile m4 +[M controller] Iteration 10, Reconcile m6 +[M controller] Iteration 11, Reconcile m1 +[MD controller] Iteration 11, Reconcile md +[MD controller] - Input to rollout planner + md, 4/6 replicas + - ms1, 4/4 replicas (m1,m4,m5,m6) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 4/6 replicas + - ms1, 4/4 replicas (m1,m4,m5,m6) + - ms2, 0/2 replicas () +[M controller] Iteration 11, Reconcile m6 +[M controller] Iteration 11, Reconcile m5 +[MS controller] Iteration 11, Reconcile ms2, 0/2 replicas () +[MS controller] - ms2 scale up to 2/2 replicas (m7,m8 created) +[M controller] Iteration 11, Reconcile m6 +[User] Iteration 11, Deleting machine m6 +[M controller] Iteration 11, Reconcile m5 +[M controller] Iteration 12, Reconcile m7 +[M controller] Iteration 12, Reconcile m4 +[M controller] Iteration 12, Reconcile m8 +[M controller] Iteration 12, Reconcile m6 +[M controller] - m6 finalizer removed +[MS controller] Iteration 12, Reconcile ms2, 2/2 replicas (m7,m8) +[MD controller] Iteration 12, Reconcile md +[MD controller] - Input to rollout planner + md, 4/6 replicas + - ms1, 4/4 replicas (m1,m4,m5) + - ms2, 2/2 replicas (m7,m8) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 4/4 replicas (m1,m4,m5) + - ms2, 2/2 replicas (m7,m8) +[M controller] Iteration 12, Reconcile m1 +[M controller] Iteration 12, Reconcile m4 +[MS controller] Iteration 12, Reconcile ms1, 4/4 replicas (m1,m4,m5) +[M controller] Iteration 13, Reconcile m7 +[M controller] Iteration 13, Reconcile m4 +[MS controller] Iteration 13, Reconcile ms2, 2/2 replicas (m7,m8) +[M controller] Iteration 13, Reconcile m8 +[M controller] Iteration 13, Reconcile m7 +[User] Iteration 13, Deleting machine m4 +[M controller] Iteration 13, Reconcile m5 +[MD controller] Iteration 14, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 3/4 replicas (m1,m4,m5) + - ms2, 2/2 replicas (m7,m8) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 3/3 replicas (m1,m4,m5) + - ms2, 2/2 replicas (m7,m8) +[M controller] Iteration 14, Reconcile m1 +[M controller] Iteration 14, Reconcile m8 +[MS controller] Iteration 14, Reconcile ms1, 3/3 replicas (m1,m4,m5) +[MS controller] Iteration 14, Reconcile ms2, 2/2 replicas (m7,m8) +[M controller] Iteration 14, Reconcile m4 +[M controller] - m4 finalizer removed +[M controller] Iteration 14, Reconcile m7 +[M controller] Iteration 15, Reconcile m8 +[MS controller] Iteration 15, Reconcile ms2, 2/2 replicas (m7,m8) +[MD controller] Iteration 15, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 3/3 replicas (m1,m5) + - ms2, 2/2 replicas (m7,m8) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 3/3 replicas (m1,m5) + - ms2, 2/3 replicas (m7,m8) +[M controller] Iteration 16, Reconcile m5 +[MS controller] Iteration 16, Reconcile ms1, 3/3 replicas (m1,m5) +[MS controller] Iteration 16, Reconcile ms2, 2/3 replicas (m7,m8) +[MS controller] - ms2 scale up to 3/3 replicas (m9 created) +[M controller] Iteration 16, Reconcile m7 +[MD controller] Iteration 16, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 2/3 replicas (m1,m5) + - ms2, 3/3 replicas (m7,m8,m9) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m1,m5) + - ms2, 3/3 replicas (m7,m8,m9) +[M controller] Iteration 16, Reconcile m1 +[MD controller] Iteration 16, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m1,m5) + - ms2, 3/3 replicas (m7,m8,m9) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m1,m5) + - ms2, 3/4 replicas (m7,m8,m9) +[M controller] Iteration 17, Reconcile m9 +[M controller] Iteration 17, Reconcile m1 +[M controller] Iteration 17, Reconcile m8 +[M controller] Iteration 17, Reconcile m7 +[User] Iteration 17, Deleting machine m1 +[M controller] Iteration 17, Reconcile m8 +[M controller] Iteration 17, Reconcile m5 +[MS controller] Iteration 17, Reconcile ms2, 3/4 replicas (m7,m8,m9) +[MS controller] - ms2 scale up to 4/4 replicas (m10 created) +[MD controller] Iteration 17, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m1,m5) + - ms2, 4/4 replicas (m7,m8,m9,m10) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m1,m5) + - ms2, 4/4 replicas (m7,m8,m9,m10) +[MS controller] Iteration 18, Reconcile ms2, 4/4 replicas (m7,m8,m9,m10) +[M controller] Iteration 18, Reconcile m8 +[M controller] Iteration 18, Reconcile m9 +[MD controller] Iteration 18, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m1,m5) + - ms2, 4/4 replicas (m7,m8,m9,m10) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m1,m5) + - ms2, 4/4 replicas (m7,m8,m9,m10) +[M controller] Iteration 18, Reconcile m5 +[M controller] Iteration 18, Reconcile m7 +[M controller] Iteration 19, Reconcile m10 +[M controller] Iteration 19, Reconcile m8 +[M controller] Iteration 19, Reconcile m1 +[M controller] - m1 finalizer removed +[M controller] Iteration 19, Reconcile m5 +[M controller] Iteration 19, Reconcile m5 +[MS controller] Iteration 19, Reconcile ms2, 4/4 replicas (m7,m8,m9,m10) +[MS controller] Iteration 19, Reconcile ms1, 2/2 replicas (m5) +[MS controller] Iteration 19, Reconcile ms1, 1/2 replicas (m5) +[M controller] Iteration 19, Reconcile m5 +[M controller] Iteration 19, Reconcile m10 +[MD controller] Iteration 19, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 1/2 replicas (m5) + - ms2, 4/4 replicas (m7,m8,m9,m10) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 1/1 replicas (m5) + - ms2, 4/4 replicas (m7,m8,m9,m10) +[M controller] Iteration 19, Reconcile m9 +[M controller] Iteration 20, Reconcile m7 +[M controller] Iteration 20, Reconcile m5 +[User] Iteration 20, Deleting machine m5 +[MS controller] Iteration 20, Reconcile ms2, 4/4 replicas (m7,m8,m9,m10) +[MS controller] Iteration 20, Reconcile ms2, 4/4 replicas (m7,m8,m9,m10) +[M controller] Iteration 20, Reconcile m5 +[M controller] - m5 finalizer removed +[MS controller] Iteration 20, Reconcile ms2, 4/4 replicas (m7,m8,m9,m10) +[M controller] Iteration 20, Reconcile m8 +[M controller] Iteration 21, Reconcile m8 +[M controller] Iteration 21, Reconcile m7 +[M controller] Iteration 21, Reconcile m7 +[MD controller] Iteration 21, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 1/1 replicas () + - ms2, 4/4 replicas (m7,m8,m9,m10) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 1/1 replicas () + - ms2, 4/5 replicas (m7,m8,m9,m10) +[M controller] Iteration 21, Reconcile m7 +[MS controller] Iteration 21, Reconcile ms1, 1/1 replicas () +[M controller] Iteration 21, Reconcile m10 +[MS controller] Iteration 21, Reconcile ms2, 4/5 replicas (m7,m8,m9,m10) +[MS controller] - ms2 scale up to 5/5 replicas (m11 created) +[M controller] Iteration 21, Reconcile m10 +[MS controller] Iteration 22, Reconcile ms2, 5/5 replicas (m7,m8,m9,m10,m11) +[M controller] Iteration 22, Reconcile m10 +[M controller] Iteration 22, Reconcile m11 +[MD controller] Iteration 22, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 0/1 replicas () + - ms2, 5/5 replicas (m7,m8,m9,m10,m11) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 5/5 replicas (m7,m8,m9,m10,m11) +[M controller] Iteration 22, Reconcile m11 +[M controller] Iteration 22, Reconcile m8 +[M controller] Iteration 22, Reconcile m7 +[M controller] Iteration 22, Reconcile m7 +[M controller] Iteration 23, Reconcile m10 +[M controller] Iteration 23, Reconcile m9 +[MD controller] Iteration 23, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 5/5 replicas (m7,m8,m9,m10,m11) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 5/6 replicas (m7,m8,m9,m10,m11) +[MS controller] Iteration 23, Reconcile ms2, 5/6 replicas (m7,m8,m9,m10,m11) +[MS controller] - ms2 scale up to 6/6 replicas (m12 created) +[MS controller] Iteration 23, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 23, Reconcile ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MS controller] Iteration 23, Reconcile ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[M controller] Iteration 23, Reconcile m10 +[M controller] Iteration 23, Reconcile m7 +[MS controller] Iteration 24, Reconcile ms1, 0/0 replicas () +[M controller] Iteration 24, Reconcile m10 +[M controller] Iteration 24, Reconcile m7 +[M controller] Iteration 24, Reconcile m8 +[MD controller] Iteration 24, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[M controller] Iteration 24, Reconcile m9 +[M controller] Iteration 24, Reconcile m12 +[Test] Final state + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) diff --git a/internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 2.test.log.golden b/internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 2.test.log.golden new file mode 100644 index 000000000000..3da216fc3550 --- /dev/null +++ b/internal/controllers/machinedeployment/testdata/ondelete/6 replicas, maxuserunavailable 2.test.log.golden @@ -0,0 +1,215 @@ +## 6 replicas, maxUserUnavailable 2 + +[Test] Initial state + md, 6/6 replicas + - ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 0/0 replicas () +[Test] Rollout 6 replicas, onDeleteStrategy +[MD controller] Iteration 1, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 0/0 replicas () +[MS controller] Iteration 1, Reconcile ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) +[MS controller] Iteration 1, Reconcile ms2, 0/0 replicas () +[User] Iteration 1, Deleting machine m1 +[M controller] Iteration 1, Reconcile m1 +[M controller] - m1 finalizer removed +[M controller] Iteration 1, Reconcile m2 +[M controller] Iteration 1, Reconcile m3 +[M controller] Iteration 1, Reconcile m4 +[M controller] Iteration 1, Reconcile m5 +[M controller] Iteration 1, Reconcile m6 +[MD controller] Iteration 2, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 6/6 replicas (m2,m3,m4,m5,m6) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 6/6 replicas (m2,m3,m4,m5,m6) + - ms2, 0/0 replicas () +[MS controller] Iteration 2, Reconcile ms1, 6/6 replicas (m2,m3,m4,m5,m6) +[MS controller] Iteration 2, Reconcile ms2, 0/0 replicas () +[User] Iteration 2, Deleting machine m2 +[M controller] Iteration 2, Reconcile m2 +[M controller] - m2 finalizer removed +[M controller] Iteration 2, Reconcile m3 +[M controller] Iteration 2, Reconcile m4 +[M controller] Iteration 2, Reconcile m5 +[M controller] Iteration 2, Reconcile m6 +[MD controller] Iteration 3, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 5/6 replicas (m3,m4,m5,m6) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 5/5 replicas (m3,m4,m5,m6) + - ms2, 0/0 replicas () +[MS controller] Iteration 3, Reconcile ms1, 5/5 replicas (m3,m4,m5,m6) +[MS controller] Iteration 3, Reconcile ms2, 0/0 replicas () +[M controller] Iteration 3, Reconcile m3 +[M controller] Iteration 3, Reconcile m4 +[M controller] Iteration 3, Reconcile m5 +[M controller] Iteration 3, Reconcile m6 +[MD controller] Iteration 4, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 4/5 replicas (m3,m4,m5,m6) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 4/6 replicas + - ms1, 4/4 replicas (m3,m4,m5,m6) + - ms2, 0/1 replicas () +[MS controller] Iteration 4, Reconcile ms1, 4/4 replicas (m3,m4,m5,m6) +[MS controller] Iteration 4, Reconcile ms2, 0/1 replicas () +[MS controller] - ms2 scale up to 1/1 replicas (m7 created) +[User] Iteration 4, Deleting machine m3 +[M controller] Iteration 4, Reconcile m3 +[M controller] - m3 finalizer removed +[M controller] Iteration 4, Reconcile m4 +[M controller] Iteration 4, Reconcile m5 +[M controller] Iteration 4, Reconcile m6 +[MD controller] Iteration 5, Reconcile md +[MD controller] - Input to rollout planner + md, 4/6 replicas + - ms1, 4/4 replicas (m4,m5,m6) + - ms2, 1/1 replicas (m7) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 4/4 replicas (m4,m5,m6) + - ms2, 1/2 replicas (m7) +[MS controller] Iteration 5, Reconcile ms1, 4/4 replicas (m4,m5,m6) +[MS controller] Iteration 5, Reconcile ms2, 1/2 replicas (m7) +[MS controller] - ms2 scale up to 2/2 replicas (m8 created) +[User] Iteration 5, Deleting machine m4 +[M controller] Iteration 5, Reconcile m4 +[M controller] - m4 finalizer removed +[M controller] Iteration 5, Reconcile m5 +[M controller] Iteration 5, Reconcile m6 +[M controller] Iteration 5, Reconcile m7 +[MD controller] Iteration 6, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 3/4 replicas (m5,m6) + - ms2, 2/2 replicas (m7,m8) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 3/3 replicas (m5,m6) + - ms2, 2/2 replicas (m7,m8) +[MS controller] Iteration 6, Reconcile ms1, 3/3 replicas (m5,m6) +[MS controller] Iteration 6, Reconcile ms2, 2/2 replicas (m7,m8) +[M controller] Iteration 6, Reconcile m5 +[M controller] Iteration 6, Reconcile m6 +[M controller] Iteration 6, Reconcile m7 +[M controller] Iteration 6, Reconcile m8 +[MD controller] Iteration 7, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 2/3 replicas (m5,m6) + - ms2, 2/2 replicas (m7,m8) +[MD controller] - Result of rollout planner + md, 4/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 2/3 replicas (m7,m8) +[MS controller] Iteration 7, Reconcile ms1, 2/2 replicas (m5,m6) +[MS controller] Iteration 7, Reconcile ms2, 2/3 replicas (m7,m8) +[MS controller] - ms2 scale up to 3/3 replicas (m9 created) +[User] Iteration 7, Deleting machine m5 +[M controller] Iteration 7, Reconcile m5 +[M controller] - m5 finalizer removed +[M controller] Iteration 7, Reconcile m6 +[M controller] Iteration 7, Reconcile m7 +[M controller] Iteration 7, Reconcile m8 +[MD controller] Iteration 8, Reconcile md +[MD controller] - Input to rollout planner + md, 4/6 replicas + - ms1, 2/2 replicas (m6) + - ms2, 3/3 replicas (m7,m8,m9) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m6) + - ms2, 3/4 replicas (m7,m8,m9) +[MS controller] Iteration 8, Reconcile ms1, 2/2 replicas (m6) +[MS controller] Iteration 8, Reconcile ms2, 3/4 replicas (m7,m8,m9) +[MS controller] - ms2 scale up to 4/4 replicas (m10 created) +[User] Iteration 8, Deleting machine m6 +[M controller] Iteration 8, Reconcile m6 +[M controller] - m6 finalizer removed +[M controller] Iteration 8, Reconcile m7 +[M controller] Iteration 8, Reconcile m8 +[M controller] Iteration 8, Reconcile m9 +[MD controller] Iteration 9, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 1/2 replicas () + - ms2, 4/4 replicas (m7,m8,m9,m10) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 1/1 replicas () + - ms2, 4/4 replicas (m7,m8,m9,m10) +[MS controller] Iteration 9, Reconcile ms1, 1/1 replicas () +[MS controller] Iteration 9, Reconcile ms2, 4/4 replicas (m7,m8,m9,m10) +[M controller] Iteration 9, Reconcile m7 +[M controller] Iteration 9, Reconcile m8 +[M controller] Iteration 9, Reconcile m9 +[M controller] Iteration 9, Reconcile m10 +[MD controller] Iteration 10, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 0/1 replicas () + - ms2, 4/4 replicas (m7,m8,m9,m10) +[MD controller] - Result of rollout planner + md, 4/6 replicas + - ms1, 0/0 replicas () + - ms2, 4/5 replicas (m7,m8,m9,m10) +[MS controller] Iteration 10, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 10, Reconcile ms2, 4/5 replicas (m7,m8,m9,m10) +[MS controller] - ms2 scale up to 5/5 replicas (m11 created) +[M controller] Iteration 10, Reconcile m7 +[M controller] Iteration 10, Reconcile m8 +[M controller] Iteration 10, Reconcile m9 +[M controller] Iteration 10, Reconcile m10 +[MD controller] Iteration 11, Reconcile md +[MD controller] - Input to rollout planner + md, 4/6 replicas + - ms1, 0/0 replicas () + - ms2, 5/5 replicas (m7,m8,m9,m10,m11) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 5/6 replicas (m7,m8,m9,m10,m11) +[MS controller] Iteration 11, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 11, Reconcile ms2, 5/6 replicas (m7,m8,m9,m10,m11) +[MS controller] - ms2 scale up to 6/6 replicas (m12 created) +[M controller] Iteration 11, Reconcile m7 +[M controller] Iteration 11, Reconcile m8 +[M controller] Iteration 11, Reconcile m9 +[M controller] Iteration 11, Reconcile m10 +[M controller] Iteration 11, Reconcile m11 +[MD controller] Iteration 12, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MS controller] Iteration 12, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 12, Reconcile ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[M controller] Iteration 12, Reconcile m7 +[M controller] Iteration 12, Reconcile m8 +[M controller] Iteration 12, Reconcile m9 +[M controller] Iteration 12, Reconcile m10 +[M controller] Iteration 12, Reconcile m11 +[M controller] Iteration 12, Reconcile m12 +[Test] Final state + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 12 replicas, maxsurge 3, maxunavailable 1, scale down to 6, random(0).test.log.golden b/internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 12 replicas, maxsurge 3, maxunavailable 1, scale down to 6, random(0).test.log.golden similarity index 100% rename from internal/controllers/machinedeployment/testdata/regular rollout, 12 replicas, maxsurge 3, maxunavailable 1, scale down to 6, random(0).test.log.golden rename to internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 12 replicas, maxsurge 3, maxunavailable 1, scale down to 6, random(0).test.log.golden diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 12 replicas, maxsurge 3, maxunavailable 1, scale down to 6.test.log.golden b/internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 12 replicas, maxsurge 3, maxunavailable 1, scale down to 6.test.log.golden similarity index 100% rename from internal/controllers/machinedeployment/testdata/regular rollout, 12 replicas, maxsurge 3, maxunavailable 1, scale down to 6.test.log.golden rename to internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 12 replicas, maxsurge 3, maxunavailable 1, scale down to 6.test.log.golden diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 3 replicas, maxsurge 0, maxunavailable 1, random(0).test.log.golden b/internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 3 replicas, maxsurge 0, maxunavailable 1, random(0).test.log.golden similarity index 100% rename from internal/controllers/machinedeployment/testdata/regular rollout, 3 replicas, maxsurge 0, maxunavailable 1, random(0).test.log.golden rename to internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 3 replicas, maxsurge 0, maxunavailable 1, random(0).test.log.golden diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 3 replicas, maxsurge 0, maxunavailable 1.test.log.golden b/internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 3 replicas, maxsurge 0, maxunavailable 1.test.log.golden similarity index 100% rename from internal/controllers/machinedeployment/testdata/regular rollout, 3 replicas, maxsurge 0, maxunavailable 1.test.log.golden rename to internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 3 replicas, maxsurge 0, maxunavailable 1.test.log.golden diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 3 replicas, maxsurge 1, maxunavailable 0, random(0).test.log.golden b/internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 3 replicas, maxsurge 1, maxunavailable 0, random(0).test.log.golden similarity index 100% rename from internal/controllers/machinedeployment/testdata/regular rollout, 3 replicas, maxsurge 1, maxunavailable 0, random(0).test.log.golden rename to internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 3 replicas, maxsurge 1, maxunavailable 0, random(0).test.log.golden diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 3 replicas, maxsurge 1, maxunavailable 0.test.log.golden b/internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 3 replicas, maxsurge 1, maxunavailable 0.test.log.golden similarity index 100% rename from internal/controllers/machinedeployment/testdata/regular rollout, 3 replicas, maxsurge 1, maxunavailable 0.test.log.golden rename to internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 3 replicas, maxsurge 1, maxunavailable 0.test.log.golden diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 0, maxunavailable 10, random(0).test.log.golden b/internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 0, maxunavailable 10, random(0).test.log.golden similarity index 100% rename from internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 0, maxunavailable 10, random(0).test.log.golden rename to internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 0, maxunavailable 10, random(0).test.log.golden diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 0, maxunavailable 10.test.log.golden b/internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 0, maxunavailable 10.test.log.golden similarity index 100% rename from internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 0, maxunavailable 10.test.log.golden rename to internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 0, maxunavailable 10.test.log.golden diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 1, maxunavailable 3, random(0).test.log.golden b/internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 1, maxunavailable 3, random(0).test.log.golden similarity index 100% rename from internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 1, maxunavailable 3, random(0).test.log.golden rename to internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 1, maxunavailable 3, random(0).test.log.golden diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 1, maxunavailable 3.test.log.golden b/internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 1, maxunavailable 3.test.log.golden similarity index 100% rename from internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 1, maxunavailable 3.test.log.golden rename to internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 1, maxunavailable 3.test.log.golden diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 10, maxunavailable 0, random(0).test.log.golden b/internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 10, maxunavailable 0, random(0).test.log.golden similarity index 100% rename from internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 10, maxunavailable 0, random(0).test.log.golden rename to internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 10, maxunavailable 0, random(0).test.log.golden diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 10, maxunavailable 0.test.log.golden b/internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 10, maxunavailable 0.test.log.golden similarity index 100% rename from internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 10, maxunavailable 0.test.log.golden rename to internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 10, maxunavailable 0.test.log.golden diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, change spec, random(0).test.log.golden b/internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, change spec, random(0).test.log.golden similarity index 100% rename from internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, change spec, random(0).test.log.golden rename to internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, change spec, random(0).test.log.golden diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, change spec.test.log.golden b/internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, change spec.test.log.golden similarity index 100% rename from internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, change spec.test.log.golden rename to internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, change spec.test.log.golden diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, random(0).test.log.golden b/internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, random(0).test.log.golden similarity index 100% rename from internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, random(0).test.log.golden rename to internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, random(0).test.log.golden diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, scale up to 12, random(0).test.log.golden b/internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, scale up to 12, random(0).test.log.golden similarity index 100% rename from internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, scale up to 12, random(0).test.log.golden rename to internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, scale up to 12, random(0).test.log.golden diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, scale up to 12.test.log.golden b/internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, scale up to 12.test.log.golden similarity index 100% rename from internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, scale up to 12.test.log.golden rename to internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, scale up to 12.test.log.golden diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1.test.log.golden b/internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1.test.log.golden similarity index 100% rename from internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1.test.log.golden rename to internal/controllers/machinedeployment/testdata/rollingupdate/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1.test.log.golden From 47c1d22f20d7a58f3509c1cc49af82be6d1f7e0d Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Thu, 9 Oct 2025 14:11:59 +0200 Subject: [PATCH 2/2] Address feedback --- .../machinedeployment_rollout_ondelete.go | 8 +++---- ...machinedeployment_rollout_ondelete_test.go | 24 +++++++++---------- ...machinedeployment_rollout_rollingupdate.go | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/internal/controllers/machinedeployment/machinedeployment_rollout_ondelete.go b/internal/controllers/machinedeployment/machinedeployment_rollout_ondelete.go index 766257613cfe..1b6b3a875075 100644 --- a/internal/controllers/machinedeployment/machinedeployment_rollout_ondelete.go +++ b/internal/controllers/machinedeployment/machinedeployment_rollout_ondelete.go @@ -102,7 +102,7 @@ func (p *rolloutPlanner) reconcileOldMachineSetsOnDelete(ctx context.Context) { return } - // Determine if there are replicas to be scaled down due to a scale down in MD. + // Determine if there are more Machines than MD.spec.replicas, e.g. due to a scale down in MD. newMSReplicas := ptr.Deref(p.newMS.Spec.Replicas, 0) if v, ok := p.scaleIntents[p.newMS.Name]; ok { newMSReplicas = v @@ -116,10 +116,10 @@ func (p *rolloutPlanner) reconcileOldMachineSetsOnDelete(ctx context.Context) { // Start scaling down old machine sets to acknowledge spec.replicas without corresponding status.replicas. // Note: spec.replicas without corresponding status.replicas exists // - after a user manually deletes a replica - // - when a newMS non yet fully provisioned suddenly becomes an oldMS. + // - when a newMS not yet fully provisioned suddenly becomes an oldMS. // In both cases spec.replicas without corresponding status.replicas should be dropped, no matter // if there are replicas to be scaled down due to a scale down in MD or not. - // However, if there are replicas to be scaled down due to a scale down in MD, deleted replicas should + // However, just in case there are replicas to be scaled down due to a scale down in MD, deleted replicas should // be deducted from the totalScaleDownCount. for _, oldMS := range p.oldMSs { // No op if this MS has been already scaled down to zero. @@ -147,7 +147,7 @@ func (p *rolloutPlanner) reconcileOldMachineSetsOnDelete(ctx context.Context) { // No op if this MS has been already scaled down to zero. scaleIntent := ptr.Deref(oldMS.Spec.Replicas, 0) if v, ok := p.scaleIntents[oldMS.Name]; ok { - scaleIntent = min(scaleIntent, v) + scaleIntent = v } if scaleIntent <= 0 { diff --git a/internal/controllers/machinedeployment/machinedeployment_rollout_ondelete_test.go b/internal/controllers/machinedeployment/machinedeployment_rollout_ondelete_test.go index a0149c7d4900..561c0f5f7435 100644 --- a/internal/controllers/machinedeployment/machinedeployment_rollout_ondelete_test.go +++ b/internal/controllers/machinedeployment/machinedeployment_rollout_ondelete_test.go @@ -25,6 +25,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" @@ -201,8 +202,8 @@ func TestReconcileOldMachineSetsOnDelete(t *testing.T) { }, expectScaleIntent: map[string]int32{ // new intent for old MS - "ms1": 0, // scale down by one,1 exceeding machine removed from ms1 - "ms2": 1, // scale down by one,1 exceeding machine removed from ms1 + "ms1": 0, // scale down by one, 1 exceeding machine removed from ms1 + "ms2": 1, // scale down by one, 1 exceeding machine removed from ms2 }, }, { @@ -259,7 +260,7 @@ func TestReconcileOldMachineSetsOnDelete(t *testing.T) { "ms3": 2, // new intent for old MS "ms1": 0, // scale down by one, 1 exceeding machine removed from ms1 - "ms2": 1, // scale down by one,1 exceeding machine removed from ms1 + "ms2": 1, // scale down by one, 1 exceeding machine removed from ms2 }, }, { @@ -390,6 +391,7 @@ type onDeleteSequenceTestCase struct { func Test_OnDeleteSequences(t *testing.T) { ctx := context.Background() ctx = ctrl.LoggerInto(ctx, klog.Background()) + klog.SetOutput(ginkgo.GinkgoWriter) tests := []onDeleteSequenceTestCase{ { // delete 1 @@ -578,8 +580,6 @@ func runOnDeleteTestCase(ctx context.Context, t *testing.T, tt onDeleteSequenceT fLogger.Logf("[MD controller] - Input to rollout planner\n%s", current) // Running a small subset of MD reconcile (the rollout logic and a bit of setReplicas) - - // create rollout planner and override the func that computeDesiredMS so it uses a predictable MS name when creating newMS. p := newRolloutPlanner() p.md = current.machineDeployment p.newMS = current.newMS() @@ -618,11 +618,11 @@ func runOnDeleteTestCase(ctx context.Context, t *testing.T, tt onDeleteSequenceT // Simulate the user deleting machines not upToDate; in order to make this realistic deletion will be performed // if the operation respects maxUserUnavailable e.g. - // - with maxUserUnavailable=1, perform deletion only when previous deletion has been completed and a replacement machines, - // which we assume it is the most common behaviour. - // - maxUserUnavailable > 1 can be used to test scenarios where the users delete machines in a more aggressive - // Also deletion cannot happen before the newMS has been created (thus preventing machines being deleted and - // re-created on oldMSs in case of random sequences, which will lead to result different than what expected) + // - with maxUserUnavailable=1, perform deletion only when previous deletion has been completed and a replacement machines + // has been created, which we assume it is the most common behaviour. + // - maxUserUnavailable > 1 can be used to test scenarios where the users delete machines in a more aggressive way. + // Also, deletion cannot happen before the newMS has been created (thus preventing machines being deleted and + // re-created on oldMSs in case of random sequences, which will lead to result different from what expected) if task == "delete-machine" { deletionsInFlight := int32(0) for _, m := range current.machines() { @@ -634,7 +634,7 @@ func runOnDeleteTestCase(ctx context.Context, t *testing.T, tt onDeleteSequenceT totAvailable := max(int32(len(current.machines()))-deletionsInFlight, 0) minAvailable := max(ptr.Deref(current.machineDeployment.Spec.Replicas, 0)-tt.maxUserUnavailable, 0) if totAvailable > minAvailable && canDelete { - // Determine the list of machines that should be deleted manually, which are the machines not yet upd to date. + // Determine the list of machines that should be deleted manually, which are the machines not yet UpToDate. machinesToDelete := []*clusterv1.Machine{} for _, m := range current.machines() { if upToDate, _ := mdutil.MachineTemplateUpToDate(&clusterv1.MachineTemplateSpec{Spec: m.Spec}, &desired.machineDeployment.Spec.Template); !upToDate { @@ -644,7 +644,7 @@ func runOnDeleteTestCase(ctx context.Context, t *testing.T, tt onDeleteSequenceT if len(machinesToDelete) > 0 { // When running with default controller order, also delete machines in order; - // conversely, when running with random controller order, also machine deletion happens in rando order. + // conversely, when running with random controller order, also machine deletion happens in random order. n := 0 if tt.randomControllerOrder { n = rng.Intn(len(machinesToDelete)) diff --git a/internal/controllers/machinedeployment/machinedeployment_rollout_rollingupdate.go b/internal/controllers/machinedeployment/machinedeployment_rollout_rollingupdate.go index 83be656bcbb8..cf2c5666aa37 100644 --- a/internal/controllers/machinedeployment/machinedeployment_rollout_rollingupdate.go +++ b/internal/controllers/machinedeployment/machinedeployment_rollout_rollingupdate.go @@ -56,7 +56,7 @@ func (r *Reconciler) rolloutRollingUpdate(ctx context.Context, md *clusterv1.Mac // - TBD is we want to implement a proper cleanup thus ensuring create machines on oldMS can always happen when using the // RolloutUpdate strategy, or if we want to always prevent machine creation on oldMS also in this case. // Note: When rollout is paused the code assumes it can always scale up oldMS (see scale in machinedeployment_sync.go). - // however current implement it is of missing removal of the DisableMachineCreateAnnotation that might exist or not + // however current implementation is missing removal of the DisableMachineCreateAnnotation that might exist or not // on oldMS (depending on strategy / proper cleanup etc.). if err := r.cleanupDisableMachineCreateAnnotation(ctx, newMS); err != nil { return err