diff --git a/internal/controllers/machinedeployment/machinedeployment_rolling.go b/internal/controllers/machinedeployment/machinedeployment_rolling.go index d4102185c616..83bd72f70d6e 100644 --- a/internal/controllers/machinedeployment/machinedeployment_rolling.go +++ b/internal/controllers/machinedeployment/machinedeployment_rolling.go @@ -18,6 +18,7 @@ package machinedeployment import ( "context" + "fmt" "sort" "github.com/pkg/errors" @@ -33,6 +34,7 @@ import ( // rolloutRolling implements the logic for rolling a new MachineSet. func (r *Reconciler) rolloutRolling(ctx context.Context, md *clusterv1.MachineDeployment, msList []*clusterv1.MachineSet, templateExists bool) error { + // TODO(in-place): move create newMS into rolloutPlanner newMS, oldMSs, err := r.getAllMachineSetsAndSyncRevision(ctx, md, msList, true, templateExists) if err != nil { return err @@ -47,18 +49,29 @@ func (r *Reconciler) rolloutRolling(ctx context.Context, md *clusterv1.MachineDe allMSs := append(oldMSs, newMS) - // Scale up, if we can. - if err := r.reconcileNewMachineSet(ctx, allMSs, newMS, md); err != nil { + // TODO(in-place): also apply/remove labels to MS should go into rolloutPlanner + if err := r.cleanupDisableMachineCreateAnnotation(ctx, newMS); err != nil { return err } - if err := r.syncDeploymentStatus(allMSs, newMS, md); err != nil { + planner := newRolloutPlanner() + planner.md = md + planner.newMS = newMS + planner.oldMSs = oldMSs + + if err := planner.Plan(ctx); err != nil { return err } - // Scale down, if we can. - if err := r.reconcileOldMachineSets(ctx, allMSs, oldMSs, newMS, md); err != nil { - return err + // TODO(in-place): this should be changed as soon as rolloutPlanner support MS creation and adding/removing labels from MS + for _, ms := range allMSs { + scaleIntent := ptr.Deref(ms.Spec.Replicas, 0) + if v, ok := planner.scaleIntents[ms.Name]; ok { + scaleIntent = v + } + if err := r.scaleMachineSet(ctx, ms, scaleIntent, md); err != nil { + return err + } } if err := r.syncDeploymentStatus(allMSs, newMS, md); err != nil { @@ -74,59 +87,95 @@ func (r *Reconciler) rolloutRolling(ctx context.Context, md *clusterv1.MachineDe return nil } -func (r *Reconciler) reconcileNewMachineSet(ctx context.Context, allMSs []*clusterv1.MachineSet, newMS *clusterv1.MachineSet, deployment *clusterv1.MachineDeployment) error { - if err := r.cleanupDisableMachineCreateAnnotation(ctx, newMS); err != nil { - return err +type rolloutPlanner struct { + md *clusterv1.MachineDeployment + newMS *clusterv1.MachineSet + oldMSs []*clusterv1.MachineSet + scaleIntents map[string]int32 +} + +func newRolloutPlanner() *rolloutPlanner { + return &rolloutPlanner{ + scaleIntents: make(map[string]int32), + } +} + +// Plan determine how to proceed with the rollout if we are not yet at the desired state. +func (p *rolloutPlanner) Plan(ctx context.Context) error { + if p.md.Spec.Replicas == nil { + return errors.Errorf("spec.replicas for MachineDeployment %v is nil, this is unexpected", client.ObjectKeyFromObject(p.md)) } - if deployment.Spec.Replicas == nil { - return errors.Errorf("spec.replicas for MachineDeployment %v is nil, this is unexpected", client.ObjectKeyFromObject(deployment)) + if p.newMS.Spec.Replicas == nil { + return errors.Errorf("spec.replicas for MachineSet %v is nil, this is unexpected", client.ObjectKeyFromObject(p.newMS)) + } + + for _, oldMS := range p.oldMSs { + if oldMS.Spec.Replicas == nil { + return errors.Errorf("spec.replicas for MachineSet %v is nil, this is unexpected", client.ObjectKeyFromObject(oldMS)) + } } - if newMS.Spec.Replicas == nil { - return errors.Errorf("spec.replicas for MachineSet %v is nil, this is unexpected", client.ObjectKeyFromObject(newMS)) + // Scale up, if we can. + if err := p.reconcileNewMachineSet(ctx); err != nil { + return err } - if *(newMS.Spec.Replicas) == *(deployment.Spec.Replicas) { + // Scale down, if we can. + return p.reconcileOldMachineSets(ctx) +} + +func (p *rolloutPlanner) reconcileNewMachineSet(ctx context.Context) error { + log := ctrl.LoggerFrom(ctx) + allMSs := append(p.oldMSs, p.newMS) + + if *(p.newMS.Spec.Replicas) == *(p.md.Spec.Replicas) { // Scaling not required. return nil } - if *(newMS.Spec.Replicas) > *(deployment.Spec.Replicas) { + if *(p.newMS.Spec.Replicas) > *(p.md.Spec.Replicas) { // Scale down. - return r.scaleMachineSet(ctx, newMS, *(deployment.Spec.Replicas), deployment) + log.V(5).Info(fmt.Sprintf("Setting scale down intent for MachineSet %s to %d replicas", p.newMS.Name, *(p.md.Spec.Replicas)), "MachineSet", klog.KObj(p.newMS)) + p.scaleIntents[p.newMS.Name] = *(p.md.Spec.Replicas) + return nil } - newReplicasCount, err := mdutil.NewMSNewReplicas(deployment, allMSs, *newMS.Spec.Replicas) + newReplicasCount, err := mdutil.NewMSNewReplicas(p.md, allMSs, *p.newMS.Spec.Replicas) if err != nil { return err } - return r.scaleMachineSet(ctx, newMS, newReplicasCount, deployment) -} -func (r *Reconciler) reconcileOldMachineSets(ctx context.Context, allMSs []*clusterv1.MachineSet, oldMSs []*clusterv1.MachineSet, newMS *clusterv1.MachineSet, deployment *clusterv1.MachineDeployment) error { - log := ctrl.LoggerFrom(ctx) - - if deployment.Spec.Replicas == nil { - return errors.Errorf("spec.replicas for MachineDeployment %v is nil, this is unexpected", - client.ObjectKeyFromObject(deployment)) + if newReplicasCount < *(p.newMS.Spec.Replicas) { + scaleDownCount := *(p.newMS.Spec.Replicas) - newReplicasCount + log.V(5).Info(fmt.Sprintf("Setting scale down intent for MachineSet %s to %d replicas (-%d)", p.newMS.Name, newReplicasCount, scaleDownCount), "MachineSet", klog.KObj(p.newMS)) + p.scaleIntents[p.newMS.Name] = newReplicasCount } - - if newMS.Spec.Replicas == nil { - return errors.Errorf("spec.replicas for MachineSet %v is nil, this is unexpected", - client.ObjectKeyFromObject(newMS)) + if newReplicasCount > *(p.newMS.Spec.Replicas) { + scaleUpCount := newReplicasCount - *(p.newMS.Spec.Replicas) + log.V(5).Info(fmt.Sprintf("Setting scale up intent for MachineSet %s to %d replicas (+%d)", p.newMS.Name, newReplicasCount, scaleUpCount), "MachineSet", klog.KObj(p.newMS)) + p.scaleIntents[p.newMS.Name] = newReplicasCount } + return nil +} + +func (p *rolloutPlanner) reconcileOldMachineSets(ctx context.Context) error { + log := ctrl.LoggerFrom(ctx) - oldMachinesCount := mdutil.GetReplicaCountForMachineSets(oldMSs) + oldMachinesCount := mdutil.GetReplicaCountForMachineSets(p.oldMSs) if oldMachinesCount == 0 { // Can't scale down further return nil } - allMachinesCount := mdutil.GetReplicaCountForMachineSets(allMSs) + newMSReplicas := ptr.Deref(p.newMS.Spec.Replicas, 0) + if v, ok := p.scaleIntents[p.newMS.Name]; ok { + newMSReplicas = v + } + allMachinesCount := oldMachinesCount + newMSReplicas log.V(4).Info("New MachineSet has available machines", - "machineset", client.ObjectKeyFromObject(newMS).String(), "available-replicas", ptr.Deref(newMS.Status.AvailableReplicas, 0)) - maxUnavailable := mdutil.MaxUnavailable(*deployment) + "machineset", client.ObjectKeyFromObject(p.newMS).String(), "available-replicas", ptr.Deref(p.newMS.Status.AvailableReplicas, 0)) + maxUnavailable := mdutil.MaxUnavailable(*p.md) // Check if we can scale down. We can scale down in the following 2 cases: // * Some old MachineSets have unhealthy replicas, we could safely scale down those unhealthy replicas since that won't further @@ -158,10 +207,10 @@ func (r *Reconciler) reconcileOldMachineSets(ctx context.Context, allMSs []*clus // * The new MachineSet created must start with 0 replicas because allMachinesCount is already at 13. // * However, newMSMachinesUnavailable would also be 0, so the 2 old MachineSets could be scaled down by 5 (13 - 8 - 0), which would then // allow the new MachineSet to be scaled up by 5. - availableReplicas := ptr.Deref(newMS.Status.AvailableReplicas, 0) + availableReplicas := ptr.Deref(p.newMS.Status.AvailableReplicas, 0) - minAvailable := *(deployment.Spec.Replicas) - maxUnavailable - newMSUnavailableMachineCount := *(newMS.Spec.Replicas) - availableReplicas + minAvailable := *(p.md.Spec.Replicas) - maxUnavailable + newMSUnavailableMachineCount := newMSReplicas - availableReplicas maxScaledDown := allMachinesCount - minAvailable - newMSUnavailableMachineCount if maxScaledDown <= 0 { return nil @@ -169,7 +218,7 @@ func (r *Reconciler) reconcileOldMachineSets(ctx context.Context, allMSs []*clus // Clean up unhealthy replicas first, otherwise unhealthy replicas will block deployment // and cause timeout. See https://github.com/kubernetes/kubernetes/issues/16737 - oldMSs, cleanupCount, err := r.cleanupUnhealthyReplicas(ctx, oldMSs, deployment, maxScaledDown) + cleanupCount, err := p.cleanupUnhealthyReplicas(ctx, maxScaledDown) if err != nil { return err } @@ -177,9 +226,7 @@ func (r *Reconciler) reconcileOldMachineSets(ctx context.Context, allMSs []*clus log.V(4).Info("Cleaned up unhealthy replicas from old MachineSets", "count", cleanupCount) // Scale down old MachineSets, need check maxUnavailable to ensure we can scale down - allMSs = oldMSs - allMSs = append(allMSs, newMS) - scaledDownCount, err := r.scaleDownOldMachineSetsForRollingUpdate(ctx, allMSs, oldMSs, deployment) + scaledDownCount, err := p.scaleDownOldMachineSetsForRollingUpdate(ctx) if err != nil { return err } @@ -189,10 +236,10 @@ func (r *Reconciler) reconcileOldMachineSets(ctx context.Context, allMSs []*clus } // cleanupUnhealthyReplicas will scale down old MachineSets with unhealthy replicas, so that all unhealthy replicas will be deleted. -func (r *Reconciler) cleanupUnhealthyReplicas(ctx context.Context, oldMSs []*clusterv1.MachineSet, deployment *clusterv1.MachineDeployment, maxCleanupCount int32) ([]*clusterv1.MachineSet, int32, error) { +func (p *rolloutPlanner) cleanupUnhealthyReplicas(ctx context.Context, maxCleanupCount int32) (int32, error) { log := ctrl.LoggerFrom(ctx) - sort.Sort(mdutil.MachineSetsByCreationTimestamp(oldMSs)) + sort.Sort(mdutil.MachineSetsByCreationTimestamp(p.oldMSs)) // Scale down all old MachineSets with any unhealthy replicas. MachineSet will honour spec.deletion.order // for deleting Machines. Machines with a deletion timestamp, with a failure message or without a nodeRef @@ -200,60 +247,62 @@ func (r *Reconciler) cleanupUnhealthyReplicas(ctx context.Context, oldMSs []*clu // This results in a best effort to remove machines backing unhealthy nodes. totalScaledDown := int32(0) - for _, targetMS := range oldMSs { - if targetMS.Spec.Replicas == nil { - return nil, 0, errors.Errorf("spec.replicas for MachineSet %v is nil, this is unexpected", client.ObjectKeyFromObject(targetMS)) + for _, oldMS := range p.oldMSs { + if oldMS.Spec.Replicas == nil { + return 0, errors.Errorf("spec.replicas for MachineSet %v is nil, this is unexpected", client.ObjectKeyFromObject(oldMS)) } if totalScaledDown >= maxCleanupCount { break } - oldMSReplicas := *(targetMS.Spec.Replicas) + oldMSReplicas := *(oldMS.Spec.Replicas) if oldMSReplicas == 0 { // cannot scale down this MachineSet. continue } - oldMSAvailableReplicas := ptr.Deref(targetMS.Status.AvailableReplicas, 0) + oldMSAvailableReplicas := ptr.Deref(oldMS.Status.AvailableReplicas, 0) log.V(4).Info("Found available Machines in old MachineSet", - "count", oldMSAvailableReplicas, "target-machineset", client.ObjectKeyFromObject(targetMS).String()) + "count", oldMSAvailableReplicas, "target-machineset", client.ObjectKeyFromObject(oldMS).String()) if oldMSReplicas == oldMSAvailableReplicas { // no unhealthy replicas found, no scaling required. continue } + // TODO(in-place): fix this logic + // It looks like that the current logic fails when the MD controller is called twice in a row, without MS controller being triggered in the between, e.g. + // - first reconcile scales down ms1, 6-->5 (-1) + // - second reconcile is not taking into account scales down already in progress, unhealthy count is wrongly computed as -1 instead of 0, this leads to increasing replica count instead of keeping it as it is (or scaling down), and then the safeguard below errors out. + remainingCleanupCount := maxCleanupCount - totalScaledDown unhealthyCount := oldMSReplicas - oldMSAvailableReplicas scaledDownCount := min(remainingCleanupCount, unhealthyCount) newReplicasCount := oldMSReplicas - scaledDownCount if newReplicasCount > oldMSReplicas { - return nil, 0, errors.Errorf("when cleaning up unhealthy replicas, got invalid request to scale down %v: %d -> %d", - client.ObjectKeyFromObject(targetMS), oldMSReplicas, newReplicasCount) + return 0, errors.Errorf("when cleaning up unhealthy replicas, got invalid request to scale down %v: %d -> %d", + client.ObjectKeyFromObject(oldMS), oldMSReplicas, newReplicasCount) } - if err := r.scaleMachineSet(ctx, targetMS, newReplicasCount, deployment); err != nil { - return nil, totalScaledDown, err - } + scaleDownCount := *(oldMS.Spec.Replicas) - newReplicasCount + log.V(5).Info(fmt.Sprintf("Setting scale down intent for MachineSet %s to %d replicas (-%d)", oldMS.Name, newReplicasCount, scaleDownCount), "MachineSet", klog.KObj(oldMS)) + p.scaleIntents[oldMS.Name] = newReplicasCount totalScaledDown += scaledDownCount } - return oldMSs, totalScaledDown, nil + return totalScaledDown, nil } // scaleDownOldMachineSetsForRollingUpdate scales down old MachineSets when deployment strategy is "RollingUpdate". // Need check maxUnavailable to ensure availability. -func (r *Reconciler) scaleDownOldMachineSetsForRollingUpdate(ctx context.Context, allMSs []*clusterv1.MachineSet, oldMSs []*clusterv1.MachineSet, deployment *clusterv1.MachineDeployment) (int32, error) { +func (p *rolloutPlanner) scaleDownOldMachineSetsForRollingUpdate(ctx context.Context) (int32, error) { log := ctrl.LoggerFrom(ctx) + allMSs := append(p.oldMSs, p.newMS) - if deployment.Spec.Replicas == nil { - return 0, errors.Errorf("spec.replicas for MachineDeployment %v is nil, this is unexpected", client.ObjectKeyFromObject(deployment)) - } - - maxUnavailable := mdutil.MaxUnavailable(*deployment) - minAvailable := *(deployment.Spec.Replicas) - maxUnavailable + maxUnavailable := mdutil.MaxUnavailable(*p.md) + minAvailable := *(p.md.Spec.Replicas) - maxUnavailable // Find the number of available machines. availableMachineCount := ptr.Deref(mdutil.GetAvailableReplicaCountForMachineSets(allMSs), 0) @@ -266,36 +315,44 @@ func (r *Reconciler) scaleDownOldMachineSetsForRollingUpdate(ctx context.Context log.V(4).Info("Found available machines in deployment, scaling down old MSes", "count", availableMachineCount) - sort.Sort(mdutil.MachineSetsByCreationTimestamp(oldMSs)) + sort.Sort(mdutil.MachineSetsByCreationTimestamp(p.oldMSs)) + + // TODO(in-place): fix this logic + // It looks like that the current logic fails when the MD controller is called twice in a row e.g. reconcile of md 6 replicas MaxSurge=3, MaxUnavailable=1 and + // ms1, 6/5 replicas << one is scaling down, but scale down not yet processed by the MS controller. + // ms2, 3/3 replicas + // Leads to: + // ms1, 6/1 replicas << it further scaled down by 4, which leads to totAvailable machines is less than MinUnavailable, which should not happen + // ms2, 3/3 replicas totalScaledDown := int32(0) totalScaleDownCount := availableMachineCount - minAvailable - for _, targetMS := range oldMSs { - if targetMS.Spec.Replicas == nil { - return 0, errors.Errorf("spec.replicas for MachineSet %v is nil, this is unexpected", client.ObjectKeyFromObject(targetMS)) - } - + for _, oldMS := range p.oldMSs { if totalScaledDown >= totalScaleDownCount { // No further scaling required. break } - if *(targetMS.Spec.Replicas) == 0 { + oldMSReplicas := ptr.Deref(oldMS.Spec.Replicas, 0) + if v, ok := p.scaleIntents[oldMS.Name]; ok { + oldMSReplicas = v + } + + if oldMSReplicas == 0 { // cannot scale down this MachineSet. continue } // Scale down. - scaleDownCount := min(*(targetMS.Spec.Replicas), totalScaleDownCount-totalScaledDown) - newReplicasCount := *(targetMS.Spec.Replicas) - scaleDownCount - if newReplicasCount > *(targetMS.Spec.Replicas) { + scaleDownCount := min(oldMSReplicas, totalScaleDownCount-totalScaledDown) + newReplicasCount := oldMSReplicas - scaleDownCount + if newReplicasCount > oldMSReplicas { return totalScaledDown, errors.Errorf("when scaling down old MachineSet, got invalid request to scale down %v: %d -> %d", - client.ObjectKeyFromObject(targetMS), *(targetMS.Spec.Replicas), newReplicasCount) + client.ObjectKeyFromObject(oldMS), oldMSReplicas, newReplicasCount) } - if err := r.scaleMachineSet(ctx, targetMS, newReplicasCount, deployment); err != nil { - return totalScaledDown, err - } + log.V(5).Info(fmt.Sprintf("Setting scale down intent for MachineSet %s to %d replicas (-%d)", oldMS.Name, newReplicasCount, scaleDownCount), "MachineSet", klog.KObj(oldMS)) + p.scaleIntents[oldMS.Name] = newReplicasCount totalScaledDown += scaleDownCount } diff --git a/internal/controllers/machinedeployment/machinedeployment_rolling_test.go b/internal/controllers/machinedeployment/machinedeployment_rolling_test.go index 49311c494523..31e555e857fd 100644 --- a/internal/controllers/machinedeployment/machinedeployment_rolling_test.go +++ b/internal/controllers/machinedeployment/machinedeployment_rolling_test.go @@ -17,19 +17,13 @@ limitations under the License. package machinedeployment import ( - "strconv" "testing" . "github.com/onsi/gomega" - "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/record" "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" - "sigs.k8s.io/cluster-api/internal/controllers/machinedeployment/mdutil" ) func TestReconcileNewMachineSet(t *testing.T) { @@ -41,36 +35,6 @@ func TestReconcileNewMachineSet(t *testing.T) { expectedNewMachineSetReplicas int error error }{ - { - name: "It fails when machineDeployment has no replicas", - machineDeployment: &clusterv1.MachineDeployment{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "foo", - Name: "bar", - }, - }, - newMachineSet: &clusterv1.MachineSet{ - Spec: clusterv1.MachineSetSpec{ - Replicas: ptr.To[int32](2), - }, - }, - error: errors.Errorf("spec.replicas for MachineDeployment foo/bar is nil, this is unexpected"), - }, - { - name: "It fails when new machineSet has no replicas", - machineDeployment: &clusterv1.MachineDeployment{ - Spec: clusterv1.MachineDeploymentSpec{ - Replicas: ptr.To[int32](2), - }, - }, - newMachineSet: &clusterv1.MachineSet{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "foo", - Name: "bar", - }, - }, - error: errors.Errorf("spec.replicas for MachineSet foo/bar is nil, this is unexpected"), - }, { name: "RollingUpdate strategy: Scale up: 0 -> 2", machineDeployment: &clusterv1.MachineDeployment{ @@ -268,21 +232,12 @@ func TestReconcileNewMachineSet(t *testing.T) { t.Run(tc.name, func(t *testing.T) { g := NewWithT(t) - resources := []client.Object{ - tc.machineDeployment, - } - - allMachineSets := append(tc.oldMachineSets, tc.newMachineSet) - for key := range allMachineSets { - resources = append(resources, allMachineSets[key]) - } - - r := &Reconciler{ - Client: fake.NewClientBuilder().WithObjects(resources...).Build(), - recorder: record.NewFakeRecorder(32), - } + planner := newRolloutPlanner() + planner.md = tc.machineDeployment + planner.newMS = tc.newMachineSet + planner.oldMSs = tc.oldMachineSets - err := r.reconcileNewMachineSet(ctx, allMachineSets, tc.newMachineSet, tc.machineDeployment) + err := planner.reconcileNewMachineSet(ctx) if tc.error != nil { g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(BeEquivalentTo(tc.error.Error())) @@ -291,22 +246,23 @@ func TestReconcileNewMachineSet(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) - freshNewMachineSet := &clusterv1.MachineSet{} - err = r.Client.Get(ctx, client.ObjectKeyFromObject(tc.newMachineSet), freshNewMachineSet) - g.Expect(err).ToNot(HaveOccurred()) - - g.Expect(*freshNewMachineSet.Spec.Replicas).To(BeEquivalentTo(tc.expectedNewMachineSetReplicas)) - - _, ok := freshNewMachineSet.GetAnnotations()[clusterv1.DisableMachineCreateAnnotation] - g.Expect(ok).To(BeFalse()) - - desiredReplicasAnnotation, ok := freshNewMachineSet.GetAnnotations()[clusterv1.DesiredReplicasAnnotation] - g.Expect(ok).To(BeTrue()) - g.Expect(strconv.Atoi(desiredReplicasAnnotation)).To(BeEquivalentTo(*tc.machineDeployment.Spec.Replicas)) + scaleIntent := ptr.Deref(tc.newMachineSet.Spec.Replicas, 0) + if v, ok := planner.scaleIntents[tc.newMachineSet.Name]; ok { + scaleIntent = v + } + g.Expect(scaleIntent).To(BeEquivalentTo(tc.expectedNewMachineSetReplicas)) - maxReplicasAnnotation, ok := freshNewMachineSet.GetAnnotations()[clusterv1.MaxReplicasAnnotation] - g.Expect(ok).To(BeTrue()) - g.Expect(strconv.Atoi(maxReplicasAnnotation)).To(BeEquivalentTo(*tc.machineDeployment.Spec.Replicas + mdutil.MaxSurge(*tc.machineDeployment))) + // TODO(in-place): Restore tests on DisableMachineCreateAnnotation and MaxReplicasAnnotation as soon as handling those annotation is moved into the rollout planner + // _, ok := freshNewMachineSet.GetAnnotations()[clusterv1.DisableMachineCreateAnnotation] + // g.Expect(ok).To(BeFalse()) + // + // desiredReplicasAnnotation, ok := freshNewMachineSet.GetAnnotations()[clusterv1.DesiredReplicasAnnotation] + // g.Expect(ok).To(BeTrue()) + // g.Expect(strconv.Atoi(desiredReplicasAnnotation)).To(BeEquivalentTo(*tc.machineDeployment.Spec.Replicas)) + // + // maxReplicasAnnotation, ok := freshNewMachineSet.GetAnnotations()[clusterv1.MaxReplicasAnnotation] + // g.Expect(ok).To(BeTrue()) + // g.Expect(strconv.Atoi(maxReplicasAnnotation)).To(BeEquivalentTo(*tc.machineDeployment.Spec.Replicas + mdutil.MaxSurge(*tc.machineDeployment))) }) } } @@ -320,36 +276,6 @@ func TestReconcileOldMachineSets(t *testing.T) { expectedOldMachineSetsReplicas int error error }{ - { - name: "It fails when machineDeployment has no replicas", - machineDeployment: &clusterv1.MachineDeployment{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "foo", - Name: "bar", - }, - }, - newMachineSet: &clusterv1.MachineSet{ - Spec: clusterv1.MachineSetSpec{ - Replicas: ptr.To[int32](2), - }, - }, - error: errors.Errorf("spec.replicas for MachineDeployment foo/bar is nil, this is unexpected"), - }, - { - name: "It fails when new machineSet has no replicas", - machineDeployment: &clusterv1.MachineDeployment{ - Spec: clusterv1.MachineDeploymentSpec{ - Replicas: ptr.To[int32](2), - }, - }, - newMachineSet: &clusterv1.MachineSet{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "foo", - Name: "bar", - }, - }, - error: errors.Errorf("spec.replicas for MachineSet foo/bar is nil, this is unexpected"), - }, { name: "RollingUpdate strategy: Scale down old MachineSets when all new replicas are available", machineDeployment: &clusterv1.MachineDeployment{ @@ -465,21 +391,12 @@ func TestReconcileOldMachineSets(t *testing.T) { t.Run(tc.name, func(t *testing.T) { g := NewWithT(t) - resources := []client.Object{ - tc.machineDeployment, - } - - allMachineSets := append(tc.oldMachineSets, tc.newMachineSet) - for key := range allMachineSets { - resources = append(resources, allMachineSets[key]) - } - - r := &Reconciler{ - Client: fake.NewClientBuilder().WithObjects(resources...).Build(), - recorder: record.NewFakeRecorder(32), - } + planner := newRolloutPlanner() + planner.md = tc.machineDeployment + planner.newMS = tc.newMachineSet + planner.oldMSs = tc.oldMachineSets - err := r.reconcileOldMachineSets(ctx, allMachineSets, tc.oldMachineSets, tc.newMachineSet, tc.machineDeployment) + err := planner.reconcileOldMachineSets(ctx) if tc.error != nil { g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(BeEquivalentTo(tc.error.Error())) @@ -487,11 +404,12 @@ func TestReconcileOldMachineSets(t *testing.T) { } g.Expect(err).ToNot(HaveOccurred()) - for key := range tc.oldMachineSets { - freshOldMachineSet := &clusterv1.MachineSet{} - err = r.Client.Get(ctx, client.ObjectKeyFromObject(tc.oldMachineSets[key]), freshOldMachineSet) - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(*freshOldMachineSet.Spec.Replicas).To(BeEquivalentTo(tc.expectedOldMachineSetsReplicas)) + for i := range tc.oldMachineSets { + scaleIntent := ptr.Deref(tc.oldMachineSets[i].Spec.Replicas, 0) + if v, ok := planner.scaleIntents[tc.oldMachineSets[i].Name]; ok { + scaleIntent = v + } + g.Expect(scaleIntent).To(BeEquivalentTo(tc.expectedOldMachineSetsReplicas)) } }) } diff --git a/internal/controllers/machinedeployment/machinedeployment_rollout_ondelete.go b/internal/controllers/machinedeployment/machinedeployment_rollout_ondelete.go index ce485797eebe..bfb8a249e461 100644 --- a/internal/controllers/machinedeployment/machinedeployment_rollout_ondelete.go +++ b/internal/controllers/machinedeployment/machinedeployment_rollout_ondelete.go @@ -23,6 +23,7 @@ import ( "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -48,7 +49,7 @@ func (r *Reconciler) rolloutOnDelete(ctx context.Context, md *clusterv1.MachineD allMSs := append(oldMSs, newMS) // Scale up, if we can. - if err := r.reconcileNewMachineSetOnDelete(ctx, allMSs, newMS, md); err != nil { + if err := r.reconcileNewMachineSetOnDelete(ctx, md, oldMSs, newMS); err != nil { return err } @@ -164,10 +165,25 @@ func (r *Reconciler) reconcileOldMachineSetsOnDelete(ctx context.Context, oldMSs } // reconcileNewMachineSetOnDelete handles reconciliation of the latest MachineSet associated with the MachineDeployment in the OnDelete rollout strategy. -func (r *Reconciler) reconcileNewMachineSetOnDelete(ctx context.Context, allMSs []*clusterv1.MachineSet, newMS *clusterv1.MachineSet, deployment *clusterv1.MachineDeployment) error { +func (r *Reconciler) reconcileNewMachineSetOnDelete(ctx context.Context, md *clusterv1.MachineDeployment, oldMSs []*clusterv1.MachineSet, newMS *clusterv1.MachineSet) error { + // TODO(in-place): also apply/remove labels should go into rolloutPlanner if err := r.cleanupDisableMachineCreateAnnotation(ctx, newMS); err != nil { return err } - return r.reconcileNewMachineSet(ctx, allMSs, newMS, deployment) + planner := newRolloutPlanner() + planner.md = md + planner.newMS = newMS + planner.oldMSs = oldMSs + + if err := planner.reconcileNewMachineSet(ctx); err != nil { + return err + } + + // TODO(in-place): this should be changed as soon as rolloutPlanner support MS creation and adding/removing labels from MS + scaleIntent := ptr.Deref(newMS.Spec.Replicas, 0) + if v, ok := planner.scaleIntents[newMS.Name]; ok { + scaleIntent = v + } + return r.scaleMachineSet(ctx, newMS, scaleIntent, md) } diff --git a/internal/controllers/machinedeployment/machinedeployment_rollout_sequence_test.go b/internal/controllers/machinedeployment/machinedeployment_rollout_sequence_test.go new file mode 100644 index 000000000000..6c890235d647 --- /dev/null +++ b/internal/controllers/machinedeployment/machinedeployment_rollout_sequence_test.go @@ -0,0 +1,926 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package machinedeployment + +import ( + "context" + "fmt" + "math/rand" + "os" + "sort" + "strconv" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/cluster-api/internal/controllers/machinedeployment/mdutil" +) + +type rolloutSequenceTestCase struct { + name string + maxSurge int32 + maxUnavailable int32 + + // currentMachineNames is the list of machines before the rollout, and provides a simplified alternative to currentScope. + // all the machines in this list are initialized as upToDate and owned by the new MS before the rollout (which is different from the new MS after the rollout). + // Please name machines as "mX" where X is a progressive number starting from 1 (do not skip numbers), + // e.g. "m1","m2","m3" + currentMachineNames []string + + // currentScope defines the current state at the beginning of the test case. + // When the test case start from a stable state (there are no previous rollout in progress), use currentMachineNames instead. + // Please name machines as "mX" where X is a progressive number starting from 1 (do not skip numbers), + // e.g. "m1","m2","m3" + // machineUID must be set to the last used number. + currentScope *rolloutScope + + // maxUnavailableBreachToleration can be used to temporarily silence MaxUnavailable breaches + // + // maxUnavailableBreachToleration: func(log *logger, i int, scope *rolloutScope, minAvailableReplicas, totAvailableReplicas int32) bool { + // if i == 5 { + // t.Log("[Toleration] tolerate minAvailable breach after scale up") + // return true + // } + // return false + // }, + maxUnavailableBreachToleration func(log *fileLogger, i int, scope *rolloutScope, minAvailableReplicas, totAvailableReplicas int32) bool + + // maxSurgeBreachToleration can be used to temporarily silence MaxSurge breaches + // (see maxUnavailableBreachToleration example) + maxSurgeBreachToleration func(log *fileLogger, i int, scope *rolloutScope, maxAllowedReplicas, totReplicas int32) bool + + // desiredMachineNames is the list of machines at the end of the rollout. + // all the machines in this list are expected to be upToDate and owned by the new MS after the rollout (which is different from the new MS before the rollout). + // if this list contains old machines names (machine names already in currentMachineNames), it implies those machine have been upgraded in places. + // if this list contains new machines names (machine names not in currentMachineNames), it implies those machines have been created during a rollout; + // please name new machines names as "mX" where X is a progressive number starting after the max number in currentMachineNames (do not skip numbers), + // e.g. desiredMachineNames "m4","m5","m6" (desired machine names after a regular rollout of a MD with currentMachineNames "m1","m2","m3") + // e.g. desiredMachineNames "m1","m2","m3" (desired machine names after rollout performed using in-place upgrade for an MD with currentMachineNames "m1","m2","m3") + desiredMachineNames []string + + // skipLogToFileAndGoldenFileCheck allows to skip storing the log to file and golden file Check. + // NOTE: this field is controlled by the test itself. + skipLogToFileAndGoldenFileCheck bool + + // name of the log to file and the golden file. + // NOTE: this field is controlled by the test itself. + logAndGoldenFileName string + + // randomControllerOrder force the tests to run controllers in random order, mimicking what happens in production. + // NOTE. We are using a pseudo randomizer, so the random order remains consistent across runs of the same groups of tests. + // NOTE: this field is controlled by the test itself. + randomControllerOrder bool + + // maxIterations defines the max number of iterations the system must attempt before assuming the logic has an issue + // in reaching the desired state. + // When the test is using default controller order, an iteration implies reconcile MD + reconcile all MS in a predictable order; + // while using randomControllerOrder the concept of iteration is less defined, but it can still be used to prevent + // the test from running indefinitely. + // NOTE: this field is controlled by the test itself. + maxIterations int + + // seed value to initialize the generator. + // NOTE: this field is controlled by the test itself. + seed int64 +} + +func Test_rolloutSequencesWithPredictableReconcileOrder(t *testing.T) { + ctx := context.Background() + ctx = ctrl.LoggerInto(ctx, klog.Background()) + + tests := []rolloutSequenceTestCase{ + // Regular rollout (no in-place) + + { // scale out by 1 + name: "Regular rollout, 3 Replicas, maxSurge 1, maxUnavailable 0", + maxSurge: 1, + maxUnavailable: 0, + currentMachineNames: []string{"m1", "m2", "m3"}, + desiredMachineNames: []string{"m4", "m5", "m6"}, + }, + { // scale in by 1 + name: "Regular rollout, 3 Replicas, maxSurge 0, maxUnavailable 1", + maxSurge: 0, + maxUnavailable: 1, + currentMachineNames: []string{"m1", "m2", "m3"}, + desiredMachineNames: []string{"m4", "m5", "m6"}, + }, + { // scale out by 3, scale in by 1 (maxSurge > maxUnavailable) + name: "Regular rollout, 6 Replicas, maxSurge 3, maxUnavailable 1", + maxSurge: 3, + maxUnavailable: 1, + currentMachineNames: []string{"m1", "m2", "m3", "m4", "m5", "m6"}, + desiredMachineNames: []string{"m7", "m8", "m9", "m10", "m11", "m12"}, + }, + { // scale out by 1, scale in by 3 (maxSurge < maxUnavailable) + name: "Regular rollout, 6 Replicas, maxSurge 1, maxUnavailable 3", + maxSurge: 1, + maxUnavailable: 3, + currentMachineNames: []string{"m1", "m2", "m3", "m4", "m5", "m6"}, + desiredMachineNames: []string{"m7", "m8", "m9", "m10", "m11", "m12"}, + }, + { // scale out by 10 (maxSurge >= replicas) + name: "Regular rollout, 6 Replicas, maxSurge 10, maxUnavailable 0", + maxSurge: 10, + maxUnavailable: 0, + currentMachineNames: []string{"m1", "m2", "m3", "m4", "m5", "m6"}, + desiredMachineNames: []string{"m7", "m8", "m9", "m10", "m11", "m12"}, + }, + { // scale in by 10 (maxUnavailable >= replicas) + name: "Regular rollout, 6 Replicas, maxSurge 0, maxUnavailable 10", + maxSurge: 0, + maxUnavailable: 10, + currentMachineNames: []string{"m1", "m2", "m3", "m4", "m5", "m6"}, + desiredMachineNames: []string{"m7", "m8", "m9", "m10", "m11", "m12"}, + }, + { // scale out by 3, scale in by 1 (maxSurge > maxUnavailable) + scale up machine deployment in the middle + name: "Regular rollout, 6 Replicas, maxSurge 3, maxUnavailable 1, scale up to 12", + maxSurge: 3, + maxUnavailable: 1, + currentScope: &rolloutScope{ // Manually providing a scope simulating a MD originally with 6 replica in the middle of a rollout, with 3 machines already created in the newMS and 3 still on the oldMS, and then MD scaled up to 12. + machineDeployment: createMD("v2", 12, 3, 1), + machineSets: []*clusterv1.MachineSet{ + createMS("ms1", "v1", 3), + createMS("ms2", "v2", 3), + }, + machineSetMachines: map[string][]*clusterv1.Machine{ + "ms1": { + // "m1", "m2", "m3" already deleted + createM("m4", "ms1", "v1"), + createM("m5", "ms1", "v1"), + createM("m6", "ms1", "v1"), + }, + "ms2": { + createM("m7", "ms2", "v2"), + createM("m8", "ms2", "v2"), + createM("m9", "ms2", "v2"), + }, + }, + machineUID: 9, + }, + desiredMachineNames: []string{"m7", "m8", "m9", "m10", "m11", "m12", "m13", "m14", "m15", "m16", "m17", "m18"}, + maxUnavailableBreachToleration: maxUnavailableBreachToleration(), + }, + { // scale out by 3, scale in by 1 (maxSurge > maxUnavailable) + scale down machine deployment in the middle + name: "Regular rollout, 12 Replicas, maxSurge 3, maxUnavailable 1, scale down to 6", + maxSurge: 3, + maxUnavailable: 1, + currentScope: &rolloutScope{ // Manually providing a scope simulating a MD originally with 12 replica in the middle of a rollout, with 3 machines already created in the newMS and 9 still on the oldMS, and then MD scaled down to 6. + machineDeployment: createMD("v2", 6, 3, 1), + machineSets: []*clusterv1.MachineSet{ + createMS("ms1", "v1", 9), + createMS("ms2", "v2", 3), + }, + machineSetMachines: map[string][]*clusterv1.Machine{ + "ms1": { + // "m1", "m2", "m3" already deleted + createM("m4", "ms1", "v1"), + createM("m5", "ms1", "v1"), + createM("m6", "ms1", "v1"), + createM("m7", "ms1", "v1"), + createM("m8", "ms1", "v1"), + createM("m9", "ms1", "v1"), + createM("m10", "ms1", "v1"), + createM("m11", "ms1", "v1"), + createM("m12", "ms1", "v1"), + }, + "ms2": { + createM("m13", "ms2", "v2"), + createM("m14", "ms2", "v2"), + createM("m15", "ms2", "v2"), + }, + }, + machineUID: 15, + }, + desiredMachineNames: []string{"m13", "m14", "m15", "m16", "m17", "m18"}, + maxSurgeBreachToleration: maxSurgeToleration(), + }, + { // scale out by 3, scale in by 1 (maxSurge > maxUnavailable) + change spec in the middle + name: "Regular rollout, 6 Replicas, maxSurge 3, maxUnavailable 1, change spec", + maxSurge: 3, + maxUnavailable: 1, + currentScope: &rolloutScope{ // Manually providing a scope simulating a MD with 6 replica in the middle of a rollout, with 3 machines already created in the newMS and 3 still on the oldMS, and then MD spec is changed. + machineDeployment: createMD("v3", 6, 3, 1), + machineSets: []*clusterv1.MachineSet{ + createMS("ms1", "v1", 3), + createMS("ms2", "v2", 3), + createMS("ms3", "v3", 0), + }, + machineSetMachines: map[string][]*clusterv1.Machine{ + "ms1": { + // "m1", "m2", "m3" already deleted + createM("m4", "ms1", "v1"), + createM("m5", "ms1", "v1"), + createM("m6", "ms1", "v1"), + }, + "ms2": { + createM("m7", "ms2", "v2"), + createM("m8", "ms2", "v2"), + createM("m9", "ms2", "v2"), + }, + }, + machineUID: 9, + }, + desiredMachineNames: []string{"m10", "m11", "m12", "m13", "m14", "m15"}, // NOTE: Machines created before the spec change are deleted + }, + } + + testWithPredictableReconcileOrder := true + // TODO(in-place): enable tests with random reconcile order as soon as the issues in reconcileOldMachineSets are fixed + testWithRandomReconcileOrderFromConstantSeed := false + testWithRandomReconcileOrderFromRandomSeed := false + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + name := tt.name + + if testWithPredictableReconcileOrder { + tt.maxIterations = 50 + tt.randomControllerOrder = false + if tt.logAndGoldenFileName == "" { + tt.logAndGoldenFileName = strings.ToLower(tt.name) + } + t.Run("default", func(t *testing.T) { + runTestCase(ctx, t, tt) + }) + } + + if testWithRandomReconcileOrderFromConstantSeed { + tt.maxIterations = 70 + tt.name = fmt.Sprintf("%s, random(0)", name) + tt.randomControllerOrder = true + tt.seed = 0 + // TODO(in-place): drop the following line as soon as issue with scale down are fixed + tt.skipLogToFileAndGoldenFileCheck = true + if tt.logAndGoldenFileName == "" { + tt.logAndGoldenFileName = strings.ToLower(tt.name) + } + t.Run("random(0)", func(t *testing.T) { + runTestCase(ctx, t, tt) + }) + } + + if testWithRandomReconcileOrderFromRandomSeed { + for range 100 { + tt.maxIterations = 150 + tt.seed = time.Now().UnixNano() + tt.name = fmt.Sprintf("%s, random(%d)", name, tt.seed) + tt.randomControllerOrder = true + tt.skipLogToFileAndGoldenFileCheck = true + t.Run(fmt.Sprintf("random(%d)", tt.seed), func(t *testing.T) { + runTestCase(ctx, t, tt) + }) + } + } + }) + } +} + +func runTestCase(ctx context.Context, t *testing.T, tt rolloutSequenceTestCase) { + t.Helper() + g := NewWithT(t) + + rng := rand.New(rand.NewSource(tt.seed)) //nolint:gosec // it is ok to use a weak randomizer here + fLogger := newFileLogger(t, tt.name, fmt.Sprintf("testdata/%s", tt.logAndGoldenFileName)) + // uncomment this line to automatically generate/update golden files: fLogger.writeGoldenFile = true + + // Init current and desired state from test case + current := tt.currentScope.Clone() + if current == nil { + current = initCurrentRolloutScope(tt) + } + desired := computeDesiredRolloutScope(current, tt.desiredMachineNames) + + // Log initial state + fLogger.Logf("[Test] Initial state\n%s", current) + random := "" + if tt.randomControllerOrder { + random = fmt.Sprintf(", random(%d)", tt.seed) + } + fLogger.Logf("[Test] Rollout %d replicas, MaxSurge=%d, MaxUnavailable=%d%s\n", len(current.machines()), tt.maxSurge, tt.maxUnavailable, random) + i := 1 + maxIterations := tt.maxIterations + for { + taskOrder := defaultTaskOrder(current) + if tt.randomControllerOrder { + taskOrder = randomTaskOrder(current, rng) + } + for _, taskID := range taskOrder { + if taskID == 0 { + fLogger.Logf("[MD controller] Iteration %d, Reconcile md", i) + fLogger.Logf("[MD controller] - Input to rollout planner\n%s", current) + + // Running a small subset of MD reconcile (the rollout logic and a bit of setReplicas) + p := newRolloutPlanner() + p.md = current.machineDeployment + p.newMS = current.newMS() + p.oldMSs = current.oldMSs() + + err := p.Plan(ctx) + g.Expect(err).ToNot(HaveOccurred()) + // Apply changes. + for _, ms := range current.machineSets { + if scaleIntent, ok := p.scaleIntents[ms.Name]; ok { + ms.Spec.Replicas = ptr.To(scaleIntent) + } + } + + // Running a small subset of setReplicas (we don't want to run the full func to avoid unnecessary noise on the test) + current.machineDeployment.Status.Replicas = mdutil.GetActualReplicaCountForMachineSets(current.machineSets) + current.machineDeployment.Status.AvailableReplicas = mdutil.GetAvailableReplicaCountForMachineSets(current.machineSets) + + // Log state after this reconcile + fLogger.Logf("[MD controller] - Result of rollout planner\n%s", current) + + // Check we are not breaching rollout constraints + minAvailableReplicas := ptr.Deref(current.machineDeployment.Spec.Replicas, 0) - mdutil.MaxUnavailable(*current.machineDeployment) + totAvailableReplicas := ptr.Deref(current.machineDeployment.Status.AvailableReplicas, 0) + if totAvailableReplicas < minAvailableReplicas { + tolerateBreach := false + if tt.maxUnavailableBreachToleration != nil { + tolerateBreach = tt.maxUnavailableBreachToleration(fLogger, i, current, minAvailableReplicas, totAvailableReplicas) + } + if !tolerateBreach { + g.Expect(totAvailableReplicas).To(BeNumerically(">=", minAvailableReplicas), "totAvailable machines is less than md.spec.replicas - maxUnavailable") + } + } + + maxAllowedReplicas := ptr.Deref(current.machineDeployment.Spec.Replicas, 0) + mdutil.MaxSurge(*current.machineDeployment) + totReplicas := mdutil.TotalMachineSetsReplicaSum(current.machineSets) + if totReplicas > maxAllowedReplicas { + tolerateBreach := false + if tt.maxSurgeBreachToleration != nil { + tolerateBreach = tt.maxSurgeBreachToleration(fLogger, i, current, maxAllowedReplicas, totReplicas) + } + if !tolerateBreach { + g.Expect(totReplicas).To(BeNumerically("<=", maxAllowedReplicas), "totReplicas machines is greater than md.spec.replicas + maxSurge") + } + } + } + + // Run mutators faking other controllers + for _, ms := range current.machineSets { + if fmt.Sprintf("ms%d", taskID) == ms.Name { + fLogger.Logf("[MS controller] Iteration %d, Reconcile ms%d, %s", i, taskID, msLog(ms, current.machineSetMachines[ms.Name])) + machineSetControllerMutator(fLogger, ms, current) + break + } + } + } + + // Check if we are at the desired state + if current.Equal(desired) { + fLogger.Logf("[Test] Final state\n%s", current) + break + } + + // Safeguard for infinite reconcile + i++ + if i > maxIterations { + // NOTE: the following can be used to set a breakpoint for debugging why the system is not reaching desired state after maxIterations (to check what is not yet equal) + current.Equal(desired) + // Log desired state we never reached + fLogger.Logf("[Test] Desired state\n%s", desired) + g.Fail(fmt.Sprintf("Failed to reach desired state in %d iterations", maxIterations)) + } + } + + if !tt.skipLogToFileAndGoldenFileCheck { + currentLog, goldenLog, err := fLogger.WriteLogAndCompareWithGoldenFile() + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(currentLog).To(Equal(goldenLog), "current test case log and golden test case log are different\n%s", cmp.Diff(currentLog, goldenLog)) + } +} + +// machineSetControllerMutator fakes a small part of the MachineSet controller, just what is required for the rollout to progress. +func machineSetControllerMutator(log *fileLogger, ms *clusterv1.MachineSet, scope *rolloutScope) { + // Update counters + // Note: this should not be implemented in production code + ms.Status.Replicas = ptr.To(int32(len(scope.machineSetMachines[ms.Name]))) + + // Sort machines to ensure stable results of move/delete operations during tests. + // Note: this should not be implemented in production code + sortMachineSetMachines(scope.machineSetMachines[ms.Name]) + + // if too few machines, create missing machine. + // new machines are created with a predictable name, so it is easier to write test case and validate rollout sequences. + // e.g. if the cluster is initialized with m1, m2, m3, new machines will be m4, m5, m6 + machinesToAdd := ptr.Deref(ms.Spec.Replicas, 0) - ptr.Deref(ms.Status.Replicas, 0) + if machinesToAdd > 0 { + machinesAdded := []string{} + for range machinesToAdd { + machineName := fmt.Sprintf("m%d", scope.GetNextMachineUID()) + scope.machineSetMachines[ms.Name] = append(scope.machineSetMachines[ms.Name], + &clusterv1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: machineName, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: clusterv1.GroupVersion.String(), + Kind: "MachineSet", + Name: ms.Name, + Controller: ptr.To(true), + }, + }, + }, + }, + ) + machinesAdded = append(machinesAdded, machineName) + } + + log.Logf("[MS controller] - %s scale up to %d/%[2]d replicas (%s created)", ms.Name, ptr.Deref(ms.Spec.Replicas, 0), strings.Join(machinesAdded, ",")) + } + + // if too many replicas, delete exceeding machines. + // exceeding machines are deleted in predictable order, so it is easier to write test case and validate rollout sequences. + // e.g. if a ms has m1,m2,m3 created in this order, m1 will be deleted first, then m2 and finally m3. + machinesToDelete := max(ptr.Deref(ms.Status.Replicas, 0)-ptr.Deref(ms.Spec.Replicas, 0), 0) + + if machinesToDelete > 0 { + machinesDeleted := []string{} + machinesSetMachines := []*clusterv1.Machine{} + for i, m := range scope.machineSetMachines[ms.Name] { + if int32(len(machinesDeleted)) >= machinesToDelete { + machinesSetMachines = append(machinesSetMachines, scope.machineSetMachines[ms.Name][i:]...) + break + } + machinesDeleted = append(machinesDeleted, m.Name) + } + scope.machineSetMachines[ms.Name] = machinesSetMachines + log.Logf("[MS controller] - %s scale down to %d/%[2]d replicas (%s deleted)", ms.Name, ptr.Deref(ms.Spec.Replicas, 0), strings.Join(machinesDeleted, ",")) + } + + // Update counters + ms.Status.Replicas = ptr.To(int32(len(scope.machineSetMachines[ms.Name]))) + ms.Status.AvailableReplicas = ptr.To(int32(len(scope.machineSetMachines[ms.Name]))) +} + +type rolloutScope struct { + machineDeployment *clusterv1.MachineDeployment + machineSets []*clusterv1.MachineSet + machineSetMachines map[string][]*clusterv1.Machine + + machineUID int32 +} + +// Init creates current state and desired state for rolling out a md from currentMachines to wantMachineNames. +func initCurrentRolloutScope(tt rolloutSequenceTestCase) (current *rolloutScope) { + // create current state, with a MD with + // - given MaxSurge, MaxUnavailable + // - replica counters assuming all the machines are at stable state + // - spec different from the MachineSets and Machines we are going to create down below (to simulate a change that triggers a rollout, but it is not yet started) + mdReplicaCount := int32(len(tt.currentMachineNames)) + current = &rolloutScope{ + machineDeployment: createMD("v2", mdReplicaCount, tt.maxSurge, tt.maxUnavailable), + } + + // Create current MS, with + // - replica counters assuming all the machines are at stable state + // - spec at stable state (rollout is not yet propagated to machines) + ms := createMS("ms1", "v1", mdReplicaCount) + current.machineSets = append(current.machineSets, ms) + + // Create current Machines, with + // - spec at stable state (rollout is not yet propagated to machines) + var totMachines int32 + currentMachines := []*clusterv1.Machine{} + for _, machineSetMachineName := range tt.currentMachineNames { + totMachines++ + currentMachines = append(currentMachines, createM(machineSetMachineName, ms.Name, ms.Spec.Template.Spec.FailureDomain)) + } + current.machineSetMachines = map[string][]*clusterv1.Machine{} + current.machineSetMachines[ms.Name] = currentMachines + + current.machineUID = totMachines + + // TODO(in-place): this should be removed as soon as rolloutPlanner will take care of creating newMS + newMS := createMS("ms2", current.machineDeployment.Spec.Template.Spec.FailureDomain, 0) + current.machineSets = append(current.machineSets, newMS) + + return current +} + +func computeDesiredRolloutScope(current *rolloutScope, desiredMachineNames []string) (desired *rolloutScope) { + var totMachineSets, totMachines int32 + totMachineSets = int32(len(current.machineSets)) + for _, msMachines := range current.machineSetMachines { + totMachines += int32(len(msMachines)) + } + + // Create current state, with a MD equal to the one we started from because: + // - spec was already changed in current to simulate a change that triggers a rollout + // - desired replica counters are the same than current replica counters (we start with all the machines at stable state v1, we should end with all the machines at stable state v2) + desired = &rolloutScope{ + machineDeployment: current.machineDeployment.DeepCopy(), + } + desired.machineDeployment.Status.Replicas = desired.machineDeployment.Spec.Replicas + desired.machineDeployment.Status.AvailableReplicas = desired.machineDeployment.Spec.Replicas + + // Add current MS to desired state, but set replica counters to zero because all the machines must be moved to the new MS. + // Note: one of the old MS could also be the NewMS, the MS that must become owner of all the desired machines. + var newMS *clusterv1.MachineSet + for _, currentMS := range current.machineSets { + oldMS := currentMS.DeepCopy() + oldMS.Spec.Replicas = ptr.To(int32(0)) + oldMS.Status.Replicas = ptr.To(int32(0)) + oldMS.Status.AvailableReplicas = ptr.To(int32(0)) + desired.machineSets = append(desired.machineSets, oldMS) + + if upToDate, _, _ := mdutil.MachineTemplateUpToDate(&oldMS.Spec.Template, &desired.machineDeployment.Spec.Template); upToDate { + if newMS != nil { + panic("there should be only one MachineSet with MachineTemplateUpToDate") + } + newMS = oldMS + } + } + + // Add or update the new MS to desired state, with + // - the new spec from the MD + // - replica counters assuming all the replicas must be here at the end of the rollout. + if newMS != nil { + newMS.Spec.Replicas = desired.machineDeployment.Spec.Replicas + newMS.Status.Replicas = desired.machineDeployment.Status.Replicas + newMS.Status.AvailableReplicas = desired.machineDeployment.Status.AvailableReplicas + } else { + totMachineSets++ + newMS = createMS(fmt.Sprintf("ms%d", totMachineSets), desired.machineDeployment.Spec.Template.Spec.FailureDomain, *desired.machineDeployment.Spec.Replicas) + desired.machineSets = append(desired.machineSets, newMS) + } + + // Add a desired machines to desired state, with + // - the new spec from the MD (steady state) + desiredMachines := []*clusterv1.Machine{} + for _, machineSetMachineName := range desiredMachineNames { + totMachines++ + desiredMachines = append(desiredMachines, createM(machineSetMachineName, newMS.Name, newMS.Spec.Template.Spec.FailureDomain)) + } + desired.machineSetMachines = map[string][]*clusterv1.Machine{} + desired.machineSetMachines[newMS.Name] = desiredMachines + return desired +} + +// GetNextMachineUID provides a predictable UID for machines. +func (r *rolloutScope) GetNextMachineUID() int32 { + r.machineUID++ + return r.machineUID +} + +func (r *rolloutScope) Clone() *rolloutScope { + if r == nil { + return nil + } + + c := &rolloutScope{ + machineDeployment: r.machineDeployment.DeepCopy(), + machineSetMachines: map[string][]*clusterv1.Machine{}, + machineUID: r.machineUID, + } + for _, ms := range r.machineSets { + c.machineSets = append(c.machineSets, ms.DeepCopy()) + } + for ms, machines := range r.machineSetMachines { + cmachines := make([]*clusterv1.Machine, 0, len(machines)) + for _, m := range machines { + cmachines = append(cmachines, m.DeepCopy()) + } + c.machineSetMachines[ms] = cmachines + } + return c +} + +func (r rolloutScope) String() string { + sb := strings.Builder{} + sb.WriteString(fmt.Sprintf("%s, %d/%d replicas\n", r.machineDeployment.Name, ptr.Deref(r.machineDeployment.Status.Replicas, 0), ptr.Deref(r.machineDeployment.Spec.Replicas, 0))) + + sort.Slice(r.machineSets, func(i, j int) bool { return r.machineSets[i].Name < r.machineSets[j].Name }) + for _, ms := range r.machineSets { + sb.WriteString(fmt.Sprintf("- %s, %s\n", ms.Name, msLog(ms, r.machineSetMachines[ms.Name]))) + } + return sb.String() +} + +func msLog(ms *clusterv1.MachineSet, machines []*clusterv1.Machine) string { + sb := strings.Builder{} + machineNames := []string{} + for _, m := range machines { + machineNames = append(machineNames, m.Name) + } + sb.WriteString(strings.Join(machineNames, ",")) + msLog := fmt.Sprintf("%d/%d replicas (%s)", ptr.Deref(ms.Status.Replicas, 0), ptr.Deref(ms.Spec.Replicas, 0), sb.String()) + return msLog +} + +func (r rolloutScope) newMS() *clusterv1.MachineSet { + for _, ms := range r.machineSets { + if upToDate, _, _ := mdutil.MachineTemplateUpToDate(&r.machineDeployment.Spec.Template, &ms.Spec.Template); upToDate { + return ms + } + } + return nil +} + +func (r rolloutScope) oldMSs() []*clusterv1.MachineSet { + var oldMSs []*clusterv1.MachineSet + for _, ms := range r.machineSets { + if upToDate, _, _ := mdutil.MachineTemplateUpToDate(&r.machineDeployment.Spec.Template, &ms.Spec.Template); !upToDate { + oldMSs = append(oldMSs, ms) + } + } + return oldMSs +} + +func (r rolloutScope) machines() []*clusterv1.Machine { + machines := []*clusterv1.Machine{} + for _, ms := range r.machineSets { + machines = append(machines, r.machineSetMachines[ms.Name]...) + } + return machines +} + +func (r *rolloutScope) Equal(s *rolloutScope) bool { + return machineDeploymentIsEqual(r.machineDeployment, s.machineDeployment) && machineSetsAreEqual(r.machineSets, s.machineSets) && machineSetMachinesAreEqual(r.machineSetMachines, s.machineSetMachines) +} + +func machineDeploymentIsEqual(a, b *clusterv1.MachineDeployment) bool { + if upToDate, _, _ := mdutil.MachineTemplateUpToDate(&a.Spec.Template, &b.Spec.Template); !upToDate || + ptr.Deref(a.Spec.Replicas, 0) != ptr.Deref(b.Spec.Replicas, 0) || + ptr.Deref(a.Status.Replicas, 0) != ptr.Deref(b.Status.Replicas, 0) || + ptr.Deref(a.Status.AvailableReplicas, 0) != ptr.Deref(b.Status.AvailableReplicas, 0) { + return false + } + return true +} + +func machineSetsAreEqual(a, b []*clusterv1.MachineSet) bool { + if len(a) != len(b) { + return false + } + + aMap := make(map[string]*clusterv1.MachineSet) + for i := range a { + aMap[a[i].Name] = a[i] + } + + for i := range b { + desiredMS := b[i] + currentMS, ok := aMap[desiredMS.Name] + if !ok { + return false + } + if upToDate, _, _ := mdutil.MachineTemplateUpToDate(&desiredMS.Spec.Template, ¤tMS.Spec.Template); !upToDate || + ptr.Deref(desiredMS.Spec.Replicas, 0) != ptr.Deref(currentMS.Spec.Replicas, 0) || + ptr.Deref(desiredMS.Status.Replicas, 0) != ptr.Deref(currentMS.Status.Replicas, 0) || + ptr.Deref(desiredMS.Status.AvailableReplicas, 0) != ptr.Deref(currentMS.Status.AvailableReplicas, 0) { + return false + } + } + return true +} + +func machineSetMachinesAreEqual(a, b map[string][]*clusterv1.Machine) bool { + for ms, aMachines := range a { + bMachines, ok := b[ms] + if !ok { + if len(aMachines) > 0 { + return false + } + continue + } + + if len(aMachines) != len(bMachines) { + return false + } + + for i := range aMachines { + if aMachines[i].Name != bMachines[i].Name { + return false + } + if len(aMachines[i].OwnerReferences) != 1 || len(bMachines[i].OwnerReferences) != 1 || aMachines[i].OwnerReferences[0].Name != bMachines[i].OwnerReferences[0].Name { + return false + } + } + } + return true +} + +type UniqueRand struct { + rng *rand.Rand + generated map[int]bool // keeps track of random numbers already generated. + max int // max number to be generated +} + +func (u *UniqueRand) Int() int { + if u.Done() { + return -1 + } + for { + i := u.rng.Intn(u.max) + if !u.generated[i] { + u.generated[i] = true + return i + } + } +} + +func (u *UniqueRand) Done() bool { + return len(u.generated) >= u.max +} + +func (u *UniqueRand) Forget(n int) { + delete(u.generated, n) +} + +type fileLogger struct { + t *testing.T + + testCase string + fileName string + testCaseStringBuilder strings.Builder + writeGoldenFile bool +} + +func newFileLogger(t *testing.T, name, fileName string) *fileLogger { + t.Helper() + + l := &fileLogger{t: t, testCaseStringBuilder: strings.Builder{}} + l.testCaseStringBuilder.WriteString(fmt.Sprintf("## %s\n\n", name)) + l.testCase = name + l.fileName = fileName + return l +} + +func (l *fileLogger) Logf(format string, args ...interface{}) { + l.t.Logf(format, args...) + + // this codes takes a log line that has been formatted for t.Logf and change it + // so it will look nice in the files. e.g. adds indentation to all lines except the fist one, which by convention starts with [. + s := strings.TrimSuffix(fmt.Sprintf(format, args...), "\n") + sb := &strings.Builder{} + if strings.Contains(s, "\n") { + lines := strings.Split(s, "\n") + for _, line := range lines { + indent := " " + if strings.HasPrefix(line, "[") { + indent = "" + } + sb.WriteString(indent + line + "\n") + } + } else { + sb.WriteString(s + "\n") + } + l.testCaseStringBuilder.WriteString(sb.String()) +} + +func (l *fileLogger) WriteLogAndCompareWithGoldenFile() (string, string, error) { + if err := os.WriteFile(fmt.Sprintf("%s.test.log", l.fileName), []byte(l.testCaseStringBuilder.String()), 0600); err != nil { + return "", "", err + } + if l.writeGoldenFile { + if err := os.WriteFile(fmt.Sprintf("%s.test.log.golden", l.fileName), []byte(l.testCaseStringBuilder.String()), 0600); err != nil { + return "", "", err + } + } + + currentBytes, _ := os.ReadFile(fmt.Sprintf("%s.test.log", l.fileName)) + current := string(currentBytes) + + goldenBytes, _ := os.ReadFile(fmt.Sprintf("%s.test.log.golden", l.fileName)) + golden := string(goldenBytes) + + return current, golden, nil +} + +func sortMachineSetMachines(machines []*clusterv1.Machine) { + sort.Slice(machines, func(i, j int) bool { + iIndex, _ := strconv.Atoi(strings.TrimPrefix(machines[i].Name, "m")) + jiIndex, _ := strconv.Atoi(strings.TrimPrefix(machines[j].Name, "m")) + return iIndex < jiIndex + }) +} + +func maxUnavailableBreachToleration() func(log *fileLogger, _ int, _ *rolloutScope, _, _ int32) bool { + return func(log *fileLogger, _ int, _ *rolloutScope, _, _ int32) bool { + log.Logf("[Toleration] tolerate maxUnavailable breach") + return true + } +} + +func maxSurgeToleration() func(log *fileLogger, _ int, _ *rolloutScope, _, _ int32) bool { + return func(log *fileLogger, _ int, _ *rolloutScope, _, _ int32) bool { + log.Logf("[Toleration] tolerate maxSurge breach") + return true + } +} + +// default task order ensure the controllers are run in a consistent and predictable way: md, ms1, ms2 and so on. +func defaultTaskOrder(current *rolloutScope) []int { + taskOrder := []int{} + for t := range len(current.machineSets) + 1 + 1 { // +1 is for the MachineSet that might be created when reconciling md, +1 is for the md itself + taskOrder = append(taskOrder, t) + } + return taskOrder +} + +func randomTaskOrder(current *rolloutScope, rng *rand.Rand) []int { + u := &UniqueRand{ + rng: rng, + generated: map[int]bool{}, + max: len(current.machineSets) + 1 + 1, // +1 is for the MachineSet that might be created when reconciling md, +1 is for the md itself + } + taskOrder := []int{} + for !u.Done() { + n := u.Int() + if rng.Intn(10) < 3 { // skip a step in the 30% of cases + continue + } + taskOrder = append(taskOrder, n) + if r := rng.Intn(10); r < 3 { // repeat a step in the 30% of cases + u.Forget(n) + } + } + return taskOrder +} + +func createMD(failureDomain string, replicas int32, maxSurge, maxUnavailable int32) *clusterv1.MachineDeployment { + return &clusterv1.MachineDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "md"}, + Spec: clusterv1.MachineDeploymentSpec{ + // Note: using failureDomain as a template field to determine upToDate + Template: clusterv1.MachineTemplateSpec{Spec: clusterv1.MachineSpec{FailureDomain: failureDomain}}, + Replicas: &replicas, + Rollout: clusterv1.MachineDeploymentRolloutSpec{ + Strategy: clusterv1.MachineDeploymentRolloutStrategy{ + Type: clusterv1.RollingUpdateMachineDeploymentStrategyType, + RollingUpdate: clusterv1.MachineDeploymentRolloutStrategyRollingUpdate{ + MaxSurge: ptr.To(intstr.FromInt32(maxSurge)), + MaxUnavailable: ptr.To(intstr.FromInt32(maxUnavailable)), + }, + }, + }, + }, + Status: clusterv1.MachineDeploymentStatus{ + Replicas: &replicas, + AvailableReplicas: &replicas, + }, + } +} + +func createMS(name, failureDomain string, replicas int32) *clusterv1.MachineSet { + return &clusterv1.MachineSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: clusterv1.MachineSetSpec{ + // Note: using failureDomain as a template field to determine upToDate + Template: clusterv1.MachineTemplateSpec{Spec: clusterv1.MachineSpec{FailureDomain: failureDomain}}, + Replicas: ptr.To(replicas), + }, + Status: clusterv1.MachineSetStatus{ + Replicas: ptr.To(replicas), + AvailableReplicas: ptr.To(replicas), + }, + } +} + +func createM(name, ownedByMS, failureDomain string) *clusterv1.Machine { + return &clusterv1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: clusterv1.GroupVersion.String(), + Kind: "MachineSet", + Name: ownedByMS, + Controller: ptr.To(true), + }, + }, + }, + Spec: clusterv1.MachineSpec{ + // Note: using failureDomain as a template field to determine upToDate + FailureDomain: failureDomain, + }, + } +} diff --git a/internal/controllers/machinedeployment/testdata/.gitignore b/internal/controllers/machinedeployment/testdata/.gitignore new file mode 100644 index 000000000000..919cfaee411b --- /dev/null +++ b/internal/controllers/machinedeployment/testdata/.gitignore @@ -0,0 +1 @@ +*.test.log diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 12 replicas, maxsurge 3, maxunavailable 1, scale down to 6.test.log.golden b/internal/controllers/machinedeployment/testdata/regular rollout, 12 replicas, maxsurge 3, maxunavailable 1, scale down to 6.test.log.golden new file mode 100644 index 000000000000..8af54587caae --- /dev/null +++ b/internal/controllers/machinedeployment/testdata/regular rollout, 12 replicas, maxsurge 3, maxunavailable 1, scale down to 6.test.log.golden @@ -0,0 +1,59 @@ +## Regular rollout, 12 Replicas, maxSurge 3, maxUnavailable 1, scale down to 6 + +[Test] Initial state + md, 6/6 replicas + - ms1, 9/9 replicas (m4,m5,m6,m7,m8,m9,m10,m11,m12) + - ms2, 3/3 replicas (m13,m14,m15) +[Test] Rollout 12 replicas, MaxSurge=3, MaxUnavailable=1 +[MD controller] Iteration 1, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 9/9 replicas (m4,m5,m6,m7,m8,m9,m10,m11,m12) + - ms2, 3/3 replicas (m13,m14,m15) +[MD controller] - Result of rollout planner + md, 12/6 replicas + - ms1, 9/2 replicas (m4,m5,m6,m7,m8,m9,m10,m11,m12) + - ms2, 3/3 replicas (m13,m14,m15) +[Toleration] tolerate maxSurge breach +[MS controller] Iteration 1, Reconcile ms1, 9/2 replicas (m4,m5,m6,m7,m8,m9,m10,m11,m12) +[MS controller] - ms1 scale down to 2/2 replicas (m4,m5,m6,m7,m8,m9,m10 deleted) +[MS controller] Iteration 1, Reconcile ms2, 3/3 replicas (m13,m14,m15) +[MD controller] Iteration 2, Reconcile md +[MD controller] - Input to rollout planner + md, 12/6 replicas + - ms1, 2/2 replicas (m11,m12) + - ms2, 3/3 replicas (m13,m14,m15) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m11,m12) + - ms2, 3/6 replicas (m13,m14,m15) +[MS controller] Iteration 2, Reconcile ms1, 2/2 replicas (m11,m12) +[MS controller] Iteration 2, Reconcile ms2, 3/6 replicas (m13,m14,m15) +[MS controller] - ms2 scale up to 6/6 replicas (m16,m17,m18 created) +[MD controller] Iteration 3, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 2/2 replicas (m11,m12) + - ms2, 6/6 replicas (m13,m14,m15,m16,m17,m18) +[MD controller] - Result of rollout planner + md, 8/6 replicas + - ms1, 2/0 replicas (m11,m12) + - ms2, 6/6 replicas (m13,m14,m15,m16,m17,m18) +[MS controller] Iteration 3, Reconcile ms1, 2/0 replicas (m11,m12) +[MS controller] - ms1 scale down to 0/0 replicas (m11,m12 deleted) +[MS controller] Iteration 3, Reconcile ms2, 6/6 replicas (m13,m14,m15,m16,m17,m18) +[MD controller] Iteration 4, Reconcile md +[MD controller] - Input to rollout planner + md, 8/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m13,m14,m15,m16,m17,m18) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m13,m14,m15,m16,m17,m18) +[MS controller] Iteration 4, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 4, Reconcile ms2, 6/6 replicas (m13,m14,m15,m16,m17,m18) +[Test] Final state + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m13,m14,m15,m16,m17,m18) diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 3 replicas, maxsurge 0, maxunavailable 1.test.log.golden b/internal/controllers/machinedeployment/testdata/regular rollout, 3 replicas, maxsurge 0, maxunavailable 1.test.log.golden new file mode 100644 index 000000000000..dc291b7404d9 --- /dev/null +++ b/internal/controllers/machinedeployment/testdata/regular rollout, 3 replicas, maxsurge 0, maxunavailable 1.test.log.golden @@ -0,0 +1,94 @@ +## Regular rollout, 3 Replicas, maxSurge 0, maxUnavailable 1 + +[Test] Initial state + md, 3/3 replicas + - ms1, 3/3 replicas (m1,m2,m3) + - ms2, 0/0 replicas () +[Test] Rollout 3 replicas, MaxSurge=0, MaxUnavailable=1 +[MD controller] Iteration 1, Reconcile md +[MD controller] - Input to rollout planner + md, 3/3 replicas + - ms1, 3/3 replicas (m1,m2,m3) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 3/3 replicas + - ms1, 3/2 replicas (m1,m2,m3) + - ms2, 0/0 replicas () +[MS controller] Iteration 1, Reconcile ms1, 3/2 replicas (m1,m2,m3) +[MS controller] - ms1 scale down to 2/2 replicas (m1 deleted) +[MS controller] Iteration 1, Reconcile ms2, 0/0 replicas () +[MD controller] Iteration 2, Reconcile md +[MD controller] - Input to rollout planner + md, 3/3 replicas + - ms1, 2/2 replicas (m2,m3) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 2/3 replicas + - ms1, 2/2 replicas (m2,m3) + - ms2, 0/1 replicas () +[MS controller] Iteration 2, Reconcile ms1, 2/2 replicas (m2,m3) +[MS controller] Iteration 2, Reconcile ms2, 0/1 replicas () +[MS controller] - ms2 scale up to 1/1 replicas (m4 created) +[MD controller] Iteration 3, Reconcile md +[MD controller] - Input to rollout planner + md, 2/3 replicas + - ms1, 2/2 replicas (m2,m3) + - ms2, 1/1 replicas (m4) +[MD controller] - Result of rollout planner + md, 3/3 replicas + - ms1, 2/1 replicas (m2,m3) + - ms2, 1/1 replicas (m4) +[MS controller] Iteration 3, Reconcile ms1, 2/1 replicas (m2,m3) +[MS controller] - ms1 scale down to 1/1 replicas (m2 deleted) +[MS controller] Iteration 3, Reconcile ms2, 1/1 replicas (m4) +[MD controller] Iteration 4, Reconcile md +[MD controller] - Input to rollout planner + md, 3/3 replicas + - ms1, 1/1 replicas (m3) + - ms2, 1/1 replicas (m4) +[MD controller] - Result of rollout planner + md, 2/3 replicas + - ms1, 1/1 replicas (m3) + - ms2, 1/2 replicas (m4) +[MS controller] Iteration 4, Reconcile ms1, 1/1 replicas (m3) +[MS controller] Iteration 4, Reconcile ms2, 1/2 replicas (m4) +[MS controller] - ms2 scale up to 2/2 replicas (m5 created) +[MD controller] Iteration 5, Reconcile md +[MD controller] - Input to rollout planner + md, 2/3 replicas + - ms1, 1/1 replicas (m3) + - ms2, 2/2 replicas (m4,m5) +[MD controller] - Result of rollout planner + md, 3/3 replicas + - ms1, 1/0 replicas (m3) + - ms2, 2/2 replicas (m4,m5) +[MS controller] Iteration 5, Reconcile ms1, 1/0 replicas (m3) +[MS controller] - ms1 scale down to 0/0 replicas (m3 deleted) +[MS controller] Iteration 5, Reconcile ms2, 2/2 replicas (m4,m5) +[MD controller] Iteration 6, Reconcile md +[MD controller] - Input to rollout planner + md, 3/3 replicas + - ms1, 0/0 replicas () + - ms2, 2/2 replicas (m4,m5) +[MD controller] - Result of rollout planner + md, 2/3 replicas + - ms1, 0/0 replicas () + - ms2, 2/3 replicas (m4,m5) +[MS controller] Iteration 6, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 6, Reconcile ms2, 2/3 replicas (m4,m5) +[MS controller] - ms2 scale up to 3/3 replicas (m6 created) +[MD controller] Iteration 7, Reconcile md +[MD controller] - Input to rollout planner + md, 2/3 replicas + - ms1, 0/0 replicas () + - ms2, 3/3 replicas (m4,m5,m6) +[MD controller] - Result of rollout planner + md, 3/3 replicas + - ms1, 0/0 replicas () + - ms2, 3/3 replicas (m4,m5,m6) +[MS controller] Iteration 7, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 7, Reconcile ms2, 3/3 replicas (m4,m5,m6) +[Test] Final state + md, 3/3 replicas + - ms1, 0/0 replicas () + - ms2, 3/3 replicas (m4,m5,m6) diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 3 replicas, maxsurge 1, maxunavailable 0.test.log.golden b/internal/controllers/machinedeployment/testdata/regular rollout, 3 replicas, maxsurge 1, maxunavailable 0.test.log.golden new file mode 100644 index 000000000000..4d22fbea8db5 --- /dev/null +++ b/internal/controllers/machinedeployment/testdata/regular rollout, 3 replicas, maxsurge 1, maxunavailable 0.test.log.golden @@ -0,0 +1,94 @@ +## Regular rollout, 3 Replicas, maxSurge 1, maxUnavailable 0 + +[Test] Initial state + md, 3/3 replicas + - ms1, 3/3 replicas (m1,m2,m3) + - ms2, 0/0 replicas () +[Test] Rollout 3 replicas, MaxSurge=1, MaxUnavailable=0 +[MD controller] Iteration 1, Reconcile md +[MD controller] - Input to rollout planner + md, 3/3 replicas + - ms1, 3/3 replicas (m1,m2,m3) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 3/3 replicas + - ms1, 3/3 replicas (m1,m2,m3) + - ms2, 0/1 replicas () +[MS controller] Iteration 1, Reconcile ms1, 3/3 replicas (m1,m2,m3) +[MS controller] Iteration 1, Reconcile ms2, 0/1 replicas () +[MS controller] - ms2 scale up to 1/1 replicas (m4 created) +[MD controller] Iteration 2, Reconcile md +[MD controller] - Input to rollout planner + md, 3/3 replicas + - ms1, 3/3 replicas (m1,m2,m3) + - ms2, 1/1 replicas (m4) +[MD controller] - Result of rollout planner + md, 4/3 replicas + - ms1, 3/2 replicas (m1,m2,m3) + - ms2, 1/1 replicas (m4) +[MS controller] Iteration 2, Reconcile ms1, 3/2 replicas (m1,m2,m3) +[MS controller] - ms1 scale down to 2/2 replicas (m1 deleted) +[MS controller] Iteration 2, Reconcile ms2, 1/1 replicas (m4) +[MD controller] Iteration 3, Reconcile md +[MD controller] - Input to rollout planner + md, 4/3 replicas + - ms1, 2/2 replicas (m2,m3) + - ms2, 1/1 replicas (m4) +[MD controller] - Result of rollout planner + md, 3/3 replicas + - ms1, 2/2 replicas (m2,m3) + - ms2, 1/2 replicas (m4) +[MS controller] Iteration 3, Reconcile ms1, 2/2 replicas (m2,m3) +[MS controller] Iteration 3, Reconcile ms2, 1/2 replicas (m4) +[MS controller] - ms2 scale up to 2/2 replicas (m5 created) +[MD controller] Iteration 4, Reconcile md +[MD controller] - Input to rollout planner + md, 3/3 replicas + - ms1, 2/2 replicas (m2,m3) + - ms2, 2/2 replicas (m4,m5) +[MD controller] - Result of rollout planner + md, 4/3 replicas + - ms1, 2/1 replicas (m2,m3) + - ms2, 2/2 replicas (m4,m5) +[MS controller] Iteration 4, Reconcile ms1, 2/1 replicas (m2,m3) +[MS controller] - ms1 scale down to 1/1 replicas (m2 deleted) +[MS controller] Iteration 4, Reconcile ms2, 2/2 replicas (m4,m5) +[MD controller] Iteration 5, Reconcile md +[MD controller] - Input to rollout planner + md, 4/3 replicas + - ms1, 1/1 replicas (m3) + - ms2, 2/2 replicas (m4,m5) +[MD controller] - Result of rollout planner + md, 3/3 replicas + - ms1, 1/1 replicas (m3) + - ms2, 2/3 replicas (m4,m5) +[MS controller] Iteration 5, Reconcile ms1, 1/1 replicas (m3) +[MS controller] Iteration 5, Reconcile ms2, 2/3 replicas (m4,m5) +[MS controller] - ms2 scale up to 3/3 replicas (m6 created) +[MD controller] Iteration 6, Reconcile md +[MD controller] - Input to rollout planner + md, 3/3 replicas + - ms1, 1/1 replicas (m3) + - ms2, 3/3 replicas (m4,m5,m6) +[MD controller] - Result of rollout planner + md, 4/3 replicas + - ms1, 1/0 replicas (m3) + - ms2, 3/3 replicas (m4,m5,m6) +[MS controller] Iteration 6, Reconcile ms1, 1/0 replicas (m3) +[MS controller] - ms1 scale down to 0/0 replicas (m3 deleted) +[MS controller] Iteration 6, Reconcile ms2, 3/3 replicas (m4,m5,m6) +[MD controller] Iteration 7, Reconcile md +[MD controller] - Input to rollout planner + md, 4/3 replicas + - ms1, 0/0 replicas () + - ms2, 3/3 replicas (m4,m5,m6) +[MD controller] - Result of rollout planner + md, 3/3 replicas + - ms1, 0/0 replicas () + - ms2, 3/3 replicas (m4,m5,m6) +[MS controller] Iteration 7, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 7, Reconcile ms2, 3/3 replicas (m4,m5,m6) +[Test] Final state + md, 3/3 replicas + - ms1, 0/0 replicas () + - ms2, 3/3 replicas (m4,m5,m6) diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 0, maxunavailable 10.test.log.golden b/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 0, maxunavailable 10.test.log.golden new file mode 100644 index 000000000000..9f557ca4cb64 --- /dev/null +++ b/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 0, maxunavailable 10.test.log.golden @@ -0,0 +1,46 @@ +## Regular rollout, 6 Replicas, maxSurge 0, maxUnavailable 10 + +[Test] Initial state + md, 6/6 replicas + - ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 0/0 replicas () +[Test] Rollout 6 replicas, MaxSurge=0, MaxUnavailable=10 +[MD controller] Iteration 1, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 6/0 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 0/0 replicas () +[MS controller] Iteration 1, Reconcile ms1, 6/0 replicas (m1,m2,m3,m4,m5,m6) +[MS controller] - ms1 scale down to 0/0 replicas (m1,m2,m3,m4,m5,m6 deleted) +[MS controller] Iteration 1, Reconcile ms2, 0/0 replicas () +[MD controller] Iteration 2, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 0/6 replicas + - ms1, 0/0 replicas () + - ms2, 0/6 replicas () +[MS controller] Iteration 2, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 2, Reconcile ms2, 0/6 replicas () +[MS controller] - ms2 scale up to 6/6 replicas (m7,m8,m9,m10,m11,m12 created) +[MD controller] Iteration 3, Reconcile md +[MD controller] - Input to rollout planner + md, 0/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MS controller] Iteration 3, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 3, Reconcile ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[Test] Final state + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 1, maxunavailable 3.test.log.golden b/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 1, maxunavailable 3.test.log.golden new file mode 100644 index 000000000000..41f177ffa99c --- /dev/null +++ b/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 1, maxunavailable 3.test.log.golden @@ -0,0 +1,73 @@ +## Regular rollout, 6 Replicas, maxSurge 1, maxUnavailable 3 + +[Test] Initial state + md, 6/6 replicas + - ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 0/0 replicas () +[Test] Rollout 6 replicas, MaxSurge=1, MaxUnavailable=3 +[MD controller] Iteration 1, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 6/3 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 0/1 replicas () +[MS controller] Iteration 1, Reconcile ms1, 6/3 replicas (m1,m2,m3,m4,m5,m6) +[MS controller] - ms1 scale down to 3/3 replicas (m1,m2,m3 deleted) +[MS controller] Iteration 1, Reconcile ms2, 0/1 replicas () +[MS controller] - ms2 scale up to 1/1 replicas (m7 created) +[MD controller] Iteration 2, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 1/1 replicas (m7) +[MD controller] - Result of rollout planner + md, 4/6 replicas + - ms1, 3/2 replicas (m4,m5,m6) + - ms2, 1/4 replicas (m7) +[MS controller] Iteration 2, Reconcile ms1, 3/2 replicas (m4,m5,m6) +[MS controller] - ms1 scale down to 2/2 replicas (m4 deleted) +[MS controller] Iteration 2, Reconcile ms2, 1/4 replicas (m7) +[MS controller] - ms2 scale up to 4/4 replicas (m8,m9,m10 created) +[MD controller] Iteration 3, Reconcile md +[MD controller] - Input to rollout planner + md, 4/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 4/4 replicas (m7,m8,m9,m10) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 2/0 replicas (m5,m6) + - ms2, 4/5 replicas (m7,m8,m9,m10) +[MS controller] Iteration 3, Reconcile ms1, 2/0 replicas (m5,m6) +[MS controller] - ms1 scale down to 0/0 replicas (m5,m6 deleted) +[MS controller] Iteration 3, Reconcile ms2, 4/5 replicas (m7,m8,m9,m10) +[MS controller] - ms2 scale up to 5/5 replicas (m11 created) +[MD controller] Iteration 4, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 5/5 replicas (m7,m8,m9,m10,m11) +[MD controller] - Result of rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 5/6 replicas (m7,m8,m9,m10,m11) +[MS controller] Iteration 4, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 4, Reconcile ms2, 5/6 replicas (m7,m8,m9,m10,m11) +[MS controller] - ms2 scale up to 6/6 replicas (m12 created) +[MD controller] Iteration 5, Reconcile md +[MD controller] - Input to rollout planner + md, 5/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MS controller] Iteration 5, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 5, Reconcile ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[Test] Final state + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 10, maxunavailable 0.test.log.golden b/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 10, maxunavailable 0.test.log.golden new file mode 100644 index 000000000000..655776816f16 --- /dev/null +++ b/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 10, maxunavailable 0.test.log.golden @@ -0,0 +1,46 @@ +## Regular rollout, 6 Replicas, maxSurge 10, maxUnavailable 0 + +[Test] Initial state + md, 6/6 replicas + - ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 0/0 replicas () +[Test] Rollout 6 replicas, MaxSurge=10, MaxUnavailable=0 +[MD controller] Iteration 1, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 0/6 replicas () +[MS controller] Iteration 1, Reconcile ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) +[MS controller] Iteration 1, Reconcile ms2, 0/6 replicas () +[MS controller] - ms2 scale up to 6/6 replicas (m7,m8,m9,m10,m11,m12 created) +[MD controller] Iteration 2, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MD controller] - Result of rollout planner + md, 12/6 replicas + - ms1, 6/0 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MS controller] Iteration 2, Reconcile ms1, 6/0 replicas (m1,m2,m3,m4,m5,m6) +[MS controller] - ms1 scale down to 0/0 replicas (m1,m2,m3,m4,m5,m6 deleted) +[MS controller] Iteration 2, Reconcile ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MD controller] Iteration 3, Reconcile md +[MD controller] - Input to rollout planner + md, 12/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MS controller] Iteration 3, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 3, Reconcile ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[Test] Final state + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, change spec.test.log.golden b/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, change spec.test.log.golden new file mode 100644 index 000000000000..256c9ac8cb30 --- /dev/null +++ b/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, change spec.test.log.golden @@ -0,0 +1,91 @@ +## Regular rollout, 6 Replicas, maxSurge 3, maxUnavailable 1, change spec + +[Test] Initial state + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 0/0 replicas () +[Test] Rollout 6 replicas, MaxSurge=3, MaxUnavailable=1 +[MD controller] Iteration 1, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 3/2 replicas (m4,m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 0/3 replicas () +[MS controller] Iteration 1, Reconcile ms1, 3/2 replicas (m4,m5,m6) +[MS controller] - ms1 scale down to 2/2 replicas (m4 deleted) +[MS controller] Iteration 1, Reconcile ms2, 3/3 replicas (m7,m8,m9) +[MS controller] Iteration 1, Reconcile ms3, 0/3 replicas () +[MS controller] - ms3 scale up to 3/3 replicas (m10,m11,m12 created) +[MD controller] Iteration 2, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) + - ms3, 3/3 replicas (m10,m11,m12) +[MD controller] - Result of rollout planner + md, 8/6 replicas + - ms1, 2/0 replicas (m5,m6) + - ms2, 3/2 replicas (m7,m8,m9) + - ms3, 3/4 replicas (m10,m11,m12) +[MS controller] Iteration 2, Reconcile ms1, 2/0 replicas (m5,m6) +[MS controller] - ms1 scale down to 0/0 replicas (m5,m6 deleted) +[MS controller] Iteration 2, Reconcile ms2, 3/2 replicas (m7,m8,m9) +[MS controller] - ms2 scale down to 2/2 replicas (m7 deleted) +[MS controller] Iteration 2, Reconcile ms3, 3/4 replicas (m10,m11,m12) +[MS controller] - ms3 scale up to 4/4 replicas (m13 created) +[MD controller] Iteration 3, Reconcile md +[MD controller] - Input to rollout planner + md, 8/6 replicas + - ms1, 0/0 replicas () + - ms2, 2/2 replicas (m8,m9) + - ms3, 4/4 replicas (m10,m11,m12,m13) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 2/1 replicas (m8,m9) + - ms3, 4/6 replicas (m10,m11,m12,m13) +[MS controller] Iteration 3, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 3, Reconcile ms2, 2/1 replicas (m8,m9) +[MS controller] - ms2 scale down to 1/1 replicas (m8 deleted) +[MS controller] Iteration 3, Reconcile ms3, 4/6 replicas (m10,m11,m12,m13) +[MS controller] - ms3 scale up to 6/6 replicas (m14,m15 created) +[MD controller] Iteration 4, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 1/1 replicas (m9) + - ms3, 6/6 replicas (m10,m11,m12,m13,m14,m15) +[MD controller] - Result of rollout planner + md, 7/6 replicas + - ms1, 0/0 replicas () + - ms2, 1/0 replicas (m9) + - ms3, 6/6 replicas (m10,m11,m12,m13,m14,m15) +[MS controller] Iteration 4, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 4, Reconcile ms2, 1/0 replicas (m9) +[MS controller] - ms2 scale down to 0/0 replicas (m9 deleted) +[MS controller] Iteration 4, Reconcile ms3, 6/6 replicas (m10,m11,m12,m13,m14,m15) +[MD controller] Iteration 5, Reconcile md +[MD controller] - Input to rollout planner + md, 7/6 replicas + - ms1, 0/0 replicas () + - ms2, 0/0 replicas () + - ms3, 6/6 replicas (m10,m11,m12,m13,m14,m15) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 0/0 replicas () + - ms3, 6/6 replicas (m10,m11,m12,m13,m14,m15) +[MS controller] Iteration 5, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 5, Reconcile ms2, 0/0 replicas () +[MS controller] Iteration 5, Reconcile ms3, 6/6 replicas (m10,m11,m12,m13,m14,m15) +[Test] Final state + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 0/0 replicas () + - ms3, 6/6 replicas (m10,m11,m12,m13,m14,m15) diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, scale up to 12.test.log.golden b/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, scale up to 12.test.log.golden new file mode 100644 index 000000000000..21014d51af39 --- /dev/null +++ b/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1, scale up to 12.test.log.golden @@ -0,0 +1,47 @@ +## Regular rollout, 6 Replicas, maxSurge 3, maxUnavailable 1, scale up to 12 + +[Test] Initial state + md, 12/12 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) +[Test] Rollout 6 replicas, MaxSurge=3, MaxUnavailable=1 +[MD controller] Iteration 1, Reconcile md +[MD controller] - Input to rollout planner + md, 12/12 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) +[MD controller] - Result of rollout planner + md, 6/12 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 3/12 replicas (m7,m8,m9) +[Toleration] tolerate maxUnavailable breach +[MS controller] Iteration 1, Reconcile ms1, 3/3 replicas (m4,m5,m6) +[MS controller] Iteration 1, Reconcile ms2, 3/12 replicas (m7,m8,m9) +[MS controller] - ms2 scale up to 12/12 replicas (m10,m11,m12,m13,m14,m15,m16,m17,m18 created) +[MD controller] Iteration 2, Reconcile md +[MD controller] - Input to rollout planner + md, 6/12 replicas + - ms1, 3/3 replicas (m4,m5,m6) + - ms2, 12/12 replicas (m7,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18) +[MD controller] - Result of rollout planner + md, 15/12 replicas + - ms1, 3/0 replicas (m4,m5,m6) + - ms2, 12/12 replicas (m7,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18) +[MS controller] Iteration 2, Reconcile ms1, 3/0 replicas (m4,m5,m6) +[MS controller] - ms1 scale down to 0/0 replicas (m4,m5,m6 deleted) +[MS controller] Iteration 2, Reconcile ms2, 12/12 replicas (m7,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18) +[MD controller] Iteration 3, Reconcile md +[MD controller] - Input to rollout planner + md, 15/12 replicas + - ms1, 0/0 replicas () + - ms2, 12/12 replicas (m7,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18) +[MD controller] - Result of rollout planner + md, 12/12 replicas + - ms1, 0/0 replicas () + - ms2, 12/12 replicas (m7,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18) +[MS controller] Iteration 3, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 3, Reconcile ms2, 12/12 replicas (m7,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18) +[Test] Final state + md, 12/12 replicas + - ms1, 0/0 replicas () + - ms2, 12/12 replicas (m7,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18) diff --git a/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1.test.log.golden b/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1.test.log.golden new file mode 100644 index 000000000000..612d601a0737 --- /dev/null +++ b/internal/controllers/machinedeployment/testdata/regular rollout, 6 replicas, maxsurge 3, maxunavailable 1.test.log.golden @@ -0,0 +1,73 @@ +## Regular rollout, 6 Replicas, maxSurge 3, maxUnavailable 1 + +[Test] Initial state + md, 6/6 replicas + - ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 0/0 replicas () +[Test] Rollout 6 replicas, MaxSurge=3, MaxUnavailable=1 +[MD controller] Iteration 1, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 6/6 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 0/0 replicas () +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 6/5 replicas (m1,m2,m3,m4,m5,m6) + - ms2, 0/3 replicas () +[MS controller] Iteration 1, Reconcile ms1, 6/5 replicas (m1,m2,m3,m4,m5,m6) +[MS controller] - ms1 scale down to 5/5 replicas (m1 deleted) +[MS controller] Iteration 1, Reconcile ms2, 0/3 replicas () +[MS controller] - ms2 scale up to 3/3 replicas (m7,m8,m9 created) +[MD controller] Iteration 2, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 5/5 replicas (m2,m3,m4,m5,m6) + - ms2, 3/3 replicas (m7,m8,m9) +[MD controller] - Result of rollout planner + md, 8/6 replicas + - ms1, 5/2 replicas (m2,m3,m4,m5,m6) + - ms2, 3/4 replicas (m7,m8,m9) +[MS controller] Iteration 2, Reconcile ms1, 5/2 replicas (m2,m3,m4,m5,m6) +[MS controller] - ms1 scale down to 2/2 replicas (m2,m3,m4 deleted) +[MS controller] Iteration 2, Reconcile ms2, 3/4 replicas (m7,m8,m9) +[MS controller] - ms2 scale up to 4/4 replicas (m10 created) +[MD controller] Iteration 3, Reconcile md +[MD controller] - Input to rollout planner + md, 8/6 replicas + - ms1, 2/2 replicas (m5,m6) + - ms2, 4/4 replicas (m7,m8,m9,m10) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 2/1 replicas (m5,m6) + - ms2, 4/6 replicas (m7,m8,m9,m10) +[MS controller] Iteration 3, Reconcile ms1, 2/1 replicas (m5,m6) +[MS controller] - ms1 scale down to 1/1 replicas (m5 deleted) +[MS controller] Iteration 3, Reconcile ms2, 4/6 replicas (m7,m8,m9,m10) +[MS controller] - ms2 scale up to 6/6 replicas (m11,m12 created) +[MD controller] Iteration 4, Reconcile md +[MD controller] - Input to rollout planner + md, 6/6 replicas + - ms1, 1/1 replicas (m6) + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MD controller] - Result of rollout planner + md, 7/6 replicas + - ms1, 1/0 replicas (m6) + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MS controller] Iteration 4, Reconcile ms1, 1/0 replicas (m6) +[MS controller] - ms1 scale down to 0/0 replicas (m6 deleted) +[MS controller] Iteration 4, Reconcile ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MD controller] Iteration 5, Reconcile md +[MD controller] - Input to rollout planner + md, 7/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MD controller] - Result of rollout planner + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[MS controller] Iteration 5, Reconcile ms1, 0/0 replicas () +[MS controller] Iteration 5, Reconcile ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12) +[Test] Final state + md, 6/6 replicas + - ms1, 0/0 replicas () + - ms2, 6/6 replicas (m7,m8,m9,m10,m11,m12)