diff --git a/test/e2e/autoscaler.go b/test/e2e/autoscaler.go index 7269edf8367a..e67e08f6251d 100644 --- a/test/e2e/autoscaler.go +++ b/test/e2e/autoscaler.go @@ -260,9 +260,7 @@ func AutoscalerSpec(ctx context.Context, inputGetter func() AutoscalerSpecInput) By("Scaling the MachineDeployment scale up deployment to zero") framework.ScaleScaleUpDeploymentAndWait(ctx, framework.ScaleScaleUpDeploymentAndWaitInput{ ClusterProxy: workloadClusterProxy, - // We need to sum up the expected number of MachineDeployment replicas and the current - // number of MachinePool replicas because otherwise the pods get scheduled on the MachinePool nodes. - Replicas: mpOriginalReplicas + 0, + Replicas: 0, }, input.E2EConfig.GetIntervals(specName, "wait-autoscaler")...) By("Checking the MachineDeployment finished scaling down to zero") diff --git a/test/e2e/autoscaler_test.go b/test/e2e/autoscaler_test.go index 490826482dc9..cfdda26a1a78 100644 --- a/test/e2e/autoscaler_test.go +++ b/test/e2e/autoscaler_test.go @@ -36,7 +36,8 @@ var _ = Describe("When using the autoscaler with Cluster API using ClusterClass InfrastructureMachinePoolTemplateKind: "dockermachinepooltemplates", InfrastructureMachinePoolKind: "dockermachinepools", Flavor: ptr.To("topology-autoscaler"), - AutoscalerVersion: "v1.33.0", + AutoscalerVersion: "v1.32.1", + ScaleToAndFromZero: true, } }) }) diff --git a/test/e2e/config/docker.yaml b/test/e2e/config/docker.yaml index df88fb8d44c1..6da9021017d0 100644 --- a/test/e2e/config/docker.yaml +++ b/test/e2e/config/docker.yaml @@ -399,7 +399,7 @@ variables: CAPI_INSECURE_DIAGNOSTICS: "true" intervals: - default/wait-controllers: ["3m", "10s"] + default/wait-controllers: ["4m", "10s"] default/wait-cluster: ["5m", "10s"] default/wait-control-plane: ["10m", "10s"] default/wait-worker-nodes: ["5m", "10s"] diff --git a/test/e2e/data/autoscaler/autoscaler-to-workload-workload.yaml b/test/e2e/data/autoscaler/autoscaler-to-workload-workload.yaml index 84cf0f9f3e7b..3ddd2341ca92 100644 --- a/test/e2e/data/autoscaler/autoscaler-to-workload-workload.yaml +++ b/test/e2e/data/autoscaler/autoscaler-to-workload-workload.yaml @@ -198,6 +198,8 @@ spec: # Note: The E2E test should only go up to 4 (assuming it starts with a min node group size of 2). # Using 6 for additional some buffer and to allow different starting min node group sizes. - --max-nodes-total=6 + # Set the log verbosity + - --v=4 env: # Per default autoscaler uses the preferred apiVersion to retrieve MachineDeployments. # If that apiVersion is v1beta2 the current autoscaler implementation is not able @@ -208,6 +210,13 @@ spec: - name: kubeconfig-management-cluster mountPath: /management-cluster readOnly: true + # Run the autoscaler on control plane Machines to avoid disruptions when scaling to 0. + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + operator: Exists serviceAccountName: cluster-autoscaler terminationGracePeriodSeconds: 10 volumes: diff --git a/test/infrastructure/container/docker.go b/test/infrastructure/container/docker.go index 6eae6c27d467..a3a43945e37a 100644 --- a/test/infrastructure/container/docker.go +++ b/test/infrastructure/container/docker.go @@ -64,6 +64,7 @@ func NewDockerClient() (Runtime, error) { if err != nil { return nil, errors.Wrapf(err, "failed to created docker runtime client") } + return &dockerRuntime{ dockerClient: dockerClient, }, nil @@ -544,6 +545,11 @@ func (d *dockerRuntime) RunContainer(ctx context.Context, runConfig *RunContaine return nil } +// GetSystemInfo will return the docker system info. +func (d *dockerRuntime) GetSystemInfo(ctx context.Context) (dockersystem.Info, error) { + return d.dockerClient.Info(ctx) +} + // needsDevMapper checks whether we need to mount /dev/mapper. // This is required when the docker storage driver is Btrfs or ZFS. // https://github.com/kubernetes-sigs/kind/pull/1464 diff --git a/test/infrastructure/container/fake.go b/test/infrastructure/container/fake.go index 7b777a759b32..c24fe1c2f9ef 100644 --- a/test/infrastructure/container/fake.go +++ b/test/infrastructure/container/fake.go @@ -19,6 +19,8 @@ package container import ( "context" "io" + + dockersystem "github.com/docker/docker/api/types/system" ) var runContainerCallLog []RunContainerArgs @@ -170,3 +172,8 @@ func (f *FakeRuntime) RunContainerCalls() []RunContainerArgs { func (f *FakeRuntime) ResetRunContainerCallLogs() { runContainerCallLog = []RunContainerArgs{} } + +// GetSystemInfo returns empty docker system info. +func (f *FakeRuntime) GetSystemInfo(_ context.Context) (dockersystem.Info, error) { + return dockersystem.Info{}, nil +} diff --git a/test/infrastructure/container/interface.go b/test/infrastructure/container/interface.go index 80869607f2d1..2a72cda7bfb5 100644 --- a/test/infrastructure/container/interface.go +++ b/test/infrastructure/container/interface.go @@ -23,6 +23,7 @@ import ( "net" dockercontainer "github.com/docker/docker/api/types/container" + dockersystem "github.com/docker/docker/api/types/system" "github.com/pkg/errors" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" @@ -46,6 +47,7 @@ type Runtime interface { ContainerDebugInfo(ctx context.Context, containerName string, w io.Writer) error DeleteContainer(ctx context.Context, containerName string) error KillContainer(ctx context.Context, containerName, signal string) error + GetSystemInfo(ctx context.Context) (dockersystem.Info, error) } // Mount contains mount details. diff --git a/test/infrastructure/docker/api/v1alpha3/conversion.go b/test/infrastructure/docker/api/v1alpha3/conversion.go index e76d87226752..baa836efff98 100644 --- a/test/infrastructure/docker/api/v1alpha3/conversion.go +++ b/test/infrastructure/docker/api/v1alpha3/conversion.go @@ -165,6 +165,7 @@ func (src *DockerMachineTemplate) ConvertTo(dstRaw conversion.Hub) error { if ok { RestoreDockerMachineTemplateSpec(&restored.Spec, &dst.Spec) + dst.Status = restored.Status } return nil @@ -350,3 +351,7 @@ func Convert_v1alpha3_DockerClusterSpec_To_v1beta2_DockerClusterSpec(in *DockerC return nil } + +func Convert_v1beta2_DockerMachineTemplate_To_v1alpha3_DockerMachineTemplate(in *infrav1.DockerMachineTemplate, out *DockerMachineTemplate, s apiconversion.Scope) error { + return autoConvert_v1beta2_DockerMachineTemplate_To_v1alpha3_DockerMachineTemplate(in, out, s) +} diff --git a/test/infrastructure/docker/api/v1alpha3/zz_generated.conversion.go b/test/infrastructure/docker/api/v1alpha3/zz_generated.conversion.go index 4782c222967e..b7b0f9d6b975 100644 --- a/test/infrastructure/docker/api/v1alpha3/zz_generated.conversion.go +++ b/test/infrastructure/docker/api/v1alpha3/zz_generated.conversion.go @@ -99,11 +99,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta2.DockerMachineTemplate)(nil), (*DockerMachineTemplate)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_DockerMachineTemplate_To_v1alpha3_DockerMachineTemplate(a.(*v1beta2.DockerMachineTemplate), b.(*DockerMachineTemplate), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*DockerMachineTemplateList)(nil), (*v1beta2.DockerMachineTemplateList)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_DockerMachineTemplateList_To_v1beta2_DockerMachineTemplateList(a.(*DockerMachineTemplateList), b.(*v1beta2.DockerMachineTemplateList), scope) }); err != nil { @@ -189,6 +184,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta2.DockerMachineTemplate)(nil), (*DockerMachineTemplate)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_DockerMachineTemplate_To_v1alpha3_DockerMachineTemplate(a.(*v1beta2.DockerMachineTemplate), b.(*DockerMachineTemplate), scope) + }); err != nil { + return err + } return nil } @@ -497,14 +497,10 @@ func autoConvert_v1beta2_DockerMachineTemplate_To_v1alpha3_DockerMachineTemplate if err := Convert_v1beta2_DockerMachineTemplateSpec_To_v1alpha3_DockerMachineTemplateSpec(&in.Spec, &out.Spec, s); err != nil { return err } + // WARNING: in.Status requires manual conversion: does not exist in peer-type return nil } -// Convert_v1beta2_DockerMachineTemplate_To_v1alpha3_DockerMachineTemplate is an autogenerated conversion function. -func Convert_v1beta2_DockerMachineTemplate_To_v1alpha3_DockerMachineTemplate(in *v1beta2.DockerMachineTemplate, out *DockerMachineTemplate, s conversion.Scope) error { - return autoConvert_v1beta2_DockerMachineTemplate_To_v1alpha3_DockerMachineTemplate(in, out, s) -} - func autoConvert_v1alpha3_DockerMachineTemplateList_To_v1beta2_DockerMachineTemplateList(in *DockerMachineTemplateList, out *v1beta2.DockerMachineTemplateList, s conversion.Scope) error { out.ListMeta = in.ListMeta if in.Items != nil { diff --git a/test/infrastructure/docker/api/v1alpha4/conversion.go b/test/infrastructure/docker/api/v1alpha4/conversion.go index accfbf111dbc..0a165f05b730 100644 --- a/test/infrastructure/docker/api/v1alpha4/conversion.go +++ b/test/infrastructure/docker/api/v1alpha4/conversion.go @@ -196,6 +196,7 @@ func (src *DockerMachineTemplate) ConvertTo(dstRaw conversion.Hub) error { if ok { RestoreDockerMachineTemplateSpec(&restored.Spec, &dst.Spec) + dst.Status = restored.Status } return nil diff --git a/test/infrastructure/docker/api/v1alpha4/zz_generated.conversion.go b/test/infrastructure/docker/api/v1alpha4/zz_generated.conversion.go index 5c66d44f3fab..37a931254e6e 100644 --- a/test/infrastructure/docker/api/v1alpha4/zz_generated.conversion.go +++ b/test/infrastructure/docker/api/v1alpha4/zz_generated.conversion.go @@ -679,6 +679,7 @@ func autoConvert_v1beta2_DockerMachineTemplate_To_v1alpha4_DockerMachineTemplate if err := Convert_v1beta2_DockerMachineTemplateSpec_To_v1alpha4_DockerMachineTemplateSpec(&in.Spec, &out.Spec, s); err != nil { return err } + // WARNING: in.Status requires manual conversion: does not exist in peer-type return nil } diff --git a/test/infrastructure/docker/api/v1beta1/conversion.go b/test/infrastructure/docker/api/v1beta1/conversion.go index 7f344f9b8d46..ec32610cc6a4 100644 --- a/test/infrastructure/docker/api/v1beta1/conversion.go +++ b/test/infrastructure/docker/api/v1beta1/conversion.go @@ -120,7 +120,22 @@ func (dst *DockerMachine) ConvertFrom(srcRaw conversion.Hub) error { func (src *DockerMachineTemplate) ConvertTo(dstRaw conversion.Hub) error { dst := dstRaw.(*infrav1.DockerMachineTemplate) - return Convert_v1beta1_DockerMachineTemplate_To_v1beta2_DockerMachineTemplate(src, dst, nil) + if err := Convert_v1beta1_DockerMachineTemplate_To_v1beta2_DockerMachineTemplate(src, dst, nil); err != nil { + return err + } + + // Manually restore data. + restored := &infrav1.DockerMachineTemplate{} + ok, err := utilconversion.UnmarshalData(src, restored) + if err != nil { + return err + } + + if ok { + dst.Status = restored.Status + } + + return nil } func (dst *DockerMachineTemplate) ConvertFrom(srcRaw conversion.Hub) error { @@ -134,7 +149,7 @@ func (dst *DockerMachineTemplate) ConvertFrom(srcRaw conversion.Hub) error { dst.Spec.Template.Spec.ProviderID = nil } - return nil + return utilconversion.MarshalData(src, dst) } func (src *DevCluster) ConvertTo(dstRaw conversion.Hub) error { @@ -226,7 +241,22 @@ func (dst *DevMachine) ConvertFrom(srcRaw conversion.Hub) error { func (src *DevMachineTemplate) ConvertTo(dstRaw conversion.Hub) error { dst := dstRaw.(*infrav1.DevMachineTemplate) - return Convert_v1beta1_DevMachineTemplate_To_v1beta2_DevMachineTemplate(src, dst, nil) + if err := Convert_v1beta1_DevMachineTemplate_To_v1beta2_DevMachineTemplate(src, dst, nil); err != nil { + return err + } + + // Manually restore data. + restored := &infrav1.DevMachineTemplate{} + ok, err := utilconversion.UnmarshalData(src, restored) + if err != nil { + return err + } + + if ok { + dst.Status = restored.Status + } + + return nil } func (dst *DevMachineTemplate) ConvertFrom(srcRaw conversion.Hub) error { @@ -240,7 +270,7 @@ func (dst *DevMachineTemplate) ConvertFrom(srcRaw conversion.Hub) error { dst.Spec.Template.Spec.ProviderID = nil } - return nil + return utilconversion.MarshalData(src, dst) } func Convert_v1beta1_ObjectMeta_To_v1beta2_ObjectMeta(in *clusterv1beta1.ObjectMeta, out *clusterv1.ObjectMeta, s apiconversion.Scope) error { @@ -620,3 +650,11 @@ func Convert_v1beta1_DockerClusterBackendSpec_To_v1beta2_DockerClusterBackendSpe return nil } + +func Convert_v1beta2_DockerMachineTemplate_To_v1beta1_DockerMachineTemplate(in *infrav1.DockerMachineTemplate, out *DockerMachineTemplate, s apiconversion.Scope) error { + return autoConvert_v1beta2_DockerMachineTemplate_To_v1beta1_DockerMachineTemplate(in, out, s) +} + +func Convert_v1beta2_DevMachineTemplate_To_v1beta1_DevMachineTemplate(in *infrav1.DevMachineTemplate, out *DevMachineTemplate, s apiconversion.Scope) error { + return autoConvert_v1beta2_DevMachineTemplate_To_v1beta1_DevMachineTemplate(in, out, s) +} diff --git a/test/infrastructure/docker/api/v1beta1/zz_generated.conversion.go b/test/infrastructure/docker/api/v1beta1/zz_generated.conversion.go index 8300204cdc06..6f737c92c21a 100644 --- a/test/infrastructure/docker/api/v1beta1/zz_generated.conversion.go +++ b/test/infrastructure/docker/api/v1beta1/zz_generated.conversion.go @@ -195,11 +195,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta2.DevMachineTemplate)(nil), (*DevMachineTemplate)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_DevMachineTemplate_To_v1beta1_DevMachineTemplate(a.(*v1beta2.DevMachineTemplate), b.(*DevMachineTemplate), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*DevMachineTemplateList)(nil), (*v1beta2.DevMachineTemplateList)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_DevMachineTemplateList_To_v1beta2_DevMachineTemplateList(a.(*DevMachineTemplateList), b.(*v1beta2.DevMachineTemplateList), scope) }); err != nil { @@ -355,11 +350,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta2.DockerMachineTemplate)(nil), (*DockerMachineTemplate)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_DockerMachineTemplate_To_v1beta1_DockerMachineTemplate(a.(*v1beta2.DockerMachineTemplate), b.(*DockerMachineTemplate), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*DockerMachineTemplateList)(nil), (*v1beta2.DockerMachineTemplateList)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_DockerMachineTemplateList_To_v1beta2_DockerMachineTemplateList(a.(*DockerMachineTemplateList), b.(*v1beta2.DockerMachineTemplateList), scope) }); err != nil { @@ -525,6 +515,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta2.DevMachineTemplate)(nil), (*DevMachineTemplate)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_DevMachineTemplate_To_v1beta1_DevMachineTemplate(a.(*v1beta2.DevMachineTemplate), b.(*DevMachineTemplate), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1beta2.DockerClusterBackendSpec)(nil), (*DockerClusterBackendSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta2_DockerClusterBackendSpec_To_v1beta1_DockerClusterBackendSpec(a.(*v1beta2.DockerClusterBackendSpec), b.(*DockerClusterBackendSpec), scope) }); err != nil { @@ -545,6 +540,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta2.DockerMachineTemplate)(nil), (*DockerMachineTemplate)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_DockerMachineTemplate_To_v1beta1_DockerMachineTemplate(a.(*v1beta2.DockerMachineTemplate), b.(*DockerMachineTemplate), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*corev1beta2.ObjectMeta)(nil), (*corev1beta1.ObjectMeta)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta2_ObjectMeta_To_v1beta1_ObjectMeta(a.(*corev1beta2.ObjectMeta), b.(*corev1beta1.ObjectMeta), scope) }); err != nil { @@ -1099,14 +1099,10 @@ func autoConvert_v1beta2_DevMachineTemplate_To_v1beta1_DevMachineTemplate(in *v1 if err := Convert_v1beta2_DevMachineTemplateSpec_To_v1beta1_DevMachineTemplateSpec(&in.Spec, &out.Spec, s); err != nil { return err } + // WARNING: in.Status requires manual conversion: does not exist in peer-type return nil } -// Convert_v1beta2_DevMachineTemplate_To_v1beta1_DevMachineTemplate is an autogenerated conversion function. -func Convert_v1beta2_DevMachineTemplate_To_v1beta1_DevMachineTemplate(in *v1beta2.DevMachineTemplate, out *DevMachineTemplate, s conversion.Scope) error { - return autoConvert_v1beta2_DevMachineTemplate_To_v1beta1_DevMachineTemplate(in, out, s) -} - func autoConvert_v1beta1_DevMachineTemplateList_To_v1beta2_DevMachineTemplateList(in *DevMachineTemplateList, out *v1beta2.DevMachineTemplateList, s conversion.Scope) error { out.ListMeta = in.ListMeta if in.Items != nil { @@ -1711,14 +1707,10 @@ func autoConvert_v1beta2_DockerMachineTemplate_To_v1beta1_DockerMachineTemplate( if err := Convert_v1beta2_DockerMachineTemplateSpec_To_v1beta1_DockerMachineTemplateSpec(&in.Spec, &out.Spec, s); err != nil { return err } + // WARNING: in.Status requires manual conversion: does not exist in peer-type return nil } -// Convert_v1beta2_DockerMachineTemplate_To_v1beta1_DockerMachineTemplate is an autogenerated conversion function. -func Convert_v1beta2_DockerMachineTemplate_To_v1beta1_DockerMachineTemplate(in *v1beta2.DockerMachineTemplate, out *DockerMachineTemplate, s conversion.Scope) error { - return autoConvert_v1beta2_DockerMachineTemplate_To_v1beta1_DockerMachineTemplate(in, out, s) -} - func autoConvert_v1beta1_DockerMachineTemplateList_To_v1beta2_DockerMachineTemplateList(in *DockerMachineTemplateList, out *v1beta2.DockerMachineTemplateList, s conversion.Scope) error { out.ListMeta = in.ListMeta if in.Items != nil { diff --git a/test/infrastructure/docker/api/v1beta2/devmachinetemplate_types.go b/test/infrastructure/docker/api/v1beta2/devmachinetemplate_types.go index 41a9a68d98eb..9843851f7d63 100644 --- a/test/infrastructure/docker/api/v1beta2/devmachinetemplate_types.go +++ b/test/infrastructure/docker/api/v1beta2/devmachinetemplate_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1beta2 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" @@ -27,7 +28,17 @@ type DevMachineTemplateSpec struct { Template DevMachineTemplateResource `json:"template"` } +// DevMachineTemplateStatus defines the observed state of a DevMachineTemplate. +type DevMachineTemplateStatus struct { + // capacity defines the resource capacity for this DevMachineTemplate. + // This value is used for autoscaling from zero operations as defined in: + // https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20210310-opt-in-autoscaling-from-zero.md + // +optional + Capacity corev1.ResourceList `json:"capacity,omitempty"` +} + // +kubebuilder:object:root=true +// +kubebuilder:subresource:status // +kubebuilder:resource:path=devmachinetemplates,scope=Namespaced,categories=cluster-api // +kubebuilder:storageversion // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of the DevMachineTemplate" @@ -37,7 +48,8 @@ type DevMachineTemplate struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec DevMachineTemplateSpec `json:"spec,omitempty"` + Spec DevMachineTemplateSpec `json:"spec,omitempty"` + Status DevMachineTemplateStatus `json:"status,omitempty"` } // +kubebuilder:object:root=true diff --git a/test/infrastructure/docker/api/v1beta2/dockermachinetemplate_types.go b/test/infrastructure/docker/api/v1beta2/dockermachinetemplate_types.go index 78142a6ec0d4..e640f42ce3fb 100644 --- a/test/infrastructure/docker/api/v1beta2/dockermachinetemplate_types.go +++ b/test/infrastructure/docker/api/v1beta2/dockermachinetemplate_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1beta2 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" @@ -27,7 +28,17 @@ type DockerMachineTemplateSpec struct { Template DockerMachineTemplateResource `json:"template"` } +// DockerMachineTemplateStatus defines the observed state of a DockerMachineTemplate. +type DockerMachineTemplateStatus struct { + // capacity defines the resource capacity for this DockerMachineTemplate. + // This value is used for autoscaling from zero operations as defined in: + // https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20210310-opt-in-autoscaling-from-zero.md + // +optional + Capacity corev1.ResourceList `json:"capacity,omitempty"` +} + // +kubebuilder:object:root=true +// +kubebuilder:subresource:status // +kubebuilder:resource:path=dockermachinetemplates,scope=Namespaced,categories=cluster-api // +kubebuilder:storageversion // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of DockerMachineTemplate" @@ -37,7 +48,8 @@ type DockerMachineTemplate struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec DockerMachineTemplateSpec `json:"spec,omitempty"` + Spec DockerMachineTemplateSpec `json:"spec,omitempty"` + Status DockerMachineTemplateStatus `json:"status,omitempty"` } // +kubebuilder:object:root=true diff --git a/test/infrastructure/docker/api/v1beta2/zz_generated.deepcopy.go b/test/infrastructure/docker/api/v1beta2/zz_generated.deepcopy.go index e822aa02c269..efa1f01244c0 100644 --- a/test/infrastructure/docker/api/v1beta2/zz_generated.deepcopy.go +++ b/test/infrastructure/docker/api/v1beta2/zz_generated.deepcopy.go @@ -551,6 +551,7 @@ func (in *DevMachineTemplate) DeepCopyInto(out *DevMachineTemplate) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevMachineTemplate. @@ -636,6 +637,28 @@ func (in *DevMachineTemplateSpec) DeepCopy() *DevMachineTemplateSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DevMachineTemplateStatus) DeepCopyInto(out *DevMachineTemplateStatus) { + *out = *in + if in.Capacity != nil { + in, out := &in.Capacity, &out.Capacity + *out = make(corev1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevMachineTemplateStatus. +func (in *DevMachineTemplateStatus) DeepCopy() *DevMachineTemplateStatus { + if in == nil { + return nil + } + out := new(DevMachineTemplateStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DevMachineV1Beta1DeprecatedStatus) DeepCopyInto(out *DevMachineV1Beta1DeprecatedStatus) { *out = *in @@ -1186,6 +1209,7 @@ func (in *DockerMachineTemplate) DeepCopyInto(out *DockerMachineTemplate) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DockerMachineTemplate. @@ -1271,6 +1295,28 @@ func (in *DockerMachineTemplateSpec) DeepCopy() *DockerMachineTemplateSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DockerMachineTemplateStatus) DeepCopyInto(out *DockerMachineTemplateStatus) { + *out = *in + if in.Capacity != nil { + in, out := &in.Capacity, &out.Capacity + *out = make(corev1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DockerMachineTemplateStatus. +func (in *DockerMachineTemplateStatus) DeepCopy() *DockerMachineTemplateStatus { + if in == nil { + return nil + } + out := new(DockerMachineTemplateStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DockerMachineV1Beta1DeprecatedStatus) DeepCopyInto(out *DockerMachineV1Beta1DeprecatedStatus) { *out = *in diff --git a/test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_devmachinetemplates.yaml b/test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_devmachinetemplates.yaml index 98a5eb6169fe..ecdf8b4d13f8 100644 --- a/test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_devmachinetemplates.yaml +++ b/test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_devmachinetemplates.yaml @@ -484,7 +484,25 @@ spec: required: - template type: object + status: + description: DevMachineTemplateStatus defines the observed state of a + DevMachineTemplate. + properties: + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + capacity defines the resource capacity for this DevMachineTemplate. + This value is used for autoscaling from zero operations as defined in: + https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20210310-opt-in-autoscaling-from-zero.md + type: object + type: object type: object served: true storage: true - subresources: {} + subresources: + status: {} diff --git a/test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_dockermachinetemplates.yaml b/test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_dockermachinetemplates.yaml index a9881c0429cf..4c9da8025766 100644 --- a/test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_dockermachinetemplates.yaml +++ b/test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_dockermachinetemplates.yaml @@ -456,7 +456,25 @@ spec: required: - template type: object + status: + description: DockerMachineTemplateStatus defines the observed state of + a DockerMachineTemplate. + properties: + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + capacity defines the resource capacity for this DockerMachineTemplate. + This value is used for autoscaling from zero operations as defined in: + https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20210310-opt-in-autoscaling-from-zero.md + type: object + type: object type: object served: true storage: true - subresources: {} + subresources: + status: {} diff --git a/test/infrastructure/docker/config/rbac/role.yaml b/test/infrastructure/docker/config/rbac/role.yaml index e5ac7be47236..4fa6890d4d0f 100644 --- a/test/infrastructure/docker/config/rbac/role.yaml +++ b/test/infrastructure/docker/config/rbac/role.yaml @@ -97,6 +97,7 @@ rules: - devmachines/finalizers - devmachines/status - devmachinetemplates + - devmachinetemplates/status - dockerclusters/finalizers - dockerclusters/status - dockerclustertemplates @@ -106,6 +107,7 @@ rules: - dockermachines/finalizers - dockermachines/status - dockermachinetemplates + - dockermachinetemplates/status verbs: - get - list diff --git a/test/infrastructure/docker/controllers/alias.go b/test/infrastructure/docker/controllers/alias.go index ebe596bddfb1..b98c933935c7 100644 --- a/test/infrastructure/docker/controllers/alias.go +++ b/test/infrastructure/docker/controllers/alias.go @@ -72,6 +72,24 @@ func (r *DockerClusterReconciler) SetupWithManager(ctx context.Context, mgr ctrl }).SetupWithManager(ctx, mgr, options) } +// DockerMachineTemplateReconciler reconciles a DockerMachineTemplate object. +type DockerMachineTemplateReconciler struct { + Client client.Client + ContainerRuntime container.Runtime + + // WatchFilterValue is the label value used to filter events prior to reconciliation. + WatchFilterValue string +} + +// SetupWithManager sets up the reconciler with the Manager. +func (r *DockerMachineTemplateReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { + return (&dockercontrollers.DockerMachineTemplateReconciler{ + Client: r.Client, + ContainerRuntime: r.ContainerRuntime, + WatchFilterValue: r.WatchFilterValue, + }).SetupWithManager(ctx, mgr, options) +} + // DevMachineReconciler reconciles a DevMachine object. type DevMachineReconciler struct { Client client.Client @@ -117,3 +135,21 @@ func (r *DevClusterReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Ma APIServerMux: r.APIServerMux, }).SetupWithManager(ctx, mgr, options) } + +// DevMachineTemplateReconciler reconciles a DevMachineTemplate object. +type DevMachineTemplateReconciler struct { + Client client.Client + ContainerRuntime container.Runtime + + // WatchFilterValue is the label value used to filter events prior to reconciliation. + WatchFilterValue string +} + +// SetupWithManager sets up the reconciler with the Manager. +func (r *DevMachineTemplateReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { + return (&dockercontrollers.DevMachineTemplateReconciler{ + Client: r.Client, + ContainerRuntime: r.ContainerRuntime, + WatchFilterValue: r.WatchFilterValue, + }).SetupWithManager(ctx, mgr, options) +} diff --git a/test/infrastructure/docker/internal/controllers/devmachinetemplate_controller.go b/test/infrastructure/docker/internal/controllers/devmachinetemplate_controller.go new file mode 100644 index 000000000000..5bb6d5d92d2c --- /dev/null +++ b/test/infrastructure/docker/internal/controllers/devmachinetemplate_controller.go @@ -0,0 +1,105 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + + "sigs.k8s.io/cluster-api/test/infrastructure/container" + infrav1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1beta2" + "sigs.k8s.io/cluster-api/util/patch" + "sigs.k8s.io/cluster-api/util/predicates" +) + +// DevMachineTemplateReconciler reconciles a DevMachineTemplate object. +type DevMachineTemplateReconciler struct { + client.Client + ContainerRuntime container.Runtime + + // WatchFilterValue is the label value used to filter events prior to reconciliation. + WatchFilterValue string +} + +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=devmachinetemplates,verbs=get;list;watch +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=devmachinetemplates/status,verbs=get;watch;list;update;patch + +// Reconcile reconciles the DevMachineTemplate to set the capcity information. +func (r *DevMachineTemplateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, rerr error) { + log := ctrl.LoggerFrom(ctx) + + // Fetch the DevMachineTemplate instance + machineTemplate := &infrav1.DevMachineTemplate{} + if err := r.Get(ctx, req.NamespacedName, machineTemplate); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // Initialize the patch helper + patchHelper, err := patch.NewHelper(machineTemplate, r.Client) + if err != nil { + return ctrl.Result{}, err + } + + capacity, err := fetchSystemResourceCapacity(ctx, r.ContainerRuntime) + if err != nil { + return ctrl.Result{}, err + } + + log.V(3).Info("Calculated capacity for DevMachineTemplate", "capacity", capacity) + machineTemplate.Status.Capacity = capacity + if err := patchHelper.Patch(ctx, machineTemplate); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil +} + +func fetchSystemResourceCapacity(ctx context.Context, containerRuntime container.Runtime) (corev1.ResourceList, error) { + systemInfo, err := containerRuntime.GetSystemInfo(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get system info using container runtime client: %w", err) + } + + return map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceMemory: *resource.NewQuantity(systemInfo.MemTotal, resource.BinarySI), + corev1.ResourceCPU: *resource.NewQuantity(int64(systemInfo.NCPU), resource.DecimalSI), + }, nil +} + +// SetupWithManager will add watches for this controller. +func (r *DevMachineTemplateReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { + if r.Client == nil || r.ContainerRuntime == nil { + return errors.New("Client and ContainerRuntime must not be nil") + } + predicateLog := ctrl.LoggerFrom(ctx).WithValues("controller", "devmachinetemplate") + err := ctrl.NewControllerManagedBy(mgr). + For(&infrav1.DevMachineTemplate{}). + WithOptions(options). + WithEventFilter(predicates.ResourceHasFilterLabel(mgr.GetScheme(), predicateLog, r.WatchFilterValue)). + Complete(r) + if err != nil { + return errors.Wrap(err, "failed setting up with a controller manager") + } + + return nil +} diff --git a/test/infrastructure/docker/internal/controllers/dockermachinetemplate_controller.go b/test/infrastructure/docker/internal/controllers/dockermachinetemplate_controller.go new file mode 100644 index 000000000000..f41e92957aa2 --- /dev/null +++ b/test/infrastructure/docker/internal/controllers/dockermachinetemplate_controller.go @@ -0,0 +1,90 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + + "github.com/pkg/errors" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + + "sigs.k8s.io/cluster-api/test/infrastructure/container" + infrav1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1beta2" + "sigs.k8s.io/cluster-api/util/patch" + "sigs.k8s.io/cluster-api/util/predicates" +) + +// DockerMachineTemplateReconciler reconciles a DockerMachineTemplate object. +type DockerMachineTemplateReconciler struct { + client.Client + ContainerRuntime container.Runtime + + // WatchFilterValue is the label value used to filter events prior to reconciliation. + WatchFilterValue string +} + +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=dockermachinetemplates,verbs=get;list;watch +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=dockermachinetemplates/status,verbs=get;watch;list;update;patch + +// Reconcile reconciles the DockerMachineTemplate to set the capcity information. +func (r *DockerMachineTemplateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, rerr error) { + log := ctrl.LoggerFrom(ctx) + + // Fetch the DockerMachineTemplate instance + machineTemplate := &infrav1.DockerMachineTemplate{} + if err := r.Get(ctx, req.NamespacedName, machineTemplate); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // Initialize the patch helper + patchHelper, err := patch.NewHelper(machineTemplate, r.Client) + if err != nil { + return ctrl.Result{}, err + } + + capacity, err := fetchSystemResourceCapacity(ctx, r.ContainerRuntime) + if err != nil { + return ctrl.Result{}, err + } + + log.V(3).Info("Calculated capacity for DockerMachineTemplate", "capacity", capacity) + machineTemplate.Status.Capacity = capacity + if err := patchHelper.Patch(ctx, machineTemplate); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil +} + +// SetupWithManager will add watches for this controller. +func (r *DockerMachineTemplateReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { + if r.Client == nil || r.ContainerRuntime == nil { + return errors.New("Client and ContainerRuntime must not be nil") + } + predicateLog := ctrl.LoggerFrom(ctx).WithValues("controller", "dockermachinetemplate") + err := ctrl.NewControllerManagedBy(mgr). + For(&infrav1.DockerMachineTemplate{}). + WithOptions(options). + WithEventFilter(predicates.ResourceHasFilterLabel(mgr.GetScheme(), predicateLog, r.WatchFilterValue)). + Complete(r) + if err != nil { + return errors.Wrap(err, "failed setting up with a controller manager") + } + + return nil +} diff --git a/test/infrastructure/docker/main.go b/test/infrastructure/docker/main.go index a1e2694c9fbd..cdf00d9c8172 100644 --- a/test/infrastructure/docker/main.go +++ b/test/infrastructure/docker/main.go @@ -450,6 +450,15 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) { os.Exit(1) } + if err := (&controllers.DockerMachineTemplateReconciler{ + Client: mgr.GetClient(), + ContainerRuntime: runtimeClient, + WatchFilterValue: watchFilterValue, + }).SetupWithManager(ctx, mgr, controller.Options{}); err != nil { + setupLog.Error(err, "Unable to create controller", "controller", "DockerMachineTemplate") + os.Exit(1) + } + if feature.Gates.Enabled(feature.MachinePool) { if err := (&expcontrollers.DockerMachinePoolReconciler{ Client: mgr.GetClient(), @@ -485,6 +494,15 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) { setupLog.Error(err, "Unable to create controller", "controller", "DevCluster") os.Exit(1) } + + if err := (&controllers.DevMachineTemplateReconciler{ + Client: mgr.GetClient(), + ContainerRuntime: runtimeClient, + WatchFilterValue: watchFilterValue, + }).SetupWithManager(ctx, mgr, controller.Options{}); err != nil { + setupLog.Error(err, "Unable to create controller", "controller", "DevMachineTemplate") + os.Exit(1) + } } func setupWebhooks(mgr ctrl.Manager) {