| layout | default |
|---|---|
| title | Kubernetes Operator Patterns - Chapter 4: Managing Owned Resources |
| nav_order | 4 |
| has_children | false |
| parent | Kubernetes Operator Patterns |
Welcome to Chapter 4: Managing Owned Resources - Creating and Managing Kubernetes Objects. In this part of Kubernetes Operator Patterns: Building Production-Grade Controllers, you will build an intuitive mental model first, then move into concrete implementation details and practical production tradeoffs.
Master creating, updating, and managing Kubernetes resources that operators control, including Deployments, Services, ConfigMaps, and Secrets.
flowchart TD
A[CR created] --> B[Reconciler creates Deployment]
B --> C[SetControllerReference owner]
C --> D[Deployment owned by CR]
D --> E{CR deleted?}
E -->|yes| F[GC deletes Deployment]
E -->|no| G[Reconciler watches Deployment]
G --> H[Sync replicas on drift]
Operators manage complex applications by creating and controlling multiple Kubernetes resources. This chapter covers creating, updating, and managing owned resources while maintaining proper relationships and lifecycle management.
// Set owner reference for automatic cleanup
func (r *MyAppReconciler) setOwnerReference(myApp *appsv1.MyApp, obj metav1.Object) error {
// Set owner reference
if err := controllerutil.SetControllerReference(myApp, obj, r.Scheme); err != nil {
return err
}
return nil
}
// Create deployment with owner reference
func (r *MyAppReconciler) createDeployment(ctx context.Context, myApp *appsv1.MyApp) error {
deployment := r.buildDeployment(myApp)
// Set owner reference
if err := r.setOwnerReference(myApp, deployment); err != nil {
return err
}
// Create the deployment
if err := r.Create(ctx, deployment); err != nil {
return err
}
r.recorder.Event(myApp, corev1.EventTypeNormal, "DeploymentCreated",
fmt.Sprintf("Created deployment %s", deployment.Name))
return nil
}// Consistent naming and labeling strategy
func (r *MyAppReconciler) getResourceName(myApp *appsv1.MyApp, suffix string) string {
return fmt.Sprintf("%s-%s", myApp.Name, suffix)
}
func (r *MyAppReconciler) getLabels(myApp *appsv1.MyApp) map[string]string {
return map[string]string{
"app.kubernetes.io/name": myApp.Name,
"app.kubernetes.io/instance": myApp.Name,
"app.kubernetes.io/version": myApp.Spec.Version,
"app.kubernetes.io/component": "application",
"app.kubernetes.io/part-of": "myapp",
"app.kubernetes.io/managed-by": "myapp-operator",
}
}
func (r *MyAppReconciler) getSelectorLabels(myApp *appsv1.MyApp) map[string]string {
return map[string]string{
"app.kubernetes.io/name": myApp.Name,
"app.kubernetes.io/instance": myApp.Name,
}
}// Create and manage Deployments
func (r *MyAppReconciler) buildDeployment(myApp *appsv1.MyApp) *appsv1.Deployment {
labels := r.getLabels(myApp)
selectorLabels := r.getSelectorLabels(myApp)
replicas := myApp.Spec.Replicas
if replicas == 0 {
replicas = 1 // Default to 1 replica
}
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: r.getResourceName(myApp, "deployment"),
Namespace: myApp.Namespace,
Labels: labels,
},
Spec: appsv1.DeploymentSpec{
Replicas: &replicas,
Selector: &metav1.LabelSelector{
MatchLabels: selectorLabels,
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: labels,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
r.buildMainContainer(myApp),
},
// Add sidecar containers if needed
// Init containers for setup
// Volumes and volume mounts
},
},
// Deployment strategy
Strategy: appsv1.DeploymentStrategy{
Type: appsv1.RollingUpdateDeploymentStrategyType,
RollingUpdate: &appsv1.RollingUpdateDeployment{
MaxUnavailable: &intstr.IntOrString{
Type: intstr.String,
StrVal: "25%",
},
MaxSurge: &intstr.IntOrString{
Type: intstr.String,
StrVal: "25%",
},
},
},
},
}
return deployment
}
func (r *MyAppReconciler) buildMainContainer(myApp *appsv1.MyApp) corev1.Container {
container := corev1.Container{
Name: "myapp",
Image: myApp.Spec.Image,
Ports: []corev1.ContainerPort{
{
Name: "http",
ContainerPort: 8080,
Protocol: corev1.ProtocolTCP,
},
},
// Environment variables
Env: r.buildEnvironmentVariables(myApp),
// Resource requirements
Resources: r.buildResourceRequirements(myApp),
// Health checks
LivenessProbe: r.buildLivenessProbe(),
ReadinessProbe: r.buildReadinessProbe(),
// Volume mounts
VolumeMounts: r.buildVolumeMounts(myApp),
}
return container
}// Create and manage Services
func (r *MyAppReconciler) buildService(myApp *appsv1.MyApp) *corev1.Service {
labels := r.getLabels(myApp)
selectorLabels := r.getSelectorLabels(myApp)
service := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: r.getResourceName(myApp, "service"),
Namespace: myApp.Namespace,
Labels: labels,
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeClusterIP, // or LoadBalancer, NodePort
Selector: selectorLabels,
Ports: []corev1.ServicePort{
{
Name: "http",
Port: 80,
TargetPort: intstr.FromString("http"),
Protocol: corev1.ProtocolTCP,
},
// Additional ports as needed
},
},
}
// Add annotations for cloud provider load balancers
if myApp.Spec.ServiceType == "LoadBalancer" {
service.Annotations = map[string]string{
"service.beta.kubernetes.io/aws-load-balancer-type": "nlb",
"service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled": "true",
}
service.Spec.Type = corev1.ServiceTypeLoadBalancer
}
return service
}
func (r *MyAppReconciler) ensureService(ctx context.Context, myApp *appsv1.MyApp) error {
service := &corev1.Service{}
err := r.Get(ctx, client.ObjectKey{
Name: r.getResourceName(myApp, "service"),
Namespace: myApp.Namespace,
}, service)
if err != nil && apierrors.IsNotFound(err) {
// Create service
svc := r.buildService(myApp)
if err := r.setOwnerReference(myApp, svc); err != nil {
return err
}
return r.Create(ctx, svc)
} else if err != nil {
return err
}
// Update service if needed
desiredSvc := r.buildService(myApp)
if r.serviceNeedsUpdate(service, desiredSvc) {
service.Spec = desiredSvc.Spec
service.Labels = desiredSvc.Labels
return r.Update(ctx, service)
}
return nil
}// ConfigMap management
func (r *MyAppReconciler) buildConfigMap(myApp *appsv1.MyApp) *corev1.ConfigMap {
labels := r.getLabels(myApp)
configMap := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: r.getResourceName(myApp, "config"),
Namespace: myApp.Namespace,
Labels: labels,
},
Data: map[string]string{
"config.yaml": r.generateConfigYAML(myApp),
"app.properties": r.generateProperties(myApp),
},
// BinaryData for binary configuration files
BinaryData: map[string][]byte{
// Binary configuration data
},
}
return configMap
}
func (r *MyAppReconciler) generateConfigYAML(myApp *appsv1.MyApp) string {
config := fmt.Sprintf(`
app:
name: %s
version: %s
replicas: %d
image: %s
server:
port: 8080
host: 0.0.0.0
database:
host: %s
port: %d
name: %s
logging:
level: %s
format: json
`,
myApp.Name,
myApp.Spec.Version,
myApp.Spec.Replicas,
myApp.Spec.Image,
myApp.Spec.Database.Host,
myApp.Spec.Database.Port,
myApp.Spec.Database.Name,
myApp.Spec.Logging.Level,
)
return config
}
// Secret management
func (r *MyAppReconciler) buildSecret(myApp *appsv1.MyApp) *corev1.Secret {
labels := r.getLabels(myApp)
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: r.getResourceName(myApp, "secret"),
Namespace: myApp.Namespace,
Labels: labels,
},
Type: corev1.SecretTypeOpaque,
Data: map[string][]byte{
// Base64 encoded secrets
"database-password": []byte(myApp.Spec.Database.Password),
"api-key": []byte(myApp.Spec.APIKey),
"tls-cert": myApp.Spec.TLSCert,
"tls-key": myApp.Spec.TLSKey,
},
StringData: map[string]string{
// Plain text secrets (auto-encoded)
"config": r.generateSecretConfig(myApp),
},
}
return secret
}
func (r *MyAppReconciler) generateSecretConfig(myApp *appsv1.MyApp) string {
// Generate sensitive configuration
return fmt.Sprintf(`{
"database": {
"username": "%s",
"password": "%s"
},
"external_api": {
"key": "%s",
"secret": "%s"
}
}`,
myApp.Spec.Database.Username,
myApp.Spec.Database.Password,
myApp.Spec.ExternalAPI.Key,
myApp.Spec.ExternalAPI.Secret,
)
}// PVC management for storage
func (r *MyAppReconciler) buildPVC(myApp *appsv1.MyApp) *corev1.PersistentVolumeClaim {
labels := r.getLabels(myApp)
pvc := &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: r.getResourceName(myApp, "storage"),
Namespace: myApp.Namespace,
Labels: labels,
},
Spec: corev1.PersistentVolumeClaimSpec{
AccessModes: []corev1.PersistentVolumeAccessMode{
corev1.ReadWriteOnce, // Single node read/write
},
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceStorage: resource.MustParse(myApp.Spec.Storage.Size),
},
},
StorageClassName: &myApp.Spec.Storage.ClassName,
},
}
// Add selector for specific PV binding
if myApp.Spec.Storage.Selector != nil {
pvc.Spec.Selector = myApp.Spec.Storage.Selector
}
return pvc
}
func (r *MyAppReconciler) ensurePVC(ctx context.Context, myApp *appsv1.MyApp) error {
pvc := &corev1.PersistentVolumeClaim{}
err := r.Get(ctx, client.ObjectKey{
Name: r.getResourceName(myApp, "storage"),
Namespace: myApp.Namespace,
}, pvc)
if err != nil && apierrors.IsNotFound(err) {
// Create PVC
pvc := r.buildPVC(myApp)
if err := r.setOwnerReference(myApp, pvc); err != nil {
return err
}
return r.Create(ctx, pvc)
} else if err != nil {
return err
}
// Check PVC status
if pvc.Status.Phase != corev1.ClaimBound {
r.recorder.Event(myApp, corev1.EventTypeNormal, "PVCNotBound",
fmt.Sprintf("PVC %s is not yet bound", pvc.Name))
return fmt.Errorf("PVC not bound")
}
return nil
}// Ingress for external access
func (r *MyAppReconciler) buildIngress(myApp *appsv1.MyApp) *networkingv1.Ingress {
labels := r.getLabels(myApp)
ingressClassName := "nginx" // or "traefik", "haproxy", etc.
ingress := &networkingv1.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: r.getResourceName(myApp, "ingress"),
Namespace: myApp.Namespace,
Labels: labels,
Annotations: map[string]string{
"nginx.ingress.kubernetes.io/rewrite-target": "/",
"cert-manager.io/cluster-issuer": "letsencrypt-prod",
"nginx.ingress.kubernetes.io/ssl-redirect": "true",
},
},
Spec: networkingv1.IngressSpec{
IngressClassName: &ingressClassName,
TLS: []networkingv1.IngressTLS{
{
Hosts: []string{myApp.Spec.Domain},
SecretName: r.getResourceName(myApp, "tls"),
},
},
Rules: []networkingv1.IngressRule{
{
Host: myApp.Spec.Domain,
IngressRuleValue: networkingv1.IngressRuleValue{
HTTP: &networkingv1.HTTPIngressRuleValue{
Paths: []networkingv1.HTTPIngressPath{
{
Path: "/",
PathType: &[]networkingv1.PathType{networkingv1.PathTypePrefix}[0],
Backend: networkingv1.IngressBackend{
Service: &networkingv1.IngressServiceBackend{
Name: r.getResourceName(myApp, "service"),
Port: networkingv1.ServiceBackendPort{
Number: 80,
},
},
},
},
},
},
},
},
},
},
}
return ingress
}// Batch job management
func (r *MyAppReconciler) buildJob(myApp *appsv1.MyApp, jobName string) *batchv1.Job {
labels := r.getLabels(myApp)
selectorLabels := r.getSelectorLabels(myApp)
job := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-%s", myApp.Name, jobName),
Namespace: myApp.Namespace,
Labels: labels,
},
Spec: batchv1.JobSpec{
BackoffLimit: &[]int32{3}[0], // Retry up to 3 times
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: labels,
},
Spec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyNever,
Containers: []corev1.Container{
{
Name: "job",
Image: myApp.Spec.Image,
Command: []string{"/bin/sh", "-c"},
Args: []string{r.getJobCommand(jobName)},
Env: r.buildEnvironmentVariables(myApp),
},
},
},
},
},
}
return job
}
func (r *MyAppReconciler) buildCronJob(myApp *appsv1.MyApp, schedule string, jobName string) *batchv1beta1.CronJob {
labels := r.getLabels(myApp)
cronJob := &batchv1beta1.CronJob{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-%s", myApp.Name, jobName),
Namespace: myApp.Namespace,
Labels: labels,
},
Spec: batchv1beta1.CronJobSpec{
Schedule: schedule, // e.g., "0 */6 * * *" for every 6 hours
JobTemplate: batchv1beta1.JobTemplateSpec{
Spec: batchv1.JobSpec{
BackoffLimit: &[]int32{1}[0],
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: labels,
},
Spec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyOnFailure,
Containers: []corev1.Container{
{
Name: "cronjob",
Image: myApp.Spec.Image,
Command: []string{"/bin/sh", "-c"},
Args: []string{r.getCronJobCommand(jobName)},
Env: r.buildEnvironmentVariables(myApp),
},
},
},
},
},
},
// Prevent concurrent runs
ConcurrencyPolicy: batchv1beta1.ForbidConcurrent,
// Keep successful jobs for 100 hours
SuccessfulJobsHistoryLimit: &[]int32{3}[0],
FailedJobsHistoryLimit: &[]int32{1}[0],
},
}
return cronJob
}// Strategic merge patching for updates
func (r *MyAppReconciler) updateDeploymentStrategic(ctx context.Context, deployment *appsv1.Deployment, myApp *appsv1.MyApp) error {
// Create patch
patch := client.MergeFrom(deployment.DeepCopy())
// Update fields
if *deployment.Spec.Replicas != myApp.Spec.Replicas {
deployment.Spec.Replicas = &myApp.Spec.Replicas
}
if deployment.Spec.Template.Spec.Containers[0].Image != myApp.Spec.Image {
deployment.Spec.Template.Spec.Containers[0].Image = myApp.Spec.Image
}
// Apply patch
return r.Patch(ctx, deployment, patch)
}
func (r *MyAppReconciler) updateDeploymentJSON(ctx context.Context, deployment *appsv1.Deployment, myApp *appsv1.MyApp) error {
// JSON patch for complex updates
patch := []map[string]interface{}{
{
"op": "replace",
"path": "/spec/replicas",
"value": myApp.Spec.Replicas,
},
{
"op": "replace",
"path": "/spec/template/spec/containers/0/image",
"value": myApp.Spec.Image,
},
}
return r.Patch(ctx, deployment, client.RawPatch(types.JSONPatchType, patch))
}// Server-side apply for better conflict resolution
func (r *MyAppReconciler) applyDeploymentSSA(ctx context.Context, myApp *appsv1.MyApp) error {
deployment := r.buildDeployment(myApp)
// Set owner reference
if err := r.setOwnerReference(myApp, deployment); err != nil {
return err
}
// Server-side apply
return r.Patch(ctx, deployment, client.Apply, client.FieldOwner("myapp-operator"))
}
func (r *MyAppReconciler) applyServiceSSA(ctx context.Context, myApp *appsv1.MyApp) error {
service := r.buildService(myApp)
// Set owner reference
if err := r.setOwnerReference(myApp, service); err != nil {
return err
}
// Server-side apply
return r.Patch(ctx, service, client.Apply, client.FieldOwner("myapp-operator"))
}// Manage resource creation order and dependencies
func (r *MyAppReconciler) ensureResourcesInOrder(ctx context.Context, myApp *appsv1.MyApp) error {
// Phase 1: Infrastructure resources (PVCs, ConfigMaps, Secrets)
if err := r.ensureInfrastructure(ctx, myApp); err != nil {
return fmt.Errorf("infrastructure setup failed: %w", err)
}
// Phase 2: Core application resources (Deployments, Services)
if err := r.ensureApplication(ctx, myApp); err != nil {
return fmt.Errorf("application setup failed: %w", err)
}
// Phase 3: Networking and access (Ingress, NetworkPolicies)
if err := r.ensureNetworking(ctx, myApp); err != nil {
return fmt.Errorf("networking setup failed: %w", err)
}
// Phase 4: Observability (ServiceMonitors, PodMonitors)
if err := r.ensureObservability(ctx, myApp); err != nil {
return fmt.Errorf("observability setup failed: %w", err)
}
return nil
}
func (r *MyAppReconciler) ensureInfrastructure(ctx context.Context, myApp *appsv1.MyApp) error {
// Create PVC first (storage foundation)
if err := r.ensurePVC(ctx, myApp); err != nil {
return err
}
// Create ConfigMap
if err := r.ensureConfigMap(ctx, myApp); err != nil {
return err
}
// Create Secret
if err := r.ensureSecret(ctx, myApp); err != nil {
return err
}
return nil
}
func (r *MyAppReconciler) ensureApplication(ctx context.Context, myApp *appsv1.MyApp) error {
// ServiceAccount and RBAC first
if err := r.ensureServiceAccount(ctx, myApp); err != nil {
return err
}
// Deployment (depends on ServiceAccount, ConfigMap, Secret)
if err := r.ensureDeployment(ctx, myApp); err != nil {
return err
}
// Service (depends on Deployment)
if err := r.ensureService(ctx, myApp); err != nil {
return err
}
return nil
}// Resource readiness and health checking
func (r *MyAppReconciler) checkResourceHealth(ctx context.Context, myApp *appsv1.MyApp) error {
// Check PVC health
if err := r.checkPVCHHealth(ctx, myApp); err != nil {
return fmt.Errorf("PVC health check failed: %w", err)
}
// Check Deployment health
if err := r.checkDeploymentHealth(ctx, myApp); err != nil {
return fmt.Errorf("Deployment health check failed: %w", err)
}
// Check Service health
if err := r.checkServiceHealth(ctx, myApp); err != nil {
return fmt.Errorf("Service health check failed: %w", err)
}
return nil
}
func (r *MyAppReconciler) checkDeploymentHealth(ctx context.Context, myApp *appsv1.MyApp) error {
deployment := &appsv1.Deployment{}
err := r.Get(ctx, client.ObjectKey{
Name: r.getResourceName(myApp, "deployment"),
Namespace: myApp.Namespace,
}, deployment)
if err != nil {
return err
}
// Check deployment conditions
for _, condition := range deployment.Status.Conditions {
if condition.Type == appsv1.DeploymentAvailable {
if condition.Status != corev1.ConditionTrue {
return fmt.Errorf("deployment not available: %s", condition.Message)
}
}
}
// Check replica status
if deployment.Status.ReadyReplicas != *deployment.Spec.Replicas {
return fmt.Errorf("deployment not fully ready: %d/%d replicas",
deployment.Status.ReadyReplicas, *deployment.Spec.Replicas)
}
return nil
}
func (r *MyAppReconciler) waitForResourceReady(ctx context.Context, obj client.Object, timeout time.Duration) error {
return wait.PollUntilContextTimeout(ctx, time.Second, timeout, true, func(ctx context.Context) (bool, error) {
err := r.Get(ctx, client.ObjectKeyFromObject(obj), obj)
if err != nil {
return false, err
}
// Check if resource is ready (implementation depends on resource type)
return r.isResourceReady(obj), nil
})
}In this chapter, we've covered:
- Resource Ownership: Owner references and proper resource relationships
- Core Resources: Deployments, Services, ConfigMaps, Secrets, PVCs
- Advanced Resources: Ingress, Jobs, CronJobs for complete applications
- Update Strategies: Strategic merge patching, server-side apply
- Dependency Management: Proper resource creation ordering
- Health Monitoring: Resource readiness and health checks
- Owner References: Essential for automatic cleanup and proper relationships
- Resource Ordering: Create dependencies before dependent resources
- Update Strategies: Choose appropriate patching strategies for different scenarios
- Health Checks: Monitor resource readiness and health status
- Consistent Naming: Use predictable naming patterns for all resources
- Proper Labeling: Consistent labeling for selection and management
Next, we'll explore status and conditions - reporting resource status and implementing condition patterns for better observability.
Ready for the next chapter? Chapter 5: Status and Conditions
Generated for Awesome Code Docs
Most teams struggle here because the hard part is not writing more code, but deciding clear boundaries for myApp, Spec, corev1 so behavior stays predictable as complexity grows.
In practical terms, this chapter helps you avoid three common failures:
- coupling core logic too tightly to one implementation path
- missing the handoff boundaries between setup, execution, and validation
- shipping changes without clear rollback or observability strategy
After working through this chapter, you should be able to reason about Chapter 4: Managing Owned Resources - Creating and Managing Kubernetes Objects as an operating subsystem inside Kubernetes Operator Patterns: Building Production-Grade Controllers, with explicit contracts for inputs, state transitions, and outputs.
Use the implementation notes around appsv1, deployment, func as your checklist when adapting these patterns to your own repository.
Under the hood, Chapter 4: Managing Owned Resources - Creating and Managing Kubernetes Objects usually follows a repeatable control path:
- Context bootstrap: initialize runtime config and prerequisites for
myApp. - Input normalization: shape incoming data so
Specreceives stable contracts. - Core execution: run the main logic branch and propagate intermediate state through
corev1. - Policy and safety checks: enforce limits, auth scopes, and failure boundaries.
- Output composition: return canonical result payloads for downstream consumers.
- Operational telemetry: emit logs/metrics needed for debugging and performance tuning.
When debugging, walk this sequence in order and confirm each stage has explicit success/failure conditions.
Use the following upstream sources to verify implementation details while reading this chapter:
- Awesome Code Docs
Why it matters: authoritative reference on
Awesome Code Docs(github.com).
Suggested trace strategy:
- search upstream code for
myAppandSpecto map concrete implementation paths - compare docs claims against actual runtime/config code before reusing patterns in production