Skip to content

Commit e2622c6

Browse files
poyazpoyaz
committed
[ADD] Supporting self certificate authority and mTls when using S3 object storage
Signed-off-by: Pooya Azarpour <[email protected]>
1 parent e13ba45 commit e2622c6

18 files changed

Lines changed: 765 additions & 75 deletions

File tree

‎cmd/restic/main.go‎

Lines changed: 66 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,12 @@ import (
2020
)
2121

2222
const (
23-
backupDirEnvKey = "BACKUP_DIR"
24-
restoreDirEnvKey = "RESTORE_DIR"
23+
backupDirEnvKey = "BACKUP_DIR"
24+
restoreDirEnvKey = "RESTORE_DIR"
25+
caCertFileEnvKey = "CA_CERT_FILE"
26+
clientCertFileEnvKey = "CLIENT_CERT_FILE"
27+
clientKeyFileEnvKey = "CLIENT_KEY_FILE"
28+
workDirEnvKey = "WORK_DIR"
2529

2630
restoreTypeArg = "restoreType"
2731
restoreS3EndpointArg = "restoreS3Endpoint"
@@ -63,6 +67,9 @@ var (
6367
&cli.StringFlag{Destination: &cfg.Config.RestoreS3AccessKey, Name: restoreS3AccessKeyIDArg, EnvVars: []string{"RESTORE_ACCESSKEYID"}, Usage: "S3 access key used to connect to the S3 endpoint when restoring"},
6468
&cli.StringFlag{Destination: &cfg.Config.RestoreS3SecretKey, Name: restoreS3SecretAccessKeyArg, EnvVars: []string{"RESTORE_SECRETACCESSKEY"}, Usage: "S3 secret key used to connect to the S3 endpoint when restoring"},
6569
&cli.StringFlag{Destination: &cfg.Config.RestoreS3Endpoint, Name: restoreS3EndpointArg, EnvVars: []string{"RESTORE_S3ENDPOINT"}, Usage: "S3 endpoint to connect to when restoring, e.g. 'https://minio.svc:9000/backup"},
70+
&cli.PathFlag{Destination: &cfg.Config.RestoreCACert, Name: "restoreCaCert", Usage: "The certificate authority file path using for restore (If isn't filled, using caCert)"},
71+
&cli.PathFlag{Destination: &cfg.Config.RestoreClientCert, Name: "restoreClientCert", Usage: "The client certificate file path using for restore (If isn't filled, using clientCert)"},
72+
&cli.PathFlag{Destination: &cfg.Config.RestoreClientKey, Name: "restoreClientKey", Usage: "The client private key file path using for restore (If isn't filled, using clientKey)"},
6673
&cli.BoolFlag{Destination: &cfg.Config.VerifyRestore, Name: "verifyRestore", Usage: "If the restore should get verified, only for PVCs restore"},
6774
&cli.BoolFlag{Destination: &cfg.Config.RestoreTrimPath, Name: "trimRestorePath", EnvVars: []string{"TRIM_RESTOREPATH"}, Value: true, DefaultText: "enabled", Usage: "If set, strips the value of --restoreDir from the lefts side of the remote restore path value"},
6875

@@ -87,6 +94,11 @@ var (
8794

8895
&cli.StringSliceFlag{Name: "targetPods", EnvVars: []string{"TARGET_PODS"}, Usage: "Filter list of pods by TARGET_PODS names"},
8996
&cli.DurationFlag{Destination: &cfg.Config.SleepDuration, Name: "sleepDuration", EnvVars: []string{"SLEEP_DURATION"}, Usage: "Sleep for specified amount until init starts"},
97+
98+
&cli.PathFlag{Destination: &cfg.Config.VarDir, Name: "varDir", Value: "/k8up", Usage: "The var directory is stored k8up metadata files and temporary files"},
99+
&cli.PathFlag{Destination: &cfg.Config.CACert, Name: "caCert", EnvVars: []string{caCertFileEnvKey}, Usage: "The certificate authority file path"},
100+
&cli.PathFlag{Destination: &cfg.Config.ClientCert, Name: "clientCert", EnvVars: []string{clientCertFileEnvKey}, Usage: "The client certificate file path"},
101+
&cli.PathFlag{Destination: &cfg.Config.ClientKey, Name: "clientKey", EnvVars: []string{clientKeyFileEnvKey}, Usage: "The client private key file path"},
90102
},
91103
}
92104
)
@@ -197,30 +209,52 @@ func doCheck(resticCLI *resticCli.Restic) error {
197209
}
198210

199211
func doRestore(resticCLI *resticCli.Restic) error {
200-
if cfg.Config.DoRestore {
201-
if err := resticCLI.Restore(cfg.Config.RestoreSnap, resticCli.RestoreOptions{
202-
RestoreType: resticCli.RestoreType(cfg.Config.RestoreType),
203-
RestoreDir: cfg.Config.RestoreDir,
204-
RestoreFilter: cfg.Config.RestoreFilter,
205-
Verify: cfg.Config.VerifyRestore,
206-
S3Destination: resticCli.S3Bucket{
207-
Endpoint: cfg.Config.RestoreS3Endpoint,
208-
AccessKey: cfg.Config.RestoreS3AccessKey,
209-
SecretKey: cfg.Config.RestoreS3SecretKey,
210-
},
211-
}, cfg.Config.Tags); err != nil {
212-
return fmt.Errorf("restore job failed: %w", err)
213-
}
212+
if !cfg.Config.DoRestore {
213+
return nil
214+
}
215+
216+
restoreOptions := resticCli.RestoreOptions{
217+
RestoreType: resticCli.RestoreType(cfg.Config.RestoreType),
218+
RestoreDir: cfg.Config.RestoreDir,
219+
RestoreFilter: cfg.Config.RestoreFilter,
220+
Verify: cfg.Config.VerifyRestore,
221+
S3Destination: resticCli.S3Bucket{
222+
Endpoint: cfg.Config.RestoreS3Endpoint,
223+
AccessKey: cfg.Config.RestoreS3AccessKey,
224+
SecretKey: cfg.Config.RestoreS3SecretKey,
225+
Cert: fillRestoreS3Cert(),
226+
},
214227
}
228+
229+
if err := resticCLI.Restore(cfg.Config.RestoreSnap, restoreOptions, cfg.Config.Tags); err != nil {
230+
return fmt.Errorf("restore job failed: %w", err)
231+
}
232+
215233
return nil
216234
}
217235

218236
func doArchive(resticCLI *resticCli.Restic) error {
219-
if cfg.Config.DoArchive {
220-
if err := resticCLI.Archive(cfg.Config.ResticBin, cfg.Config.VerifyRestore, cfg.Config.Tags); err != nil {
221-
return fmt.Errorf("archive job failed: %w", err)
222-
}
237+
if !cfg.Config.DoArchive {
238+
return nil
223239
}
240+
241+
restoreOptions := resticCli.RestoreOptions{
242+
RestoreType: resticCli.RestoreType(cfg.Config.RestoreType),
243+
RestoreDir: cfg.Config.RestoreDir,
244+
RestoreFilter: cfg.Config.RestoreFilter,
245+
Verify: cfg.Config.VerifyRestore,
246+
S3Destination: resticCli.S3Bucket{
247+
Endpoint: cfg.Config.RestoreS3Endpoint,
248+
AccessKey: cfg.Config.RestoreS3AccessKey,
249+
SecretKey: cfg.Config.RestoreS3SecretKey,
250+
Cert: fillRestoreS3Cert(),
251+
},
252+
}
253+
254+
if err := resticCLI.Archive(restoreOptions, cfg.Config.Tags); err != nil {
255+
return fmt.Errorf("archive job failed: %w", err)
256+
}
257+
224258
return nil
225259
}
226260

@@ -289,3 +323,15 @@ func cancelOnTermination(cancel context.CancelFunc, mainLogger logr.Logger) {
289323
cancel()
290324
}()
291325
}
326+
327+
func fillRestoreS3Cert() (cert resticCli.S3Cert) {
328+
if cfg.Config.RestoreCACert != "" {
329+
cert.CACert = cfg.Config.RestoreCACert
330+
}
331+
if cfg.Config.RestoreClientCert != "" && cfg.Config.RestoreClientKey != "" {
332+
cert.ClientCert = cfg.Config.RestoreClientCert
333+
cert.ClientKey = cfg.Config.RestoreClientKey
334+
}
335+
336+
return cert
337+
}

‎envtest/envsuite.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,5 +235,6 @@ func defaultConfiguration() *cfg.Configuration {
235235
MetricsBindAddress: ":8080",
236236
PodFilter: "backupPod=true",
237237
EnableLeaderElection: true,
238+
PodVarDir: "/k8up",
238239
}
239240
}

‎operator/archivecontroller/executor.go‎

Lines changed: 118 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ package archivecontroller
22

33
import (
44
"context"
5-
65
"github.com/k8up-io/k8up/v2/operator/executor"
6+
"github.com/k8up-io/k8up/v2/operator/utils"
77
batchv1 "k8s.io/api/batch/v1"
88
corev1 "k8s.io/api/core/v1"
99
controllerruntime "sigs.k8s.io/controller-runtime"
@@ -14,17 +14,22 @@ import (
1414
"github.com/k8up-io/k8up/v2/operator/job"
1515
)
1616

17-
const archivePath = "/archive"
17+
const (
18+
archivePath = "/archive"
19+
_dataDirName = "k8up-dir"
20+
)
1821

1922
// ArchiveExecutor will execute the batch.job for archive.
2023
type ArchiveExecutor struct {
2124
executor.Generic
25+
archive *k8upv1.Archive
2226
}
2327

2428
// NewArchiveExecutor will return a new executor for archive jobs.
2529
func NewArchiveExecutor(config job.Config) *ArchiveExecutor {
2630
return &ArchiveExecutor{
2731
Generic: executor.Generic{Config: config},
32+
archive: config.Obj.(*k8upv1.Archive),
2833
}
2934
}
3035

@@ -36,22 +41,26 @@ func (a *ArchiveExecutor) GetConcurrencyLimit() int {
3641
// Execute creates the actual batch.job on the k8s api.
3742
func (a *ArchiveExecutor) Execute(ctx context.Context) error {
3843
log := controllerruntime.LoggerFrom(ctx)
39-
archive := a.Obj.(*k8upv1.Archive)
4044

4145
batchJob := &batchv1.Job{}
4246
batchJob.Name = a.jobName()
43-
batchJob.Namespace = archive.Namespace
47+
batchJob.Namespace = a.archive.Namespace
4448

4549
_, err := controllerutil.CreateOrUpdate(ctx, a.Client, batchJob, func() error {
46-
mutateErr := job.MutateBatchJob(batchJob, archive, a.Config)
50+
mutateErr := job.MutateBatchJob(batchJob, a.archive, a.Config)
4751
if mutateErr != nil {
4852
return mutateErr
4953
}
5054

51-
batchJob.Spec.Template.Spec.Containers[0].Env = a.setupEnvVars(ctx, archive)
52-
archive.Spec.AppendEnvFromToContainer(&batchJob.Spec.Template.Spec.Containers[0])
53-
batchJob.Spec.Template.Spec.Containers[0].Args = a.setupArgs(archive)
54-
return nil
55+
batchJob.Spec.Template.Spec.Containers[0].Env = a.setupEnvVars(ctx, a.archive)
56+
a.archive.Spec.AppendEnvFromToContainer(&batchJob.Spec.Template.Spec.Containers[0])
57+
batchJob.Spec.Template.Spec.Containers[0].VolumeMounts = a.attachMoreVolumeMounts()
58+
batchJob.Spec.Template.Spec.Volumes = a.attachMoreVolumes()
59+
60+
args, argsErr := a.setupArgs()
61+
batchJob.Spec.Template.Spec.Containers[0].Args = args
62+
63+
return argsErr
5564
})
5665
if err != nil {
5766
log.Error(err, "could not create job")
@@ -67,16 +76,17 @@ func (a *ArchiveExecutor) jobName() string {
6776
return k8upv1.ArchiveType.String() + "-" + a.Obj.GetName()
6877
}
6978

70-
func (a *ArchiveExecutor) setupArgs(archive *k8upv1.Archive) []string {
71-
args := []string{"-archive", "-restoreType", "s3"}
79+
func (a *ArchiveExecutor) setupArgs() ([]string, error) {
80+
args := a.appendOptionsArgs()
7281

73-
if archive.Spec.RestoreSpec != nil {
74-
if len(archive.Spec.RestoreSpec.Tags) > 0 {
75-
args = append(args, executor.BuildTagArgs(archive.Spec.RestoreSpec.Tags)...)
82+
args = append(args, []string{"-archive", "-restoreType", "s3"}...)
83+
if a.archive.Spec.RestoreSpec != nil {
84+
if len(a.archive.Spec.RestoreSpec.Tags) > 0 {
85+
args = append(args, executor.BuildTagArgs(a.archive.Spec.RestoreSpec.Tags)...)
7686
}
7787
}
7888

79-
return args
89+
return args, nil
8090
}
8191

8292
func (a *ArchiveExecutor) setupEnvVars(ctx context.Context, archive *k8upv1.Archive) []corev1.EnvVar {
@@ -118,3 +128,96 @@ func (a *ArchiveExecutor) setupEnvVars(ctx context.Context, archive *k8upv1.Arch
118128
func (a *ArchiveExecutor) cleanupOldArchives(ctx context.Context, archive *k8upv1.Archive) {
119129
a.CleanupOldResources(ctx, &k8upv1.ArchiveList{}, archive.Namespace, archive)
120130
}
131+
132+
func (a *ArchiveExecutor) appendOptionsArgs() []string {
133+
var args []string
134+
135+
args = append(args, []string{"--varDir", cfg.Config.PodVarDir}...)
136+
137+
if a.archive.Spec.Backend.Options != nil {
138+
if a.archive.Spec.Backend.Options.CACert != "" {
139+
args = append(args, []string{"--caCert", a.archive.Spec.Backend.Options.CACert}...)
140+
}
141+
if a.archive.Spec.Backend.Options.ClientCert != "" && a.archive.Spec.Backend.Options.ClientKey != "" {
142+
args = append(
143+
args,
144+
[]string{
145+
"--clientCert",
146+
a.archive.Spec.Backend.Options.ClientCert,
147+
"--clientKey",
148+
a.archive.Spec.Backend.Options.ClientKey,
149+
}...,
150+
)
151+
}
152+
}
153+
154+
if a.archive.Spec.RestoreMethod.Options != nil {
155+
if a.archive.Spec.RestoreMethod.Options.CACert != "" {
156+
args = append(args, []string{"--restoreCaCert", a.archive.Spec.RestoreMethod.Options.CACert}...)
157+
}
158+
if a.archive.Spec.RestoreMethod.Options.ClientCert != "" && a.archive.Spec.RestoreMethod.Options.ClientKey != "" {
159+
args = append(
160+
args,
161+
[]string{
162+
"--restoreClientCert",
163+
a.archive.Spec.RestoreMethod.Options.ClientCert,
164+
"--restoreClientKey",
165+
a.archive.Spec.RestoreMethod.Options.ClientKey,
166+
}...,
167+
)
168+
}
169+
}
170+
171+
return args
172+
}
173+
174+
func (a *ArchiveExecutor) attachMoreVolumes() []corev1.Volume {
175+
ku8pVolume := corev1.Volume{
176+
Name: _dataDirName,
177+
VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}},
178+
}
179+
180+
if utils.ZeroLen(a.archive.Spec.Volumes) {
181+
return []corev1.Volume{ku8pVolume}
182+
}
183+
184+
moreVolumes := make([]corev1.Volume, 0, len(*a.archive.Spec.Volumes)+1)
185+
moreVolumes = append(moreVolumes, ku8pVolume)
186+
for _, v := range *a.archive.Spec.Volumes {
187+
vol := v
188+
189+
var volumeSource corev1.VolumeSource
190+
if vol.PersistentVolumeClaim != nil {
191+
volumeSource.PersistentVolumeClaim = vol.PersistentVolumeClaim
192+
} else if vol.Secret != nil {
193+
volumeSource.Secret = vol.Secret
194+
} else if vol.ConfigMap != nil {
195+
volumeSource.ConfigMap = vol.ConfigMap
196+
} else {
197+
continue
198+
}
199+
200+
moreVolumes = append(moreVolumes, corev1.Volume{
201+
Name: vol.Name,
202+
VolumeSource: volumeSource,
203+
})
204+
}
205+
206+
return moreVolumes
207+
}
208+
209+
func (a *ArchiveExecutor) attachMoreVolumeMounts() []corev1.VolumeMount {
210+
var volumeMount []corev1.VolumeMount
211+
212+
if a.archive.Spec.Backend.S3 != nil && !utils.ZeroLen(a.archive.Spec.Backend.S3.VolumeMounts) {
213+
volumeMount = *a.archive.Spec.Backend.S3.VolumeMounts
214+
}
215+
if a.archive.Spec.Backend.Rest != nil && !utils.ZeroLen(a.archive.Spec.Backend.Rest.VolumeMounts) {
216+
volumeMount = *a.archive.Spec.Backend.Rest.VolumeMounts
217+
}
218+
219+
ku8pVolumeMount := corev1.VolumeMount{Name: _dataDirName, MountPath: cfg.Config.PodVarDir}
220+
volumeMount = append(volumeMount, ku8pVolumeMount)
221+
222+
return volumeMount
223+
}

0 commit comments

Comments
 (0)