Skip to content

Commit 6d9bf2e

Browse files
tenzen-ytenzen-y
committed
fix suite test
To avoid the `timeout waiting for process kube-apiserver to stop` error, we must use the `context.WithCancel`. Ref: kubernetes-sigs/controller-runtime#1571 (comment)
1 parent a61185e commit 6d9bf2e

File tree

7 files changed

+137
-106
lines changed

7 files changed

+137
-106
lines changed

‎pkg/controller.v1beta1/experiment/experiment_controller_suite_test.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,18 @@ import (
3434

3535
var (
3636
cfg *rest.Config
37+
ctx context.Context
38+
cancel context.CancelFunc
3739
controlPlaneStartTimeout = 60 * time.Second
3840
controlPlaneStopTimeout = 60 * time.Second
3941
)
4042

4143
func TestMain(m *testing.M) {
44+
// To avoid the `timeout waiting for process kube-apiserver to stop` error,
45+
// we must use the `context.WithCancel`.
46+
// Ref: https://github.com/kubernetes-sigs/controller-runtime/issues/1571#issuecomment-945535598
47+
ctx, cancel = context.WithCancel(context.TODO())
48+
4249
t := &envtest.Environment{
4350
ControlPlaneStartTimeout: controlPlaneStartTimeout,
4451
ControlPlaneStopTimeout: controlPlaneStopTimeout,
@@ -56,6 +63,7 @@ func TestMain(m *testing.M) {
5663
}
5764

5865
code := m.Run()
66+
cancel()
5967
if err = t.Stop(); err != nil {
6068
stdlog.Fatal(err)
6169
}

‎pkg/controller.v1beta1/experiment/experiment_controller_test.go‎

Lines changed: 33 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ limitations under the License.
1717
package experiment
1818

1919
import (
20-
"context"
2120
"fmt"
2221
"sync"
2322
"testing"
@@ -184,7 +183,7 @@ func TestReconcile(t *testing.T) {
184183
// Try to update status until it be succeeded
185184
for err != nil {
186185
updatedInstance := &experimentsv1beta1.Experiment{}
187-
if err = c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: experimentName}, updatedInstance); err != nil {
186+
if err = c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: experimentName}, updatedInstance); err != nil {
188187
continue
189188
}
190189
updatedInstance.Status = instance.Status
@@ -201,7 +200,7 @@ func TestReconcile(t *testing.T) {
201200
wg.Add(1)
202201
go func() {
203202
defer wg.Done()
204-
g.Expect(mgr.Start(context.TODO())).NotTo(gomega.HaveOccurred())
203+
g.Expect(mgr.Start(ctx)).NotTo(gomega.HaveOccurred())
205204
}()
206205

207206
returnedBatchJob := newFakeBatchJob()
@@ -245,11 +244,11 @@ func TestReconcile(t *testing.T) {
245244
suggestion := &suggestionsv1beta1.Suggestion{}
246245
// We should Get suggestion because resource version can be modified
247246
for err != nil {
248-
if err = c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: experimentName}, suggestion); err != nil {
247+
if err = c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: experimentName}, suggestion); err != nil {
249248
continue
250249
}
251250
suggestion.Status.Suggestions = suggestion.Status.Suggestions[1:]
252-
err = c.Status().Update(context.TODO(), suggestion)
251+
err = c.Status().Update(ctx, suggestion)
253252
}
254253
})
255254

@@ -259,11 +258,11 @@ func TestReconcile(t *testing.T) {
259258
var err error = errors.NewBadRequest("fake-error")
260259
suggestion := &suggestionsv1beta1.Suggestion{}
261260
for err != nil {
262-
if err = c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: experimentName}, suggestion); err != nil {
261+
if err = c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: experimentName}, suggestion); err != nil {
263262
continue
264263
}
265264
suggestion.MarkSuggestionStatusSucceeded(reasonRestart, msgRestartNo)
266-
err = c.Status().Update(context.TODO(), suggestion)
265+
err = c.Status().Update(ctx, suggestion)
267266
}
268267
})
269268

@@ -273,11 +272,11 @@ func TestReconcile(t *testing.T) {
273272
var err error = errors.NewBadRequest("fake-error")
274273
suggestion := &suggestionsv1beta1.Suggestion{}
275274
for err != nil {
276-
if err = c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: experimentName}, suggestion); err != nil {
275+
if err = c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: experimentName}, suggestion); err != nil {
277276
continue
278277
}
279278
suggestion.MarkSuggestionStatusSucceeded(reasonRestart, msgRestartYes)
280-
err = c.Status().Update(context.TODO(), suggestion)
279+
err = c.Status().Update(ctx, suggestion)
281280
}
282281
})
283282

@@ -287,11 +286,11 @@ func TestReconcile(t *testing.T) {
287286
suggestion := &suggestionsv1beta1.Suggestion{}
288287
var err error = errors.NewBadRequest("fake-error")
289288
for err != nil {
290-
if err = c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: experimentName}, suggestion); err != nil {
289+
if err = c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: experimentName}, suggestion); err != nil {
291290
continue
292291
}
293292
suggestion.MarkSuggestionStatusRunning(corev1.ConditionFalse, suggestionsv1beta1.SuggestionRestartReason, msgRestarting)
294-
err = c.Status().Update(context.TODO(), suggestion)
293+
err = c.Status().Update(ctx, suggestion)
295294
}
296295
})
297296

@@ -306,27 +305,27 @@ func TestReconcile(t *testing.T) {
306305

307306
// Create the suggestion with NeverResume
308307
suggestionInstance := newFakeSuggestion()
309-
g.Expect(c.Create(context.TODO(), suggestionInstance)).NotTo(gomega.HaveOccurred())
308+
g.Expect(c.Create(ctx, suggestionInstance)).NotTo(gomega.HaveOccurred())
310309
// Manually update suggestion's status with 3 suggestions
311310
// Ones redundant trial is deleted, suggestion status must be updated
312311
g.Eventually(func() error {
313312
suggestion := &suggestionsv1beta1.Suggestion{}
314-
if err = c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: experimentName}, suggestion); err != nil {
313+
if err = c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: experimentName}, suggestion); err != nil {
315314
return err
316315
}
317316
suggestion.Status.Suggestions = newFakeSuggestion().Status.Suggestions
318-
errStatus := c.Status().Update(context.TODO(), suggestion)
317+
errStatus := c.Status().Update(ctx, suggestion)
319318
return errStatus
320319
}, timeout).ShouldNot(gomega.HaveOccurred())
321320

322321
// Create the experiment
323322
instance := newFakeInstance()
324-
g.Expect(c.Create(context.TODO(), instance)).NotTo(gomega.HaveOccurred())
323+
g.Expect(c.Create(ctx, instance)).NotTo(gomega.HaveOccurred())
325324

326325
// Expect that experiment status is running
327326
experiment := &experimentsv1beta1.Experiment{}
328327
g.Eventually(func() bool {
329-
if err = c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: experimentName}, experiment); err != nil {
328+
if err = c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: experimentName}, experiment); err != nil {
330329
return false
331330
}
332331
return experiment.IsRunning()
@@ -338,7 +337,7 @@ func TestReconcile(t *testing.T) {
338337
label := labels.Set{
339338
consts.LabelExperimentName: experimentName,
340339
}
341-
g.Expect(c.List(context.TODO(), trials, &client.ListOptions{LabelSelector: label.AsSelector()})).NotTo(gomega.HaveOccurred())
340+
g.Expect(c.List(ctx, trials, &client.ListOptions{LabelSelector: label.AsSelector()})).NotTo(gomega.HaveOccurred())
342341
return len(trials.Items)
343342
}, timeout).Should(gomega.Equal(2))
344343

@@ -348,7 +347,7 @@ func TestReconcile(t *testing.T) {
348347
g.Eventually(func() bool {
349348
suggestion := &suggestionsv1beta1.Suggestion{}
350349
isDeleted := true
351-
if err = c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: experimentName}, suggestion); err != nil {
350+
if err = c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: experimentName}, suggestion); err != nil {
352351
return false
353352
}
354353
for _, s := range suggestion.Status.Suggestions {
@@ -365,38 +364,38 @@ func TestReconcile(t *testing.T) {
365364
g.Eventually(func() bool {
366365
// Update experiment
367366
experiment = &experimentsv1beta1.Experiment{}
368-
if err = c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: experimentName}, experiment); err != nil {
367+
if err = c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: experimentName}, experiment); err != nil {
369368
return false
370369
}
371370
experiment.MarkExperimentStatusFailed(experimentUtil.ExperimentMaxTrialsReachedReason, "Experiment is failed")
372-
if err = c.Status().Update(context.TODO(), experiment); err != nil {
371+
if err = c.Status().Update(ctx, experiment); err != nil {
373372
return false
374373
}
375374

376375
// Get Suggestion
377376
suggestion := &suggestionsv1beta1.Suggestion{}
378-
if err = c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: experimentName}, suggestion); err != nil {
377+
if err = c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: experimentName}, suggestion); err != nil {
379378
return false
380379
}
381380
return suggestion.IsSucceeded()
382381
}, timeout).Should(gomega.BeTrue())
383382

384383
// Delete the suggestion
385-
g.Expect(c.Delete(context.TODO(), suggestionInstance)).NotTo(gomega.HaveOccurred())
384+
g.Expect(c.Delete(ctx, suggestionInstance)).NotTo(gomega.HaveOccurred())
386385

387386
// Expect that suggestion with ResumePolicy = NeverResume is deleted
388387
g.Eventually(func() bool {
389-
return errors.IsNotFound(c.Get(context.TODO(),
388+
return errors.IsNotFound(c.Get(ctx,
390389
types.NamespacedName{Namespace: namespace, Name: experimentName}, &suggestionsv1beta1.Suggestion{}))
391390
}, timeout).Should(gomega.BeTrue())
392391

393392
// Create the suggestion with ResumePolicy = FromVolume
394393
suggestionInstance = newFakeSuggestion()
395394
suggestionInstance.Spec.ResumePolicy = experimentsv1beta1.FromVolume
396-
g.Expect(c.Create(context.TODO(), suggestionInstance)).NotTo(gomega.HaveOccurred())
395+
g.Expect(c.Create(ctx, suggestionInstance)).NotTo(gomega.HaveOccurred())
397396
// Expect that suggestion is created
398397
g.Eventually(func() bool {
399-
return errors.IsNotFound(c.Get(context.TODO(),
398+
return errors.IsNotFound(c.Get(ctx,
400399
types.NamespacedName{Namespace: namespace, Name: experimentName}, &suggestionsv1beta1.Suggestion{}))
401400
}, timeout).ShouldNot(gomega.BeTrue())
402401

@@ -405,25 +404,25 @@ func TestReconcile(t *testing.T) {
405404
g.Eventually(func() bool {
406405
experiment := &experimentsv1beta1.Experiment{}
407406
// Update ResumePolicy and maxTrialCount for resume
408-
if err = c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: experimentName}, experiment); err != nil {
407+
if err = c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: experimentName}, experiment); err != nil {
409408
return false
410409
}
411410
experiment.Spec.ResumePolicy = experimentsv1beta1.FromVolume
412411
var max int32 = 5
413412
experiment.Spec.MaxTrialCount = &max
414-
errUpdate := c.Update(context.TODO(), experiment)
413+
errUpdate := c.Update(ctx, experiment)
415414
return errUpdate == nil
416415
}, timeout).Should(gomega.BeTrue())
417416

418417
// Expect that experiment status is updated
419418
g.Eventually(func() bool {
420419
experiment := &experimentsv1beta1.Experiment{}
421420
// Update status to succeeded
422-
if err = c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: experimentName}, experiment); err != nil {
421+
if err = c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: experimentName}, experiment); err != nil {
423422
return false
424423
}
425424
experiment.MarkExperimentStatusSucceeded(experimentUtil.ExperimentMaxTrialsReachedReason, "Experiment is succeeded")
426-
errStatus := c.Status().Update(context.TODO(), experiment)
425+
errStatus := c.Status().Update(ctx, experiment)
427426
return errStatus == nil
428427
}, timeout).Should(gomega.BeTrue())
429428

@@ -432,27 +431,27 @@ func TestReconcile(t *testing.T) {
432431
// UpdateSuggestionStatus with restartYesCall call and UpdateSuggestionStatus with experimentRestartingCall call.
433432
g.Eventually(func() bool {
434433
experiment := &experimentsv1beta1.Experiment{}
435-
if err = c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: experimentName}, experiment); err != nil {
434+
if err = c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: experimentName}, experiment); err != nil {
436435
return false
437436
}
438437
return experiment.IsRestarting() && !experiment.IsSucceeded() && !experiment.IsFailed()
439438
}, timeout).Should(gomega.BeTrue())
440439

441440
// Delete the suggestion
442-
g.Expect(c.Delete(context.TODO(), suggestionInstance)).NotTo(gomega.HaveOccurred())
441+
g.Expect(c.Delete(ctx, suggestionInstance)).NotTo(gomega.HaveOccurred())
443442

444443
// Expect that suggestion with ResumePolicy = FromVolume is deleted
445444
g.Eventually(func() bool {
446-
return errors.IsNotFound(c.Get(context.TODO(),
445+
return errors.IsNotFound(c.Get(ctx,
447446
types.NamespacedName{Namespace: namespace, Name: experimentName}, &suggestionsv1beta1.Suggestion{}))
448447
}, timeout).Should(gomega.BeTrue())
449448

450449
// Delete the experiment
451-
g.Expect(c.Delete(context.TODO(), instance)).NotTo(gomega.HaveOccurred())
450+
g.Expect(c.Delete(ctx, instance)).NotTo(gomega.HaveOccurred())
452451

453452
// Expect that experiment is deleted
454453
g.Eventually(func() bool {
455-
return errors.IsNotFound(c.Get(context.TODO(),
454+
return errors.IsNotFound(c.Get(ctx,
456455
types.NamespacedName{Namespace: namespace, Name: experimentName}, &experimentsv1beta1.Experiment{}))
457456
}, timeout).Should(gomega.BeTrue())
458457

‎pkg/controller.v1beta1/suggestion/composer/composer_test.go‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ import (
5757
var (
5858
cfg *rest.Config
5959
timeout = time.Second * 40
60+
ctx context.Context
61+
cancel context.CancelFunc
6062

6163
suggestionName = "test-suggestion"
6264
suggestionAlgorithm = "random"
@@ -97,6 +99,11 @@ var (
9799
)
98100

99101
func TestMain(m *testing.M) {
102+
// To avoid the `timeout waiting for process kube-apiserver to stop` error,
103+
// we must use the `context.WithCancel`.
104+
// Ref: https://github.com/kubernetes-sigs/controller-runtime/issues/1571#issuecomment-945535598
105+
ctx, cancel = context.WithCancel(context.TODO())
106+
100107
// Start test k8s server
101108
t := &envtest.Environment{
102109
CRDDirectoryPaths: []string{
@@ -113,6 +120,7 @@ func TestMain(m *testing.M) {
113120
}
114121

115122
code := m.Run()
123+
cancel()
116124
if err = t.Stop(); err != nil {
117125
stdlog.Fatal(err)
118126
}
@@ -130,7 +138,7 @@ func TestDesiredDeployment(t *testing.T) {
130138
wg.Add(1)
131139
go func() {
132140
defer wg.Done()
133-
g.Expect(mgr.Start(context.TODO())).NotTo(gomega.HaveOccurred())
141+
g.Expect(mgr.Start(ctx)).NotTo(gomega.HaveOccurred())
134142
}()
135143

136144
c := mgr.GetClient()
@@ -141,7 +149,7 @@ func TestDesiredDeployment(t *testing.T) {
141149
Name: namespace,
142150
},
143151
}
144-
g.Expect(c.Create(context.TODO(), kubeflowNS)).NotTo(gomega.HaveOccurred())
152+
g.Expect(c.Create(ctx, kubeflowNS)).NotTo(gomega.HaveOccurred())
145153

146154
tcs := []struct {
147155
suggestion *suggestionsv1beta1.Suggestion
@@ -237,11 +245,11 @@ func TestDesiredDeployment(t *testing.T) {
237245

238246
for idx, tc := range tcs {
239247
// Create configMap with Katib config
240-
g.Expect(c.Create(context.TODO(), tc.configMap)).NotTo(gomega.HaveOccurred())
248+
g.Expect(c.Create(ctx, tc.configMap)).NotTo(gomega.HaveOccurred())
241249

242250
// Wait that Config Map is created
243251
g.Eventually(func() error {
244-
return c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: configMap}, &corev1.ConfigMap{})
252+
return c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: configMap}, &corev1.ConfigMap{})
245253
}, timeout).ShouldNot(gomega.HaveOccurred())
246254

247255
// Get deployment
@@ -269,12 +277,12 @@ func TestDesiredDeployment(t *testing.T) {
269277
}
270278

271279
// Delete configMap with Katib config
272-
g.Expect(c.Delete(context.TODO(), tc.configMap)).NotTo(gomega.HaveOccurred())
280+
g.Expect(c.Delete(ctx, tc.configMap)).NotTo(gomega.HaveOccurred())
273281

274282
// Wait that Config Map is deleted
275283
g.Eventually(func() bool {
276284
return errors.IsNotFound(
277-
c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: configMap}, &corev1.ConfigMap{}))
285+
c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: configMap}, &corev1.ConfigMap{}))
278286
}, timeout).Should(gomega.BeTrue())
279287

280288
}
@@ -377,7 +385,7 @@ func TestDesiredVolume(t *testing.T) {
377385
wg.Add(1)
378386
go func() {
379387
defer wg.Done()
380-
g.Expect(mgr.Start(context.TODO())).NotTo(gomega.HaveOccurred())
388+
g.Expect(mgr.Start(ctx)).NotTo(gomega.HaveOccurred())
381389
}()
382390

383391
c := mgr.GetClient()
@@ -443,11 +451,11 @@ func TestDesiredVolume(t *testing.T) {
443451

444452
if tc.configMap != nil {
445453
// Create ConfigMap with Katib config
446-
g.Expect(c.Create(context.TODO(), tc.configMap)).NotTo(gomega.HaveOccurred())
454+
g.Expect(c.Create(ctx, tc.configMap)).NotTo(gomega.HaveOccurred())
447455

448456
// Expect that ConfigMap is created
449457
g.Eventually(func() error {
450-
return c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: configMap}, &corev1.ConfigMap{})
458+
return c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: configMap}, &corev1.ConfigMap{})
451459
}, timeout).ShouldNot(gomega.HaveOccurred())
452460
}
453461

@@ -488,11 +496,11 @@ func TestDesiredVolume(t *testing.T) {
488496

489497
if tc.configMap != nil {
490498
// Delete ConfigMap with Katib config
491-
g.Expect(c.Delete(context.TODO(), tc.configMap)).NotTo(gomega.HaveOccurred())
499+
g.Expect(c.Delete(ctx, tc.configMap)).NotTo(gomega.HaveOccurred())
492500
// Expect that ConfigMap is deleted
493501
g.Eventually(func() bool {
494502
return errors.IsNotFound(
495-
c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: configMap}, &corev1.ConfigMap{}))
503+
c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: configMap}, &corev1.ConfigMap{}))
496504
}, timeout).Should(gomega.BeTrue())
497505
}
498506
}

0 commit comments

Comments
 (0)