diff --git a/ctrd/client.go b/ctrd/client.go index c14fbb39f..e31d197fa 100644 --- a/ctrd/client.go +++ b/ctrd/client.go @@ -302,6 +302,10 @@ func (c *Client) Cleanup() error { return err } + // Note(ziren): notify containerd is dead before containerd + // is really dead + c.watch.setContainerdDead(true) + // Ask the daemon to quit syscall.Kill(c.daemonPid, syscall.SIGTERM) diff --git a/ctrd/watch.go b/ctrd/watch.go index bfeff1905..876599a4c 100644 --- a/ctrd/watch.go +++ b/ctrd/watch.go @@ -34,9 +34,26 @@ func (m *Message) ExitTime() time.Time { type watch struct { sync.Mutex - //client *containerd.Client containers map[string]*containerPack hooks []func(string, *Message) error + + // containerdDead to specify whether containerd process is dead + containerdDead bool +} + +func (w *watch) setContainerdDead(isDead bool) error { + w.Lock() + defer w.Unlock() + + w.containerdDead = isDead + return nil +} + +func (w *watch) isContainerdDead() bool { + w.Lock() + defer w.Unlock() + + return w.containerdDead } func (w *watch) add(pack *containerPack) { @@ -49,9 +66,16 @@ func (w *watch) add(pack *containerPack) { w.containers[pack.id] = pack - go func(pack *containerPack) { + go func(w *watch, pack *containerPack) { status := <-pack.sch + // if containerd is dead, the task.Wait channel return is not because + // container' task quit, but the channel has broken. + // so we just return. + if w.isContainerdDead() { + return + } + logrus.Infof("the task has quit, id: %s, err: %v, exitcode: %d, time: %v", pack.id, status.Error(), status.ExitCode(), status.ExitTime()) @@ -85,7 +109,7 @@ func (w *watch) add(pack *containerPack) { pack.ch <- msg - }(pack) + }(w, pack) logrus.Infof("success to add container, id: %s", pack.id) }