bugfix: containerd instance exit may cause container exit

ctrd/client.go

@@ -302,6 +302,10 @@ func (c *Client) Cleanup() error {
 		return err
 	}
 
+	// Note(ziren): notify containerd is dead before containerd
+	// is really dead
+	c.watch.setContainerdDead(true)
+
 	// Ask the daemon to quit
 	syscall.Kill(c.daemonPid, syscall.SIGTERM)
 

ctrd/watch.go

@@ -34,9 +34,26 @@ func (m *Message) ExitTime() time.Time {
 
 type watch struct {
 	sync.Mutex
-	//client     *containerd.Client
 	containers map[string]*containerPack
 	hooks      []func(string, *Message) error
+
+	// containerdDead to specify whether containerd process is dead
+	containerdDead bool
+}
+
+func (w *watch) setContainerdDead(isDead bool) error {
+	w.Lock()
+	defer w.Unlock()
+
+	w.containerdDead = isDead
+	return nil
+}
+
+func (w *watch) isContainerdDead() bool {
+	w.Lock()
+	defer w.Unlock()
+
+	return w.containerdDead
 }
 
 func (w *watch) add(pack *containerPack) {
@@ -49,9 +66,16 @@ func (w *watch) add(pack *containerPack) {
 
 	w.containers[pack.id] = pack
 
-	go func(pack *containerPack) {
+	go func(w *watch, pack *containerPack) {
 		status := <-pack.sch
 
+		// if containerd is dead, the task.Wait channel return is not because
+		// container' task quit, but the channel has broken.
+		// so we just return.
+		if w.isContainerdDead() {
+			return
+		}
+
 		logrus.Infof("the task has quit, id: %s, err: %v, exitcode: %d, time: %v",
 			pack.id, status.Error(), status.ExitCode(), status.ExitTime())
 
@@ -85,7 +109,7 @@ func (w *watch) add(pack *containerPack) {
 
 		pack.ch <- msg
 
-	}(pack)
+	}(w, pack)
 
 	logrus.Infof("success to add container, id: %s", pack.id)
 }