feature: support containerd shim v2

Signed-off-by: zhuangqh <[email protected]>

Author: zhuangqh

apis/swagger.yml

@@ -2011,6 +2011,13 @@ definitions:
       and SELinux.
     type: "object"
     properties:
+      type:
+        description: The runtime type used in containerd.
+        type: "string"
+        example: "io.containerd.runtime.v1.linux"
+      options:
+        description: Options are config options for specific runtime.
+        x-go-type: "interface{}"
       path:
         description: |
           Name and, optional, path, of the OCI executable binary.
@@ -2022,6 +2029,7 @@ definitions:
       runtimeArgs:
         description: |
           List of command-line arguments to pass to the runtime when invoked.
+          DEPRECATED: Use Options instead. Remove when shim v1 is deprecated.
         type: "array"
         x-nullable: true
         items:
@@ -2452,6 +2460,12 @@ definitions:
           Runtime:
             type: "string"
             description: "Runtime to use with this container."
+          RuntimeType:
+            type: "string"
+            description: "The runtime type used in containerd."
+          RuntimeOptions:
+            description: "Runtime specific information."
+            x-go-type: "interface{}"
           # Applicable to Windows
           ConsoleSize:
             type: "array"

apis/types/host_config.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"path"
 	"runtime"
 	"strings"
 	"sync"
@@ -26,14 +25,15 @@ import (
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/leases"
 	"github.com/containerd/containerd/oci"
-	"github.com/containerd/containerd/runtime/linux/runctypes"
 	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
 var (
-	runtimeRoot = "/run"
+	RuntimeRoot       = "/run"
+	RuntimeTypeRuncV1 = fmt.Sprintf("io.containerd.runtime.v1.%s", runtime.GOOS)
+	RuntimeTypeRuncV2 = "io.containerd.runc.v1"
 )
 
 type containerPack struct {
@@ -539,11 +539,7 @@ func (c *Client) createContainer(ctx context.Context, ref, id, checkpointDir str
 	options := []containerd.NewContainerOpts{
 		containerd.WithSnapshotter(CurrentSnapshotterName(ctx)),
 		containerd.WithContainerLabels(container.Labels),
-		containerd.WithRuntime(fmt.Sprintf("io.containerd.runtime.v1.%s", runtime.GOOS), &runctypes.RuncOptions{
-			Runtime:       container.Runtime,
-			RuntimeRoot:   path.Join(runtimeRoot, container.Runtime),
-			SystemdCgroup: container.UseSystemd,
-		}),
+		containerd.WithRuntime(container.RuntimeType, container.RuntimeOptions),
 	}
 
 	rootFSPath := "rootfs"

apis/types/runtime.go

@@ -11,13 +11,14 @@ import (
 // then create container by specifying the snapshot;
 // The other is create container by specify container rootfs, we use `RootFSProvided` flag to mark it,
 type Container struct {
-	ID         string
-	Image      string
-	Runtime    string
-	Labels     map[string]string
-	IO         *containerio.IO
-	Spec       *specs.Spec
-	SnapshotID string
+	ID             string
+	Image          string
+	RuntimeType    string
+	RuntimeOptions interface{}
+	Labels         map[string]string
+	IO             *containerio.IO
+	Spec           *specs.Spec
+	SnapshotID     string
 
 	// BaseFS is rootfs used by containerd container
 	BaseFS string

ctrd/container.go

@@ -1,13 +1,18 @@
 package daemon
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/alibaba/pouch/apis/types"
+	"github.com/alibaba/pouch/ctrd"
+
+	"github.com/containerd/containerd/runtime/linux/runctypes"
+	runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
 )
 
 var (
@@ -20,6 +25,7 @@ var (
 // we will make a executable script as a path, or runtime.path is record as path.
 // NOTE: containerd not support runtime args directly, so we make executable
 // script include runtime path and args as a runtime execute binary.
+// this solution would be deprecated after shim v1 is deprecated.
 func initialRuntime(baseDir string, runtimes map[string]types.Runtime) error {
 	dir := filepath.Join(baseDir, runtimeDir)
 
@@ -35,20 +41,48 @@ func initialRuntime(baseDir string, runtimes map[string]types.Runtime) error {
 
 	// create script for runtime who has args
 	for name, r := range runtimes {
-		if len(r.RuntimeArgs) == 0 {
-			continue
-		}
-
-		script := filepath.Join(dir, name)
 		if r.Path == "" {
 			r.Path = name
 		}
-		data := fmt.Sprintf("#!/bin/sh\n%s %s $@\n", r.Path, strings.Join(r.RuntimeArgs, " "))
 
-		if err := ioutil.WriteFile(script, []byte(data), runtimeScriptPerm); err != nil {
-			return fmt.Errorf("failed to create runtime script %s: %s", script, err)
+		// setup a fake path
+		if len(r.RuntimeArgs) != 0 {
+			data := fmt.Sprintf("#!/bin/sh\n%s %s $@\n", r.Path, strings.Join(r.RuntimeArgs, " "))
+			r.Path = filepath.Join(dir, name)
+
+			if err := ioutil.WriteFile(r.Path, []byte(data), runtimeScriptPerm); err != nil {
+				return fmt.Errorf("failed to create runtime script %s: %s", r.Path, err)
+			}
 		}
+
+		if r.Type == "" {
+			r.Type = ctrd.RuntimeTypeRuncV1
+		}
+
+		// convert general json map to specific options type
+		b, err := json.Marshal(r.Options)
+		if err != nil {
+			return fmt.Errorf("failed to marshal options, runtime: %s: %v", name, err)
+		}
+		options := getRuntimeOptionsType(r.Type)
+		if err := json.Unmarshal(b, options); err != nil {
+			return fmt.Errorf("failed to unmarshal to type %+v: %v", options, err)
+		}
+
+		r.Options = options
 	}
 
 	return nil
 }
+
+func getRuntimeOptionsType(runtimeType string) interface{} {
+	switch runtimeType {
+	case ctrd.RuntimeTypeRuncV2:
+		return &runcoptions.Options{}
+	// using type RuncOptions for runc v1 shim or other v2 shim.
+	// TODO: using more general options type if containerd does
+	//       or support more v2 shim options type.
+	default:
+		return &runctypes.RuncOptions{}
+	}
+}

ctrd/container_types.go

@@ -400,9 +400,12 @@ func (mgr *ContainerManager) Create(ctx context.Context, name string, config *ty
 		config.HostConfig.Runtime = mgr.Config.DefaultRuntime
 	}
 
-	if _, exist := mgr.Config.Runtimes[config.HostConfig.Runtime]; !exist {
+	runtimeConfig, err := mgr.getRuntime(config.HostConfig.Runtime)
+	if err != nil {
 		return nil, errors.Wrapf(errtypes.ErrInvalidParam, "unknown runtime %s", config.HostConfig.Runtime)
 	}
+	config.HostConfig.RuntimeOptions = runtimeConfig.Options
+	config.HostConfig.RuntimeType = runtimeConfig.Type
 
 	snapID := id
 	// create a snapshot with image.
@@ -780,17 +783,13 @@ func (mgr *ContainerManager) createContainerdContainer(ctx context.Context, c *C
 	// set container's LogPath
 	mgr.SetContainerLogPath(c)
 
-	runtime, err := mgr.getRuntime(c.HostConfig.Runtime)
-	if err != nil {
-		return err
-	}
-
 	c.Lock()
 	ctrdContainer := &ctrd.Container{
 		ID:             c.ID,
 		Image:          c.Config.Image,
 		Labels:         c.Config.Labels,
-		Runtime:        runtime,
+		RuntimeType:    c.HostConfig.RuntimeType,
+		RuntimeOptions: c.HostConfig.RuntimeOptions,
 		Spec:           sw.s,
 		IO:             mgr.IOs.Get(c.ID),
 		RootFSProvided: c.RootFSProvided,
@@ -1919,7 +1918,7 @@ func (mgr *ContainerManager) setBaseFS(ctx context.Context, c *Container) {
 
 	// io.containerd.runtime.v1.linux as a const used by runc
 	c.Lock()
-	c.BaseFS = filepath.Join(mgr.Config.HomeDir, "containerd/state", "io.containerd.runtime.v1.linux", mgr.Config.DefaultNamespace, c.ID, "rootfs")
+	c.BaseFS = filepath.Join(mgr.Config.HomeDir, "containerd/state", c.HostConfig.RuntimeType, mgr.Config.DefaultNamespace, c.ID, "rootfs")
 	c.Unlock()
 }
 

daemon/daemon_utils.go

@@ -4,18 +4,20 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"path"
 	"path/filepath"
-	"runtime"
 	"strconv"
 	"strings"
 
 	"github.com/alibaba/pouch/apis/opts"
 	"github.com/alibaba/pouch/apis/types"
+	"github.com/alibaba/pouch/ctrd"
 	networktypes "github.com/alibaba/pouch/network/types"
 	"github.com/alibaba/pouch/pkg/errtypes"
 	"github.com/alibaba/pouch/pkg/meta"
 	"github.com/alibaba/pouch/pkg/randomid"
 
+	"github.com/containerd/containerd/runtime/linux/runctypes"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
@@ -105,35 +107,30 @@ func (mgr *ContainerManager) generateName(id string) string {
 	return name
 }
 
-// getRuntime returns runtime real path.
-// TODO(huamin): do we need validate runtime is executable ?
-func (mgr *ContainerManager) getRuntime(runtime string) (string, error) {
+// getRuntime returns runtime config and generate options.
+func (mgr *ContainerManager) getRuntime(runtime string) (*types.Runtime, error) {
 	r, exist := mgr.Config.Runtimes[runtime]
 	if !exist {
-		return "", fmt.Errorf("failed to find runtime %s in daemon config", runtime)
+		return nil, fmt.Errorf("failed to find runtime %s in daemon config", runtime)
 	}
 
-	// it is ok to use runtime name as a path.
-	rPath := runtime
-	// generally speaking, path is not be empty, but we not forbid empty path
-	// in config set, since name can be a path too.
-	if r.Path != "" {
-		rPath = r.Path
+	switch o := r.Options.(type) {
+	case *runctypes.RuncOptions:
+		o.Runtime = r.Path
+		o.RuntimeRoot = ctrd.RuntimeRoot
+		o.SystemdCgroup = mgr.Config.UseSystemd()
+		r.Options = o
+	// TODO: support other v2 shim options.
+	default:
+		return nil, fmt.Errorf("unsupported option type")
 	}
 
-	// if Runtime has args, use script path as runtime path.
-	if len(r.RuntimeArgs) > 0 {
-		rPath = filepath.Join(mgr.Config.HomeDir, RuntimeDir, runtime)
-	}
-
-	return rPath, nil
+	return &r, nil
 }
 
 // getContainerSpec returns container runtime spec, unmarshal spec from config.json
-// TODO: when runtime type can be specified, it need fix
 func (mgr *ContainerManager) getContainerSpec(c *Container) (*specs.Spec, error) {
-	runtimeType := fmt.Sprintf("io.containerd.runtime.v1.%s", runtime.GOOS)
-	configFile := filepath.Join(mgr.Config.HomeDir, "containerd/state", runtimeType, mgr.Config.DefaultNamespace, c.ID, "config.json")
+	configFile := filepath.Join(path.Dir(c.BaseFS), "config.json")
 	var spec specs.Spec
 	data, err := ioutil.ReadFile(configFile)
 	if err != nil {