libcontainer: initial support for cgroups v2

libcontainer/cgroups/fs/apply_raw.go

@@ -18,7 +18,7 @@ import (
 )
 
 var (
-	subsystems = subsystemSet{
+	subsystemsLegacy = subsystemSet{
 		&CpusetGroup{},
 		&DevicesGroup{},
 		&MemoryGroup{},
@@ -33,6 +33,14 @@ var (
 		&FreezerGroup{},
 		&NameGroup{GroupName: "name=systemd", Join: true},
 	}
+	subsystemsUnified = subsystemSet{
+		&CpusetGroupV2{},
+		&FreezerGroupV2{},
+		&CpuGroupV2{},
+		&MemoryGroupV2{},
+		&IOGroupV2{},
+		&PidsGroupV2{},
+	}
 	HugePageSizes, _ = cgroups.GetHugePageSize()
 )
 
@@ -129,6 +137,13 @@ func isIgnorableError(rootless bool, err error) bool {
 	return errno == unix.EROFS || errno == unix.EPERM || errno == unix.EACCES
 }
 
+func (m *Manager) getSubsystems() subsystemSet {
+	if cgroups.IsCgroup2UnifiedMode() {
+		return subsystemsUnified
+	}
+	return subsystemsLegacy
+}
+
 func (m *Manager) Apply(pid int) (err error) {
 	if m.Cgroups == nil {
 		return nil
@@ -158,7 +173,7 @@ func (m *Manager) Apply(pid int) (err error) {
 		return cgroups.EnterPid(m.Paths, pid)
 	}
 
-	for _, sys := range subsystems {
+	for _, sys := range m.getSubsystems() {
 		// TODO: Apply should, ideally, be reentrant or be broken up into a separate
 		// create and join phase so that the cgroup hierarchy for a container can be
 		// created then join consists of writing the process pids to cgroup.procs
@@ -214,7 +229,7 @@ func (m *Manager) GetStats() (*cgroups.Stats, error) {
 	defer m.mu.Unlock()
 	stats := cgroups.NewStats()
 	for name, path := range m.Paths {
-		sys, err := subsystems.Get(name)
+		sys, err := m.getSubsystems().Get(name)
 		if err == errSubsystemDoesNotExist || !cgroups.PathExists(path) {
 			continue
 		}
@@ -237,7 +252,7 @@ func (m *Manager) Set(container *configs.Config) error {
 	}
 
 	paths := m.GetPaths()
-	for _, sys := range subsystems {
+	for _, sys := range m.getSubsystems() {
 		path := paths[sys.Name()]
 		if err := sys.Set(path, container.Cgroups); err != nil {
 			if m.Rootless && sys.Name() == "devices" {
@@ -274,7 +289,7 @@ func (m *Manager) Freeze(state configs.FreezerState) error {
 	dir := paths["freezer"]
 	prevState := m.Cgroups.Resources.Freezer
 	m.Cgroups.Resources.Freezer = state
-	freezer, err := subsystems.Get("freezer")
+	freezer, err := m.getSubsystems().Get("freezer")
 	if err != nil {
 		return err
 	}

libcontainer/cgroups/fs/cpu_v2.go

@@ -0,0 +1,92 @@
+// +build linux
+
+package fs
+
+import (
+	"bufio"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+type CpuGroupV2 struct {
+}
+
+func (s *CpuGroupV2) Name() string {
+	return "cpu"
+}
+
+func (s *CpuGroupV2) Apply(d *cgroupData) error {
+	// We always want to join the cpu group, to allow fair cpu scheduling
+	// on a container basis
+	path, err := d.path("cpu")
+	if err != nil && !cgroups.IsNotFound(err) {
+		return err
+	}
+	return s.ApplyDir(path, d.config, d.pid)
+}
+
+func (s *CpuGroupV2) ApplyDir(path string, cgroup *configs.Cgroup, pid int) error {
+	// This might happen if we have no cpu cgroup mounted.
+	// Just do nothing and don't fail.
+	if path == "" {
+		return nil
+	}
+	if err := os.MkdirAll(path, 0755); err != nil {
+		return err
+	}
+	return cgroups.WriteCgroupProc(path, pid)
+}
+
+func (s *CpuGroupV2) Set(path string, cgroup *configs.Cgroup) error {
+	if cgroup.Resources.CpuWeight != 0 {
+		if err := writeFile(path, "cpu.weight", strconv.FormatUint(cgroup.Resources.CpuWeight, 10)); err != nil {
+			return err
+		}
+	}
+
+	if cgroup.Resources.CpuMax != "" {
+		if err := writeFile(path, "cpu.max", cgroup.Resources.CpuMax); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *CpuGroupV2) Remove(d *cgroupData) error {
+	return removePath(d.path("cpu"))
+}
+
+func (s *CpuGroupV2) GetStats(path string, stats *cgroups.Stats) error {
+	f, err := os.Open(filepath.Join(path, "cpu.stat"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+	defer f.Close()
+
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		t, v, err := getCgroupParamKeyValue(sc.Text())
+		if err != nil {
+			return err
+		}
+		switch t {
+		case "usage_usec":
+			stats.CpuStats.CpuUsage.TotalUsage = v * 1000
+
+		case "user_usec":
+			stats.CpuStats.CpuUsage.UsageInUsermode = v * 1000
+
+		case "system_usec":
+			stats.CpuStats.CpuUsage.UsageInKernelmode = v * 1000
+		}
+	}
+	return nil
+}

libcontainer/cgroups/fs/cpuset_v2.go

@@ -0,0 +1,159 @@
+// +build linux
+
+package fs
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+	libcontainerUtils "github.com/opencontainers/runc/libcontainer/utils"
+)
+
+type CpusetGroupV2 struct {
+}
+
+func (s *CpusetGroupV2) Name() string {
+	return "cpuset"
+}
+
+func (s *CpusetGroupV2) Apply(d *cgroupData) error {
+	dir, err := d.path("cpuset")
+	if err != nil && !cgroups.IsNotFound(err) {
+		return err
+	}
+	return s.ApplyDir(dir, d.config, d.pid)
+}
+
+func (s *CpusetGroupV2) Set(path string, cgroup *configs.Cgroup) error {
+	if cgroup.Resources.CpusetCpus != "" {
+		if err := writeFile(path, "cpuset.cpus", cgroup.Resources.CpusetCpus); err != nil {
+			return err
+		}
+	}
+	if cgroup.Resources.CpusetMems != "" {
+		if err := writeFile(path, "cpuset.mems", cgroup.Resources.CpusetMems); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *CpusetGroupV2) Remove(d *cgroupData) error {
+	return removePath(d.path("cpuset"))
+}
+
+func (s *CpusetGroupV2) GetStats(path string, stats *cgroups.Stats) error {
+	return nil
+}
+
+func (s *CpusetGroupV2) ApplyDir(dir string, cgroup *configs.Cgroup, pid int) error {
+	// This might happen if we have no cpuset cgroup mounted.
+	// Just do nothing and don't fail.
+	if dir == "" {
+		return nil
+	}
+	mountInfo, err := ioutil.ReadFile("/proc/self/mountinfo")
+	if err != nil {
+		return err
+	}
+	root := filepath.Dir(cgroups.GetClosestMountpointAncestor(dir, string(mountInfo)))
+	// 'ensureParent' start with parent because we don't want to
+	// explicitly inherit from parent, it could conflict with
+	// 'cpuset.cpu_exclusive'.
+	if err := s.ensureParent(filepath.Dir(dir), root); err != nil {
+		return err
+	}
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return err
+	}
+	// We didn't inherit cpuset configs from parent, but we have
+	// to ensure cpuset configs are set before moving task into the
+	// cgroup.
+	// The logic is, if user specified cpuset configs, use these
+	// specified configs, otherwise, inherit from parent. This makes
+	// cpuset configs work correctly with 'cpuset.cpu_exclusive', and
+	// keep backward compatibility.
+	if err := s.ensureCpusAndMems(dir, cgroup); err != nil {
+		return err
+	}
+
+	// because we are not using d.join we need to place the pid into the procs file
+	// unlike the other subsystems
+	return cgroups.WriteCgroupProc(dir, pid)
+}
+
+func (s *CpusetGroupV2) getSubsystemSettings(parent string) (cpus []byte, mems []byte, err error) {
+	if cpus, err = ioutil.ReadFile(filepath.Join(parent, "cpuset.cpus.effective")); err != nil {
+		return
+	}
+	if mems, err = ioutil.ReadFile(filepath.Join(parent, "cpuset.mems.effective")); err != nil {
+		return
+	}
+	return cpus, mems, nil
+}
+
+// ensureParent makes sure that the parent directory of current is created
+// and populated with the proper cpus and mems files copied from
+// it's parent.
+func (s *CpusetGroupV2) ensureParent(current, root string) error {
+	parent := filepath.Dir(current)
+	if libcontainerUtils.CleanPath(parent) == root {
+		return nil
+	}
+	// Avoid infinite recursion.
+	if parent == current {
+		return fmt.Errorf("cpuset: cgroup parent path outside cgroup root")
+	}
+	if err := s.ensureParent(parent, root); err != nil {
+		return err
+	}
+	if err := os.MkdirAll(current, 0755); err != nil {
+		return err
+	}
+	return s.copyIfNeeded(current, parent)
+}
+
+// copyIfNeeded copies the cpuset.cpus and cpuset.mems from the parent
+// directory to the current directory if the file's contents are 0
+func (s *CpusetGroupV2) copyIfNeeded(current, parent string) error {
+	var (
+		err                      error
+		currentCpus, currentMems []byte
+		parentCpus, parentMems   []byte
+	)
+
+	if currentCpus, currentMems, err = s.getSubsystemSettings(current); err != nil {
+		return err
+	}
+	if parentCpus, parentMems, err = s.getSubsystemSettings(parent); err != nil {
+		return err
+	}
+
+	if s.isEmpty(currentCpus) {
+		if err := writeFile(current, "cpuset.cpus", string(parentCpus)); err != nil {
+			return err
+		}
+	}
+	if s.isEmpty(currentMems) {
+		if err := writeFile(current, "cpuset.mems", string(parentMems)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *CpusetGroupV2) isEmpty(b []byte) bool {
+	return len(bytes.Trim(b, "\n")) == 0
+}
+
+func (s *CpusetGroupV2) ensureCpusAndMems(path string, cgroup *configs.Cgroup) error {
+	if err := s.Set(path, cgroup); err != nil {
+		return err
+	}
+	return s.copyIfNeeded(path, filepath.Dir(path))
+}