Merge pull request #1 from jschwinger23/master

OCI Wrapper for Integrating into CNI

Author: CMGS

.github/workflows/goreleaser.yml

@@ -0,0 +1,33 @@
+name: goreleaser
+
+on:
+  push:
+    tags:
+      - v*
+
+jobs:
+  goreleaser:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up environment variables
+        run: |
+          echo "VERSION=$(git describe --tags $(git rev-list --tags --max-count=1))" >> $GITHUB_ENV
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.15
+
+      - name: Run GoReleaser
+        uses: goreleaser/goreleaser-action@v2
+        with:
+          version: latest
+          args: release --rm-dist
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+

.gitignore

@@ -0,0 +1,3 @@
+vendor/
+docker-cni
+dist/

.goreleaser.yml

@@ -0,0 +1,37 @@
+before:
+  hooks:
+    - go mod download
+
+builds:
+  - binary: docker-cni
+    env:
+      - CGO_ENABLED=0
+    ldflags:
+      - -X main.REVISION={{.Commit}}
+      - -X main.VERSION={{.Env.VERSION}}
+      - -X main.BUILTAT={{.Date}}
+    goos:
+      - linux
+    goarch:
+      - amd64
+
+archives:
+  - replacements:
+      linux: Linux
+      amd64: x86_64
+
+checksum:
+  name_template: 'checksums.txt'
+
+release:
+  prerelease: auto
+
+snapshot:
+  name_template: "{{ .Tag }}-next"
+
+changelog:
+  sort: asc
+  filters:
+    exclude:
+    - '^docs:'
+    - '^test:'

Makefile

@@ -0,0 +1,17 @@
+.PHONY: binary
+
+REVISION := $(shell git rev-parse HEAD || unknown)
+BUILTAT := $(shell date +%Y-%m-%dT%H:%M:%S)
+VERSION := $(shell git describe --tags $(shell git rev-list --tags --max-count=1))
+GO_LDFLAGS ?= -X main.REVISION=$(REVISION) \
+			  -X main.BUILTAT=$(BUILTAT) \
+			  -X main.VERSION=$(VERSION)
+
+deps:
+	env GO111MODULE=on go mod download
+	env GO111MODULE=on go mod vendor
+
+binary:
+	CGO_ENABLED=0 go build -ldflags "$(GO_LDFLAGS)" -o docker-cni
+
+build: deps binary

README.md

@@ -2,12 +2,78 @@
 
 This repo aims to integrate [CNI](https://github.com/containernetworking/cni) with Dockerd.
 
-There is, [according to CNI repo](https://github.com/containernetworking/cni/blob/master/scripts/docker-run.sh), an approach to integrate by running a [pause] equivalent container ahead of the application container, but that's too pod-like for those who resent pod models.
+There is, [according to CNI repo](https://github.com/containernetworking/cni/blob/master/scripts/docker-run.sh), an approach to integrate by running a [pause](https://groups.google.com/g/kubernetes-users/c/jVjv0QK4b_o) equivalent container ahead of the application container, but that's too pod-like for those who resent pod models.
 
 Let's figure out yet another solution.
 
 # Usage
 
+## 0. Prepare your CNI plugin
+
+Make sure you have everything ready:
+
+1. CNI binaries in the right place: for example, `/opt/cni/bin/calico` and `/opt/cni/bin/calico-ipam` binaries
+2. CNI configures in the right place: for exmaple, `/etc/cni/net.d/10-calico.conf`
+3. Other services needed: for example, `calico-node` container
+
+Notes:
+
+1. Provided there are multiple CNI configures in the dir, `docker-cni` will only use the first config in alphabet order.
+
 ## 1. Configure docker-cni
 
+### 1.1 install docker-cni binary
+
+Download the latest binary from [release](https://github.com/projecteru2/docker-cni/releases).
+
+### 1.2 docker-cni configuration
+
+```shell
+mkdir -p /etc/docker/
+cat <<!
+cni_conf_dir: /etc/cni/net.d/
+cni_bin_dir: /opt/cni/bin/
+log_driver: file:///var/run/log/docker-cni.log
+log_level: debug
+!
+```
+
+You may revise the aforementioned configure with YOUR `cni_conf_dir` and `cni_bin_dir`.
+
 ## 2. Configure dockerd
+
+### 2.1 dockerd daemon configuration
+
+Add the additional `runtime` in docker daemon configure, which is usually located at `/etc/docker/daemon.json`:
+
+```
+{
+    ...
+    "runtimes": {
+        "cni": {
+            "path": "/usr/local/bin/docker-cni",
+            "runtimeArgs": [
+                "--config",
+                "/etc/docker/cni.yaml",
+                "--runtime-path",
+                "/usr/bin/runc",
+                "--"
+            ]
+        }
+    }
+}
+```
+
+### 2.2 restart dockerd
+
+```
+systemctl restart dockerd
+```
+
+## 3. Create docker container with CNI
+
+```
+docker run -td --runtime cni --net none bash bash
+```
+
+That's everything.

cni.yaml.example

@@ -0,0 +1,4 @@
+cni_conf_dir: /etc/cni/net.d/
+cni_bin_dir: /opt/cni/bin/
+log_driver: file:///var/run/log/docker-cni.log
+log_level: debug

cni/netns.go

@@ -0,0 +1,33 @@
+package cni
+
+import (
+	"os"
+
+	"github.com/pkg/errors"
+	"github.com/projecteru2/docker-cni/utils"
+)
+
+func (p netnsManager) GetNetns(ID string) (netnsPath string, err error) {
+	_, err = os.Stat(p.getNetnsPath(ID))
+	return p.getNetnsPath(ID), errors.WithStack(err)
+}
+
+func (p netnsManager) AddNetns(ID string) (netnsPath string, add, del *utils.Process, err error) {
+	if add, err = utils.NewProcess("ip", []string{"net", "a", p.getID(ID)}, nil, nil); err != nil {
+		return
+	}
+	del, err = p.DelNetns(ID)
+	return p.getNetnsPath(ID), add, del, err
+}
+
+func (p netnsManager) DelNetns(ID string) (*utils.Process, error) {
+	return utils.NewProcess("ip", []string{"net", "d", p.getID(ID)}, nil, nil)
+}
+
+func (p netnsManager) getNetnsPath(ID string) string {
+	return "/var/run/netns/" + p.getID(ID)
+}
+
+func (p netnsManager) getID(ID string) string {
+	return ID[:12]
+}

cni/network.go

@@ -0,0 +1,58 @@
+package cni
+
+import (
+	"github.com/pkg/errors"
+	"github.com/projecteru2/docker-cni/config"
+	"github.com/projecteru2/docker-cni/oci"
+	"github.com/projecteru2/docker-cni/utils"
+	log "github.com/sirupsen/logrus"
+)
+
+// checkupNetwork checks the netns and invokes "CNI check" (not implemented yet)
+func (p *CNIPlugin) CheckupNetwork(conf config.Config, containerMeta *oci.ContainerMeta) (exist bool, cleanup []*utils.Process, err error) {
+	netnsPath, err := p.GetNetns(containerMeta.ID)
+	if err != nil {
+		return false, nil, err
+	}
+
+	delCNI, err := p.DelCNI(containerMeta.ID, netnsPath, DefaultIfname)
+	if err != nil {
+		return false, nil, errors.WithStack(err)
+	}
+	delNetns, err := p.DelNetns(containerMeta.ID)
+	if err != nil {
+		return false, nil, errors.WithStack(err)
+	}
+	return true, []*utils.Process{delCNI, delNetns}, nil
+}
+
+func (p *CNIPlugin) SetupNetwork(conf config.Config, containerMeta *oci.ContainerMeta) (netnsPath string, cleanup []*utils.Process, err error) {
+	netnsPath, addNetns, delNetns, err := p.AddNetns(containerMeta.ID)
+	if err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	if _, _, err = addNetns.Run(); err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	defer func() {
+		if err != nil {
+			if _, _, e := delNetns.Run(); e != nil {
+				log.Errorf("failed to rollback netns: %+v", e)
+			}
+		}
+	}()
+
+	addCNI, delCNI, err := p.AddCNI(containerMeta.ID, netnsPath, DefaultIfname)
+	if err != nil {
+		return
+	}
+	stdoutBytes, stderrBytes, err := addCNI.Run()
+	log.Infof("add cni stdout: %s", string(stdoutBytes))
+	log.Debugf("add cni stderr: %s", string(stderrBytes))
+	return netnsPath, []*utils.Process{
+		delCNI,
+		delNetns,
+	}, err
+}

cni/operate.go

@@ -0,0 +1,43 @@
+package cni
+
+import (
+	"github.com/projecteru2/docker-cni/utils"
+)
+
+func (p *CNIPlugin) addCNI(ID, netnsPath, ifname string) (*utils.Process, error) {
+	return utils.NewProcess(
+		p.binPath, // path
+		nil,       // args
+		[]string{
+			"CNI_COMMAND=ADD",
+			"CNI_CONTAINERID=" + ID,
+			"CNI_NETNS=" + netnsPath,
+			"CNI_IFNAME=" + ifname,
+			"CNI_PATH=" + p.binDir,
+		}, // env
+		utils.NewStdio(p.specBytes), // stdio
+	)
+}
+
+func (p *CNIPlugin) DelCNI(ID, netnsPath, ifname string) (*utils.Process, error) {
+	return utils.NewProcess(
+		p.binPath, // path
+		nil,       // args
+		[]string{
+			"CNI_COMMAND=DEL",
+			"CNI_CONTAINERID=" + ID,
+			"CNI_NETNS=" + netnsPath,
+			"CNI_IFNAME=" + ifname,
+			"CNI_PATH=" + p.binDir,
+		}, // env
+		utils.NewStdio(p.specBytes), // stdio
+	)
+}
+
+func (p *CNIPlugin) AddCNI(ID, netnsPath, ifname string) (add, del *utils.Process, err error) {
+	if add, err = p.addCNI(ID, netnsPath, ifname); err != nil {
+		return
+	}
+	del, err = p.DelCNI(ID, netnsPath, ifname)
+	return
+}

cni/types.go

@@ -0,0 +1,58 @@
+package cni
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"path/filepath"
+	"sort"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	DefaultIfname = "eth0"
+)
+
+type CNIPlugin struct {
+	netnsManager
+
+	binDir    string
+	binPath   string
+	specBytes []byte
+}
+
+type CNISpec struct {
+	Type string `json:"type"`
+}
+
+func NewCNIPlugin(specDir, binDir string) (_ *CNIPlugin, err error) {
+	// walk thu the config_dir and get the first configure file in lexicographic order, the same behavior as kubelet: https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/#cni
+
+	files, err := ioutil.ReadDir(specDir)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	if len(files) == 0 {
+		return nil, errors.Errorf("cni configure not found: %s", specDir)
+	}
+
+	sort.Slice(files, func(i, j int) bool { return files[i].Name() < files[j].Name() })
+	content, err := ioutil.ReadFile(filepath.Join(specDir, files[0].Name()))
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	spec := &CNISpec{}
+	if err = json.Unmarshal(content, spec); err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	return &CNIPlugin{
+		binDir:    binDir,
+		binPath:   filepath.Join(binDir, spec.Type),
+		specBytes: content,
+	}, nil
+}
+
+type netnsManager struct{}