[WIP] Add provision support as a simplenfs plugin

.gitignore

@@ -1 +1,2 @@
 /bin
+simplenfs/bin

Dockerfile

@@ -2,6 +2,8 @@ FROM centos:7.4.1708
 
 # Copy nfsplugin from build _output directory
 COPY bin/nfsplugin /nfsplugin
+RUN mkdir -p /simplenfs/bin
+COPY simplenfs/bin/plugin.so /simplenfs/plugin.so
 
 RUN yum -y install nfs-utils && yum -y install epel-release && yum -y install jq && yum clean all
 

cmd/nfsplugin/main.go

@@ -27,8 +27,9 @@ import (
 )
 
 var (
-	endpoint string
-	nodeID   string
+	endpoint         string
+	nodeID           string
+	controllerPlugin string
 )
 
 func init() {
@@ -55,6 +56,8 @@ func main() {
 	cmd.PersistentFlags().StringVar(&endpoint, "endpoint", "", "CSI endpoint")
 	cmd.MarkPersistentFlagRequired("endpoint")
 
+	cmd.PersistentFlags().StringVar(&controllerPlugin, "controllerPlugin", "", "Controller plugin")
+
 	cmd.ParseFlags(os.Args[1:])
 	if err := cmd.Execute(); err != nil {
 		fmt.Fprintf(os.Stderr, "%s", err.Error())
@@ -65,6 +68,6 @@ func main() {
 }
 
 func handle() {
-	d := nfs.NewNFSdriver(nodeID, endpoint)
+	d := nfs.NewNFSdriver(nodeID, endpoint, controllerPlugin)
 	d.Run()
 }

deploy/kubernetes-simplenfs/csi-attacher-nfsplugin.yaml

@@ -0,0 +1,79 @@
+# This YAML file contains attacher & csi driver API objects that are necessary
+# to run external CSI attacher for nfs
+
+kind: Service
+apiVersion: v1
+metadata:
+  name: csi-attacher-nfsplugin
+  labels:
+    app: csi-attacher-nfsplugin
+spec:
+  selector:
+    app: csi-attacher-nfsplugin
+  ports:
+    - name: dummy
+      port: 12345
+
+---
+kind: StatefulSet
+apiVersion: apps/v1beta1
+metadata:
+  name: csi-attacher-nfsplugin
+spec:
+  serviceName: "csi-attacher"
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: csi-attacher-nfsplugin
+    spec:
+      serviceAccount: csi-attacher
+      containers:
+        - name: csi-attacher
+          image: quay.io/k8scsi/csi-attacher:v1.0.1
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: csi-provisioner
+          image: quay.io/k8scsi/csi-provisioner:v1.2.0
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+          env:
+            - name: ADDRESS
+              value: unix:///csi/csi.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: nfs
+          image: nfsplugin:latest
+          args :
+            - "--nodeid=$(NODE_ID)"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--controllerPlugin=/simplenfs/plugin.so"
+          env:
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CSI_ENDPOINT
+              value: unix://plugin/csi.sock
+          imagePullPolicy: "IfNotPresent"
+          securityContext:
+            capabilities:
+              add:
+                - SYS_ADMIN
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /plugin
+      volumes:
+        - name: socket-dir
+          emptyDir:

deploy/kubernetes-simplenfs/csi-attacher-rbac.yaml

@@ -0,0 +1,46 @@
+# This YAML file contains RBAC API objects that are necessary to run external
+# CSI attacher for nfs flex adapter
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-attacher
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: external-attacher-runner
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "update", "delete"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-attacher-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-attacher
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: external-attacher-runner
+  apiGroup: rbac.authorization.k8s.io

deploy/kubernetes-simplenfs/csi-nodeplugin-nfsplugin.yaml

@@ -0,0 +1,75 @@
+# This YAML file contains driver-registrar & csi driver nodeplugin API objects
+# that are necessary to run CSI nodeplugin for nfs
+kind: DaemonSet
+apiVersion: apps/v1beta2
+metadata:
+  name: csi-nodeplugin-nfsplugin
+spec:
+  selector:
+    matchLabels:
+      app: csi-nodeplugin-nfsplugin
+  template:
+    metadata:
+      labels:
+        app: csi-nodeplugin-nfsplugin
+    spec:
+      serviceAccount: csi-nodeplugin
+      hostNetwork: true
+      containers:
+        - name: node-driver-registrar
+          image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "-c", "rm -rf /registration/csi-nfsplugin /registration/csi-nfsplugin-reg.sock"]
+          args:
+            - --v=5
+            - --csi-address=/plugin/csi.sock
+            - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-nfsplugin/csi.sock
+          env:
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /plugin
+            - name: registration-dir
+              mountPath: /registration
+        - name: nfs
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: nfsplugin:latest
+          args :
+            - "--nodeid=$(NODE_ID)"
+            - "--endpoint=$(CSI_ENDPOINT)"
+          env:
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CSI_ENDPOINT
+              value: unix://plugin/csi.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /plugin
+            - name: pods-mount-dir
+              mountPath: /var/lib/kubelet/pods
+              mountPropagation: "Bidirectional"
+      volumes:
+        - name: plugin-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/csi-nfsplugin
+            type: DirectoryOrCreate
+        - name: pods-mount-dir
+          hostPath:
+            path: /var/lib/kubelet/pods
+            type: Directory
+        - hostPath:
+            path: /var/lib/kubelet/plugins_registry
+            type: Directory
+          name: registration-dir

deploy/kubernetes-simplenfs/csi-nodeplugin-rbac.yaml

@@ -0,0 +1,34 @@
+# This YAML defines all API objects to create RBAC roles for CSI node plugin
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-nodeplugin
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-nodeplugin
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-nodeplugin
+subjects:
+  - kind: ServiceAccount
+    name: csi-nodeplugin
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: csi-nodeplugin
+  apiGroup: rbac.authorization.k8s.io

examples/kubernetes-simplenfs/nginx.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-simplenfsplugin
+spec:
+  containers:
+  - image: maersk/nginx
+    imagePullPolicy: Always
+    name: nginx
+    ports:
+    - containerPort: 80
+      protocol: TCP
+    volumeMounts:
+      - mountPath: /var/www
+        name: data-simplenfsplugin 
+  volumes:
+  - name: data-simplenfsplugin
+    persistentVolumeClaim:
+      claimName: data-simplenfsplugin 

examples/kubernetes-simplenfs/pvc.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: data-simplenfsplugin
+spec:
+  accessModes:
+  - ReadWriteMany
+  resources:
+    requests:
+      storage: 1Gi
+  storageClassName: csi-simplenfs-sc

examples/kubernetes-simplenfs/storageclass.yaml

@@ -0,0 +1,9 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+   name: csi-simplenfs-sc
+provisioner: csi-nfsplugin
+parameters:
+  server: 192.168.122.57
+  rootpath: /exports
+reclaimPolicy: Delete

pkg/nfs/controllerserver.go

@@ -1,6 +1,8 @@
 package nfs
 
 import (
+	"plugin"
+
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"github.com/golang/glog"
 	"golang.org/x/net/context"
@@ -12,11 +14,51 @@ type ControllerServer struct {
 	Driver *nfsDriver
 }
 
+func isSupported(pluginName string, symbolName string) bool {
+	symbol, err := lookupSymbol(pluginName, symbolName)
+	return err == nil && symbol != nil
+}
+
+func lookupSymbol(pluginName string, symbolName string) (interface{}, error) {
+	if pluginName != "" {
+		plug, err := plugin.Open(pluginName)
+		if err != nil {
+			glog.Infof("Failed to load plugin: %s error: %v", pluginName, err)
+			return nil, err
+		}
+		symbol, err := plug.Lookup(symbolName)
+		if err != nil {
+			glog.Infof("Failed to lookup symbol: %s error: %v", symbolName, err)
+			return nil, err
+		}
+		return symbol, nil
+	}
+	return nil, nil
+}
+
 func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) {
+	glog.Infof("CreateVolume called")
+	symbol, err := lookupSymbol(cs.Driver.controllerPlugin, "CreateVolume")
+	if err == nil && symbol != nil {
+		createVolume, ok := symbol.(func(cs *ControllerServer, ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error))
+		if ok {
+			return createVolume(cs, ctx, req)
+		}
+	}
+
 	return nil, status.Error(codes.Unimplemented, "")
 }
 
 func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) {
+	glog.Infof("DeleteVolume called")
+	symbol, err := lookupSymbol(cs.Driver.controllerPlugin, "DeleteVolume")
+	if err == nil && symbol != nil {
+		deleteVolume, ok := symbol.(func(cs *ControllerServer, ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error))
+		if ok {
+			return deleteVolume(cs, ctx, req)
+		}
+	}
+
 	return nil, status.Error(codes.Unimplemented, "")
 }
 
@@ -29,6 +71,14 @@ func (cs *ControllerServer) ControllerUnpublishVolume(ctx context.Context, req *
 }
 
 func (cs *ControllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) {
+	symbol, err := lookupSymbol(cs.Driver.controllerPlugin, "ValidateVolumeCapabilities")
+	if err == nil && symbol != nil {
+		validateVolumeCapabilities, ok := symbol.(func(cs *ControllerServer, ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error))
+		if ok {
+			return validateVolumeCapabilities(cs, ctx, req)
+		}
+	}
+
 	return nil, status.Error(codes.Unimplemented, "")
 }