[baseimage]: Install Kubernetes packages if enabled in image

Makefile.work

@@ -9,6 +9,7 @@
 #    through http.
 #  * ENABLE_ZTP: Enables zero touch provisioning.
 #  * SHUTDOWN_BGP_ON_START: Sets admin-down state for all bgp peerings after restart.
+#  * INSTALL_KUBERNETES: Allows including Kubernetes
 #  * ENABLE_PFCWD_ON_START: Enable PFC Watchdog (PFCWD) on server-facing ports
 #  * by default for TOR switch.
 #  * ENABLE_SYNCD_RPC: Enables rpc-based syncd builds.
@@ -176,6 +177,10 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            ENABLE_DHCP_GRAPH_SERVICE=$(ENABLE_DHCP_GRAPH_SERVICE) \
                            ENABLE_ZTP=$(ENABLE_ZTP) \
                            SHUTDOWN_BGP_ON_START=$(SHUTDOWN_BGP_ON_START) \
+                           INSTALL_KUBERNETES=$(INSTALL_KUBERNETES) \
+                           KUBERNETES_VERSION=$(KUBERNETES_VERSION) \
+                           K8s_GCR_IO_PAUSE_VERSION=$(K8s_GCR_IO_PAUSE_VERSION) \
+                           K8s_CNI_CALICO_VERSION=$(K8s_CNI_CALICO_VERSION) \
                            SONIC_ENABLE_PFCWD_ON_START=$(ENABLE_PFCWD_ON_START) \
                            SONIC_ENABLE_SYNCD_RPC=$(ENABLE_SYNCD_RPC) \
                            SONIC_INSTALL_DEBUG_TOOLS=$(INSTALL_DEBUG_TOOLS) \

build_debian.sh

@@ -208,6 +208,24 @@ sudo LANG=C chroot $FILESYSTEM_ROOT apt-get update
 sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install docker-ce=${DOCKER_VERSION}
 sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y remove software-properties-common gnupg2
 
+if [ "$INSTALL_KUBERNETES" == "y" ]
+then
+    ## Install Kubernetes
+    echo '[INFO] Install kubernetes'
+    sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -o /tmp/k8s.gpg -fsSL \
+        https://packages.cloud.google.com/apt/doc/apt-key.gpg
+    sudo LANG=C chroot $FILESYSTEM_ROOT apt-key add /tmp/k8s.gpg
+    sudo LANG=C chroot $FILESYSTEM_ROOT rm /tmp/k8s.gpg
+    ## Check out the sources list update matches current Debian version
+    sudo cp files/image_config/kubernetes/kubernetes.list $FILESYSTEM_ROOT/etc/apt/sources.list.d/
+    sudo LANG=C chroot $FILESYSTEM_ROOT apt-get update
+    sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -qy install kubeadm=${KUBERNETES_VERSION}-00
+    sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -qy install kubectl=${KUBERNETES_VERSION}-00
+    sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -qy install kubelet=${KUBERNETES_VERSION}-00
+else
+    echo '[INFO] Skipping Install kubernetes'
+fi
+
 ## Add docker config drop-in to specify dockerd command line
 sudo mkdir -p $FILESYSTEM_ROOT/etc/systemd/system/docker.service.d/
 ## Note: $_ means last argument of last command

files/build_templates/sonic_debian_extension.j2

@@ -38,6 +38,7 @@ FILESYSTEM_ROOT_USR_SHARE_SONIC="$FILESYSTEM_ROOT_USR_SHARE/sonic"
 FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES="$FILESYSTEM_ROOT_USR_SHARE_SONIC/templates"
 FILESYSTEM_ROOT_ETC="$FILESYSTEM_ROOT/etc"
 FILESYSTEM_ROOT_ETC_SONIC="$FILESYSTEM_ROOT_ETC/sonic"
+FILESYSTEM_ROOT_ETC_SONIC_PODS="$FILESYSTEM_ROOT_ETC_SONIC/pods"
 
 GENERATED_SERVICE_FILE="$FILESYSTEM_ROOT/etc/sonic/generated_services.conf"
 
@@ -72,7 +73,8 @@ sudo cp $IMAGE_CONFIGS/environment/environment $FILESYSTEM_ROOT/etc/
 sudo cp $IMAGE_CONFIGS/environment/motd $FILESYSTEM_ROOT/etc/
 
 # Create all needed directories
-sudo mkdir -p $FILESYSTEM_ROOT/etc/sonic/
+sudo mkdir -p $FILESYSTEM_ROOT_ETC_SONIC/
+sudo mkdir -p $FILESYSTEM_ROOT_ETC_SONIC_PODS/
 sudo mkdir -p $FILESYSTEM_ROOT/etc/modprobe.d/
 sudo mkdir -p $FILESYSTEM_ROOT/var/cache/sonic/
 sudo mkdir -p $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/
@@ -245,6 +247,12 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install azure-s
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install watchdog
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install futures
 
+{% if install_kubernetes == "y" %}
+# Copy kubelet service files
+sudo cp $IMAGE_CONFIGS/kubernetes/kubelet/10-kubeadm.conf $FILESYSTEM_ROOT/etc/systemd/system/kubelet.service.d/
+sudo LANG=C chroot $FILESYSTEM_ROOT systemctl disable kubelet.service
+{% endif %}
+
 # Copy the buffer configuration template
 sudo cp $BUILD_TEMPLATES/buffers_config.j2 $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/
 
@@ -401,6 +409,23 @@ sudo LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS ta
 sudo LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS tag {{imagename}}:latest {{imagebasename}}:latest
 {% endif %}
 {% endfor %}
+
+{% if install_kubernetes == "y" %}
+## Pull in kubernetes docker images
+echo "pulling universal k8s images ..."
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker pull k8s.gcr.io/pause:${K8s_GCR_IO_PAUSE_VERSION}
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker pull k8s.gcr.io/kube-proxy:v${KUBERNETES_VERSION}
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker pull calico/node:v${K8s_CNI_CALICO_VERSION}
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker pull calico/pod2daemon-flexvol:v${K8s_CNI_CALICO_VERSION}
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker pull calico/cni:v${K8s_CNI_CALICO_VERSION}
+echo "docker images pull complete"
+## Until we bootstrap scripts to bring up kubernetes
+sudo cp $IMAGE_CONFIGS/kubernetes/scripts/kube_join.sh $FILESYSTEM_ROOT/usr/bin/
+sudo cp $IMAGE_CONFIGS/kubernetes/scripts/kube_reset.sh $FILESYSTEM_ROOT/usr/bin/
+## Pod's hooks on PostStart & PreStop
+sudo cp $IMAGE_CONFIGS/kubernetes/scripts/pod_hook.sh $FILESYSTEM_ROOT_ETC_SONIC_PODS/
+{% endif %}
+
 sudo umount $FILESYSTEM_ROOT/target
 sudo rm -r $FILESYSTEM_ROOT/target
 if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then

files/image_config/kubernetes/kubelet/10-kubeadm.conf

@@ -0,0 +1,18 @@
+# Note: This dropin only works with kubeadm and kubelet v1.11+
+
+[Service]
+Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
+
+Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
+
+# This is a file that "kubeadm init" and "kubeadm join" generates at runtime
+# Populating the KUBELET_KUBEADM_ARGS variable dynamically
+EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
+
+# This is a file that the user can use for overrides of the kubelet args as a last resort.
+# Preferably, the user should use  the .NodeRegistration.KubeletExtraArgs object
+# in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
+EnvironmentFile=-/etc/default/kubelet
+
+ExecStart=
+ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS

files/image_config/kubernetes/kubernetes.list

@@ -0,0 +1,4 @@
+# The following is as recommended by https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/
+# Whenever an OS update is done, make sure to find the matching k8s sources list
+#
+deb https://apt.kubernetes.io/ kubernetes-xenial main

files/image_config/kubernetes/scripts/kube_join.sh

@@ -0,0 +1,44 @@
+if [ ! -f /etc/sonic/kubeadm_join.sh ]
+then
+    echo "Run 'kubeadm token create --print-join-command' in Master."
+    echo "please get join command in /etc/sonic/kubeadm_join.sh"
+    exit -1
+fi
+
+if [ ! -f /etc/sonic/kube_admin.conf ]
+then
+    echo "please get kube context in /etc/sonic/kube_admin.conf"
+    echo "It can be found @ master: /etc/kubernetes/admin.conf"
+    exit -1
+fi
+
+# Reset
+modprobe br_netfilter
+# Reset
+kubeadm reset -f
+rm -rf /etc/cni/net.d
+mkdir -p /var/lib/kubelet/
+
+# Copy appropriate service config file 
+systemctl enable kubelet.service
+systemctl restart kubelet.service
+
+# Give a pause before join
+sleep 2
+
+/etc/sonic/kubeadm_join.sh
+
+# if join fails, ensure the following
+# a) Ensure both master & node run same or compatible k8s versions
+# b) If node already exists, delete it
+#       run "kubectl get nodes" to list nodes.
+#       "kubectl drain <node name> --ignore-daemonsets; kubectl delete node <node name>"
+# c) If you are using calico, you *may* need to set the following in master.
+#    Ensure calico.yaml has the following and apply at master.
+#    # Auto-detection method.
+#    - name: IP_AUTODETECTION_METHOD
+#      value: "interface=<master route dev name>,eth0"
+#     e.g. value: "interface=ens192,eth0"
+# 
+#    kubectl apply -f calico.yaml
+# 

files/image_config/kubernetes/scripts/kube_reset.sh

@@ -0,0 +1,2 @@
+kubeadm reset -f
+rm -rf /etc/cni/net.d

files/image_config/kubernetes/scripts/pod_hook.sh

@@ -0,0 +1,30 @@
+#! /bin/bash
+
+
+start() {
+    # create /etc/sonic/pods/<container name>.pod with container id
+    cat /proc/self/cgroup   | head -n1 | rev | cut -f1 -d'/' | rev > /etc/sonic/pods/$1.pod
+}
+
+stop() {
+    rm -f /etc/sonic/pods/$1.pod
+    mkdir -p /etc/sonic/pods/stopped
+    rm -f /etc/sonic/pods/stopped/$1
+    date -u > /etc/sonic/pods/stopped/$1
+    date -u +%s >> /etc/sonic/pods/stopped/$1
+    logger -p local0.notice -t "pod_stopped_$1" "pod $1 stopped"
+}
+
+case "$1" in
+    start|stop)
+        $1 $2
+        if test -f ./$2.sh
+        then
+            ./$2_hook.sh $1
+        fi
+        ;;
+    *)
+        echo "Usage: $0 {start|stop}"
+        exit 1
+        ;;
+esac

rules/config

@@ -131,3 +131,17 @@ ENABLE_RESTAPI = n
 # ENABLE_NAT - build docker-sonic-nat for nat support
 ENABLE_NAT = y
 
+# INSTALL_KUBERNETES - if set to y kubernetes packages are installed to be able to
+# run as worker node in kubernetes cluster.
+# INSTALL_KUBERNETES = y
+
+# KUBERNETES_VERSION - Set to the required version.
+# K8s_GCR_IO_PAUSE_VERSION - Version of k8s universal pause container image
+# K8s_CNI_CALICO_VERSION - Calico used as CNI; Appropriate version for this Kubernetes version
+# These are Used *only* when INSTALL_KUBERNETES=y
+# NOTE: As a worker node it has to run version compatible to kubernetes master.
+#
+KUBERNETES_VERSION = 1.18.0
+K8s_GCR_IO_PAUSE_VERSION = 3.2
+K8s_CNI_CALICO_VERSION = 3.12.0
+

slave.mk

@@ -191,6 +191,7 @@ $(info "USERNAME"                        : "$(USERNAME)")
 $(info "PASSWORD"                        : "$(PASSWORD)")
 $(info "ENABLE_DHCP_GRAPH_SERVICE"       : "$(ENABLE_DHCP_GRAPH_SERVICE)")
 $(info "SHUTDOWN_BGP_ON_START"           : "$(SHUTDOWN_BGP_ON_START)")
+$(info "INSTALL_KUBERNETES"              : "$(INSTALL_KUBERNETES)")
 $(info "ENABLE_PFCWD_ON_START"           : "$(ENABLE_PFCWD_ON_START)")
 $(info "INSTALL_DEBUG_TOOLS"             : "$(INSTALL_DEBUG_TOOLS)")
 $(info "ROUTING_STACK"                   : "$(SONIC_ROUTING_STACK)")
@@ -769,6 +770,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
 	export enable_ztp="$(ENABLE_ZTP)"
 	export enable_nat="$(ENABLE_NAT)"
 	export shutdown_bgp_on_start="$(SHUTDOWN_BGP_ON_START)"
+	export install_kubernetes="$(INSTALL_KUBERNETES)"
 	export enable_pfcwd_on_start="$(ENABLE_PFCWD_ON_START)"
 	export installer_debs="$(addprefix $(STRETCH_DEBS_PATH)/,$($*_INSTALLS))"
 	export lazy_installer_debs="$(foreach deb, $($*_LAZY_INSTALLS),$(foreach device, $($(deb)_PLATFORM),$(addprefix $(device)@, $(STRETCH_DEBS_PATH)/$(deb))))"