[mellanox] Facilitate automatic integration of sdk kernel patches

platform/mellanox/integration-scripts.mk

@@ -28,7 +28,9 @@ PTCH_LIST  = $(TEMP_HW_MGMT_DIR)/series
 KCFG_LIST = $(TEMP_HW_MGMT_DIR)/kconfig
 HWMGMT_NONUP_LIST = $(BUILD_WORKDIR)/$($(MLNX_HW_MANAGEMENT)_SRC_PATH)/hwmgmt_nonup_patches
 HWMGMT_USER_OUTFILE = $(BUILD_WORKDIR)/integrate-mlnx-hw-mgmt_user.out
+SDK_USER_OUTFILE = $(BUILD_WORKDIR)/integrate-mlnx-sdk_user.out
 TMPFILE_OUT := $(shell mktemp)
+SDK_TMPDIR := $(shell mktemp -d)
 SB_HEAD = $(shell git rev-parse --short HEAD)
 SLK_HEAD = $(shell cd src/sonic-linux-kernel; git rev-parse --short HEAD)
 
@@ -132,4 +134,52 @@ endif
 
 	popd $(LOG_SIMPLE)
 
-SONIC_PHONY_TARGETS += integrate-mlnx-hw-mgmt
+integrate-mlnx-sdk:
+	$(FLUSH_LOG)
+	rm -rf $(MLNX_SDK_VERSION).zip sx_kernel-$(MLNX_SDK_VERSION)-$(MLNX_SDK_ISSU_VERSION).tar.gz
+
+ifeq ($(SDK_FROM_SRC),y)
+	wget $(MLNX_SDK_SOURCE_BASE_URL)/sx_kernel-$(MLNX_SDK_VERSION)-$(MLNX_SDK_ISSU_VERSION).tar.gz $(LOG_SIMPLE)
+	tar -xf sx_kernel-$(MLNX_SDK_VERSION)-$(MLNX_SDK_ISSU_VERSION).tar.gz --strip-components=1 -C $(SDK_TMPDIR) $(LOG_SIMPLE)
+else
+	# Download from upstream repository
+	wget $(MLNX_SDK_DRIVERS_GITHUB_URL)/archive/refs/heads/$(MLNX_SDK_VERSION).zip $(LOG_SIMPLE)
+	unzip $(MLNX_SDK_VERSION).zip -d $(SDK_TMPDIR) $(LOG_SIMPLE)
+	mv $(SDK_TMPDIR)/Spectrum-SDK-Drivers-$(MLNX_SDK_VERSION)/* $(SDK_TMPDIR) $(LOG_SIMPLE)
+endif
+
+	pushd $(BUILD_WORKDIR)/src/sonic-linux-kernel; git clean -f -- patch/; git stash -- patch/
+ifeq ($(CREATE_BRANCH), y)
+	git checkout -B "$(BRANCH_SONIC)_$(SLK_HEAD)_integrate_$(MLNX_SDK_VERSION)" HEAD
+	echo $(BRANCH_SONIC)_$(SLK_HEAD)_integrate_$(MLNX_SDK_VERSION) branch created in sonic-linux-kernel $(LOG_SIMPLE)
+endif
+	popd
+
+	echo "#### Integrate SDK $(MLNX_SDK_VERSION) Kernel Patches into SONiC" > ${SDK_USER_OUTFILE}
+
+	pushd $(BUILD_WORKDIR)/$(PLATFORM_PATH) $(LOG_SIMPLE)
+
+    # Run tests
+	pushd integration-scripts/tests; pytest-3 -v; popd
+
+	integration-scripts/sdk_kernel_patches.py \
+                            --sonic_kernel_ver $(KERNEL_VERSION) \
+                            --patches $(SDK_TMPDIR) \
+                            --build_root $(BUILD_WORKDIR) $(LOG_SIMPLE)
+
+    # Commit the changes in linux kernel and and log the diff
+	pushd $(BUILD_WORKDIR)/src/sonic-linux-kernel
+	git add -- patch/
+
+	echo -en "\n###-> series file changes in sonic-linux-kernel <-###\n" >> ${SDK_USER_OUTFILE}
+	git diff --no-color --staged -- patch/series >> ${SDK_USER_OUTFILE}
+
+	echo -en "\n###-> summary of files updated in sonic-linux-kernel <-###\n" >> ${SDK_USER_OUTFILE}
+	git diff --no-color --staged --stat >> ${SDK_USER_OUTFILE}
+
+	git diff --staged --quiet || git commit -m "Intgerate MLNX SDK ${MLNX_SDK_VERSION} Kernel Patches";
+	popd
+
+	popd $(LOG_SIMPLE)
+
+SONIC_PHONY_TARGETS += integrate-mlnx-hw-mgmt integrate-mlnx-sdk

platform/mellanox/integration-scripts/helper.py

@@ -4,6 +4,7 @@
 
 MARK_ID = "###->"
 MLNX_KFG_MARKER = "mellanox"
+SDK_MARKER = "mellanox_sdk"
 HW_MGMT_MARKER = "mellanox_hw_mgmt"
 SLK_PATCH_LOC = "src/sonic-linux-kernel/patch/"
 SLK_KCONFIG = SLK_PATCH_LOC + "kconfig-inclusions"
@@ -13,6 +14,7 @@
 NON_UP_PATCH_LOC = NON_UP_PATCH_DIR + "patches"
 NON_UP_PATCH_DIFF = NON_UP_PATCH_DIR + "series.patch"
 KCFG_HDR_RE = "\[(.*)\]"
+KERNEL_BACKPORTS = "kernel_backports"
  # kconfig_inclusion headers to consider
 HDRS = ["common", "amd64"]
 

platform/mellanox/integration-scripts/sdk_kernel_patches.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import shutil
+import argparse
+import copy
+import difflib
+
+from helper import *
+
+class Data:
+    # old sonic-linux-kernel series file 
+    old_series = list()
+    # updated sonic-linux-kernel series file
+    new_series = list()
+    # list of any new SDK patches to add
+    new_patches = list()
+    # list of current patch list in sonic-linux-kernel
+    old_patches = list()
+    # index of the mlnx_hw_mgmt patches start marker in old_series
+    i_sdk_start = -1 
+    # index of the mlnx_hw_mgmt patches end marker in old_series
+    i_sdk_end = -1
+    # kernel directory to consider the patches
+    k_dir = ""
+
+class SDKAction(Action):
+    def __init__(self, args):
+        super().__init__(args)
+
+    def check(self):
+        if not (self.args.patches and os.path.exists(self.args.patches)):
+            print("-> ERR: patch directory is missing ")
+            return False
+
+        if not (self.args.build_root and os.path.exists(self.args.build_root)):
+            print("-> ERR: build_root is missing")
+            return False
+
+        return True
+
+    def read_data(self):
+        Data.old_series = FileHandler.read_raw(os.path.join(self.args.build_root, SLK_SERIES))
+
+    def find_sdk_patches(self):
+        (Data.i_sdk_start, Data.i_sdk_end) = FileHandler.find_marker_indices(Data.old_series, SDK_MARKER)
+        if Data.i_sdk_start < 0 or Data.i_sdk_end > len(Data.old_series):
+            print("-> FATAL mellanox_sdk marker not found. Couldn't continue.. exiting")
+            sys.exit(1)
+
+        print("-> INFO mellanox_sdk markers found. start: {}, end: {}".format(Data.i_sdk_start, Data.i_sdk_end))
+        for index in range(Data.i_sdk_start+1, Data.i_sdk_end):
+            patch = Data.old_series[index].strip()
+            if patch:
+                Data.old_patches.append(Data.old_series[index].strip())
+        print("-> INFO Current mellanox sdk upstream patches. \n{}".format("\n".join(Data.old_patches)))
+
+    def get_kernel_dir(self):
+        # Get the kernel dir name to get the patches
+        try:
+            (kernel, major, minor) = self.args.sonic_kernel_ver.split(".")
+        except Exception as e:
+            print("-> FATAL Kernel formatting error: " + str(e))
+            sys.exit(1)
+
+        Data.k_dir = os.path.join(KERNEL_BACKPORTS, "{}.{}.{}".format(kernel, major, minor))
+        if os.path.exists(os.path.join(self.args.patches, Data.k_dir)):
+            return 
+
+        # if the k_dir with actual minor doesn't exit, use the patches generated against kernel.major
+        Data.k_dir = os.path.join(KERNEL_BACKPORTS, "{}.{}.{}".format(kernel, major, "0"))
+        path_to_check = os.path.join(self.args.patches, Data.k_dir)
+        if os.path.exists(path_to_check):
+            return 
+        else:
+            print("-> FATAL Kernel dir with patches doesn't exist: {}".format(path_to_check))
+            sys.exit(1)
+
+    def process_patches(self):
+        print(Data.new_patches, Data.old_patches)
+        if set(Data.new_patches) == set(Data.old_patches):
+            return (False, [], [])
+        new_patches = [patch + "\n" for patch in Data.new_patches]
+        Data.new_series = Data.old_series[0:Data.i_sdk_start+1] + new_patches + Data.old_series[Data.i_sdk_end:]
+        return (True, Data.new_patches, Data.old_patches)
+
+    def process_update(self, patches_add, patches_del):
+        src_path = os.path.join(self.args.patches, Data.k_dir)
+        dst_path = os.path.join(self.args.build_root, SLK_PATCH_LOC)
+
+        for patch in patches_del:
+            print("-> Deleting patch: " + patch)
+            os.remove(os.path.join(dst_path, patch))
+
+        for patch in patches_add:
+            print("-> Adding patch: " + patch)
+            shutil.copy(os.path.join(src_path, patch), dst_path)
+
+        FileHandler.write_lines(os.path.join(self.args.build_root, SLK_SERIES), Data.new_series, True)
+        print("-> INFO Updated sonic-linux-kernel series file \n{}".format("".join(Data.new_series)))
+
+    def get_new_patches(self):
+        patches_path = os.path.join(self.args.patches, Data.k_dir)
+        Data.new_patches = FileHandler.read_dir(patches_path, "*.patch")
+        Data.new_patches.sort()
+
+    def perform(self):
+        self.read_data()
+        self.find_sdk_patches()
+        self.get_kernel_dir()
+        self.get_new_patches()
+        (update_required, patches_add, patches_del) = self.process_patches()
+        if update_required:
+            self.process_update(patches_add, patches_del)
+
+def create_parser():
+    # Create argument parser
+    parser = argparse.ArgumentParser()
+
+    # Optional arguments
+    parser.add_argument("--sonic_kernel_ver", type=str)
+    parser.add_argument("--patches", type=str)
+    parser.add_argument("--build_root", type=str)
+    return parser
+
+if __name__ == '__main__':
+    parser = create_parser()
+    action = SDKAction(parser.parse_args())
+    action.check()
+    action.perform()

platform/mellanox/integration-scripts/tests/test_sdkaction.py

@@ -0,0 +1,157 @@
+import sys
+from unittest import mock, TestCase
+from pyfakefs.fake_filesystem_unittest import Patcher
+sys.path.append('../')
+from sdk_kernel_patches import *
+
+MOCK_SLK_SERIES = """\
+# Trtnetlink-catch-EOPNOTSUPP-errors.patch
+
+# Mellanox patches for 5.10
+###-> mellanox_sdk-start
+###-> mellanox_sdk-end
+
+###-> mellanox_hw_mgmt-start
+0001-i2c-mlxcpld-Update-module-license.patch
+###-> mellanox_hw_mgmt-end
+"""
+
+MOCK_SLK_SERIES_1 = """\
+# Trtnetlink-catch-EOPNOTSUPP-errors.patch
+
+# Mellanox patches for 5.10
+###-> mellanox_sdk-start
+0001-psample-Encapsulate-packet-metadata-in-a-struct.patch
+###-> mellanox_sdk-end
+
+###-> mellanox_hw_mgmt-start
+0001-i2c-mlxcpld-Update-module-license.patch
+###-> mellanox_hw_mgmt-end
+"""
+
+MOCK_FINAL_SLK_SERIES = """\
+# Trtnetlink-catch-EOPNOTSUPP-errors.patch
+
+# Mellanox patches for 5.10
+###-> mellanox_sdk-start
+0001-psample-Encapsulate-packet-metadata-in-a-struct.patch
+0002-psample-new-attr-tc-tc-occ-latency.patch
+###-> mellanox_sdk-end
+
+###-> mellanox_hw_mgmt-start
+0001-i2c-mlxcpld-Update-module-license.patch
+###-> mellanox_hw_mgmt-end
+"""
+
+MOCK_SLK_VER = "5.10.104"
+
+def mock_sdk_args():
+    with mock.patch("sys.argv", ["sdk_kernel_patches.py",
+                                "--sonic_kernel_ver", MOCK_SLK_VER,
+                                "--patches", "/tmp",
+                                "--build_root", "/sonic"]):
+        parser = create_parser()
+        return parser.parse_args()
+
+
+def check_lists(exp, rec):
+    print(" ------- Expected ----------- ")
+    print("".join(exp))
+    print(" ------- Recieved ----------- ")
+    print("".join(rec))
+    if len(exp) != len(rec):
+        return False
+    for i in range(0, len(exp)):
+        if exp[i] != rec[i]:
+            return False
+    return True
+
+class TestSDKAction(TestCase):
+    def setUp(self):
+        self.action = SDKAction(mock_sdk_args())
+        self.action.check()
+
+    def tearDown(self):
+        Data.new_series = list()
+        Data.new_patches = list()
+        Data.old_series = list()
+        Data.old_patches = list()
+        Data.k_dir = ""
+        Data.i_sdk_start = -1
+        Data.i_sdk_end = -1
+
+    def test_get_kernel_dir(self):
+        with Patcher() as patcher:
+            dir_ = os.path.join(KERNEL_BACKPORTS, MOCK_SLK_VER)
+            patcher.fs.create_file(os.path.join("/tmp", dir_))
+            self.action.get_kernel_dir()
+            assert dir_ == Data.k_dir
+            assert "/tmp/kernel_backports/5.10.0" != Data.k_dir
+
+        with Patcher() as patcher:
+            dir_ = os.path.join(KERNEL_BACKPORTS, "5.10.0")
+            patcher.fs.create_file(os.path.join("/tmp", dir_))
+            self.action.get_kernel_dir()
+            assert dir_ == Data.k_dir
+            assert "/tmp/kernel_backports/5.10.140" != Data.k_dir
+
+    def test_find_sdk_patches(self):
+        Data.old_series = MOCK_SLK_SERIES.splitlines(True)
+        self.action.find_sdk_patches()
+        assert "mellanox_sdk-start" in Data.old_series[Data.i_sdk_start]
+        assert "mellanox_sdk-end" in Data.old_series[Data.i_sdk_end]
+        assert not Data.old_patches
+
+        Data.old_series = MOCK_SLK_SERIES_1.splitlines(True)
+        self.action.find_sdk_patches()
+        assert "mellanox_sdk-start" in Data.old_series[Data.i_sdk_start]
+        assert "mellanox_sdk-end" in Data.old_series[Data.i_sdk_end]
+        assert len(Data.old_patches) == 1
+        assert Data.old_patches[-1] == "0001-psample-Encapsulate-packet-metadata-in-a-struct.patch"
+
+    def test_get_new_patches(self):
+        with Patcher() as patcher:
+            root_dir = "/tmp/kernel_backports/5.10.0/"
+            patcher.fs.create_file(os.path.join(root_dir, "0001-psample-Encapsulate-packet-metadata-in-a-struct.patch"))
+            patcher.fs.create_file(os.path.join(root_dir, "0002-psample-new-attr-tc-tc-occ-latency.patch"))
+            self.action.get_kernel_dir()
+            self.action.get_new_patches()
+            assert len(Data.new_patches) == 2
+            assert Data.new_patches[0] == "0001-psample-Encapsulate-packet-metadata-in-a-struct.patch"
+            assert Data.new_patches[1] == "0002-psample-new-attr-tc-tc-occ-latency.patch"
+
+    def test_process_patches_1(self):
+        Data.old_series = MOCK_SLK_SERIES.splitlines(True)
+        with Patcher() as patcher:
+            root_dir = "/tmp/kernel_backports/5.10.0/"
+            patcher.fs.create_file(os.path.join(root_dir, "0001-psample-Encapsulate-packet-metadata-in-a-struct.patch"))
+            patcher.fs.create_file(os.path.join(root_dir, "0002-psample-new-attr-tc-tc-occ-latency.patch"))
+            self.action.find_sdk_patches()
+            self.action.get_kernel_dir()
+            self.action.get_new_patches()
+            assert not Data.old_patches
+            (update_required, patches_add, patches_del) = self.action.process_patches()
+            assert update_required == True
+            assert len(patches_add) == 2
+            assert patches_add[0] == "0001-psample-Encapsulate-packet-metadata-in-a-struct.patch"
+            assert patches_add[1] == "0002-psample-new-attr-tc-tc-occ-latency.patch"
+            assert check_lists(MOCK_FINAL_SLK_SERIES.splitlines(True), Data.new_series)
+            assert len(patches_del) == 0
+
+    def test_process_patches_2(self):
+        Data.old_series = MOCK_SLK_SERIES_1.splitlines(True)
+        with Patcher() as patcher:
+            root_dir = "/tmp/kernel_backports/5.10.0/"
+            patcher.fs.create_file(os.path.join(root_dir, "0001-psample-Encapsulate-packet-metadata-in-a-struct.patch"))
+            patcher.fs.create_file(os.path.join(root_dir, "0002-psample-new-attr-tc-tc-occ-latency.patch"))
+            self.action.find_sdk_patches()
+            self.action.get_kernel_dir()
+            self.action.get_new_patches()
+            (update_required, patches_add, patches_del) = self.action.process_patches()
+            assert update_required == True
+            assert len(patches_add) == 2
+            assert patches_add[0] == "0001-psample-Encapsulate-packet-metadata-in-a-struct.patch"
+            assert patches_add[1] == "0002-psample-new-attr-tc-tc-occ-latency.patch"
+            assert check_lists(MOCK_FINAL_SLK_SERIES.splitlines(True), Data.new_series)
+            assert len(patches_del) == 1
+            assert patches_del[0] == "0001-psample-Encapsulate-packet-metadata-in-a-struct.patch"

platform/mellanox/sdk.mk

@@ -17,6 +17,7 @@
 MLNX_SDK_VERSION = 4.5.4206
 MLNX_SDK_ISSU_VERSION = 101
 
+MLNX_SDK_DRIVERS_GITHUB_URL = https://github.com/Mellanox/Spectrum-SDK-Drivers
 MLNX_ASSETS_GITHUB_URL = https://github.com/Mellanox/Spectrum-SDK-Drivers-SONiC-Bins
 MLNX_SDK_ASSETS_RELEASE_TAG = sdk-$(MLNX_SDK_VERSION)-$(BLDENV)-$(CONFIGURED_ARCH)
 MLNX_SDK_ASSETS_URL = $(MLNX_ASSETS_GITHUB_URL)/releases/download/$(MLNX_SDK_ASSETS_RELEASE_TAG)