Modify reboot cause on Kernel Panic to give preference to SW cause (sonic-net#296)

Currently, if there's a Kernel Panic and LC reboots, random HW cause
is taking precedence over Kernel Panic SW cause. To overcome such
scenarios, give preference to SW reboot cause over HW reboot cause

Author: ymd-arista

scripts/determine-reboot-cause

@@ -45,6 +45,7 @@ REBOOT_CAUSE_UNKNOWN = "Unknown"
 REBOOT_CAUSE_NON_HARDWARE = "Non-Hardware"
 REBOOT_CAUSE_HARDWARE_OTHER = "Hardware - Other"
 REBOOT_CAUSE_HEARTBEAT_LOSS = "Heartbeat with the Supervisor card lost"
+REBOOT_CAUSE_KERNEL_PANIC = "Kernel Panic"
 
 # Global logger class instance
 sonic_logger = syslogger.SysLogger(SYSLOG_IDENTIFIER)
@@ -188,7 +189,7 @@ def determine_reboot_cause():
     software_reboot_cause = find_software_reboot_cause()
 
     # The main decision logic of the reboot cause:
-    # If software reboot cause is not heartbeat loss and there is a valid hardware reboot cause indicated by platform API,
+    # If software reboot cause is not Kernel Panic or heartbeat loss and there is a valid hardware reboot cause indicated by platform API,
     #    check the software reboot cause to add additional reboot cause.
     # If there is a reboot cause indicated by /proc/cmdline, and/or warmreboot/fastreboot/softreboot
     #   the software_reboot_cause which is the content of /hosts/reboot-cause/reboot-cause.txt
@@ -197,7 +198,9 @@ def determine_reboot_cause():
     #   the software_reboot_cause will be treated as the reboot cause if it's not unknown
     #   otherwise, the cmdline_reboot_cause will be treated as the reboot cause if it's not none
     # Else the software_reboot_cause will be treated as the reboot cause
-    if REBOOT_CAUSE_HEARTBEAT_LOSS not in software_reboot_cause and REBOOT_CAUSE_NON_HARDWARE not in hardware_reboot_cause:
+    if (REBOOT_CAUSE_KERNEL_PANIC not in software_reboot_cause and
+        REBOOT_CAUSE_HEARTBEAT_LOSS not in software_reboot_cause and
+        REBOOT_CAUSE_NON_HARDWARE not in hardware_reboot_cause):
         previous_reboot_cause = hardware_reboot_cause
         # Check if any software reboot was issued before this hardware reboot happened
         if software_reboot_cause is not REBOOT_CAUSE_UNKNOWN:

tests/determine-reboot-cause_test.py

@@ -72,6 +72,7 @@
 EXPECTED_FIND_FIRSTBOOT_VERSION = " (First boot of SONiC version 20191130.52)"
 EXPECTED_FIND_SOFTWARE_REBOOT_CAUSE_FIRSTBOOT = "Unknown (First boot of SONiC version 20191130.52)"
 EXPECTED_FIND_SOFTWARE_HEATBEAT_LOSS = "Heartbeat with the Supervisor card lost"
+EXPECTED_FIND_SOFTWARE_KERNEL_PANIC = "Kernel Panic [Time: Sun Mar 28 13:45:12 UTC 2021]"
 
 EXPECTED_WATCHDOG_REBOOT_CAUSE_DICT = {'comment': '', 'gen_time': '2020_10_22_03_15_08', 'cause': 'Watchdog', 'user': 'N/A', 'time': 'N/A'}
 EXPECTED_USER_REBOOT_CAUSE_DICT = {'comment': '', 'gen_time': '2020_10_22_03_14_07', 'cause': 'reboot', 'user': 'admin', 'time': 'Thu Oct 22 03:11:08 UTC 2020'}
@@ -209,6 +210,14 @@ def test_determine_reboot_cause_software_heartbeatloss_hardware_other(self):
                     assert previous_reboot_cause == EXPECTED_FIND_SOFTWARE_HEATBEAT_LOSS
                     assert additional_info == "N/A"
 
+    def test_determine_reboot_cause_software_kernelpanic_hardware_other(self):
+        with mock.patch("determine_reboot_cause.find_proc_cmdline_reboot_cause", return_value=EXPECTED_PARSE_WARMFAST_REBOOT_FROM_PROC_CMDLINE):
+            with mock.patch("determine_reboot_cause.find_software_reboot_cause", return_value=EXPECTED_FIND_SOFTWARE_KERNEL_PANIC):
+                with mock.patch("determine_reboot_cause.find_hardware_reboot_cause", return_value=EXPECTED_HARDWARE_REBOOT_CAUSE):
+                    previous_reboot_cause, additional_info = determine_reboot_cause.determine_reboot_cause()
+                    assert previous_reboot_cause == EXPECTED_FIND_SOFTWARE_KERNEL_PANIC
+                    assert additional_info == "N/A"
+
     @mock.patch('determine_reboot_cause.REBOOT_CAUSE_DIR', os.path.join(os.getcwd(), REBOOT_CAUSE_DIR))
     @mock.patch('determine_reboot_cause.REBOOT_CAUSE_HISTORY_DIR', os.path.join(os.getcwd(), 'host/reboot-cause/history/'))
     @mock.patch('determine_reboot_cause.PREVIOUS_REBOOT_CAUSE_FILE', os.path.join(os.getcwd(), 'host/reboot-cause/previous-reboot-cause.json'))