Add check_pfc_storm_active() to fast-reboot script (sonic-net#3969)

Author: hdwhdw

pfcwd/main.py

@@ -163,9 +163,15 @@ def collect_stats(self, empty, queues):
 
         self.table += table
 
-    def show_stats(self, empty, queues):
+    def show_stats(self, empty, queues, check_storm=False):
         del self.table[:]
         self.collect_stats(empty, queues)
+
+        if check_storm:
+            # Check for storms and exit accordingly - no output needed
+            storms_detected = any(row[1] == 'stormed' for row in self.table if len(row) > 1)
+            sys.exit(1 if storms_detected else 0)
+
         click.echo(tabulate(
             self.table, STATS_HEADER, stralign='right', numalign='right',
             tablefmt='simple'
@@ -468,13 +474,14 @@ def show():
     @show.command()
     @multi_asic_util.multi_asic_click_options
     @click.option('-e', '--empty', is_flag=True)
+    @click.option('--check-storm', is_flag=True, help='Exit 1 if any storms detected, 0 otherwise')
     @click.argument('queues', nargs=-1)
     @clicommon.pass_db
-    def stats(db, namespace, display, empty, queues):
+    def stats(db, namespace, display, empty, check_storm, queues):
         """ Show PFC Watchdog stats per queue """
         if (len(queues)):
             display = constants.DISPLAY_ALL
-        PfcwdCli(db, namespace, display).show_stats(empty, queues)
+        PfcwdCli(db, namespace, display).show_stats(empty, queues, check_storm)
 
     # Show config
     @show.command()

scripts/fast-reboot

@@ -51,6 +51,7 @@ EXIT_SONIC_INSTALLER_VERIFY_REBOOT=21
 EXIT_PLATFORM_FW_AU_FAILURE=22
 EXIT_TEAMD_RETRY_COUNT_FAILURE=23
 EXIT_NO_MIRROR_SESSION_ACLS=24
+EXIT_PFC_STORM_DETECTED=25
 EXIT_LEFTOVER_CPA_TUNNEL=30
 
 function error()
@@ -514,6 +515,18 @@ function check_docker_exec()
     done
 }
 
+function check_pfc_storm_active()
+{
+    debug "Checking for active PFC storms..."
+    
+    if pfcwd show stats --check-storm >/dev/null 2>&1; then
+        debug "No active PFC storms detected. Safe to proceed with warm-reboot..."
+    else
+        error "PFC storm detected. Aborting warm-reboot to prevent failure in recovery path..."
+        exit ${EXIT_PFC_STORM_DETECTED}
+    fi
+}
+
 function check_db_integrity()
 {
     if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" || "$REBOOT_TYPE" = "express-reboot" || "$REBOOT_TYPE" = "fast-reboot" ]]; then
@@ -533,6 +546,7 @@ function check_db_integrity()
 
 function reboot_pre_check()
 {
+    check_pfc_storm_active
     check_docker_exec
     # Make sure that the file system is normal: read-write able
     filename="/host/test-$(date +%Y%m%d-%H%M%S)"

tests/pfcwd_test.py

@@ -406,6 +406,114 @@ def test_pfcwd_enable_history_ports_invalid(self):
         # same as original config
         assert result.output == test_vectors.pfcwd_show_config_output
 
+    def test_pfcwd_show_stats_check_storm_no_storms(self):
+        """ Test --check-storm flag when no storms are present """
+        import pfcwd.main as pfcwd
+        from unittest.mock import patch
+        runner = CliRunner()
+        db = Db()
+
+        # Mock the collect_stats method to simulate no storms
+        def mock_collect_stats_no_storm(self, empty, queues, storm_only=False):
+            # Create fake table data with only operational queues
+            self.table = [
+                ['Ethernet0:3', 'operational', '2/2', '100/100', '100/100', '0/0', '0/0'],
+                ['Ethernet4:3', 'operational', '3/3', '150/150', '150/150', '0/0', '0/0'],
+                ['Ethernet8:4', 'operational', '1/1', '50/50', '50/50', '0/0', '0/0']
+            ]
+
+        with patch.object(pfcwd.PfcwdCli, 'collect_stats', mock_collect_stats_no_storm):
+            # Test with no storms - should exit 0
+            result = runner.invoke(
+                pfcwd.cli.commands["show"].commands["stats"],
+                ["--check-storm"],
+                obj=db
+            )
+            print("No storms test - exit code:", result.exit_code)
+            print("No storms test - output:", result.output)
+            assert result.exit_code == 0
+            assert result.output == ""  # Should be silent when checking storms
+
+    def test_pfcwd_show_stats_check_storm_with_storms(self):
+        """ Test --check-storm flag when storms are present """
+        import pfcwd.main as pfcwd
+        from unittest.mock import patch
+        runner = CliRunner()
+        db = Db()
+
+        # Mock the collect_stats method to simulate storm detection
+        def mock_collect_stats_with_storm(self, empty, queues, storm_only=False):
+            # Create fake table data with a stormed queue
+            self.table = [
+                ['Ethernet0:3', 'stormed', '1/0', '100/300', '100/300', '0/200', '0/200']
+            ]
+
+        with patch.object(pfcwd.PfcwdCli, 'collect_stats', mock_collect_stats_with_storm):
+            result = runner.invoke(
+                pfcwd.cli.commands["show"].commands["stats"],
+                ["--check-storm"],
+                obj=db
+            )
+            print("With storms test - exit code:", result.exit_code)
+            print("With storms test - output:", result.output)
+            assert result.exit_code == 1
+            assert result.output == ""  # Should be silent when checking storms
+
+    def test_pfcwd_show_stats_check_storm_mixed_status(self):
+        """ Test --check-storm flag with mixed operational and stormed queues """
+        import pfcwd.main as pfcwd
+        from unittest.mock import patch
+        runner = CliRunner()
+        db = Db()
+
+        # Mock the collect_stats method to simulate mixed queue states
+        def mock_collect_stats_mixed(self, empty, queues, storm_only=False):
+            # Create fake table data with mixed states - should still exit 1 if ANY storm detected
+            self.table = [
+                ['Ethernet0:3', 'operational', '2/2', '100/100', '100/100', '0/0', '0/0'],
+                ['Ethernet4:3', 'stormed', '1/0', '100/300', '100/300', '0/200', '0/200'],
+                ['Ethernet8:4', 'operational', '0/0', '50/50', '50/50', '0/0', '0/0']
+            ]
+
+        with patch.object(pfcwd.PfcwdCli, 'collect_stats', mock_collect_stats_mixed):
+            result = runner.invoke(
+                pfcwd.cli.commands["show"].commands["stats"],
+                ["--check-storm"],
+                obj=db
+            )
+            print("Mixed states test - exit code:", result.exit_code)
+            print("Mixed states test - output:", result.output)
+            assert result.exit_code == 1  # Should exit 1 if ANY storms detected
+            assert result.output == ""
+
+    def test_pfcwd_show_stats_normal_output_unchanged(self):
+        """ Test that normal stats output is unchanged when not using --check-storm """
+        import pfcwd.main as pfcwd
+        from unittest.mock import patch
+        runner = CliRunner()
+        db = Db()
+
+        # Mock the collect_stats method to ensure consistent test output
+        def mock_collect_stats_normal(self, empty, queues, storm_only=False):
+            # Create fake table data with mixed states (like normal operation)
+            self.table = [
+                ['Ethernet0:3', 'operational', '2/2', '100/100', '100/100', '0/0', '0/0'],
+                ['Ethernet4:3', 'operational', '3/3', '150/150', '150/150', '0/0', '0/0']
+            ]
+
+        with patch.object(pfcwd.PfcwdCli, 'collect_stats', mock_collect_stats_normal):
+            # Test normal stats command (without --check-storm) - should work as before
+            result = runner.invoke(
+                pfcwd.cli.commands["show"].commands["stats"],
+                obj=db
+            )
+            print("Normal output test - exit code:", result.exit_code)
+            print("Normal output test - output length:", len(result.output))
+            assert result.exit_code == 0
+            # Should have normal tabulated output (not empty)
+            assert len(result.output) > 0
+            assert "QUEUE" in result.output  # Should contain table headers
+
     @classmethod
     def teardown_class(cls):
         os.environ["PATH"] = os.pathsep.join(os.environ["PATH"].split(os.pathsep)[:-1])