[warm upgrade] Catch the regression before it becomes a problem - mine and police the protocol convergence timings (sonic-net#23114)

This PR refactors and improves the control plane session recovery gating logic during SONiC image upgrades. The logic is now modular, data-driven, and privacy-compliant, with all thresholds managed in a dedicated JSON file. Debug and legacy code are removed, and a minimal dummy data example is provided for public documentation.

How did you do it?
Centralized all control plane session recovery gating logic in a helper file
Created a structured JSON file for thresholds, supporting multiple HwSKUs and version pairs
Removed all debug and legacy code
Added logging for unknown HwSKUs and missing thresholds
Provided a minimal dummy JSON for public PRs
How did you verify/test it?
Unit and integration tested on multiple HwSKUs and version pairs
Validated JSON structure and gating logic with both real and dummy data
Confirmed correct logging and fallback behavior for unknown/missing data
Test result: CI pipeline ID - 1066532

Signed-off-by: Ravali Yeluri (WIPRO LIMITED) <[email protected]>

Author: ravaliyel

tests/common/platform/controlplane_gating.py

@@ -0,0 +1,72 @@
+import os
+import re
+import json
+import logging
+
+
+def controlplane_gating(reboot_timing_dict):
+    THRESHOLDS_FILE = os.path.join(os.path.dirname(__file__), 'hwsku_session_thresholds.json')
+    LACP_WIGGLE_ROOM = 10.0  # seconds
+    BGP_WIGGLE_ROOM = 10.0   # seconds
+
+    def _get_float(d, key):
+        val = d.get(key)
+        try:
+            return float(val) if val is not None and str(val).strip() != "" else None
+        except (ValueError, TypeError):
+            return None
+
+    def _extract_version(val):
+        m = re.search(r"(\d{6})", str(val))
+        return m.group(1) if m else None
+
+    hwsku = reboot_timing_dict.get("HwSku")
+
+    try:
+        with open(THRESHOLDS_FILE, 'r') as f:
+            thresholds = json.load(f)
+    except Exception:
+        thresholds = {}
+
+    if hwsku not in thresholds:
+        logging.warning(
+            "HwSku=%s not found in thresholds file. Skipping controlplane gating.",
+            hwsku
+        )
+        return []
+
+    lacp_val = _get_float(reboot_timing_dict, "lacp_session_max_wait")
+    bgp_val = _get_float(reboot_timing_dict, "bgp")
+    base_version = _extract_version(reboot_timing_dict.get("BaseImage"))
+    target_version = _extract_version(reboot_timing_dict.get("TargetImage"))
+
+    try:
+        lacp_avg = float(thresholds[hwsku][base_version][target_version]["LACP"]["AVG"])
+    except Exception:
+        lacp_avg = None
+    try:
+        bgp_avg = float(thresholds[hwsku][base_version][target_version]["BGP"]["AVG"])
+    except Exception:
+        bgp_avg = None
+
+    if lacp_avg is None or bgp_avg is None:
+        logging.warning(
+            "No thresholds found for HwSku=%s, BaseImage=%s, TargetImage=%s. "
+            "Skipping controlplane gating.", hwsku, base_version, target_version
+        )
+        return []
+
+    checks = [
+        ("LACP", lacp_val, lacp_avg, LACP_WIGGLE_ROOM),
+        ("BGP", bgp_val, bgp_avg, BGP_WIGGLE_ROOM),
+    ]
+    gating_failures = []
+    for label, val, avg, wiggle in checks:
+        if val is not None and avg is not None and val > (avg + wiggle):
+            gating_failures.append(
+                f"{label} session recovery {val:.2f}s exceeded allowed threshold "
+                f"(AVG + wiggle room): {avg:.2f}s + {wiggle:.2f}s "
+                f"for {hwsku} {base_version}->{target_version}"
+            )
+            logging.info("Gating failure: %s", gating_failures[-1])
+    return gating_failures

tests/common/platform/device_utils.py

@@ -20,6 +20,7 @@
     OFFSET_ITEMS, TIME_SPAN_ITEMS, REQUIRED_PATTERNS
 from tests.common.devices.duthosts import DutHosts
 from tests.common.plugins.ansible_fixtures import ansible_adhoc  # noqa: F401
+from tests.common.platform.controlplane_gating import controlplane_gating
 
 """
 Helper script for fanout switch operations
@@ -1023,6 +1024,17 @@ def post_reboot_analysis(marker, event_counters=None, reboot_oper=None, log_dir=
             report_file_name = request.node.name + "_report.json"
             summary_file_name = request.node.name + "_summary.json"
 
+        # Prepare minimal dict for control plane gating logic
+        gating_input = {
+            "lacp_session_max_wait": result_summary.get("controlplane", {}).get("lacp_session_max_wait"),
+            "bgp": result_summary.get("time_span", {}).get("bgp"),
+            "HwSku": result_summary.get("hwsku"),
+            "BaseImage": result_summary.get("base_ver"),
+            "TargetImage": result_summary.get("target_ver")
+        }
+        # Run control plane gating
+        gating_failures = controlplane_gating(gating_input)
+
         report_file_dir = os.path.realpath((os.path.join(os.path.dirname(__file__),
                                            "../../logs/platform_tests/")))
         report_file_path = report_file_dir + "/" + report_file_name
@@ -1036,6 +1048,9 @@ def post_reboot_analysis(marker, event_counters=None, reboot_oper=None, log_dir=
 
         # After generating timing data report, do some checks on the timing data
         verification_errors = list()
+        # Append the gating failures
+        if gating_failures:
+            verification_errors.extend(gating_failures)
         verify_mac_jumping(test_name, analyze_result, verification_errors)
         if duthost.facts['platform'] != 'x86_64-kvm_x86_64-r0':
             # TBD: expand this verification to KVM - extra port events in KVM which need to be filtered

tests/common/platform/hwsku_session_thresholds.json

@@ -0,0 +1,24 @@
+{
+  "Example-HwSKU-Model-A": {
+    "202405": {
+      "202411": {
+        "LACP": {"AVG": 210.0, "P95": 225.0, "MAX": 240.0},
+        "BGP": {"AVG": 205.0, "P95": 220.0, "MAX": 235.0}
+      }
+    },
+    "202411": {
+      "202505": {
+        "LACP": {"AVG": 215.0, "P95": 230.0, "MAX": 245.0},
+        "BGP": {"AVG": 208.0, "P95": 222.0, "MAX": 238.0}
+      }
+    }
+  },
+  "Example-HwSKU-Model-B": {
+    "202405": {
+      "202411": {
+        "LACP": {"AVG": 200.0, "P95": 218.0, "MAX": 232.0},
+        "BGP": {"AVG": 203.0, "P95": 215.0, "MAX": 228.0}
+      }
+    }
+  }
+}