[warm upgrade] Catch the regression before it becomes a problem - mine and police the protocol convergence timings

tests/common/platform/controlplane_gating.py

@@ -0,0 +1,72 @@
+import os
+import re
+import json
+import logging
+
+
+def controlplane_gating(reboot_timing_dict):
+    THRESHOLDS_FILE = os.path.join(os.path.dirname(__file__), 'hwsku_session_thresholds.json')
+    LACP_WIGGLE_ROOM = 10.0  # seconds
+    BGP_WIGGLE_ROOM = 10.0   # seconds
+
+    def _get_float(d, key):
+        val = d.get(key)
+        try:
+            return float(val) if val is not None and str(val).strip() != "" else None
+        except (ValueError, TypeError):
+            return None
+
+    def _extract_version(val):
+        m = re.search(r"(\d{6})", str(val))
+        return m.group(1) if m else None
+
+    hwsku = reboot_timing_dict.get("HwSku")
+
+    try:
+        with open(THRESHOLDS_FILE, 'r') as f:
+            thresholds = json.load(f)
+    except Exception:
+        thresholds = {}
+
+    if hwsku not in thresholds:
+        logging.warning(
+            "HwSku=%s not found in thresholds file. Skipping controlplane gating.",
+            hwsku
+        )
+        return []
+
+    lacp_val = _get_float(reboot_timing_dict, "lacp_session_max_wait")
+    bgp_val = _get_float(reboot_timing_dict, "bgp")
+    base_version = _extract_version(reboot_timing_dict.get("BaseImage"))
+    target_version = _extract_version(reboot_timing_dict.get("TargetImage"))
+
+    try:
+        lacp_avg = float(thresholds[hwsku][base_version][target_version]["LACP"]["AVG"])
+    except Exception:
+        lacp_avg = None
+    try:
+        bgp_avg = float(thresholds[hwsku][base_version][target_version]["BGP"]["AVG"])
+    except Exception:
+        bgp_avg = None
+
+    if lacp_avg is None or bgp_avg is None:
+        logging.warning(
+            "No thresholds found for HwSku=%s, BaseImage=%s, TargetImage=%s. "
+            "Skipping controlplane gating.", hwsku, base_version, target_version
+        )
+        return []
+
+    checks = [
+        ("LACP", lacp_val, lacp_avg, LACP_WIGGLE_ROOM),
+        ("BGP", bgp_val, bgp_avg, BGP_WIGGLE_ROOM),
+    ]
+    gating_failures = []
+    for label, val, avg, wiggle in checks:
+        if val is not None and avg is not None and val > (avg + wiggle):
+            gating_failures.append(
+                f"{label} session recovery {val:.2f}s exceeded allowed threshold "
+                f"(AVG + wiggle room): {avg:.2f}s + {wiggle:.2f}s "
+                f"for {hwsku} {base_version}->{target_version}"
+            )
+            logging.info("Gating failure: %s", gating_failures[-1])
+    return gating_failures

tests/common/platform/device_utils.py

@@ -20,6 +20,7 @@
     OFFSET_ITEMS, TIME_SPAN_ITEMS, REQUIRED_PATTERNS
 from tests.common.devices.duthosts import DutHosts
 from tests.common.plugins.ansible_fixtures import ansible_adhoc  # noqa: F401
+from tests.common.platform.controlplane_gating import controlplane_gating
 
 """
 Helper script for fanout switch operations
@@ -1023,6 +1024,17 @@ def post_reboot_analysis(marker, event_counters=None, reboot_oper=None, log_dir=
             report_file_name = request.node.name + "_report.json"
             summary_file_name = request.node.name + "_summary.json"
 
+        # Prepare minimal dict for control plane gating logic
+        gating_input = {
+            "lacp_session_max_wait": result_summary.get("controlplane", {}).get("lacp_session_max_wait"),
+            "bgp": result_summary.get("time_span", {}).get("bgp"),
+            "HwSku": result_summary.get("hwsku"),
+            "BaseImage": result_summary.get("base_ver"),
+            "TargetImage": result_summary.get("target_ver")
+        }
+        # Run control plane gating
+        gating_failures = controlplane_gating(gating_input)
+
         report_file_dir = os.path.realpath((os.path.join(os.path.dirname(__file__),
                                            "../../logs/platform_tests/")))
         report_file_path = report_file_dir + "/" + report_file_name
@@ -1036,6 +1048,9 @@ def post_reboot_analysis(marker, event_counters=None, reboot_oper=None, log_dir=
 
         # After generating timing data report, do some checks on the timing data
         verification_errors = list()
+        # Append the gating failures
+        if gating_failures:
+            verification_errors.extend(gating_failures)
         verify_mac_jumping(test_name, analyze_result, verification_errors)
         if duthost.facts['platform'] != 'x86_64-kvm_x86_64-r0':
             # TBD: expand this verification to KVM - extra port events in KVM which need to be filtered

tests/common/platform/hwsku_session_thresholds.json

@@ -0,0 +1,24 @@
+{
+  "Example-HwSKU-Model-A": {
+    "202405": {
+      "202411": {
+        "LACP": {"AVG": 210.0, "P95": 225.0, "MAX": 240.0},
+        "BGP": {"AVG": 205.0, "P95": 220.0, "MAX": 235.0}
+      }
+    },
+    "202411": {
+      "202505": {
+        "LACP": {"AVG": 215.0, "P95": 230.0, "MAX": 245.0},
+        "BGP": {"AVG": 208.0, "P95": 222.0, "MAX": 238.0}
+      }
+    }
+  },
+  "Example-HwSKU-Model-B": {
+    "202405": {
+      "202411": {
+        "LACP": {"AVG": 200.0, "P95": 218.0, "MAX": 232.0},
+        "BGP": {"AVG": 203.0, "P95": 215.0, "MAX": 228.0}
+      }
+    }
+  }
+}