[ssw][ha] vnetorch supporting DPU live re-pairing (sonic-net#4042)

What I did
Support route update from vnet tunnel route config change for custom_bfd monitoring. In DPU repairing scenario (to form HA sets), VNetOrch needs to support live update of endpoints, monitoring endpoints, primary and secondary groups.

sign-off: Jing Zhang [email protected]

Why I did it
This change is required for DPU repair in HA.

How I did it
When there is a route config change (i.e. primary endpoint changes), pass a BFD state update with Init state to trigger the monitoring session creation & deletion, and route update. Notice that monitoring orch will pass Down and Up state to vnet orch.

Author: zjswhhh

orchagent/vnetorch.cpp

@@ -1052,13 +1052,21 @@ bool VNetRouteOrch::selectNextHopGroup(const string& vnet,
         }
         else
         {
-            if (it_route->second.primary != nexthops_primary)
+            if (isCustomMonitorEndpointUpdated(vnet, ipPrefix, monitors))
             {
                 setEndpointMonitor(vnet, monitors, nexthops_primary, monitoring, rx_monitor_timer, tx_monitor_timer, ipPrefix);
+                setEndpointMonitor(vnet, monitors, nexthops_secondary, monitoring, rx_monitor_timer, tx_monitor_timer, ipPrefix);
             }
-            if (it_route->second.secondary != nexthops_secondary)
+            else
             {
-                setEndpointMonitor(vnet, monitors, nexthops_secondary, monitoring, rx_monitor_timer, tx_monitor_timer, ipPrefix);
+                if (it_route->second.primary != nexthops_primary)
+                {
+                    setEndpointMonitor(vnet, monitors, nexthops_primary, monitoring, rx_monitor_timer, tx_monitor_timer, ipPrefix);
+                }
+                if (it_route->second.secondary != nexthops_secondary)
+                {
+                    setEndpointMonitor(vnet, monitors, nexthops_secondary, monitoring, rx_monitor_timer, tx_monitor_timer, ipPrefix);
+                }
             }
             nexthops_selected = it_route->second.nhg_key;
             return true;
@@ -1173,6 +1181,16 @@ bool VNetRouteOrch::doRouteTask<VNetVrfObject>(const string& vnet, IpPrefix& ipP
 
     if (op == SET_COMMAND)
     {
+        bool custom_monitor_ep_updated = isCustomMonitorEndpointUpdated(vnet, ipPrefix, monitors);
+        std::map<NextHopKey, swss::IpAddress> origin_primary_monitors;
+        std::map<NextHopKey, swss::IpAddress> origin_secondary_monitors;
+        if (custom_monitor_ep_updated)
+        {
+            auto it_route =  syncd_tunnel_routes_[vnet].find(ipPrefix);
+            getCustomMonitors(vnet, ipPrefix, it_route->second.primary, origin_primary_monitors);
+            getCustomMonitors(vnet, ipPrefix, it_route->second.secondary, origin_secondary_monitors);
+        }
+
         sai_object_id_t nh_id = SAI_NULL_OBJECT_ID;
         NextHopGroupKey active_nhg("", true);
         if (!selectNextHopGroup(vnet, nexthops, nexthops_secondary, monitoring, rx_monitor_timer, tx_monitor_timer, ipPrefix, vrf_obj, active_nhg, monitors))
@@ -1250,63 +1268,73 @@ bool VNetRouteOrch::doRouteTask<VNetVrfObject>(const string& vnet, IpPrefix& ipP
         }
         bool route_updated = false;
         bool priority_route_updated = false;
-        if (it_route != syncd_tunnel_routes_[vnet].end() &&
-            ((monitoring == "" && it_route->second.nhg_key != nexthops) ||
-            ((monitoring == VNET_MONITORING_TYPE_CUSTOM || monitoring == VNET_MONITORING_TYPE_CUSTOM_BFD) && (it_route->second.primary != nexthops || it_route->second.secondary != nexthops_secondary))))
-        {
-            route_updated = true;
-            NextHopGroupKey nhg = it_route->second.nhg_key;
-            if (monitoring == VNET_MONITORING_TYPE_CUSTOM || monitoring == VNET_MONITORING_TYPE_CUSTOM_BFD)
-            {
-                // if the previously active NHG is same as the newly created active NHG.case of primary secondary swap or
-                //when primary is active and secondary is changed or vice versa. In these cases we dont remove the NHG
-                // but only remove the monitors for the set which has changed.
-                if (it_route->second.primary != nexthops)
-                {
-                    delEndpointMonitor(vnet, it_route->second.primary, ipPrefix);
-                }
-                if (it_route->second.secondary != nexthops_secondary)
-                {
-                    delEndpointMonitor(vnet, it_route->second.secondary, ipPrefix);
-                }
-                if (monitor_info_[vnet][ipPrefix].empty())
-                {
-                    monitor_info_[vnet].erase(ipPrefix);
-                }
-                priority_route_updated = true;
+        if (it_route != syncd_tunnel_routes_[vnet].end())
+        {
+            if (custom_monitor_ep_updated)
+            {
+                route_updated = true;
+
+                delEndpointMonitor(vnet, origin_primary_monitors, ipPrefix);
+                delEndpointMonitor(vnet, origin_secondary_monitors, ipPrefix);
             }
-            else
+            else if ((monitoring == "" && it_route->second.nhg_key != nexthops) ||
+                ((monitoring == VNET_MONITORING_TYPE_CUSTOM || monitoring == VNET_MONITORING_TYPE_CUSTOM_BFD) &&
+                 (it_route->second.primary != nexthops || it_route->second.secondary != nexthops_secondary)))
             {
-                // In case of updating an existing route, decrease the reference count for the previous nexthop group
-                if (--syncd_nexthop_groups_[vnet][nhg].ref_count == 0)
+                route_updated = true;
+                NextHopGroupKey nhg = it_route->second.nhg_key;
+                if (monitoring == VNET_MONITORING_TYPE_CUSTOM || monitoring == VNET_MONITORING_TYPE_CUSTOM_BFD)
                 {
-                    if (nhg.getSize() > 1)
+                    // if the previously active NHG is same as the newly created active NHG.case of primary secondary swap or
+                    //when primary is active and secondary is changed or vice versa. In these cases we dont remove the NHG
+                    // but only remove the monitors for the set which has changed.
+                    if (it_route->second.primary != nexthops)
                     {
-                        removeNextHopGroup(vnet, nhg, vrf_obj);
+                        delEndpointMonitor(vnet, it_route->second.primary, ipPrefix);
                     }
-                    else
+                    if (it_route->second.secondary != nexthops_secondary)
+                    {
+                        delEndpointMonitor(vnet, it_route->second.secondary, ipPrefix);
+                    }
+                    if (monitor_info_[vnet][ipPrefix].empty())
                     {
-                        syncd_nexthop_groups_[vnet].erase(nhg);
-                        if(nhg.getSize() == 1)
+                        monitor_info_[vnet].erase(ipPrefix);
+                    }
+                    priority_route_updated = true;
+                }
+                else
+                {
+                    // In case of updating an existing route, decrease the reference count for the previous nexthop group
+                    if (--syncd_nexthop_groups_[vnet][nhg].ref_count == 0)
+                    {
+                        if (nhg.getSize() > 1)
                         {
-                            NextHopKey nexthop = *nhg.getNextHops().begin();
-                            if (!isLocalEndpoint(vnet, nexthop.ip_address))
+                            removeNextHopGroup(vnet, nhg, vrf_obj);
+                        }
+                        else
+                        {
+                            syncd_nexthop_groups_[vnet].erase(nhg);
+                            if(nhg.getSize() == 1)
                             {
-                                vrf_obj->removeTunnelNextHop(nexthop);
+                                NextHopKey nexthop = *nhg.getNextHops().begin();
+                                if (!isLocalEndpoint(vnet, nexthop.ip_address))
+                                {
+                                    vrf_obj->removeTunnelNextHop(nexthop);
+                                }
                             }
                         }
+                        if (monitoring != VNET_MONITORING_TYPE_CUSTOM && monitoring != VNET_MONITORING_TYPE_CUSTOM_BFD)
+                        {
+                            delEndpointMonitor(vnet, nhg, ipPrefix);
+                        }
                     }
-                    if (monitoring != VNET_MONITORING_TYPE_CUSTOM && monitoring != VNET_MONITORING_TYPE_CUSTOM_BFD)
+                    else
                     {
-                        delEndpointMonitor(vnet, nhg, ipPrefix);
+                        syncd_nexthop_groups_[vnet][nhg].tunnel_routes.erase(ipPrefix);
                     }
+                    vrf_obj->removeRoute(ipPrefix);
+                    vrf_obj->removeProfile(ipPrefix);
                 }
-                else
-                {
-                    syncd_nexthop_groups_[vnet][nhg].tunnel_routes.erase(ipPrefix);
-                }
-                vrf_obj->removeRoute(ipPrefix);
-                vrf_obj->removeProfile(ipPrefix);
             }
         }
         if (!profile.empty())
@@ -1323,11 +1351,13 @@ bool VNetRouteOrch::doRouteTask<VNetVrfObject>(const string& vnet, IpPrefix& ipP
             syncd_tunnel_routes_[vnet][ipPrefix] = tunnel_route_entry;
             syncd_nexthop_groups_[vnet][active_nhg].ref_count++;
 
-            if (priority_route_updated)
+            if (priority_route_updated || custom_monitor_ep_updated)
             {
                 MonitorUpdate update;
+                update.monitoring_type = monitoring;
                 update.prefix = ipPrefix;
                 update.state = MONITOR_SESSION_STATE_UNKNOWN;
+                update.custom_bfd_state = SAI_BFD_SESSION_STATE_INIT;
                 update.vnet = vnet;
                 updateVnetTunnelCustomMonitor(update);
                 return true;
@@ -2203,6 +2233,11 @@ void VNetRouteOrch::removeMonitoringSession(const string& vnet, const NextHopKey
     }
 
     monitor_info_[vnet][ipPrefix].erase(monitor_addr);
+
+    if (monitor_info_[vnet][ipPrefix].empty())
+    {
+        monitor_info_[vnet].erase(ipPrefix);
+    }
 }
 
 void VNetRouteOrch::setEndpointMonitor(const string& vnet, const map<NextHopKey, IpAddress>& monitors, NextHopGroupKey& nexthops, const string& monitoring, const int32_t rx_monitor_timer, const int32_t tx_monitor_timer, IpPrefix& ipPrefix)
@@ -2299,6 +2334,43 @@ void VNetRouteOrch::delEndpointMonitor(const string& vnet, NextHopGroupKey& next
     }
 }
 
+void VNetRouteOrch::delEndpointMonitor(const string& vnet, const std::map<NextHopKey, IpAddress>& monitors, IpPrefix& ipPrefix)
+{
+    SWSS_LOG_ENTER();
+
+    bool is_custom_monitoring = false;
+    if (monitor_info_[vnet].find(ipPrefix) != monitor_info_[vnet].end())
+    {
+        is_custom_monitoring = true;
+    }
+    for(auto monitor : monitors)
+    {
+        NextHopKey nhk = monitor.first;
+        IpAddress monitor_ip = monitor.second;
+        if (is_custom_monitoring)
+        {
+            if (monitor_info_[vnet][ipPrefix].find(monitor_ip) != monitor_info_[vnet][ipPrefix].end() &&
+                monitor_info_[vnet][ipPrefix][monitor_ip].endpoint == nhk)
+            {
+                if (--monitor_info_[vnet][ipPrefix][monitor_ip].ref_count == 0)
+                {
+                    removeMonitoringSession(vnet, monitor_info_[vnet][ipPrefix][monitor_ip].endpoint, monitor_ip, ipPrefix);
+                }
+            }
+        }
+        else
+        {
+            if (nexthop_info_[vnet].find(nhk.ip_address) != nexthop_info_[vnet].end())
+            {
+                if (--nexthop_info_[vnet][nhk.ip_address].ref_count == 0)
+                {
+                    removeBfdSession(vnet, nhk, monitor_ip);
+                }
+            }
+        }
+    }
+}
+
 void VNetRouteOrch::updateCustomBfdState(const IpAddress& monitoring_ip, const string& state)
 {
     SWSS_LOG_ENTER();
@@ -2321,7 +2393,8 @@ void VNetRouteOrch::updateCustomBfdState(const IpAddress& monitoring_ip, const s
     }
     else
     {
-        SWSS_LOG_WARN("Unknown BFD state: %s", state.c_str());
+        /* VNetRouteOrch only processes Up/Down states from MonitorOrch */
+        SWSS_LOG_DEBUG("Ignoring BFD state: %s", state.c_str());
         return;
     }
 
@@ -2812,7 +2885,15 @@ void VNetRouteOrch::updateVnetTunnelCustomMonitor(const MonitorUpdate& update)
 
     if(monitoring_type == VNET_MONITORING_TYPE_CUSTOM_BFD)
     {
-        monitor_info_[vnet][prefix][monitor].custom_bfd_state = custom_bfd_state;
+        if (custom_bfd_state == SAI_BFD_SESSION_STATE_INIT)
+        {
+            updateRoute = true;
+            config_update = true;
+        }
+        else
+        {
+            monitor_info_[vnet][prefix][monitor].custom_bfd_state = custom_bfd_state;
+        }
     }
 
     auto route = syncd_tunnel_routes_[vnet].find(prefix);
@@ -3329,6 +3410,63 @@ bool VNetRouteOrch::isPartiallyLocal(const std::vector<swss::IpAddress>& ip_list
     return !(all_true || all_false);
 }
 
+bool VNetRouteOrch::isCustomMonitorEndpointUpdated(const string& vnet, IpPrefix& ipPrefix,
+                                             const std::map<NextHopKey, swss::IpAddress>& monitors)
+{
+    SWSS_LOG_ENTER();
+
+    if (monitor_info_.find(vnet) == monitor_info_.end() ||
+    monitor_info_[vnet].find(ipPrefix) == monitor_info_[vnet].end())
+    {
+        return false;
+    }
+
+    // If the nh endpoint exists in the monitor_info map but the monitor ip is different, then return true.
+    // Note that the key of monitor_info is monitoring endpoint ip address, while monitors is nexthop key
+    // to monitor endpoint ip mapping. 
+    bool monitor_endpoint_updated = false;
+    for (auto it = monitor_info_[vnet][ipPrefix].begin(); it != monitor_info_[vnet][ipPrefix].end(); it++)
+    {
+        if (it->second.monitoring_type != VNET_MONITORING_TYPE_CUSTOM_BFD)
+        {
+            continue;
+        }
+
+        if (monitors.find(it->second.endpoint) != monitors.end() &&
+            it->first != monitors.at(it->second.endpoint))
+        {
+            monitor_endpoint_updated = true;
+
+            SWSS_LOG_NOTICE("Custom monitor endpoint updated for vnet %s, prefix %s, old monitor %s, new monitor %s",
+                          vnet.c_str(), ipPrefix.to_string().c_str(),
+                          it->first.to_string().c_str(),
+                          monitors.at(it->second.endpoint).to_string().c_str());
+
+        }
+    }
+
+    return monitor_endpoint_updated;
+}
+
+void VNetRouteOrch::getCustomMonitors(const string& vnet, const IpPrefix& ipPrefix, const NextHopGroupKey& nexthops, 
+    std::map<NextHopKey, swss::IpAddress>& monitors)
+{
+    if (monitor_info_.find(vnet) != monitor_info_.end() && monitor_info_[vnet].find(ipPrefix) != monitor_info_[vnet].end())
+    {
+        for (auto nhk:nexthops.getNextHops())
+        {
+            for (const auto& monitor : monitor_info_[vnet][ipPrefix])
+            {
+                if (monitor.second.endpoint == nhk && (monitor.second.monitoring_type == VNET_MONITORING_TYPE_CUSTOM ||
+                    monitor.second.monitoring_type == VNET_MONITORING_TYPE_CUSTOM_BFD))
+                {
+                    monitors[nhk] = monitor.first;
+                    break;
+                }
+            }
+        }
+    }
+}
 
 VNetCfgRouteOrch::VNetCfgRouteOrch(DBConnector *db, DBConnector *appDb, vector<string> &tableNames)
                                   : Orch(db, tableNames),

orchagent/vnetorch.h

@@ -540,6 +540,12 @@ class VNetRouteOrch : public Orch2, public Subject, public Observer
                             const string& monitoring, const int32_t rx_monitor_timer, const int32_t tx_monitor_timer,
                             IpPrefix& ipPrefix);
     void delEndpointMonitor(const string& vnet, NextHopGroupKey& nexthops, IpPrefix& ipPrefix);
+    void delEndpointMonitor(const string& vnet, const std::map<NextHopKey, IpAddress>& monitors, IpPrefix& ipPrefix);
+
+    bool isCustomMonitorEndpointUpdated(const string& vnet, IpPrefix& ipPrefix,
+                                  const std::map<NextHopKey, IpAddress>& monitors);
+    void getCustomMonitors(const string& vnet, const IpPrefix& ipPrefix, const NextHopGroupKey& nexthops, std::map<NextHopKey, IpAddress>& monitors);
+
     void postRouteState(const string& vnet, IpPrefix& ipPrefix, NextHopGroupKey& nexthops, string& profile);
     void removeRouteState(const string& vnet, IpPrefix& ipPrefix);
     void addRouteAdvertisement(IpPrefix& ipPrefix, string& profile);