[muxorch] handling multiple mux nexthops for route

orchagent/muxorch.cpp

@@ -163,6 +163,34 @@ static sai_status_t remove_route(IpPrefix &pfx)
     return status;
 }
 
+/**
+ * @brief sets the given route to point to the given nexthop
+ * @param pfx IpPrefix of the route
+ * @param nexthop NextHopKey of the nexthop
+ * @return SAI_STATUS_SUCCESS on success
+ */
+static sai_status_t set_route(const IpPrefix& pfx, sai_object_id_t next_hop_id)
+{
+    /* set route entry to point to nh */
+    sai_route_entry_t route_entry;
+    sai_attribute_t route_attr;
+
+    route_entry.vr_id = gVirtualRouterId;
+    route_entry.switch_id = gSwitchId;
+    copy(route_entry.destination, pfx);
+
+    route_attr.id = SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID;
+    route_attr.value.oid = next_hop_id;
+
+    sai_status_t status = sai_route_api->set_route_entry_attribute(&route_entry, &route_attr);
+    if (status != SAI_STATUS_SUCCESS)
+    {
+        SWSS_LOG_ERROR("Failed to set route entry %s nh %" PRIx64 " rv:%d",
+                pfx.to_string().c_str(), next_hop_id, status);
+    }
+    return status;
+}
+
 static sai_object_id_t create_tunnel(
     const IpAddress* p_dst_ip,
     const IpAddress* p_src_ip,
@@ -534,11 +562,13 @@ bool MuxCable::isIpInSubnet(IpAddress ip)
 
 bool MuxCable::nbrHandler(bool enable, bool update_rt)
 {
+    bool ret;
     SWSS_LOG_NOTICE("Processing neighbors for mux %s, enable %d, state %d",
                      mux_name_.c_str(), enable, state_);
     if (enable)
     {
-        return nbr_handler_->enable(update_rt);
+        ret = nbr_handler_->enable(update_rt);
+        updateRoutes();
     }
     else
     {
@@ -548,9 +578,10 @@ bool MuxCable::nbrHandler(bool enable, bool update_rt)
             SWSS_LOG_INFO("Null NH object id, retry for %s", peer_ip4_.to_string().c_str());
             return false;
         }
-
-        return nbr_handler_->disable(tnh);
+        updateRoutes();
+        ret = nbr_handler_->disable(tnh);
     }
+    return ret;
 }
 
 void MuxCable::updateNeighbor(NextHopKey nh, bool add)
@@ -567,6 +598,28 @@ void MuxCable::updateNeighbor(NextHopKey nh, bool add)
     {
         mux_orch_->removeNexthop(nh);
     }
+    updateRoutes();
+}
+
+/**
+ * @brief updates all routes pointing to the cables neighbor list
+ */
+void MuxCable::updateRoutes()
+{
+    MuxNeighbor neighbors = nbr_handler_->getNeighbors();
+    string alias = nbr_handler_->getAlias();
+    for (auto nh = neighbors.begin(); nh != neighbors.end(); nh ++)
+    {
+        std::set<RouteKey> routes;
+        NextHopKey nhkey(nh->first, alias);
+        if (gRouteOrch->getRoutesForNexthop(routes, nhkey))
+        {
+            for (auto rt = routes.begin(); rt != routes.end(); rt++)
+            {
+                mux_orch_->updateRoute(rt->prefix, true);
+            }
+        }
+    }
 }
 
 void MuxNbrHandler::update(NextHopKey nh, sai_object_id_t tunnelId, bool add, MuxState state)
@@ -983,6 +1036,111 @@ sai_object_id_t MuxOrch::getNextHopTunnelId(std::string tunnelKey, IpAddress& ip
     return it->second.nh_id;
 }
 
+/**
+ * @brief updates the given route to point to a single active NH or tunnel
+ * @param pfx IpPrefix of route to update
+ * @param remove bool only true when route is getting removed
+ */
+void MuxOrch::updateRoute(const IpPrefix &pfx, bool add)
+{
+    NextHopGroupKey nhg_key;
+    NextHopGroupEntry nhg_entry;
+
+    if (!add)
+    {
+        mux_multi_active_nh_table.erase(pfx);
+        return;
+    }
+
+    /* get nexthop group key from syncd */
+    nhg_key = gRouteOrch->getSyncdRouteNhgKey(gVirtualRouterId, pfx);
+
+    /* check for multi-nh neighbors.
+     * if none are present, ignore
+     */
+    if (nhg_key.getSize() <= 1)
+    {
+        return;
+    }
+
+    std::set<NextHopKey> nextHops;
+    sai_object_id_t next_hop_id;
+    sai_status_t status;
+    bool active_found = false;
+
+    /* get nexthops from nexthop group */
+    nextHops = nhg_key.getNextHops();
+
+    auto it = mux_multi_active_nh_table.find(pfx);
+    if (it != mux_multi_active_nh_table.end())
+    {
+        /* This will only work for configured MUX neighbors (most cases)
+         * TODO: add way to find MUX from neighbor
+         */
+        MuxCable* cable = findMuxCableInSubnet(it->second.ip_address);
+        auto standalone = standalone_tunnel_neighbors_.find(it->second.ip_address);
+
+        if ((cable == nullptr && standalone == standalone_tunnel_neighbors_.end()) ||
+             cable->isActive())
+        {
+            SWSS_LOG_INFO("Route %s pointing to active neighbor %s",
+                pfx.to_string().c_str(), it->second.to_string().c_str());
+            return;
+        }
+    }
+
+    SWSS_LOG_NOTICE("Updating route %s pointing to Mux nexthops %s",
+                pfx.to_string().c_str(), nhg_key.to_string().c_str());
+
+    for (auto it = nextHops.begin(); it != nextHops.end(); it++)
+    {
+        NextHopKey nexthop = *it;
+        /* This will only work for configured MUX neighbors (most cases)
+         * TODO: add way to find MUX from neighbor
+         */
+        MuxCable* cable = findMuxCableInSubnet(nexthop.ip_address);
+        auto standalone = standalone_tunnel_neighbors_.find(nexthop.ip_address);
+
+        if ((cable == nullptr && standalone == standalone_tunnel_neighbors_.end()) ||
+             cable->isActive())
+        {
+            /* Here we pull from local nexthop ID because neighbor update occurs during state change
+             * before nexthopID is updated in neighorch. This ensures that if a neighbor is Active
+             * only that neighbor's nexthop ID is added, and not the tunnel nexthop
+             */
+            next_hop_id = gNeighOrch->getLocalNextHopId(nexthop);
+            /* set route entry to point to nh */
+            status = set_route(pfx, next_hop_id);
+            if (status != SAI_STATUS_SUCCESS)
+            {
+                SWSS_LOG_ERROR("Failed to set route entry %s to nexthop %s",
+                        pfx.to_string().c_str(), nexthop.to_string().c_str());
+                continue;
+            }
+            SWSS_LOG_INFO("setting route %s with nexthop %s %" PRIx64 "",
+                pfx.to_string().c_str(), nexthop.to_string().c_str(), next_hop_id);
+            mux_multi_active_nh_table[pfx] = nexthop;
+            active_found = true;
+            break;
+        }
+    }
+
+    if (!active_found)
+    {
+        next_hop_id = getNextHopTunnelId(MUX_TUNNEL, mux_peer_switch_);
+        /* no active nexthop found, point to first */
+        SWSS_LOG_INFO("No Active neighbors found, setting route %s to point to tun",
+                    pfx.getIp().to_string().c_str());
+        status = set_route(pfx, next_hop_id);
+        if (status != SAI_STATUS_SUCCESS)
+        {
+            SWSS_LOG_ERROR("Failed to set route entry %s to tunnel",
+                    pfx.getIp().to_string().c_str());
+        }
+        mux_multi_active_nh_table.erase(pfx);
+    }
+}
+
 MuxCable* MuxOrch::findMuxCableInSubnet(IpAddress ip)
 {
     for (auto it = mux_cable_tb_.begin(); it != mux_cable_tb_.end(); it++)
@@ -1211,6 +1369,35 @@ void MuxOrch::removeNexthop(NextHopKey nh)
     mux_nexthop_tb_.erase(nh);
 }
 
+/**
+ * @brief checks if mux nexthop tb contains nexthop
+ * @param nexthop NextHopKey
+ * @return true if a mux contains the nexthop
+ */
+bool MuxOrch::containsNextHop(const NextHopKey& nexthop)
+{
+    return mux_nexthop_tb_.find(nexthop) != mux_nexthop_tb_.end();
+}
+
+/**
+ * @brief checks if a given nexthop group belongs to a mux
+ * @param nextHops NextHopGroupKey
+ * @return true if a mux contains any of the nexthops in the group
+ *         false if none of the nexthops belong to a mux
+ */
+bool MuxOrch::isMuxNexthops(const NextHopGroupKey& nextHops)
+{
+    const std::set<NextHopKey> s_nexthops = nextHops.getNextHops();
+    for (auto it = s_nexthops.begin(); it != s_nexthops.end(); it ++)
+    {
+        if (this->containsNextHop(*it))
+        {
+            return true;
+        }
+    }
+    return false;
+}
+
 string MuxOrch::getNexthopMuxName(NextHopKey nh)
 {
     if (mux_nexthop_tb_.find(nh) == mux_nexthop_tb_.end())

orchagent/muxorch.h

@@ -73,6 +73,8 @@ class MuxNbrHandler
     void update(NextHopKey nh, sai_object_id_t, bool = true, MuxState = MuxState::MUX_STATE_INIT);
 
     sai_object_id_t getNextHopId(const NextHopKey);
+    MuxNeighbor getNeighbors() const { return neighbors_; };
+    string getAlias() const { return alias_; };
 
 private:
     inline void updateTunnelRoute(NextHopKey, bool = true);
@@ -103,6 +105,7 @@ class MuxCable
 
     bool isIpInSubnet(IpAddress ip);
     void updateNeighbor(NextHopKey nh, bool add);
+    void updateRoutes();
     sai_object_id_t getNextHopId(const NextHopKey nh)
     {
         return nbr_handler_->getNextHopId(nh);
@@ -159,6 +162,7 @@ typedef std::unique_ptr<MuxCable> MuxCable_T;
 typedef std::map<std::string, MuxCable_T> MuxCableTb;
 typedef std::map<IpAddress, NHTunnel> MuxTunnelNHs;
 typedef std::map<NextHopKey, std::string> NextHopTb;
+typedef std::map<IpPrefix, NextHopKey> MuxRouteTb;
 
 class MuxCfgRequest : public Request
 {
@@ -190,19 +194,27 @@ class MuxOrch : public Orch2, public Observer, public Subject
         return (skip_neighbors_.find(nbr) != skip_neighbors_.end());
     }
 
+    bool isMultiNexthopRoute(const IpPrefix& pfx)
+    {
+        return (mux_multi_active_nh_table.find(pfx) != mux_multi_active_nh_table.end());
+    }
+
     MuxCable* findMuxCableInSubnet(IpAddress);
     bool isNeighborActive(const IpAddress&, const MacAddress&, string&);
     void update(SubjectType, void *);
 
     void addNexthop(NextHopKey, string = "");
     void removeNexthop(NextHopKey);
+    bool containsNextHop(const NextHopKey&);
+    bool isMuxNexthops(const NextHopGroupKey&);
     string getNexthopMuxName(NextHopKey);
     sai_object_id_t getNextHopId(const NextHopKey&);
 
     sai_object_id_t createNextHopTunnel(std::string tunnelKey, IpAddress& ipAddr);
     bool removeNextHopTunnel(std::string tunnelKey, IpAddress& ipAddr);
     sai_object_id_t getNextHopTunnelId(std::string tunnelKey, IpAddress& ipAddr);
 
+    void updateRoute(const IpPrefix &pfx, bool add);
     bool isStandaloneTunnelRouteInstalled(const IpAddress& neighborIp);
 
 private:
@@ -244,6 +256,9 @@ class MuxOrch : public Orch2, public Observer, public Subject
     MuxTunnelNHs mux_tunnel_nh_;
     NextHopTb mux_nexthop_tb_;
 
+    /* contains reference of programmed routes by updateRoute */
+    MuxRouteTb mux_multi_active_nh_table;
+
     handler_map handler_map_;
 
     TunnelDecapOrch *decap_orch_;