[neighorch] VOQ encap index change handling

Signed-off-by: vedganes <[email protected]>

For the remote neighbors in the VOQ systems when there is change in the
encap index (due to syncd restart), the previously programmed remote
neighs should be re-written with the new changed encap index. The
current voq system neigh handling api in neighorch only checks for the
change of mac address. Because of this the change in the encap index is
ignored. This causes mismatch of encap index in the neighbor records in
owner asic and the remote asics after syncd restart situations like
config reload or failure recovery. This results in packet forwarding
failures for the packets forwarded across fabric and egressing in
different asics than ingress asics. The encap index that was previously
allocated by SAI and synced to the CHASSIS_APP_DB may change for the
same neighbor when syncd is restarted after config reload or any failure
recovery situations. When syncd restarts, the SAI may undergo full
re-initialization and may loose the memory of previous encap index
allocation for the neighbors. During reprogramming of the neighbors they
may not get the same encap index that were allocated before restart.

This fix is to store the allocated encap index in the orchagent
(neighorch) and compare with received encap index if the neighbor entry
already exits. This comparison of encap index is new to voq chassis.
This is done in addition to checking for mac change. For any received
(synced) neigh (received from CHASSIS_APP_DB) if neigh entry already
exists and if there is change in encap  index, the current neigh entry
is removed and re-added. As of now, since current SAI does not support
change of encap index (i.e, set of operaton of encap index attribute for
neigh records) del and re-add is done.

Author: vedganes

orchagent/neighorch.cpp

@@ -783,16 +783,16 @@ bool NeighOrch::addNeighbor(const NeighborEntry &neighborEntry, const MacAddress
     MuxOrch* mux_orch = gDirectory.get<MuxOrch*>();
     bool hw_config = isHwConfigured(neighborEntry);
 
-    if (!hw_config && mux_orch->isNeighborActive(ip_address, macAddress, alias))
+    if (gMySwitchType == "voq")
     {
-        if (gMySwitchType == "voq")
+        if (!addVoqEncapIndex(alias, ip_address, neighbor_attrs))
         {
-            if (!addVoqEncapIndex(alias, ip_address, neighbor_attrs))
-            {
-                return false;
-            }
+            return false;
         }
+    }
 
+    if (!hw_config && mux_orch->isNeighborActive(ip_address, macAddress, alias))
+    {
         status = sai_neighbor_api->create_neighbor_entry(&neighbor_entry,
                                    (uint32_t)neighbor_attrs.size(), neighbor_attrs.data());
         if (status != SAI_STATUS_SUCCESS)
@@ -1230,8 +1230,30 @@ void NeighOrch::doVoqSystemNeighTask(Consumer &consumer)
             }
 
             if (m_syncdNeighbors.find(neighbor_entry) == m_syncdNeighbors.end() ||
-                    m_syncdNeighbors[neighbor_entry].mac != mac_address)
+                    m_syncdNeighbors[neighbor_entry].mac != mac_address ||
+                    m_syncdNeighbors[neighbor_entry].voq_encap_index != encap_index)
             {
+                // Handle encap index change. SAI does not support change of encap index for
+                // existing neighbors. Remove the neighbor but do not errase from consumer sync
+                // buffer. The next iteration will add the neighbor back with new encap index
+
+                if (m_syncdNeighbors.find(neighbor_entry) != m_syncdNeighbors.end() &&
+                    m_syncdNeighbors[neighbor_entry].voq_encap_index != encap_index)
+                {
+                    //Remove neigh from SAI
+                    if (removeNeighbor(neighbor_entry))
+                    {
+                        //neigh successfully deleted from SAI. Set STATE DB to signal to remove entries from kernel
+                        m_stateSystemNeighTable->del(state_key);
+                    }
+                    else
+                    {
+                        SWSS_LOG_ERROR("Failed to remove voq neighbor %s from SAI during encap index change", kfvKey(t).c_str());
+                    }
+                    it++;
+                    continue;
+                }
+
                 //Add neigh to SAI
                 if (addNeighbor(neighbor_entry, mac_address))
                 {
@@ -1467,6 +1489,27 @@ void NeighOrch::voqSyncAddNeigh(string &alias, IpAddress &ip_address, const MacA
     sai_attribute_t attr;
     sai_status_t status;
 
+    // Get the encap index and store it for handling change of
+    // encap index for remote neighbors synced via CHASSIS_APP_DB
+
+    attr.id = SAI_NEIGHBOR_ENTRY_ATTR_ENCAP_INDEX;
+
+    status = sai_neighbor_api->get_neighbor_entry_attribute(&neighbor_entry, 1, &attr);
+    if (status != SAI_STATUS_SUCCESS)
+    {
+        SWSS_LOG_ERROR("Failed to get neighbor attribute for %s on %s, rv:%d", ip_address.to_string().c_str(), alias.c_str(), status);
+        return;
+    }
+
+    if (!attr.value.u32)
+    {
+        SWSS_LOG_ERROR("Invalid neighbor encap_index for %s on %s", ip_address.to_string().c_str(), alias.c_str());
+        return;
+    }
+
+    NeighborEntry nbrEntry = {ip_address, alias};
+    m_syncdNeighbors[nbrEntry].voq_encap_index = attr.value.u32;
+
     //Sync only local neigh. Confirm for the local neigh and
     //get the system port alias for key for syncing to CHASSIS_APP_DB
     Port port;
@@ -1495,21 +1538,6 @@ void NeighOrch::voqSyncAddNeigh(string &alias, IpAddress &ip_address, const MacA
         return;
     }
 
-    attr.id = SAI_NEIGHBOR_ENTRY_ATTR_ENCAP_INDEX;
-
-    status = sai_neighbor_api->get_neighbor_entry_attribute(&neighbor_entry, 1, &attr);
-    if (status != SAI_STATUS_SUCCESS)
-    {
-        SWSS_LOG_ERROR("Failed to get neighbor attribute for %s on %s, rv:%d", ip_address.to_string().c_str(), alias.c_str(), status);
-        return;
-    }
-
-    if (!attr.value.u32)
-    {
-        SWSS_LOG_ERROR("Invalid neighbor encap_index for %s on %s", ip_address.to_string().c_str(), alias.c_str());
-        return;
-    }
-
     vector<FieldValueTuple> attrs;
 
     FieldValueTuple eiFv ("encap_index", to_string(attr.value.u32));

orchagent/neighorch.h

@@ -27,6 +27,7 @@ struct NeighborData
 {
     MacAddress    mac;
     bool          hw_configured = false; // False means, entry is not written to HW
+    uint32_t      voq_encap_index = 0;
 };
 
 /* NeighborTable: NeighborEntry, neighbor MAC address */