Orchestration changes for Error Handling Framework

- Receives notifications from syncd
- Process the notifications from syncd and logs the errors into error database
- Notify the status to registered applications

Author: sivamukka

orchagent/Makefile.am

@@ -53,7 +53,8 @@ orchagent_SOURCES = \
             flexcounterorch.cpp \
             watermarkorch.cpp \
             policerorch.cpp \
-            chassisorch.cpp
+            chassisorch.cpp \
+            errororch.cpp
 
 orchagent_CFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_SAI)
 orchagent_CPPFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_SAI)

orchagent/main.cpp

@@ -294,8 +294,11 @@ int main(int argc, char **argv)
     DBConnector appl_db(APPL_DB, DBConnector::DEFAULT_UNIXSOCKET, 0);
     DBConnector config_db(CONFIG_DB, DBConnector::DEFAULT_UNIXSOCKET, 0);
     DBConnector state_db(STATE_DB, DBConnector::DEFAULT_UNIXSOCKET, 0);
+    DBConnector counters_db(COUNTERS_DB, DBConnector::DEFAULT_UNIXSOCKET, 0);
+    DBConnector asic_db(ASIC_DB, DBConnector::DEFAULT_UNIXSOCKET, 0);
+    DBConnector error_db(ERROR_DB, DBConnector::DEFAULT_UNIXSOCKET, 0);
 
-    auto orchDaemon = make_shared<OrchDaemon>(&appl_db, &config_db, &state_db);
+    auto orchDaemon = make_shared<OrchDaemon>(&appl_db, &config_db, &state_db, &counters_db, &asic_db, &error_db);
 
     if (!orchDaemon->init())
     {

orchagent/neighorch.cpp

@@ -4,6 +4,13 @@
 #include "swssnet.h"
 #include "crmorch.h"
 #include "routeorch.h"
+#include "errororch.h"
+#include "tokenize.h"
+#include "sai_serialize.h"
+#include "redisclient.h"
+
+#include "swss/json.hpp"
+using json = nlohmann::json;
 
 extern sai_neighbor_api_t*         sai_neighbor_api;
 extern sai_next_hop_api_t*         sai_next_hop_api;
@@ -12,13 +19,22 @@ extern PortsOrch *gPortsOrch;
 extern sai_object_id_t gSwitchId;
 extern CrmOrch *gCrmOrch;
 extern RouteOrch *gRouteOrch;
+extern ErrorOrch *gErrorOrch;
+extern std::shared_ptr<swss::RedisClient>   g_redisClientAsicDb;
+extern std::shared_ptr<swss::RedisClient>   g_redisClientCountersDb;
 
 const int neighorch_pri = 30;
 
 NeighOrch::NeighOrch(DBConnector *db, string tableName, IntfsOrch *intfsOrch) :
         Orch(db, tableName, neighorch_pri), m_intfsOrch(intfsOrch)
 {
     SWSS_LOG_ENTER();
+
+    if(gErrorOrch->mappingHandlerRegister(APP_NEIGH_TABLE_NAME, this) == false)
+    {
+        SWSS_LOG_ERROR("Failed to register with Error Handling Framework for %s",
+                APP_NEIGH_TABLE_NAME);
+    }
 }
 
 bool NeighOrch::hasNextHop(const NextHopKey &nexthop)
@@ -581,3 +597,129 @@ bool NeighOrch::removeNeighbor(const NeighborEntry &neighborEntry)
 
     return true;
 }
+
+bool NeighOrch::mapToErrorDbFormat(sai_object_type_t& object_type, std::vector<FieldValueTuple> &asicValues,
+        std::vector<FieldValueTuple> &appValues)
+{
+    SWSS_LOG_ENTER();
+
+    if(object_type != SAI_OBJECT_TYPE_NEIGHBOR_ENTRY)
+    {
+        return false;
+    }
+
+    /*
+       127.0.0.1:6379> hgetall "NEIGH_TABLE:Ethernet0:2.2.2.2"
+       1) "neigh"
+       2) "00:00:3a:3e:9e:a7"
+       3) "family"
+       4) "IPv4"
+
+       127.0.0.1:6379[1]> hgetall "ASIC_STATE:SAI_OBJECT_TYPE_NEIGHBOR_ENTRY:{\"ip\":\"2.2.2.2\",\"rif\":\"oid:0x60000000006f3\",\"switch_id\":\"oid:0x21000000000000\"}"
+       1) "SAI_NEIGHBOR_ENTRY_ATTR_DST_MAC_ADDRESS"
+       2) "00:00:3A:3E:9E:A7"
+       127.0.0.1:6379[1]>
+
+       127.0.0.1:6379[2]> hgetall "COUNTERS_PORT_NAME_MAP"
+       25) "Ethernet0"
+       26) "oid:0x100000000000e"
+    */
+
+    const auto& values = asicValues;
+    std::string asicKV, strNbrIP, strRifOid, strMac;
+    std::string strIntfName, strRtrIntfType;
+    for (size_t i = 0; i < values.size(); i++)
+    {
+        if(fvField(values[i]) == "key")
+        {
+            /* Extract Neighbor IP and Router Interface ID from the "key" field */
+            asicKV = fvValue(values[i]);
+            auto tokens = tokenize(asicKV, ':', 1);
+            json j = json::parse(tokens[1]);
+            strNbrIP = j["ip"];
+            strRifOid = j["rif"];
+            SWSS_LOG_DEBUG("Neighbor IP is %s, router interface ID is %s",
+                    strNbrIP.c_str(), strRifOid.c_str());
+
+            /* Extract Port OID from Router Interface OID */
+            std::string strRtrIfKey = "ASIC_STATE:SAI_OBJECT_TYPE_ROUTER_INTERFACE:" + strRifOid;
+            std::string strIntfOid;
+            /* OID in neighbor entry points to ROUTER_INTERFACE
+             * Port based routing interface
+             * ASIC_STATE:SAI_OBJECT_TYPE_ROUTER_INTERFACE:oid:0x60000000006d9
+             *   SAI_ROUTER_INTERFACE_ATTR_TYPE = SAI_ROUTER_INTERFACE_TYPE_PORT
+             *   SAI_ROUTER_INTERFACE_ATTR_PORT_ID = oid:0x100000000000e
+             *   Check above oid in COUNTERS_PORT_NAME_MAP in COUNTERS_DB to get the physical interface name
+             *
+             * VLAN based routing interface
+             * ASIC_STATE:SAI_OBJECT_TYPE_ROUTER_INTERFACE:oid:0x60000000006f6
+             *  SAI_ROUTER_INTERFACE_ATTR_TYPE -> SAI_ROUTER_INTERFACE_TYPE_VLAN
+             *  SAI_ROUTER_INTERFACE_ATTR_VLAN_ID -> oid:0x260000000006f3
+             *  Check above oid in "ASIC_STATE:SAI_OBJECT_TYPE_VLAN:oid:0x260000000006f3"
+             */
+
+            auto hashRif = g_redisClientAsicDb->hgetall(strRtrIfKey);
+            for (auto &kv: hashRif)
+            {
+                const std::string &skey = kv.first;
+                const std::string &svalue = kv.second;
+
+                if(skey == "SAI_ROUTER_INTERFACE_ATTR_TYPE")
+                {
+                    strRtrIntfType = svalue;
+                }
+                if(skey == "SAI_ROUTER_INTERFACE_ATTR_PORT_ID" || skey == "SAI_ROUTER_INTERFACE_ATTR_VLAN_ID")
+                {
+                    strIntfOid = svalue;
+                }
+            }
+            SWSS_LOG_DEBUG("Router interface type is %s, interface ID is %s",
+                    strRtrIntfType.c_str(), strIntfOid.c_str());
+
+            /* Extract Port name from the Port OID */
+            if(strRtrIntfType == "SAI_ROUTER_INTERFACE_TYPE_PORT")
+            {
+                auto hashCntr = g_redisClientCountersDb->hgetall("COUNTERS_PORT_NAME_MAP");
+                for (auto &kv: hashCntr)
+                {
+                    const std::string &skey = kv.first;
+                    const std::string &svalue = kv.second;
+                    if(svalue == strIntfOid)
+                    {
+                        strIntfName = skey;
+                        break;
+                    }
+                }
+            }
+            else if (strRtrIntfType == "SAI_ROUTER_INTERFACE_TYPE_VLAN")
+            {
+                std::string strVlanKey = "ASIC_STATE:SAI_OBJECT_TYPE_VLAN:" + strIntfOid;
+                auto hashVlan = g_redisClientAsicDb->hgetall(strVlanKey);
+                for (auto &kv: hashVlan)
+                {
+                    const std::string &skey = kv.first;
+                    const std::string &svalue = kv.second;
+                    if(skey == "SAI_VLAN_ATTR_VLAN_ID")
+                    {
+                        strIntfName = "Vlan" + svalue;
+                        break;
+                    }
+                }
+            }
+            SWSS_LOG_DEBUG("Interface name is %s", strIntfName.c_str());
+        } /* End of if(fvField(values[i]) == "key") */
+
+        /* Extract MAC address */
+        if(fvField(values[i]) == "SAI_NEIGHBOR_ENTRY_ATTR_DST_MAC_ADDRESS")
+        {
+            strMac = fvValue(values[i]);
+        }
+    } /* End of for (size_t i = 0; i < values.size(); i++) */
+
+    std::string appKey = strIntfName + ":" + strNbrIP;
+    appValues.emplace_back("key", appKey);
+    appValues.emplace_back("neigh", strMac);
+
+    return true;
+}
+

orchagent/neighorch.h

@@ -67,6 +67,8 @@ class NeighOrch : public Orch, public Subject
     bool clearNextHopFlag(const NextHopKey &, const uint32_t);
 
     void doTask(Consumer &consumer);
+    bool mapToErrorDbFormat(sai_object_type_t& object_type, std::vector<FieldValueTuple> &asicValues,
+            std::vector<FieldValueTuple> &appValues);
 };
 
 #endif /* SWSS_NEIGHORCH_H */

orchagent/orch.cpp

@@ -432,6 +432,12 @@ string Orch::dumpTuple(Consumer &consumer, KeyOpFieldsValuesTuple &tuple)
     return s;
 }
 
+bool Orch::mapToErrorDbFormat(sai_object_type_t& object_type, std::vector<FieldValueTuple> &asicValues,
+        std::vector<FieldValueTuple> &appValues)
+{
+    return false;
+}
+
 ref_resolve_status Orch::resolveFieldRefArray(
     type_map &type_maps,
     const string &field_name,

orchagent/orch.h

@@ -195,6 +195,8 @@ class Orch
     static void recordTuple(Consumer &consumer, swss::KeyOpFieldsValuesTuple &tuple);
 
     void dumpPendingTasks(std::vector<std::string> &ts);
+    virtual bool mapToErrorDbFormat(sai_object_type_t& object_type, std::vector<FieldValueTuple> &asicValues,
+            std::vector<FieldValueTuple> &appValues);
 protected:
     ConsumerMap m_consumerMap;
 

orchagent/orchdaemon.cpp

@@ -33,14 +33,25 @@ AclOrch *gAclOrch;
 CrmOrch *gCrmOrch;
 BufferOrch *gBufferOrch;
 SwitchOrch *gSwitchOrch;
+ErrorOrch *gErrorOrch;
 Directory<Orch*> gDirectory;
 
-OrchDaemon::OrchDaemon(DBConnector *applDb, DBConnector *configDb, DBConnector *stateDb) :
+std::shared_ptr<swss::RedisClient>          g_redisClientAppDb;
+std::shared_ptr<swss::RedisClient>          g_redisClientAsicDb;
+std::shared_ptr<swss::RedisClient>          g_redisClientCountersDb;
+
+OrchDaemon::OrchDaemon(DBConnector *applDb, DBConnector *configDb, DBConnector *stateDb, DBConnector *countersDb, DBConnector *asicDb, DBConnector *errorDb) :
         m_applDb(applDb),
         m_configDb(configDb),
-        m_stateDb(stateDb)
+        m_stateDb(stateDb),
+        m_countersDb(countersDb),
+        m_asicDb(asicDb),
+        m_errorDb(errorDb)
 {
     SWSS_LOG_ENTER();
+    g_redisClientAppDb = std::make_shared<swss::RedisClient>(applDb);
+    g_redisClientAsicDb = std::make_shared<swss::RedisClient>(asicDb);
+    g_redisClientCountersDb = std::make_shared<swss::RedisClient>(countersDb);
 }
 
 OrchDaemon::~OrchDaemon()
@@ -68,6 +79,11 @@ bool OrchDaemon::init()
 
     string platform = getenv("platform") ? getenv("platform") : "";
 
+    vector<string> error_tables = {
+        { ERROR_ROUTE_TABLE_NAME},
+        { ERROR_NEIGH_TABLE_NAME}
+    };
+    gErrorOrch = new ErrorOrch(m_asicDb, m_errorDb, error_tables);
     gSwitchOrch = new SwitchOrch(m_applDb, APP_SWITCH_TABLE_NAME);
 
     const int portsorch_base_pri = 40;
@@ -193,7 +209,7 @@ bool OrchDaemon::init()
      * when iterating ConsumerMap.
      * That is ensured implicitly by the order of map key, "LAG_TABLE" is smaller than "VLAN_TABLE" in lexicographic order.
      */
-    m_orchList = { gSwitchOrch, gCrmOrch, gBufferOrch, gPortsOrch, gIntfsOrch, gNeighOrch, gRouteOrch, copp_orch, tunnel_decap_orch, qos_orch, wm_orch, policer_orch };
+    m_orchList = { gErrorOrch, gSwitchOrch, gCrmOrch, gBufferOrch, gPortsOrch, gIntfsOrch, gNeighOrch, gRouteOrch, copp_orch, tunnel_decap_orch, qos_orch, wm_orch, policer_orch };
 
 
     bool initialize_dtel = false;

orchagent/orchdaemon.h

@@ -23,6 +23,7 @@
 #include "vrforch.h"
 #include "vxlanorch.h"
 #include "vnetorch.h"
+#include "errororch.h"
 #include "countercheckorch.h"
 #include "flexcounterorch.h"
 #include "watermarkorch.h"
@@ -34,7 +35,7 @@ using namespace swss;
 class OrchDaemon
 {
 public:
-    OrchDaemon(DBConnector *, DBConnector *, DBConnector *);
+    OrchDaemon(DBConnector *, DBConnector *, DBConnector *, DBConnector *, DBConnector *, DBConnector *);
     ~OrchDaemon();
 
     bool init();
@@ -48,6 +49,9 @@ class OrchDaemon
     DBConnector *m_applDb;
     DBConnector *m_configDb;
     DBConnector *m_stateDb;
+    DBConnector *m_countersDb;
+    DBConnector *m_asicDb;
+    DBConnector *m_errorDb;
 
     std::vector<Orch *> m_orchList;
     Select *m_select;

orchagent/routeorch.cpp

@@ -4,6 +4,14 @@
 #include "logger.h"
 #include "swssnet.h"
 #include "crmorch.h"
+#include "tokenize.h"
+#include  "errororch.h"
+#include "sai_serialize.h"
+#include "redisclient.h"
+
+#include "swss/json.hpp"
+using json = nlohmann::json;
+using namespace swss;
 
 extern sai_object_id_t gVirtualRouterId;
 extern sai_object_id_t gSwitchId;
@@ -15,6 +23,10 @@ extern sai_switch_api_t*            sai_switch_api;
 extern PortsOrch *gPortsOrch;
 extern IntfsOrch *gIntfsOrch;
 extern CrmOrch *gCrmOrch;
+extern ErrorOrch *gErrorOrch;
+extern std::shared_ptr<swss::RedisClient>   g_redisClientAppDb;
+extern std::shared_ptr<swss::RedisClient>   g_redisClientAsicDb;
+extern std::shared_ptr<swss::RedisClient>   g_redisClientCountersDb;
 
 /* Default maximum number of next hop groups */
 #define DEFAULT_NUMBER_OF_ECMP_GROUPS   128
@@ -30,6 +42,12 @@ RouteOrch::RouteOrch(DBConnector *db, string tableName, NeighOrch *neighOrch) :
 {
     SWSS_LOG_ENTER();
 
+    if(gErrorOrch->mappingHandlerRegister(APP_ROUTE_TABLE_NAME, this) == false)
+    {
+        SWSS_LOG_ERROR("Failed to register with Error Handling Framework for %s",
+                APP_ROUTE_TABLE_NAME);
+    }
+
     sai_attribute_t attr;
     attr.id = SAI_SWITCH_ATTR_NUMBER_OF_ECMP_GROUPS;
 
@@ -1097,3 +1115,59 @@ bool RouteOrch::removeRoute(const IpPrefix &ipPrefix)
 
     return true;
 }
+
+bool RouteOrch::mapToErrorDbFormat(sai_object_type_t& object_type, std::vector<FieldValueTuple> &asicValues,
+        std::vector<FieldValueTuple> &appValues)
+{
+    SWSS_LOG_ENTER();
+
+    if(object_type != SAI_OBJECT_TYPE_ROUTE_ENTRY)
+    {
+        return false;
+    }
+
+    /*
+       127.0.0.1:6379> hgetall "ROUTE_TABLE:1.1.1.0/24"
+       1) "nexthop"
+       2) "2.2.2.2"
+       3) "ifname"
+       4) "Ethernet0"
+
+       127.0.0.1:6379[1]> hgetall "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY:{\"dest\":\"1.1.1.0/24\",\"switch_id\":\"oid:0x21000000000000\",\"vr\":\"oid:0x3000000000028\"}"
+       1) "SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"
+       2) "oid:0x40000000006fd"
+       127.0.0.1:6379[1]>
+
+     */
+
+    const auto& values = asicValues;
+    std::string asicKV, strNhopKey, strNextHopIP;
+    std::string strRifOid, strPrefix, strIntfName, strRtrIntfType;
+    for (size_t i = 0; i < values.size(); i++)
+    {
+        /* Extract IP prefix from the key */
+        if(fvField(values[i]) == "key")
+        {
+            asicKV = fvValue(values[i]);
+            auto tokens = tokenize(asicKV, ':', 1);
+            json j = json::parse(tokens[1]);
+            strPrefix = j["dest"];
+            SWSS_LOG_DEBUG("Prefix is %s", strPrefix.c_str());
+            break;
+        }
+    } /* End of for (i < values.size(); i++) */
+
+    appValues.emplace_back("key", strPrefix);
+    string strRouteTable = APP_ROUTE_TABLE_NAME;
+    string strAppKey = strRouteTable + ":" + strPrefix;
+    auto hashApp = g_redisClientAppDb->hgetall(strAppKey);
+    for (auto &kv: hashApp)
+    {
+        const std::string &skey = kv.first;
+        const std::string &svalue = kv.second;
+        appValues.emplace_back(skey, svalue);
+    }
+
+    return true;
+}
+