Warm reboot: Orchagent state restore

orchagent/Makefile.am

@@ -1,4 +1,4 @@
-INCLUDES = -I $(top_srcdir)
+INCLUDES = -I $(top_srcdir) -I $(top_srcdir)/warmrestart
 
 CFLAGS_SAI = -I /usr/include/sai
 
@@ -66,12 +66,14 @@ orchagent_SOURCES = \
 	            switchorch.h \
 		    swssnet.h \
 		    tunneldecaporch.h \
-		    crmorch.h
+		    crmorch.h        \
 		    request_parser.h \
 		    vrforch.h \
 		    dtelorch.h \
 		    countercheckorch.h \
-		    flexcounterorch.h
+		    flexcounterorch.h   \
+		    $(top_srcdir)/warmrestart/warm_restart.cpp \
+		    $(top_srcdir)/warmrestart/warm_restart.h
 
 orchagent_CFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_SAI)
 orchagent_CPPFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_SAI)

orchagent/aclorch.cpp

@@ -1694,6 +1694,7 @@ void AclOrch::init(vector<TableConnector>& connectors, PortsOrch *portOrch, Mirr
     auto interv = timespec { .tv_sec = COUNTERS_READ_INTERVAL, .tv_nsec = 0 };
     auto timer = new SelectableTimer(interv);
     auto executor = new ExecutableTimer(timer, this);
+    executor->setName("ACL_POLL_TIMER");
     Orch::addExecutor("", executor);
     timer->start();
 }

orchagent/countercheckorch.cpp

@@ -31,6 +31,7 @@ CounterCheckOrch::CounterCheckOrch(DBConnector *db, vector<string> &tableNames):
     auto interv = timespec { .tv_sec = COUNTER_CHECK_POLL_TIMEOUT_SEC, .tv_nsec = 0 };
     auto timer = new SelectableTimer(interv);
     auto executor = new ExecutableTimer(timer, this);
+    executor->setName("MC_COUNTERS_POLL");
     Orch::addExecutor("MC_COUNTERS_POLL", executor);
     timer->start();
 }

orchagent/crmorch.cpp

@@ -162,7 +162,11 @@ CrmOrch::CrmOrch(DBConnector *db, string tableName):
         m_resourcesMap.emplace(res.first, CrmResourceEntry(res.second, CRM_THRESHOLD_TYPE_DEFAULT, CRM_THRESHOLD_LOW_DEFAULT, CRM_THRESHOLD_HIGH_DEFAULT));
     }
 
+    // The CRM stats needs to be populated again
+    m_countersCrmTable->del(CRM_COUNTERS_TABLE_KEY);
+
     auto executor = new ExecutableTimer(m_timer.get(), this);
+    executor->setName("CRM_COUNTERS_POLL");
     Orch::addExecutor("CRM_COUNTERS_POLL", executor);
     m_timer->start();
 }
@@ -333,7 +337,7 @@ void CrmOrch::decCrmAclUsedCounter(CrmResourceType resource, sai_acl_stage_t sta
     {
         m_resourcesMap.at(resource).countersMap[getCrmAclKey(stage, point)].usedCounter--;
 
-        // Remove ACL table related counters 
+        // Remove ACL table related counters
         if (resource == CrmResourceType::CRM_ACL_TABLE)
         {
             auto & cntMap = m_resourcesMap.at(CrmResourceType::CRM_ACL_TABLE).countersMap;

orchagent/fdborch.cpp

@@ -27,13 +27,17 @@ FdbOrch::FdbOrch(DBConnector *db, string tableName, PortsOrch *port) :
     m_portsOrch->attach(this);
     m_flushNotificationsConsumer = new NotificationConsumer(db, "FLUSHFDBREQUEST");
     auto flushNotifier = new Notifier(m_flushNotificationsConsumer, this);
+    flushNotifier->setName("FLUSHFDBREQUEST");
     Orch::addExecutor("", flushNotifier);
 
     /* Add FDB notifications support from ASIC */
     DBConnector *notificationsDb = new DBConnector(ASIC_DB, DBConnector::DEFAULT_UNIXSOCKET, 0);
     m_fdbNotificationConsumer = new swss::NotificationConsumer(notificationsDb, "NOTIFICATIONS");
     auto fdbNotifier = new Notifier(m_fdbNotificationConsumer, this);
+    fdbNotifier->setName("FDB_NOTIFICATIONS");
     Orch::addExecutor("FDB_NOTIFICATIONS", fdbNotifier);
+
+    addExistingData(tableName);
 }
 
 void FdbOrch::update(sai_fdb_event_t type, const sai_fdb_entry_t* entry, sai_object_id_t bridge_port_id)
@@ -53,6 +57,13 @@ void FdbOrch::update(sai_fdb_event_t type, const sai_fdb_entry_t* entry, sai_obj
             return;
         }
 
+        if (m_entries.count(update.entry) != 0) // we already have such entries
+        {
+             SWSS_LOG_INFO("FdbOrch notification: mac %s is already in bv_id 0x%lx",
+                    update.entry.mac.to_string().c_str(), entry->bv_id);
+             break;
+        }
+
         update.add = true;
 
         {

orchagent/intfsorch.cpp

@@ -29,6 +29,10 @@ IntfsOrch::IntfsOrch(DBConnector *db, string tableName) :
         Orch(db, tableName, intfsorch_pri)
 {
     SWSS_LOG_ENTER();
+
+    // Read the pre-existing data for INTF in appDB.
+    addExistingData(APP_INTF_TABLE_NAME);
+
 }
 
 sai_object_id_t IntfsOrch::getRouterIntfsId(const string &alias)
@@ -86,7 +90,16 @@ void IntfsOrch::doTask(Consumer &consumer)
         {
             if (alias == "lo")
             {
-                addIp2MeRoute(ip_prefix);
+                // set request for lo may come after warm start restore
+                auto it_intfs = m_syncdIntfses.find(alias);
+                if (it_intfs == m_syncdIntfses.end())
+                {
+                    IntfsEntry intfs_entry;
+                    intfs_entry.ref_count = 0;
+                    m_syncdIntfses[alias] = intfs_entry;
+                    addIp2MeRoute(ip_prefix);
+                }
+
                 it = consumer.m_toSync.erase(it);
                 continue;
             }
@@ -171,6 +184,7 @@ void IntfsOrch::doTask(Consumer &consumer)
         {
             if (alias == "lo")
             {
+                m_syncdIntfses.erase(alias);
                 removeIp2MeRoute(ip_prefix);
                 it = consumer.m_toSync.erase(it);
                 continue;

orchagent/main.cpp

@@ -22,6 +22,7 @@ extern "C" {
 #include "saihelper.h"
 #include "notifications.h"
 #include <signal.h>
+#include <warm_restart.h>
 
 using namespace std;
 using namespace swss;
@@ -77,6 +78,23 @@ void sighup_handler(int signo)
     }
 }
 
+void syncd_apply_view()
+{
+    SWSS_LOG_NOTICE("Notify syncd APPLY_VIEW");
+
+    sai_status_t status;
+    sai_attribute_t attr;
+    attr.id = SAI_REDIS_SWITCH_ATTR_NOTIFY_SYNCD;
+    attr.value.s32 = SAI_REDIS_NOTIFY_SYNCD_APPLY_VIEW;
+    status = sai_switch_api->set_switch_attribute(gSwitchId, &attr);
+
+    if (status != SAI_STATUS_SUCCESS)
+    {
+        SWSS_LOG_ERROR("Failed to notify syncd APPLY_VIEW %d", status);
+        exit(EXIT_FAILURE);
+    }
+}
+
 int main(int argc, char **argv)
 {
     swss::Logger::linkToDbNative("orchagent");
@@ -251,6 +269,8 @@ int main(int argc, char **argv)
     DBConnector config_db(CONFIG_DB, DBConnector::DEFAULT_UNIXSOCKET, 0);
     DBConnector state_db(STATE_DB, DBConnector::DEFAULT_UNIXSOCKET, 0);
 
+    WarmStart::checkWarmStart("orchagent");
+
     OrchDaemon *orchDaemon = new OrchDaemon(&appl_db, &config_db, &state_db);
     if (!orchDaemon->init())
     {
@@ -261,17 +281,9 @@ int main(int argc, char **argv)
 
     try
     {
-        SWSS_LOG_NOTICE("Notify syncd APPLY_VIEW");
-
-        attr.id = SAI_REDIS_SWITCH_ATTR_NOTIFY_SYNCD;
-        attr.value.s32 = SAI_REDIS_NOTIFY_SYNCD_APPLY_VIEW;
-        status = sai_switch_api->set_switch_attribute(gSwitchId, &attr);
-
-        if (status != SAI_STATUS_SUCCESS)
+        if (!WarmStart::isWarmStart())
         {
-            SWSS_LOG_ERROR("Failed to notify syncd APPLY_VIEW %d", status);
-            delete orchDaemon;
-            exit(EXIT_FAILURE);
+            syncd_apply_view();
         }
 
         orchDaemon->start();

orchagent/neighorch.cpp

@@ -19,6 +19,10 @@ NeighOrch::NeighOrch(DBConnector *db, string tableName, IntfsOrch *intfsOrch) :
         Orch(db, tableName, neighorch_pri), m_intfsOrch(intfsOrch)
 {
     SWSS_LOG_ENTER();
+
+    // Read the pre-existing data for neigh in appDB.
+    addExistingData(tableName);
+
 }
 
 bool NeighOrch::hasNextHop(IpAddress ipAddress)

orchagent/orch.cpp

@@ -171,6 +171,23 @@ void Consumer::drain()
         m_orch->doTask(*this);
 }
 
+void Consumer::dumpTasks(vector<string> &ts)
+{
+    for (auto &tm :m_toSync)
+    {
+        KeyOpFieldsValuesTuple& tuple = tm.second;
+
+        string s = getTableName() + ":" + kfvKey(tuple)
+               + "|" + kfvOp(tuple);
+        for (auto i = kfvFieldsValues(tuple).begin(); i != kfvFieldsValues(tuple).end(); i++)
+        {
+            s += "|" + fvField(*i) + ":" + fvValue(*i);
+        }
+
+        ts.push_back(s);
+    }
+}
+
 bool Orch::addExistingData(const string& tableName)
 {
     Consumer* consumer = dynamic_cast<Consumer *>(getExecutor(tableName));
@@ -293,6 +310,21 @@ void Orch::doTask()
     }
 }
 
+void Orch::dumpTasks(vector<string> &ts)
+{
+    for(auto &it : m_consumerMap)
+    {
+        Consumer* consumer = dynamic_cast<Consumer *>(it.second.get());
+        if (consumer == NULL)
+        {
+            SWSS_LOG_DEBUG("Executor is not a Consumer");
+            continue;
+        }
+
+        consumer->dumpTasks(ts);
+    }
+}
+
 void Orch::logfileReopen()
 {
     gRecordOfs.close();

orchagent/orch.h

@@ -91,10 +91,22 @@ class Executor : public Selectable
     virtual void execute() { }
     virtual void drain() { }
 
+    virtual string getName() const
+    {
+        return m_name;
+    }
+    virtual void setName(string name)
+    {
+        m_name = name;
+    }
+
 protected:
     Selectable *m_selectable;
     Orch *m_orch;
 
+    // Name for Executor
+    string m_name;
+
     // Get the underlying selectable
     Selectable *getSelectable() const { return m_selectable; }
 };
@@ -116,10 +128,24 @@ class Consumer : public Executor {
         return getConsumerTable()->getTableName();
     }
 
+
+    string getName() const
+    {
+        return getConsumerTable()->getTableName();
+    }
+
+    int getDbId() const
+    {
+        return getConsumerTable()->getDbId();
+    }
+
+    void dumpTasks(vector<string> &ts);
+
     void addToSync(std::deque<KeyOpFieldsValuesTuple> &entries);
     void refillToSync();
     void refillToSync(Table* table);
     void execute();
+
     void drain();
 
     /* Store the latest 'golden' status */
@@ -166,6 +192,8 @@ class Orch
 
     /* TODO: refactor recording */
     static void recordTuple(Consumer &consumer, KeyOpFieldsValuesTuple &tuple);
+
+    void dumpTasks(vector<string> &ts);
 protected:
     ConsumerMap m_consumerMap;