[flex-counters] Delay flex counters stats init for faster boot time (sonic-net#1749)

**What I did**
Update flex counters DB with counters stats only when counters are enabled.
As long as the polling counters are not enabled, flex counters information will stored internally on PortsOrch.

**Why I did it**
Creating flex counters objects on the DB will trigger 'SYNCD' to access the HW for query statistics capabilities.
This HW access takes time and will be better to finish boot before doing this (mainly for fast-reboot but good to have in general).
The flex counters are not crucial at boot time, we can delay it to the end of the boot process.

**How I verified it**
Reboot a switch and observer the flex counters DB populated after counters are enabled.

Author: shlomibitton

orchagent/flexcounterorch.cpp

@@ -18,6 +18,11 @@ extern IntfsOrch *gIntfsOrch;
 extern BufferOrch *gBufferOrch;
 
 #define BUFFER_POOL_WATERMARK_KEY   "BUFFER_POOL_WATERMARK"
+#define PORT_KEY                    "PORT"
+#define PORT_BUFFER_DROP_KEY        "PORT_BUFFER_DROP"
+#define QUEUE_KEY                   "QUEUE"
+#define PG_WATERMARK_KEY            "PG_WATERMARK"
+#define RIF_KEY                     "RIF"
 
 unordered_map<string, string> flexCounterGroupMap =
 {
@@ -102,26 +107,31 @@ void FlexCounterOrch::doTask(Consumer &consumer)
                     // which is automatically satisfied upon the creation of the orch object that requires
                     // the syncd flex counter polling service
                     // This postponement is introduced by design to accelerate the initialization process
-                    //
-                    // generateQueueMap() is called as long as a field "FLEX_COUNTER_STATUS" event is heard,
-                    // regardless of whether the key is "QUEUE" or whether the value is "enable" or "disable"
-                    // This can be because generateQueueMap() installs a fundamental list of queue stats
-                    // that need to be polled. So my doubt here is if queue watermark stats shall be piggybacked
-                    // into the same function as they may not be counted as fundamental
-                    if(gPortsOrch)
+                    if(gPortsOrch && (value == "enable"))
                     {
-                        gPortsOrch->generateQueueMap();
-                        gPortsOrch->generatePriorityGroupMap();
+                        if(key == PORT_KEY)
+                        {
+                            gPortsOrch->generatePortCounterMap();
+                            m_port_counter_enabled = true;
+                        }
+                        else if(key == PORT_BUFFER_DROP_KEY)
+                        {
+                            gPortsOrch->generatePortBufferDropCounterMap();
+                            m_port_buffer_drop_counter_enabled = true;
+                        }
+                        else if(key == QUEUE_KEY)
+                        {
+                            gPortsOrch->generateQueueMap();
+                        }
+                        else if(key == PG_WATERMARK_KEY)
+                        {
+                            gPortsOrch->generatePriorityGroupMap();
+                        }
                     }
-                    if(gPortsOrch)
-                    {
-                        gPortsOrch->generatePriorityGroupMap();
-                    }
-                    if(gIntfsOrch)
+                    if(gIntfsOrch && (key == RIF_KEY) && (value == "enable"))
                     {
                         gIntfsOrch->generateInterfaceMap();
                     }
-                    // Install COUNTER_ID_LIST/ATTR_ID_LIST only when hearing buffer pool watermark enable event
                     if (gBufferOrch && (key == BUFFER_POOL_WATERMARK_KEY) && (value == "enable"))
                     {
                         gBufferOrch->generateBufferPoolWatermarkCounterIdList();
@@ -144,3 +154,13 @@ void FlexCounterOrch::doTask(Consumer &consumer)
         consumer.m_toSync.erase(it++);
     }
 }
+
+bool FlexCounterOrch::getPortCountersState() const
+{
+    return m_port_counter_enabled;
+}
+
+bool FlexCounterOrch::getPortBufferDropCountersState() const
+{
+    return m_port_buffer_drop_counter_enabled;
+}

orchagent/flexcounterorch.h

@@ -15,10 +15,14 @@ class FlexCounterOrch: public Orch
     void doTask(Consumer &consumer);
     FlexCounterOrch(swss::DBConnector *db, std::vector<std::string> &tableNames);
     virtual ~FlexCounterOrch(void);
+    bool getPortCountersState() const;
+    bool getPortBufferDropCountersState() const;
 
 private:
     std::shared_ptr<swss::DBConnector> m_flexCounterDb = nullptr;
     std::shared_ptr<swss::ProducerTable> m_flexCounterGroupTable = nullptr;
+    bool m_port_counter_enabled = false;
+    bool m_port_buffer_drop_counter_enabled = false;
 };
 
 #endif

orchagent/orchdaemon.cpp

@@ -352,7 +352,11 @@ bool OrchDaemon::init()
         CFG_FLEX_COUNTER_TABLE_NAME
     };
 
-    m_orchList.push_back(new FlexCounterOrch(m_configDb, flex_counter_tables));
+    auto* flexCounterOrch = new FlexCounterOrch(m_configDb, flex_counter_tables);
+    m_orchList.push_back(flexCounterOrch);
+
+    gDirectory.set(flexCounterOrch);
+    gDirectory.set(gPortsOrch);
 
     vector<string> pfc_wd_tables = {
         CFG_PFC_WD_TABLE_NAME

orchagent/portsorch.cpp

@@ -284,9 +284,9 @@ static char* hostif_vlan_tag[] = {
 PortsOrch::PortsOrch(DBConnector *db, DBConnector *stateDb, vector<table_name_with_pri_t> &tableNames, DBConnector *chassisAppDb) :
         Orch(db, tableNames),
         m_portStateTable(stateDb, STATE_PORT_TABLE_NAME),
-        port_stat_manager(PORT_STAT_COUNTER_FLEX_COUNTER_GROUP, StatsMode::READ, PORT_STAT_FLEX_COUNTER_POLLING_INTERVAL_MS, true),
-        port_buffer_drop_stat_manager(PORT_BUFFER_DROP_STAT_FLEX_COUNTER_GROUP, StatsMode::READ, PORT_BUFFER_DROP_STAT_POLLING_INTERVAL_MS, true),
-        queue_stat_manager(QUEUE_STAT_COUNTER_FLEX_COUNTER_GROUP, StatsMode::READ, QUEUE_STAT_FLEX_COUNTER_POLLING_INTERVAL_MS, true)
+        port_stat_manager(PORT_STAT_COUNTER_FLEX_COUNTER_GROUP, StatsMode::READ, PORT_STAT_FLEX_COUNTER_POLLING_INTERVAL_MS, false),
+        port_buffer_drop_stat_manager(PORT_BUFFER_DROP_STAT_FLEX_COUNTER_GROUP, StatsMode::READ, PORT_BUFFER_DROP_STAT_POLLING_INTERVAL_MS, false),
+        queue_stat_manager(QUEUE_STAT_COUNTER_FLEX_COUNTER_GROUP, StatsMode::READ, QUEUE_STAT_FLEX_COUNTER_POLLING_INTERVAL_MS, false)
 {
     SWSS_LOG_ENTER();
 
@@ -2273,19 +2273,21 @@ bool PortsOrch::initPort(const string &alias, const string &role, const int inde
                 vector<FieldValueTuple> fields;
                 fields.push_back(tuple);
                 m_counterTable->set("", fields);
+
                 // Install a flex counter for this port to track stats
-                std::unordered_set<std::string> counter_stats;
-                for (const auto& it: port_stat_ids)
+                auto flex_counters_orch = gDirectory.get<FlexCounterOrch*>();
+                /* Delay installing the counters if they are yet enabled
+                If they are enabled, install the counters immediately */
+                if (flex_counters_orch->getPortCountersState())
                 {
-                    counter_stats.emplace(sai_serialize_port_stat(it));
+                    auto port_counter_stats = generateCounterStats(PORT_STAT_COUNTER_FLEX_COUNTER_GROUP);
+                    port_stat_manager.setCounterIdList(p.m_port_id, CounterType::PORT, port_counter_stats);
                 }
-                port_stat_manager.setCounterIdList(p.m_port_id, CounterType::PORT, counter_stats);
-                std::unordered_set<std::string> port_buffer_drop_stats;
-                for (const auto& it: port_buffer_drop_stat_ids)
+                if (flex_counters_orch->getPortBufferDropCountersState())
                 {
-                    port_buffer_drop_stats.emplace(sai_serialize_port_stat(it));
+                    auto port_buffer_drop_stats = generateCounterStats(PORT_BUFFER_DROP_STAT_FLEX_COUNTER_GROUP);
+                    port_buffer_drop_stat_manager.setCounterIdList(p.m_port_id, CounterType::PORT, port_buffer_drop_stats);
                 }
-                port_buffer_drop_stat_manager.setCounterIdList(p.m_port_id, CounterType::PORT, port_buffer_drop_stats);
 
                 PortUpdate update = { p, true };
                 notify(SUBJECT_TYPE_PORT_CHANGE, static_cast<void *>(&update));
@@ -2318,8 +2320,11 @@ void PortsOrch::deInitPort(string alias, sai_object_id_t port_id)
     p.m_port_id = port_id;
 
     /* remove port from flex_counter_table for updating counters  */
-    port_stat_manager.clearCounterIdList(p.m_port_id);
-
+    auto flex_counters_orch = gDirectory.get<FlexCounterOrch*>();
+    if ((flex_counters_orch->getPortCountersState()))
+    {
+        port_stat_manager.clearCounterIdList(p.m_port_id);
+    }
     /* remove port name map from counter table */
     m_counter_db->hdel(COUNTERS_PORT_NAME_MAP, alias);
 
@@ -5072,6 +5077,42 @@ void PortsOrch::generatePriorityGroupMapPerPort(const Port& port)
     CounterCheckOrch::getInstance().addPort(port);
 }
 
+void PortsOrch::generatePortCounterMap()
+{
+    if (m_isPortCounterMapGenerated)
+    {
+        return;
+    }
+
+    auto port_counter_stats = generateCounterStats(PORT_STAT_COUNTER_FLEX_COUNTER_GROUP);
+    for (const auto& it: m_portList)
+    {
+        if (it.first == "CPU")
+        {
+            continue;
+        }
+        port_stat_manager.setCounterIdList(it.second.m_port_id, CounterType::PORT, port_counter_stats);
+    }
+
+    m_isPortCounterMapGenerated = true;
+}
+
+void PortsOrch::generatePortBufferDropCounterMap()
+{
+    if (m_isPortBufferDropCounterMapGenerated)
+    {
+        return;
+    }
+
+    auto port_buffer_drop_stats = generateCounterStats(PORT_BUFFER_DROP_STAT_FLEX_COUNTER_GROUP);
+    for (const auto& it: m_portList)
+    {
+        port_buffer_drop_stat_manager.setCounterIdList(it.second.m_port_id, CounterType::PORT, port_buffer_drop_stats);
+    }
+
+    m_isPortBufferDropCounterMapGenerated = true;
+}
+
 void PortsOrch::doTask(NotificationConsumer &consumer)
 {
     SWSS_LOG_ENTER();
@@ -6228,3 +6269,23 @@ void PortsOrch::voqSyncDelLagMember(Port &lag, Port &port)
     string key = lag.m_system_lag_info.alias + ":" + port.m_system_port_info.alias;
     m_tableVoqSystemLagMemberTable->del(key);
 }
+
+std::unordered_set<std::string> PortsOrch::generateCounterStats(const string& type)
+{
+    std::unordered_set<std::string> counter_stats;
+    if (type == PORT_STAT_COUNTER_FLEX_COUNTER_GROUP)
+    {
+        for (const auto& it: port_stat_ids)
+        {
+            counter_stats.emplace(sai_serialize_port_stat(it));
+        }
+    }
+    else if (type == PORT_BUFFER_DROP_STAT_FLEX_COUNTER_GROUP)
+    {
+        for (const auto& it: port_buffer_drop_stat_ids)
+        {
+            counter_stats.emplace(sai_serialize_port_stat(it));
+        }
+    }
+    return counter_stats;
+}

orchagent/portsorch.h

@@ -13,6 +13,7 @@
 #include "gearboxutils.h"
 #include "saihelper.h"
 #include "lagid.h"
+#include "flexcounterorch.h"
 
 
 #define FCS_LEN 4
@@ -125,6 +126,8 @@ class PortsOrch : public Orch, public Subject
 
     void generateQueueMap();
     void generatePriorityGroupMap();
+    void generatePortCounterMap();
+    void generatePortBufferDropCounterMap();
 
     void refreshPortStatus();
     bool removeAclTableGroup(const Port &p);
@@ -290,6 +293,9 @@ class PortsOrch : public Orch, public Subject
     bool m_isPriorityGroupMapGenerated = false;
     void generatePriorityGroupMapPerPort(const Port& port);
 
+    bool m_isPortCounterMapGenerated = false;
+    bool m_isPortBufferDropCounterMapGenerated = false;
+
     bool setPortAutoNeg(sai_object_id_t id, int an);
     bool setPortFecMode(sai_object_id_t id, int fec);
     bool setPortInterfaceType(sai_object_id_t id, sai_port_interface_type_t interface_type);
@@ -333,6 +339,8 @@ class PortsOrch : public Orch, public Subject
     void voqSyncDelLagMember(Port &lag, Port &port);
     unique_ptr<LagIdAllocator> m_lagIdAllocator;
 
+    std::unordered_set<std::string> generateCounterStats(const string& type);
+
 };
 #endif /* SWSS_PORTSORCH_H */
 

tests/mock_tests/mock_orchagent_main.h

@@ -15,6 +15,8 @@
 #include "vxlanorch.h"
 #include "policerorch.h"
 #include "fgnhgorch.h"
+#include "flexcounterorch.h"
+#include "directory.h"
 
 extern int gBatchSize;
 extern bool gSwssRecord;
@@ -42,6 +44,7 @@ extern FdbOrch *gFdbOrch;
 extern MirrorOrch *gMirrorOrch;
 extern BufferOrch *gBufferOrch;
 extern VRFOrch *gVrfOrch;
+extern Directory<Orch*> gDirectory;
 
 extern sai_acl_api_t *sai_acl_api;
 extern sai_switch_api_t *sai_switch_api;

tests/mock_tests/portsorch_ut.cpp

@@ -146,8 +146,13 @@ namespace portsorch_test
 
         ASSERT_EQ(gPortsOrch, nullptr);
 
-        gPortsOrch = new PortsOrch(m_app_db.get(), m_state_db.get(), ports_tables, m_chassis_app_db.get());
+        vector<string> flex_counter_tables = {
+            CFG_FLEX_COUNTER_TABLE_NAME
+        };
+        auto* flexCounterOrch = new FlexCounterOrch(m_config_db.get(), flex_counter_tables);
+        gDirectory.set(flexCounterOrch);
 
+	    gPortsOrch = new PortsOrch(m_app_db.get(), m_state_db.get(), ports_tables, m_chassis_app_db.get());
         vector<string> buffer_tables = { APP_BUFFER_POOL_TABLE_NAME,
                                          APP_BUFFER_PROFILE_TABLE_NAME,
                                          APP_BUFFER_QUEUE_TABLE_NAME,