[swss] Support FIPS MACSec POST (sonic-net#3836)

Add SAI MACSec POST support in SWSS

When FIPS is enabled in SONiC, enable MACSecPOST in switch creation.
If MACSec POST is only supported in MACSec init, create MACSec objects and enable POST when initializing MACSecOrch.
Set SAI POST status in StateDB accordingly based on SAI POST status notificaiton.
MACSecMgr does not process any MACSec configuration if SAI POST fails.
With the PR, the only case that Orchagent declares SAI MACSec POST to fail is:

FIPS is enabled in SONiC; AND
SAI supports MACSec POST; AND
SAI returns failure on MACSec POST.
We particularly verified the following on switch with current BRCM SAI (13.2.1.0) that does not support MACSec POST yet:

MACSec ports came up fine when FIPS is not enabled in SONiC;
MACSec ports came up fine when FIPS is enabled in SONiC and SAI does not support MACSec POST.

Author: ysmanman

cfgmgr/Makefile.am

@@ -96,7 +96,7 @@ tunnelmgrd_CFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_SAI) $(CF
 tunnelmgrd_CPPFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_SAI) $(CFLAGS_ASAN)
 tunnelmgrd_LDADD = $(LDFLAGS_ASAN) $(COMMON_LIBS) $(SAIMETA_LIBS)
 
-macsecmgrd_SOURCES = macsecmgrd.cpp macsecmgr.cpp $(COMMON_ORCH_SOURCE) shellcmd.h
+macsecmgrd_SOURCES = macsecmgrd.cpp macsecmgr.cpp $(COMMON_ORCH_SOURCE) shellcmd.h $(top_srcdir)/orchagent/macsecpost.cpp
 macsecmgrd_CFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_SAI) $(CFLAGS_ASAN)
 macsecmgrd_CPPFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_SAI) $(CFLAGS_ASAN)
 macsecmgrd_LDADD = $(LDFLAGS_ASAN) $(COMMON_LIBS) $(SAIMETA_LIBS)

cfgmgr/macsecmgrd.cpp

@@ -17,6 +17,7 @@
 #include <select.h>
 
 #include "macsecmgr.h"
+#include "macsecpost.h"
 
 using namespace std;
 using namespace swss;
@@ -76,9 +77,28 @@ int main(int argc, char **argv)
             s.addSelectables(o->getSelectables());
         }
 
+        bool isPostStateReady = false;
+
         SWSS_LOG_NOTICE("starting main loop");
         while (!received_sigterm)
         {
+            /* Don't process any config until POST state is ready */
+            if (!isPostStateReady)
+            {
+                std::string state = getMacsecPostState(&stateDb);
+                if (state == "pass" || state == "disabled")
+                {
+                    SWSS_LOG_NOTICE("FIPS MACSec POST ready: state %s", state.c_str());
+                    isPostStateReady = true;
+                }
+                else
+                {
+                    /* Yield before retry */
+                    sleep(1);
+                    continue;
+                }
+            }
+
             Selectable *sel;
             int ret;
 

orchagent/Makefile.am

@@ -132,6 +132,7 @@ orchagent_SOURCES = \
             twamporch.cpp \
             stporch.cpp \
             nexthopkey.cpp \
+            macsecpost.cpp \
             high_frequency_telemetry/hftelorch.cpp \
             high_frequency_telemetry/hftelprofile.cpp \
             high_frequency_telemetry/counternameupdater.cpp \

orchagent/macsecorch.cpp

@@ -1,4 +1,6 @@
 #include "macsecorch.h"
+#include "macsecpost.h"
+#include "notifier.h"
 
 #include <macaddress.h>
 #include <sai_serialize.h>
@@ -599,6 +601,7 @@ MACsecOrch::MACsecOrch(
     const std::vector<std::string> &tables,
     PortsOrch *port_orch) : Orch(app_db, tables),
                             m_port_orch(port_orch),
+                            m_state_db(state_db),
                             m_state_macsec_port(state_db, STATE_MACSEC_PORT_TABLE_NAME),
                             m_state_macsec_egress_sc(state_db, STATE_MACSEC_EGRESS_SC_TABLE_NAME),
                             m_state_macsec_ingress_sc(state_db, STATE_MACSEC_INGRESS_SC_TABLE_NAME),
@@ -649,6 +652,53 @@ MACsecOrch::MACsecOrch(
             saiAclFieldSciMatchSupported = false;
         }
     }
+
+    // Add handler for POST completion callback/notification.
+    string post_state = getMacsecPostState(m_state_db);
+    if (post_state == "switch-level-post-in-progress" ||
+        post_state == "macsec-level-post-in-progress" )
+    {
+        m_notificationsDb = make_shared<DBConnector>("ASIC_DB", 0);
+        m_postCompletionNotificationConsumer = new swss::NotificationConsumer(m_notificationsDb.get(), "NOTIFICATIONS");
+        auto postCompletionNotificatier = new Notifier(m_postCompletionNotificationConsumer, this, "POST_COMPLETION__NOTIFICATIONS");
+        Orch::addExecutor(postCompletionNotificatier);
+    }
+
+    if (post_state == "switch-level-post-in-progress")
+    {
+        // POST was already enabled in switch init. The completion notification may have already been sent
+        // before MACSecOrch is initialized. So query if POST is completed or not. 
+        sai_attribute_t attr;
+        attr.id = SAI_SWITCH_ATTR_MACSEC_POST_STATUS;
+        if (sai_switch_api->get_switch_attribute(gSwitchId, 1, &attr) == SAI_STATUS_SUCCESS)
+        {
+            if (attr.value.s32 == SAI_SWITCH_MACSEC_POST_STATUS_PASS)
+            {
+                setMacsecPostState(m_state_db, "pass");
+                SWSS_LOG_NOTICE("Switch MACSec POST passed");
+            }
+            else if (attr.value.s32 == SAI_SWITCH_MACSEC_POST_STATUS_FAIL)
+            {
+                setMacsecPostState(m_state_db, "fail");
+                SWSS_LOG_ERROR("Switch MACSec POST failed: oid %" PRIu64, gSwitchId);
+            }
+            else
+            {
+                SWSS_LOG_NOTICE("Switch MACSec POST status: %d", attr.value.s32);
+            }
+        }
+        else
+        {
+            SWSS_LOG_ERROR("Failed to get MACSec POST status");
+        }
+    }
+    else if (post_state == "macsec-level-post-in-progress")
+    {
+        // POST is only supported in MACSec init. Create MACSec and enable POST.
+        m_enable_post = true;
+        initMACsecObject(gSwitchId);
+        SWSS_LOG_NOTICE("Init MACSec objects and enable POST");
+    }
 }
 
 MACsecOrch::~MACsecOrch()
@@ -661,6 +711,127 @@ MACsecOrch::~MACsecOrch()
     }
 }
 
+void MACsecOrch::doTask(NotificationConsumer &consumer)
+{
+    SWSS_LOG_ENTER();
+
+    if (&consumer != m_postCompletionNotificationConsumer)
+    {
+        return;
+    }
+
+    std::deque<KeyOpFieldsValuesTuple> entries;
+    consumer.pops(entries);
+    for (auto& entry : entries)
+    {
+        handleNotification(consumer, entry);
+    }
+}
+
+void MACsecOrch::handleNotification(NotificationConsumer &consumer, KeyOpFieldsValuesTuple& entry)
+{
+    SWSS_LOG_ENTER();
+
+    if (&consumer != m_postCompletionNotificationConsumer)
+    {
+        return;
+    }
+
+    auto op = kfvOp(entry);
+    auto data = kfvKey(entry);
+    SWSS_LOG_NOTICE("Received SAI notification: op %s, data %s", op.c_str(), data.c_str());
+
+    if (op == "switch_macsec_post_status")
+    {
+        sai_object_id_t switch_id;
+        sai_switch_macsec_post_status_t switch_macsec_post_status;
+        sai_deserialize_switch_macsec_post_status_ntf(data, switch_id, switch_macsec_post_status);
+
+        string post_state = getMacsecPostState(m_state_db);
+        if (post_state == "switch-level-post-in-progress")
+        {
+            // MACSec POST was enabled in switch init. SAI enables POST in all HW MACSec engines.
+            // The returned POST status is the aggregated result for all HW MACSec engines.
+
+            if (switch_macsec_post_status == SAI_SWITCH_MACSEC_POST_STATUS_PASS)
+            {
+                setMacsecPostState(m_state_db, "pass");
+                SWSS_LOG_NOTICE("Switch MACSec POST passed");
+            }
+            else if (switch_macsec_post_status == SAI_SWITCH_MACSEC_POST_STATUS_FAIL)
+            {
+                setMacsecPostState(m_state_db, "fail");
+                SWSS_LOG_ERROR("Switch MACSec POST failed");
+            }
+        }
+        else if (post_state == "macsec-level-post-in-progress")
+        {
+            SWSS_LOG_ERROR("POST enabled in MACSec init, but got notification from switch init");
+        }
+    }
+
+    if (op == "macsec_post_status")
+    {
+        sai_object_id_t macsec_id;
+        sai_macsec_post_status_t macsec_post_status;
+        sai_deserialize_macsec_post_status_ntf(data, macsec_id, macsec_post_status);
+
+        if (m_enable_post)
+        {
+            // MACSec POST was enabled in MACSec object init. Since two MACSec objects were created
+            // (one for each direction), two POST status must be returend from SAI. POST is considered
+            // pass only if POST passes in both MACSec objects.
+
+            string direction = "unknown";
+            auto macsec_obj = m_macsec_objs.find(gSwitchId);
+            if (macsec_obj->second.m_ingress_id == macsec_id)
+            {
+                direction = "ingress";
+            }
+            else if (macsec_obj->second.m_egress_id == macsec_id)
+            {
+                direction = "egress";
+            }
+
+            if (macsec_post_status == SAI_MACSEC_POST_STATUS_PASS)
+            {
+                if (direction == "ingress")
+                {
+                    SWSS_LOG_NOTICE("Ingress MACSec POST passed");
+                    macsec_obj->second.m_ingress_post_passed = true;
+                }
+                else if (direction == "egress")
+                {
+                    SWSS_LOG_NOTICE("Egress MACSec POST passed");
+                    macsec_obj->second.m_egress_post_passed = true;
+                }
+
+                // Check if POST passed on both MACSec objects.
+                if (macsec_obj->second.m_ingress_post_passed && macsec_obj->second.m_egress_post_passed)
+                {
+                    setMacsecPostState(m_state_db, "pass");
+                    SWSS_LOG_NOTICE("Ingress and egress MACSec POST passed");
+                }
+            }
+            else if(macsec_post_status == SAI_MACSEC_POST_STATUS_FAIL)
+            {
+                if (direction == "ingress")
+                {
+                    SWSS_LOG_ERROR("Ingress MACSec POST failed");
+                }
+                else if (direction == "egress")
+                {
+                    SWSS_LOG_ERROR("Egress MACSec POST failed");
+                }
+
+                // Consider POST failed since it failed on one MACSec object.
+                setMacsecPostState(m_state_db, "fail");
+                SWSS_LOG_ERROR("MACSec POST failed");
+            }
+        }
+    }
+}
+
 void MACsecOrch::doTask(Consumer &consumer)
 {
     SWSS_LOG_ENTER();
@@ -1044,6 +1215,13 @@ bool MACsecOrch::initMACsecObject(sai_object_id_t switch_id)
     attr.value.booldata = true;
     attrs.push_back(attr);
 
+    if (m_enable_post)
+    {
+        attr.id = SAI_MACSEC_ATTR_ENABLE_POST;
+        attr.value.booldata = true;
+        attrs.push_back(attr);
+    }
+
     sai_status_t status = sai_macsec_api->create_macsec(
                                 &macsec_obj.first->second.m_egress_id,
                                 switch_id,
@@ -1069,6 +1247,13 @@ bool MACsecOrch::initMACsecObject(sai_object_id_t switch_id)
     attr.value.booldata = true;
     attrs.push_back(attr);
 
+    if (m_enable_post)
+    {
+        attr.id = SAI_MACSEC_ATTR_ENABLE_POST;
+        attr.value.booldata = true;
+        attrs.push_back(attr);
+    }
+
     status = sai_macsec_api->create_macsec(
                                 &macsec_obj.first->second.m_ingress_id,
                                 switch_id,

orchagent/macsecorch.h

@@ -40,6 +40,8 @@ class MACsecOrch : public Orch
 
 private:
     void doTask(Consumer &consumer);
+    void doTask(NotificationConsumer &consumer);
+    void handleNotification(NotificationConsumer &consumer, KeyOpFieldsValuesTuple& entry);
 
 public:
     using TaskArgs = std::vector<FieldValueTuple>;
@@ -57,6 +59,11 @@ class MACsecOrch : public Orch
     task_process_status taskUpdateIngressSA(const std::string & port_sci_an, const TaskArgs & sa_attr);
     task_process_status taskDeleteIngressSA(const std::string & port_sci_an, const TaskArgs & sa_attr);
 
+    DBConnector * m_state_db;
+    shared_ptr<DBConnector> m_notificationsDb;
+    NotificationConsumer* m_postCompletionNotificationConsumer;
+    bool m_enable_post;
+
     PortsOrch * m_port_orch;
 
     Table m_state_macsec_port;
@@ -117,6 +124,8 @@ class MACsecOrch : public Orch
         map<std::string, std::shared_ptr<MACsecPort> >  m_macsec_ports;
         bool                                            m_sci_in_ingress_macsec_acl;
         sai_uint8_t                                     m_max_sa_per_sc;
+        bool                                            m_egress_post_passed;
+        bool                                            m_ingress_post_passed;
     };
     map<sai_object_id_t, MACsecObject>              m_macsec_objs;
     map<std::string, std::shared_ptr<MACsecPort> >  m_macsec_ports;

orchagent/macsecpost.cpp

@@ -0,0 +1,42 @@
+#include "dbconnector.h"
+#include "macsecpost.h"
+#include "redisutility.h"
+#include "schema.h"
+#include "table.h"
+
+namespace swss {
+
+void setMacsecPostState(DBConnector *stateDb, string postState)
+{
+    Table macsecPostStateTable = Table(stateDb, STATE_FIPS_MACSEC_POST_TABLE_NAME);
+    vector<FieldValueTuple> fvts;
+    FieldValueTuple postStateFvt("post_state", postState);
+    fvts.push_back(postStateFvt);
+
+    auto now = std::chrono::system_clock::now();
+    std::time_t now_c = std::chrono::system_clock::to_time_t(now);
+    char buffer[32];
+    std::strftime(buffer, sizeof(buffer), "%a %b %d %H:%M:%S %Y", std::gmtime(&now_c));
+    FieldValueTuple lastUpdateTimeFvt("last_update_time", buffer);
+    fvts.push_back(lastUpdateTimeFvt);
+
+    macsecPostStateTable.set("sai", fvts);
+}
+
+string getMacsecPostState(DBConnector *stateDb)
+{
+    std::string postState = "";
+    std::vector<FieldValueTuple> fvts;
+    Table macsecPostStateTable = Table(stateDb, STATE_FIPS_MACSEC_POST_TABLE_NAME);
+    if (macsecPostStateTable.get("sai", fvts))
+    {
+        auto state = fvsGetValue(fvts, "post_state", true);
+        if (state)
+        {
+            postState = *state;
+        }
+    }
+    return postState;
+}
+
+}

orchagent/macsecpost.h

@@ -0,0 +1,13 @@
+#ifndef ORCHAGENT_MACSECPOST_H
+#define ORCHAGENT_MACSECPOST_H
+
+using namespace std;
+
+namespace swss {
+
+void setMacsecPostState(DBConnector *stateDb, string postState);
+string getMacsecPostState(DBConnector *stateDb);
+
+}
+
+#endif // ORCHAGENT_MACSECPOST_H