Linkmgrd subscribing State DB route event (sonic-net#13)

Summary:
Fixes # (issue)

This PR is to make linkmgrd subscribe events from ROUTE_TABLE in State DB, and react accordingly:  
- If any of the two default route state appears to be 'na', linkmgrd should switch to standby. 
- If both are 'ok', there will be a mux state probing and what happens next depends on linkmgrd state machine and the probing response.  

Type of change:
New feature

Motivation for this PR:
To make linkmgrd subscribe state DB route event, and handle the switchovers. 

Documentation: 
Related PR: sonic-net/sonic-swss#2009

Author: zjswhhh

src/DbInterface.cpp

@@ -730,6 +730,59 @@ void DbInterface::handleMuxStateNotifiction(swss::SubscriberStateTable &statedbP
     processMuxStateNotifiction(entries);
 }
 
+//
+// ---> processDefaultRouteStateNotification(std::deque<swss::KeyOpFieldsValuesTuple> &entries)
+// 
+// process default route state notification from orchagent
+//
+void DbInterface::processDefaultRouteStateNotification(std::deque<swss::KeyOpFieldsValuesTuple> &entries)
+{
+    for (auto &entry: entries) {
+        std::string key = kfvKey(entry);
+        std::string op = kfvOp(entry);
+        std::vector<swss::FieldValueTuple> fieldValues = kfvFieldsValues(entry);
+
+        std::vector<swss::FieldValueTuple>::const_iterator cit = std::find_if(
+            fieldValues.cbegin(),
+            fieldValues.cend(),
+            [] (const swss::FieldValueTuple &fv) {return fvField(fv) == "state";}
+        );
+
+        if (cit != fieldValues.cend()) {
+            const std::string field = cit->first;
+            const std::string value = cit->second;
+
+            MUXLOGDEBUG(boost::format("key: %s, operation: %s, field: %s, value: %s") %
+                key %
+                op %
+                field %
+                value
+            );
+
+            if (key == "0.0.0.0/0") {
+                mMuxManagerPtr->addOrUpdateDefaultRouteState(true, value);
+            } else if (key == "::/0") {
+                mMuxManagerPtr->addOrUpdateDefaultRouteState(false, value);
+            } else {
+                MUXLOGFATAL(boost::format("Received Invalid IP: %s") % key );
+            }
+        }
+    }
+}
+
+//
+// ---> handleDefaultRouteStateNotification(swss::SubscriberStateTable &statedbRouteTable);
+//
+// handle Default Route State notification from orchagent
+//
+void DbInterface::handleDefaultRouteStateNotification(swss::SubscriberStateTable &statedbRouteTable)
+{
+    std::deque<swss::KeyOpFieldsValuesTuple> entries;
+
+    statedbRouteTable.pops(entries);
+    processDefaultRouteStateNotification(entries);
+}
+
 //
 // ---> handleSwssNotification();
 //
@@ -752,6 +805,8 @@ void DbInterface::handleSwssNotification()
     swss::SubscriberStateTable appDbMuxResponseTable(appDbPtr.get(), APP_MUX_CABLE_RESPONSE_TABLE_NAME);
     // for getting state db MUX state when orchagent updates it
     swss::SubscriberStateTable stateDbPortTable(stateDbPtr.get(), STATE_MUX_CABLE_TABLE_NAME);
+    // for getting state db default route state 
+    swss::SubscriberStateTable stateDbRouteTable(stateDbPtr.get(), STATE_ROUTE_TABLE_NAME);
 
     getTorMacAddress(configDbPtr);
     getLoopback2InterfaceInfo(configDbPtr);
@@ -771,6 +826,7 @@ void DbInterface::handleSwssNotification()
     swssSelect.addSelectable(&appDbPortTable);
     swssSelect.addSelectable(&appDbMuxResponseTable);
     swssSelect.addSelectable(&stateDbPortTable);
+    swssSelect.addSelectable(&stateDbRouteTable);
     swssSelect.addSelectable(&netlinkNeighbor);
 
     while (mPollSwssNotifcation) {
@@ -797,6 +853,8 @@ void DbInterface::handleSwssNotification()
             handleMuxResponseNotifiction(appDbMuxResponseTable);
         } else if (selectable == static_cast<swss::Selectable *> (&stateDbPortTable)) {
             handleMuxStateNotifiction(stateDbPortTable);
+        } else if (selectable == static_cast<swss::Selectable *> (&stateDbRouteTable)) {
+            handleDefaultRouteStateNotification(stateDbRouteTable);
         } else if (selectable == static_cast<swss::Selectable *> (&netlinkNeighbor)) {
             continue;
         } else {

src/DbInterface.h

@@ -462,6 +462,28 @@ class DbInterface
     */
     void handleSwssNotification();
 
+    /**
+     * @method processDefaultRouteStateNotification
+     * 
+     * @brief process default route state notification from orchagent
+     * 
+     * @param entries reference to state db default route state entries
+     * 
+     * @return none
+    */
+    void processDefaultRouteStateNotification(std::deque<swss::KeyOpFieldsValuesTuple> &entries);
+
+    /**
+     * @method handleDefaultRouteStateNotification
+     * 
+     * @brief handle Default Route State notification from orchagent
+     * 
+     * @param statedbRouteTable reference to state db route table 
+     * 
+     * @return none
+    */
+    void handleDefaultRouteStateNotification(swss::SubscriberStateTable &statedbRouteTable);
+
 private:
     static std::vector<std::string> mMuxState;
     static std::vector<std::string> mMuxLinkmgrState;

src/MuxManager.cpp

@@ -234,6 +234,34 @@ void MuxManager::processProbeMuxState(const std::string &portName, const std::st
     }
 }
 
+//
+// ---> addOrUpdateDefaultRouteState(boost::asio::ip::address& address, const std::string &routeState);
+//
+// update default route state based on state db notification
+//
+void MuxManager::addOrUpdateDefaultRouteState(bool is_v4, const std::string &routeState) 
+{
+    if (is_v4) {
+        mIpv4DefaultRouteState = routeState;
+    } else {
+        mIpv6DefaultRouteState = routeState;
+    }
+
+    std::string nextState = "na";
+    // For now we only need IPv4 default route state to be "ok". If we switch to IPv6 in the furture, this will cause an issue. 
+    if (mIpv4DefaultRouteState == "ok") {
+        nextState = "ok";
+    }
+
+    MUXLOGINFO(boost::format("Default route state: %s") % nextState);
+
+    PortMapIterator portMapIterator = mPortMap.begin();
+    while (portMapIterator != mPortMap.end()) {
+        portMapIterator->second->handleDefaultRouteState(nextState);
+        portMapIterator ++;
+    }
+}
+
 //
 // ---> getMuxPortPtrOrThrow(const std::string &portName);
 //

src/MuxManager.h

@@ -305,6 +305,19 @@ class MuxManager
     */
     void processProbeMuxState(const std::string &portName, const std::string &muxState);
 
+    /**
+     * @method addOrUpdateDefaultRouteState
+     * 
+     * @brief update default route state based on state db notification
+     * 
+     * @param ipAddress
+     * @param routeState
+     *  
+     * @return none
+     * 
+    */
+    void addOrUpdateDefaultRouteState(bool is_v4, const std::string &routeState);
+
 private:
     /**
     *@method getMuxPortPtrOrThrow
@@ -360,6 +373,9 @@ class MuxManager
     std::shared_ptr<mux::DbInterface> mDbInterfacePtr;
 
     PortMap mPortMap;
+
+    std::string mIpv4DefaultRouteState = "na";
+    std::string mIpv6DefaultRouteState = "na";
 };
 
 } /* namespace mux */

src/MuxPort.cpp

@@ -228,4 +228,20 @@ void MuxPort::handleMuxConfig(const std::string &config)
     )));
 }
 
+// 
+// ---> handleDefaultRouteState(const std::string &routeState);
+// 
+// handles default route state notification
+//
+void MuxPort::handleDefaultRouteState(const std::string &routeState)
+{
+    MUXLOGDEBUG(boost::format("port: %s, state db default route state: %s") % mMuxPortConfig.getPortName() % routeState);
+
+    boost::asio::io_service &ioService = mStrand.context();
+    ioService.post(mStrand.wrap(boost::bind(
+        &link_manager::LinkManagerStateMachine::handleDefaultRouteStateNotification,
+        &mLinkManagerStateMachine,
+        routeState
+    )));
+}
 } /* namespace mux */

src/MuxPort.h

@@ -252,6 +252,17 @@ class MuxPort: public std::enable_shared_from_this<MuxPort>
     */
     void handleMuxConfig(const std::string &config);
 
+    /**
+     * @method handleDefaultRouteState(const std::string &routeState)
+     * 
+     * @brief handles default route state notification
+     * 
+     * @param routeState 
+     * 
+     * @return none
+    */
+    void handleDefaultRouteState(const std::string &routeState);
+
 protected:
     friend class test::MuxManagerTest;
     friend class test::FakeMuxPort;

src/link_manager/LinkManagerStateMachine.cpp

@@ -815,6 +815,28 @@ void LinkManagerStateMachine::handleSwitchActiveRequestEvent()
     }
 }
 
+// 
+// ---> handleDefaultRouteStateNotification(const std::string &routeState);
+// 
+// handle default route state notification from routeorch
+//
+void LinkManagerStateMachine::handleDefaultRouteStateNotification(const std::string &routeState)
+{
+    MUXLOGWARNING(boost::format("%s: state db default route state: %s") % mMuxPortConfig.getPortName() % routeState);
+
+    if (mComponentInitState.test(MuxStateComponent)) {
+        if (ms(mCompositeState) != mux_state::MuxState::Label::Standby && routeState == "na") {
+            CompositeState nextState = mCompositeState;
+            enterLinkProberState(nextState, link_prober::LinkProberState::Wait);
+            switchMuxState(nextState, mux_state::MuxState::Label::Standby, true);
+            LOGWARNING_MUX_STATE_TRANSITION(mMuxPortConfig.getPortName(), mCompositeState, nextState);
+            mCompositeState = nextState;
+        } else {
+            enterMuxWaitState(mCompositeState);
+        }
+    } 
+}
+
 //
 // ---> updateMuxLinkmgrState();
 //

src/link_manager/LinkManagerStateMachine.h

@@ -476,6 +476,17 @@ class LinkManagerStateMachine: public common::StateMachine,
     */
     void handleSwitchActiveRequestEvent();
 
+    /**
+     * @method handleDefaultRouteStateNotification(const std::string &routeState)
+     * 
+     * @brief handle default route state notification from routeorch
+     * 
+     * @param routeState
+     * 
+     * @return none
+    */
+    void handleDefaultRouteStateNotification(const std::string &routeState);
+
 private:
     /**
     *@method updateMuxLinkmgrState

test/LinkManagerStateMachineTest.cpp

@@ -230,6 +230,12 @@ void LinkManagerStateMachineTest::setMuxStandby()
     VALIDATE_STATE(Standby, Standby, Up);
 }
 
+void LinkManagerStateMachineTest::postDefaultRouteEvent(std::string routeState, uint32_t count)
+{
+    mFakeMuxPort.handleDefaultRouteState(routeState);
+    runIoService(count);
+}
+
 TEST_F(LinkManagerStateMachineTest, MuxActiveSwitchOver)
 {
     setMuxActive();
@@ -1013,4 +1019,38 @@ TEST_F(LinkManagerStateMachineTest, MuxStandby2Unknown2Error)
     VALIDATE_STATE(Standby, Error, Down);
 }
 
+TEST_F(LinkManagerStateMachineTest, MuxActivDefaultRouteStateNA) 
+{
+    setMuxActive();
+
+    EXPECT_EQ(mDbInterfacePtr->mSetMuxStateInvokeCount, 0);
+    postDefaultRouteEvent("na", 3);
+
+    VALIDATE_STATE(Wait, Wait, Up);
+    EXPECT_EQ(mDbInterfacePtr->mSetMuxStateInvokeCount, 1);
+
+    postLinkProberEvent(link_prober::LinkProberState::Standby, 3);
+    VALIDATE_STATE(Standby, Wait, Up);
+
+    handleMuxState("standby", 3);
+    VALIDATE_STATE(Standby, Standby, Up);
+}
+
+TEST_F(LinkManagerStateMachineTest, MuxStandbyDefaultRouteStateOK) 
+{
+    setMuxStandby();
+
+    EXPECT_EQ(mDbInterfacePtr->mProbeMuxStateInvokeCount, 0);
+    postDefaultRouteEvent("ok", 2);
+
+    VALIDATE_STATE(Standby, Wait, Up);
+    EXPECT_EQ(mDbInterfacePtr->mProbeMuxStateInvokeCount, 1);
+
+    postLinkProberEvent(link_prober::LinkProberState::Standby, 3);
+    VALIDATE_STATE(Standby, Wait, Up);
+
+    handleMuxState("standby", 3);
+    VALIDATE_STATE(Standby, Standby, Up);
+}
+
 } /* namespace test */

test/LinkManagerStateMachineTest.h

@@ -51,6 +51,7 @@ class LinkManagerStateMachineTest: public ::testing::Test
     void activateStateMachine();
     void setMuxActive();
     void setMuxStandby();
+    void postDefaultRouteEvent(std::string routeState, uint32_t count = 0);
 
 public:
     boost::asio::io_service mIoService;