Add orchagent heart beat message for watchdog. (#2737)

**What I did**
Improve orch agent: output heartbeat message to systemd.

**Why I did it**
Currently SONiC monit system only monit orchagent process exist or not. If orchagent process stuck and stop processing, current monit can't find and report it.

**How I verified it**
Pass all UT.
Manually validate the heartbeat message works correctly.

**Details if related**
Another inprogress PR will add watchdog for this heartbeat message:
sonic-net/sonic-buildimage#14686

sonic-mgmt UT PR: sonic-net/sonic-mgmt#8306

Author: liuh-80

orchagent/orchdaemon.cpp

@@ -6,6 +6,7 @@
 #include "logger.h"
 #include <sairedis.h>
 #include "warm_restart.h"
+#include <iostream>
 
 #define SAI_SWITCH_ATTR_CUSTOM_RANGE_BASE SAI_SWITCH_ATTR_CUSTOM_RANGE_START
 #include "sairedis.h"
@@ -18,6 +19,9 @@ using namespace swss;
 #define SELECT_TIMEOUT 1000
 #define PFC_WD_POLL_MSECS 100
 
+/* orchagent heart beat message interval */
+#define HEART_BEAT_INTERVAL_MSECS 10 * 1000
+
 extern sai_switch_api_t*           sai_switch_api;
 extern sai_object_id_t             gSwitchId;
 extern bool                        gSaiRedisLogRotate;
@@ -72,6 +76,7 @@ OrchDaemon::OrchDaemon(DBConnector *applDb, DBConnector *configDb, DBConnector *
 {
     SWSS_LOG_ENTER();
     m_select = new Select();
+    m_lastHeartBeat = std::chrono::high_resolution_clock::now();
 }
 
 OrchDaemon::~OrchDaemon()
@@ -722,6 +727,7 @@ void OrchDaemon::start()
         ret = m_select->select(&s, SELECT_TIMEOUT);
 
         auto tend = std::chrono::high_resolution_clock::now();
+        heartBeat(tend);
 
         auto diff = std::chrono::duration_cast<std::chrono::milliseconds>(tend - tstart);
 
@@ -957,6 +963,18 @@ void OrchDaemon::addOrchList(Orch *o)
     m_orchList.push_back(o);
 }
 
+void OrchDaemon::heartBeat(std::chrono::time_point<std::chrono::high_resolution_clock> tcurrent)
+{
+    // output heart beat message to SYSLOG
+    auto diff = std::chrono::duration_cast<std::chrono::milliseconds>(tcurrent - m_lastHeartBeat);
+    if (diff.count() >= HEART_BEAT_INTERVAL_MSECS)
+    {
+        m_lastHeartBeat = tcurrent;
+        // output heart beat message to supervisord with 'PROCESS_COMMUNICATION_STDOUT' event: http://supervisord.org/events.html
+        cout << "<!--XSUPERVISOR:BEGIN-->heartbeat<!--XSUPERVISOR:END-->" << endl;
+    }
+}
+
 FabricOrchDaemon::FabricOrchDaemon(DBConnector *applDb, DBConnector *configDb, DBConnector *stateDb, DBConnector *chassisAppDb) :
     OrchDaemon(applDb, configDb, stateDb, chassisAppDb),
     m_applDb(applDb),

orchagent/orchdaemon.h

@@ -90,8 +90,12 @@ class OrchDaemon
 
     std::vector<Orch *> m_orchList;
     Select *m_select;
+    
+    std::chrono::time_point<std::chrono::high_resolution_clock> m_lastHeartBeat;
 
     void flush();
+
+    void heartBeat(std::chrono::time_point<std::chrono::high_resolution_clock> tcurrent);
 };
 
 class FabricOrchDaemon : public OrchDaemon