Stream structured log data to telemetry's UDP listener

dockers/docker-fpm-frr/00-rsyslog-plugin.conf

@@ -0,0 +1,26 @@
+## omprog streaming log data
+
+template(name="prog_msg" type="list") {
+     property(name="msg")
+     constant(value="\n")
+     }
+
+$ModLoad omprog
+
+if ($programname startswith "bgpcfgd") then {
+    action(type="omprog"
+        binary="/usr/share/sonic/scripts/rsyslog_plugin.py -p bgpcfgd"
+        output="/var/log/rsyslog_plugin.log"
+        confirmMessages="on"
+        template="prog_msg")
+}
+
+if ($programname startswith "bgpd") then {
+    action(type="omprog"
+        binary="/usr/share/sonic/scripts/rsyslog_plugin.py -p bgpd"
+        output="/var/log/rsyslog_plugin.log"
+        confirmMessages="on"
+        template="prog_msg")
+}
+
+

dockers/docker-fpm-frr/Dockerfile.j2

@@ -56,6 +56,8 @@ COPY ["TSC", "/usr/bin/TSC"]
 COPY ["TS", "/usr/bin/TS"]
 COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
 COPY ["zsocket.sh", "/usr/bin/"]
+COPY ["00-rsyslog-plugin.conf", "/etc/rsyslog.d"]
+COPY ["*_parse.rc.json", "/etc/rsyslog.d/"]
 RUN chmod a+x /usr/bin/TSA && \
     chmod a+x /usr/bin/TSB && \
     chmod a+x /usr/bin/TSC && \

dockers/docker-fpm-frr/bgpcfgd_parse.rc.json

@@ -0,0 +1,12 @@
+[
+    {
+        "tag": "bgp_admin_up",
+        "regex": "^ Peer 'default.([0-9a-f:.]*)' admin state is set to 'up'",
+        "params": [ "peer_ip" ]
+    },
+    {
+        "tag": "bgp_admin_down",
+        "regex": "^ Peer 'default.([0-9a-f:.]*)' admin state is set to 'down'",
+        "params": [ "peer_ip" ]
+    }
+]

dockers/docker-fpm-frr/bgpd_parse.rc.json

@@ -0,0 +1,17 @@
+[
+    {
+        "tag": "bgp_admin_down",
+        "regex": "^ %ADJCHANGE: neighbor ([0-9a-f:.]*).*vrf default Down Admin",
+        "params": [ "peer_ip" ]
+    },
+    {
+        "tag": "bgp_admin_up",
+        "regex": "^ %ADJCHANGE: neighbor ([0-9a-f:.]*).*vrf default Up",
+        "params": [ "peer_ip" ]
+    },
+    {
+        "tag": "bgp_timer",
+        "regex": "^ %NOTIFICATION: sent to neighbor ([0-9a-f:.]*) .{3} .Hold Timer Expired",
+        "params": [ "peer_ip" ]
+    }
+]

dockers/docker-fpm-frr/zebra_parse.rc.json

@@ -0,0 +1,7 @@
+[
+    {
+        "tag": "no_space",
+        "regex": "^ netlink-listen recvmsg overrun: No buffer space available",
+        "params": [ "peer_ip" ]
+    }
+]

dockers/docker-orchagent/00-rsyslog-plugin.conf

@@ -0,0 +1,17 @@
+## omprog streaming log data
+
+template(name="prog_msg" type="list") {
+     property(name="msg")
+     constant(value="\n")
+     }
+
+$ModLoad omprog
+
+if ($programname startswith "orchagent") then {
+    action(type="omprog"
+        binary="/usr/share/sonic/scripts/rsyslog_plugin.py -p orchagent"
+        output="/var/log/rsyslog_plugin.log"
+        confirmMessages="on"
+        template="prog_msg")
+}
+

dockers/docker-orchagent/orchagent_parse.rc.json

@@ -0,0 +1,12 @@
+[
+    {
+        "tag": "redis_generic",
+        "regex": "^ :- internal_redis_generic_get: generic get failed to get response",
+        "params": []
+    },
+    {
+        "tag": "pfc_storm",
+        "regex": "^ :- startWdActionOnQueue: PFC Watchdog detected PFC storm on port (Ethernet[0-9]*)",
+        "params": []
+    }
+]

files/build_templates/sonic_debian_extension.j2

@@ -436,6 +436,12 @@ fi
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install azure-storage==0.36.0
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install watchdog==0.10.3
 
+# Copy rsyslog plugin files.
+#
+sudo mkdir -p ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS}
+sudo LANG=C cp $FILESYSTEM_ROOT/usr/local/bin/rsyslog_plugin.py ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS}/
+sudo chmod a+x ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS}/rsyslog_plugin.py
+
 {% if include_kubernetes == "y" %}
 # Point to kubelet to /etc/resolv.conf
 #
@@ -456,7 +462,6 @@ sudo rm -rf $FILESYSTEM_ROOT/$SONIC_CTRMGMT_WHEEL_NAME
 
 # Copy remote container mangement files
 # File called from each container upon start/stop to record the state
-sudo mkdir -p ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS}
 sudo cp ${files_path}/container_startup.py ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS}/
 sudo chmod a+x ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS}/container_startup.py
 

files/image_config/rsyslog/rsyslog.d/00-rsyslog-plugin.conf

@@ -0,0 +1,33 @@
+## omprog streaming log data
+
+template(name="prog_msg" type="list") {
+     property(name="msg")
+     }
+
+$ModLoad omprog
+
+if ($programname startswith "sshd") then {
+    action(type="omprog"
+        binary="/usr/share/sonic/scripts/rsyslog_plugin.py -p sshd"
+        output="/var/log/rsyslog_plugin.log"
+        confirmMessages="on"
+        template="prog_msg")
+}
+
+if ($programname startswith "kernel") then {
+    action(type="omprog"
+        binary="/usr/share/sonic/scripts/rsyslog_plugin.py -p kernel"
+        output="/var/log/rsyslog_plugin.log"
+        confirmMessages="on"
+        template="prog_msg")
+}
+
+if ($programname startswith "monit") then {
+    action(type="omprog"
+        binary="/usr/share/sonic/scripts/rsyslog_plugin.py -p monit"
+        output="/var/log/rsyslog_plugin.log"
+        confirmMessages="on"
+        template="prog_msg")
+}
+
+

files/image_config/rsyslog/rsyslog.d/kernel_parse.rc.json

@@ -0,0 +1,17 @@
+[
+    {
+        "tag": "space_usage",
+        "regex": "'(.*)' space usage (.*)% matches resource limit \[space usage>(.*)%\]",
+        "params": [fs, usage, limit]
+    },
+    {
+        "tag": "mem_usage",
+        "regex": "mem usage of (.*)% matches resource limit \[mem usage>(.*)%\]",
+        "params": [usage, limit]
+    },
+    {
+        "tag": "read_lock",
+        "regex": "aufs_read_lock",
+        "params": []
+    }
+]

files/image_config/rsyslog/rsyslog.d/monit_parse.rc.json

@@ -0,0 +1,12 @@
+[
+    {
+        "tag": "kernel_write",
+        "regex": "[wW]rite (failed|protected)",
+        "params": []
+    },
+    {
+        "tag": "kernel_ro",
+        "regex": "Remounting filesystem read.only",
+        "params": []
+    }
+]

files/image_config/rsyslog/rsyslog.d/sshd_parse.rc.json

@@ -0,0 +1,7 @@
+[
+    {
+        "tag": "INCORRECT_PASSWD",
+        "regex": "^ auth fail: Password incorrect. user: (.*)",
+        "params": [ "user" ]
+    }
+]

rules/docker-fpm-frr.mk

@@ -30,6 +30,7 @@ SONIC_DOCKER_DBG_IMAGES += $(DOCKER_FPM_FRR_DBG)
 $(DOCKER_FPM_FRR)_CONTAINER_NAME = bgp
 $(DOCKER_FPM_FRR)_RUN_OPT += --privileged -t
 $(DOCKER_FPM_FRR)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
+$(DOCKER_FPM_FRR)_RUN_OPT += -v /usr/share/sonic/scripts:/usr/share/sonic/scripts:ro
 
 $(DOCKER_FPM_FRR)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)
 

rules/docker-platform-monitor.mk

@@ -51,7 +51,6 @@ SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_PLATFORM_MONITOR_DBG)
 $(DOCKER_PLATFORM_MONITOR)_CONTAINER_NAME = pmon
 $(DOCKER_PLATFORM_MONITOR)_RUN_OPT += --privileged -t
 $(DOCKER_PLATFORM_MONITOR)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
-$(DOCKER_PLATFORM_MONITOR)_RUN_OPT += -v /usr/share/sonic/scripts:/usr/share/sonic/scripts:ro
 $(DOCKER_PLATFORM_MONITOR)_RUN_OPT += -v /var/run/platform_cache:/var/run/platform_cache:ro
 $(DOCKER_PLATFORM_MONITOR)_RUN_OPT += -v /usr/share/sonic/device/pddf:/usr/share/sonic/device/pddf:ro