[Auto Techsupport] Event driven Techsupport Changes

Makefile.work

@@ -39,6 +39,9 @@
 #  * SONIC_DPKG_CACHE_SOURCE: Debian package cache location when cache enabled for debian packages
 #  * BUILD_LOG_TIMESTAMP: Set timestamp in the build log (simple/none)
 #  * DOCKER_EXTRA_OPTS: Extra command line arguments for dockerd running in slave container.
+#  * ENABLE_AUTO_TECH_SUPPORT: Enable the configuration for event-driven techsupport & coredump mgmt feature
+#  *                 Default: y
+#  *                 Values: y,n
 #
 ###############################################################################
 
@@ -280,6 +283,7 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            SONIC_ENABLE_IMAGE_SIGNATURE=$(ENABLE_IMAGE_SIGNATURE) \
                            ENABLE_HOST_SERVICE_ON_START=$(ENABLE_HOST_SERVICE_ON_START) \
                            SLAVE_DIR=$(SLAVE_DIR) \
+                           ENABLE_AUTO_TECH_SUPPORT=$(ENABLE_AUTO_TECH_SUPPORT) \
                            $(SONIC_OVERRIDE_BUILD_VARS)
 
 .PHONY: sonic-slave-build sonic-slave-bash init reset

files/build_templates/init_cfg.json.j2

@@ -53,5 +53,26 @@
             "high_mem_alert": "disabled"
         }{% if not loop.last %},{% endif -%}
 {% endfor %}
+    },
+    "AUTO_TECHSUPPORT": {
+        "GLOBAL": {
+{%- if enable_auto_tech_support == "y" %}
+            "state" : "enabled", {% else %}
+            "state" : "disabled", {% endif %}
+            "rate_limit_interval" : "180",
+            "max_techsupport_size" : "10.0",
+            "max_core_size" : "5.0",
+            "since" : "2 days ago"
+        }
+    },
+    "AUTO_TECHSUPPORT_FEATURE": {
+{%- for feature, _, _, _ in features %}
+        "{{feature}}": {
+{%- if enable_auto_tech_support == "y" %}
+            "state" : "enabled", {% else %}
+            "state" : "disabled", {% endif %}
+            "rate_limit_interval" : "600"
+        }{%if not loop.last %},{% endif -%}
+{% endfor %} 
     }
 }

files/scripts/supervisor-proc-exit-listener

@@ -9,9 +9,10 @@ import sys
 import syslog
 import time
 from collections import defaultdict
+import subprocess
+from swsscommon.swsscommon import DBConnector
 
 import swsssdk
-
 from supervisor import childutils
 
 # Each line of this file should specify either one critical process or one
@@ -31,6 +32,50 @@ SELECT_TIMEOUT_SECS = 1.0
 # Alerting message will be written into syslog in the following interval
 ALERTING_INTERVAL_SECS = 60
 
+# Saves the mapping between PID and the command/executable name
+PID_COMM_FILE_MAP = dict()
+
+# STATE_DB
+STATE_DB = "STATE_DB"
+
+
+def subprocess_exec(cmd):
+    rc, stdout, stderr = 0, "", ""
+    try:
+        stdout = subprocess.check_output(cmd).decode("utf-8").strip()
+    except CalledProcessError as err:
+        rc = err.returncode
+        stderr = err.output.decode("utf-8").strip()
+    return rc, stdout, stderr
+
+
+def handle_proc_event(dbconn, event, supervisor_proc_name, pid, container_name):
+    """
+    @summary:
+    1) Set/Del the entry in the AUTO_TECHSUPPORT|FEATURE_PROC_INFO table of STATE DB
+    2) Update the PID_COMM_FILE_MAP dict
+    Table Schema:
+    key = "AUTO_TECHSUPPORT|FEATURE_PROC_INFO"
+    <feature_name;supervisor_proc_name> = <pid:comm>
+    """
+    field = "{};{}".format(container_name, supervisor_proc_name)
+    table = "AUTO_TECHSUPPORT|FEATURE_PROC_INFO"
+    try:
+        if event == "PROCESS_STATE_EXITED":
+            comm_name = PID_COMM_FILE_MAP.get(pid, "<unknown>")
+            # Create an entry in the Table
+            value = "{};{}".format(pid, comm_name)
+            dbconn.hset(table, field, value)
+            PID_COMM_FILE_MAP.pop(pid, None)
+        elif event == "PROCESS_STATE_RUNNING":
+            rc, stdout, stderr = subprocess_exec(["cat", "/proc/{}/comm".format(pid)])
+            if not rc:
+                PID_COMM_FILE_MAP[pid] = stdout  # Update the comm<->pid map
+            # Delete the Event Entry, since the process has started
+            dbconn.hdel(table, field)
+    except Exception as e:
+        syslog.syslog(syslog.LOG_ERR, "handle_proc_event method failed, ERR: {}".format(e))
+
 
 def get_critical_group_and_process_list():
     """
@@ -124,20 +169,26 @@ def main(argv):
     # Transition from ACKNOWLEDGED to READY
     childutils.listener.ready()
 
+    # STATE_DB dbconnector
+    dbconn = DBConnector(STATE_DB, 0)
+
     while True:
         file_descriptor_list = select.select([sys.stdin], [], [], SELECT_TIMEOUT_SECS)[0]
         if len(file_descriptor_list) > 0:
             line = file_descriptor_list[0].readline()
             headers = childutils.get_headers(line)
             payload = sys.stdin.read(int(headers['len']))
+            payload_headers, payload_data = childutils.eventdata(payload + '\n')
+            process_name = payload_headers['processname']
+            group_name = payload_headers['groupname']
+            pid = payload_headers['pid']
+
+            if process_name in critical_process_list or group_name in critical_group_list:
+                handle_proc_event(dbconn, headers['eventname'], process_name, pid, container_name)
 
             # Handle the PROCESS_STATE_EXITED event
             if headers['eventname'] == 'PROCESS_STATE_EXITED':
-                payload_headers, payload_data = childutils.eventdata(payload + '\n')
-
                 expected = int(payload_headers['expected'])
-                process_name = payload_headers['processname']
-                group_name = payload_headers['groupname']
 
                 if (process_name in critical_process_list or group_name in critical_group_list) and expected == 0:
                     is_auto_restart = get_autorestart_state(container_name)
@@ -152,9 +203,6 @@ def main(argv):
 
             # Handle the PROCESS_STATE_RUNNING event
             elif headers['eventname'] == 'PROCESS_STATE_RUNNING':
-                payload_headers, payload_data = childutils.eventdata(payload + '\n')
-                process_name = payload_headers['processname']
-
                 if process_name in process_under_alerting:
                     process_under_alerting.pop(process_name)
 

rules/config

@@ -144,6 +144,9 @@ INCLUDE_NAT = y
 # INCLUDE_DHCP_RELAY - build and install dhcp-relay package
 INCLUDE_DHCP_RELAY = y
 
+# ENABLE_AUTO_TECH_SUPPORT - Enable the configuration for event-driven techsupport & coredump mgmt feature
+ENABLE_AUTO_TECH_SUPPORT = y
+
 # TELEMETRY_WRITABLE - Enable write/config operations via the gNMI interface.
 # Uncomment to enable:
 # TELEMETRY_WRITABLE = y

slave.mk

@@ -160,6 +160,10 @@ ifeq ($(SONIC_INCLUDE_MACSEC),y)
 INCLUDE_MACSEC = y
 endif
 
+ifeq ($(ENABLE_AUTO_TECH_SUPPORT),y)
+ENABLE_AUTO_TECH_SUPPORT = y
+endif
+
 include $(RULES_PATH)/functions
 
 ifeq ($(SONIC_USE_PDDF_FRAMEWORK),y)
@@ -278,6 +282,7 @@ $(info "INCLUDE_DHCP_RELAY"              : "$(INCLUDE_DHCP_RELAY)")
 $(info "INCLUDE_KUBERNETES"              : "$(INCLUDE_KUBERNETES)")
 $(info "INCLUDE_MACSEC"                  : "$(INCLUDE_MACSEC)")
 $(info "TELEMETRY_WRITABLE"              : "$(TELEMETRY_WRITABLE)")
+$(info "ENABLE_AUTO_TECH_SUPPORT"        : "$(ENABLE_AUTO_TECH_SUPPORT)")
 $(info "PDDF_SUPPORT"                    : "$(PDDF_SUPPORT)")
 $(info "MULTIARCH_QEMU_ENVIRON"          : "$(MULTIARCH_QEMU_ENVIRON)")
 $(info "SONIC_VERSION_CONTROL_COMPONENTS": "$(SONIC_VERSION_CONTROL_COMPONENTS)")
@@ -944,6 +949,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
 	export include_restapi="$(INCLUDE_RESTAPI)"
 	export include_nat="$(INCLUDE_NAT)"
 	export include_sflow="$(INCLUDE_SFLOW)"
+	export enable_auto_tech_support="$(ENABLE_AUTO_TECH_SUPPORT)"
 	export include_macsec="$(INCLUDE_MACSEC)"
 	export include_mgmt_framework="$(INCLUDE_MGMT_FRAMEWORK)"
 	export include_iccpd="$(INCLUDE_ICCPD)"

src/sonic-yang-models/setup.py

@@ -42,6 +42,7 @@
     version='1.0',
     data_files=[
         ('yang-models', ['./yang-models/sonic-acl.yang',
+                         './yang-models/sonic-auto_techsupport.yang',
                          './yang-models/sonic-bgp-common.yang',
                          './yang-models/sonic-bgp-global.yang',
                          './yang-models/sonic-bgp-neighbor.yang',

src/sonic-yang-models/tests/files/sample_config_db.json

@@ -1051,6 +1051,33 @@
                 "trap_group": "queue1_group1"
             }
         },
+        "AUTO_TECHSUPPORT": {
+            "GLOBAL": {
+                "state" : "enabled",
+                "rate_limit_interval" : "180",
+                "max_techsupport_size" : "10.0",
+                "max_core_size" : "0.0",
+                "since" : "2 days ago"
+            }
+        },
+        "AUTO_TECHSUPPORT_FEATURE": {
+            "bgp" :{
+                "state" : "disabled",
+                "rate_limit_interval" : "600"
+            },
+            "swss" : {
+                "state" : "enabled",
+                "rate_limit_interval" : "600"
+            },
+            "snmp" : {
+                "state" : "enabled",
+                "rate_limit_interval" : "500"
+            },
+            "dhcp_relay" : {
+                "state" : "disabled",
+                "rate_limit_interval" : "1000"
+            }
+        },
         "LLDP": {
             "GLOBAL": {
                 "mode": "TRANSMIT",
@@ -1071,7 +1098,6 @@
         }
 
     },
-
     "SAMPLE_CONFIG_DB_UNKNOWN": {
         "UNKNOWN_TABLE": {
             "Error": "This Table is for testing, This Table does not have YANG models."

src/sonic-yang-models/tests/yang_model_tests/tests/auto_techsupport.json

@@ -0,0 +1,27 @@
+{
+    "AUTO_TECHSUPPORT_TEST":{
+	       "desc": "Configure auto techsupport params in AUTO_TECHSUPPORT Table"
+    },
+    "AUTO_TECHSUPPORT_WRONG_STATE_VALUE": {
+           "desc": "Configure state key with invalid value",
+           "eStrKey": "InvalidValue"
+    },
+    "AUTO_TECHSUPPORT_INVALID_RATE_LIMIT_FORMAT": {
+            "desc" : "Configure cooloff with a value of invalid format",
+            "eStrKey": "InvalidValue" 
+    },
+    "AUTO_TECHSUPPORT_OUT_OF_RANGE_DECIMAL": {
+            "desc" : "Configure a value for core-uage outside the range [0, 100)",
+            "eStr": "Value \"100.00\" does not satisfy the constraint \"0..99.99\" (range, length, or pattern)."
+    },
+    "AUTO_TECHSUPPORT_VALID_DECIMAL_VALUE": {
+            "desc" : "Configure a value for max_techsupport_size inside the range [0, 100)"
+    },
+    "AUTO_TECHSUPPORT_INVALID_FRACTION_DIGITS": {
+            "desc" : "Configure a value for max_techsupport_size inside the range [0, 100) but with 3 fractional digits",
+            "eStrKey": "InvalidValue" 
+    },
+    "AUTO_TECHSUPPORT_RATE_LIMIT_INTERVAL_TEST": {
+            "desc" : "Configure and test the valid configuration"
+    }
+}

src/sonic-yang-models/tests/yang_model_tests/tests_config/auto_techsupport.json

@@ -0,0 +1,85 @@
+{
+    "AUTO_TECHSUPPORT_TEST": {
+        "sonic-auto_techsupport:sonic-auto_techsupport": {
+            "sonic-auto_techsupport:AUTO_TECHSUPPORT": {
+                "sonic-auto_techsupport:GLOBAL": {
+                    "state" : "enabled",
+                    "rate_limit_interval" : "180",
+                    "max_techsupport_size" : "10.0",
+                    "max_core_size" : "5.0",
+                    "since" : "2 days ago"
+                } 
+            }
+        }
+    },
+    "AUTO_TECHSUPPORT_WRONG_STATE_VALUE": {
+        "sonic-auto_techsupport:sonic-auto_techsupport": {
+            "sonic-auto_techsupport:AUTO_TECHSUPPORT": {
+                "sonic-auto_techsupport:GLOBAL": {
+                    "state" : "start",
+                    "rate_limit_interval" : "180",
+                    "max_techsupport_size" : "10.0",
+                    "max_core_size" : "5.0",
+                    "since" : "2 days ago"                
+                } 
+            }
+        }
+    },
+    "AUTO_TECHSUPPORT_INVALID_RATE_LIMIT_FORMAT": {
+        "sonic-auto_techsupport:sonic-auto_techsupport": {
+            "sonic-auto_techsupport:AUTO_TECHSUPPORT": {
+                "sonic-auto_techsupport:GLOBAL": {
+                    "rate_limit_interval" : "whatever"
+                } 
+            }
+        }
+    },
+    "AUTO_TECHSUPPORT_OUT_OF_RANGE_DECIMAL": {
+        "sonic-auto_techsupport:sonic-auto_techsupport": {
+            "sonic-auto_techsupport:AUTO_TECHSUPPORT": {
+                "sonic-auto_techsupport:GLOBAL": {
+                    "max_core_size" : "100.00",
+                    "rate_limit_interval" : "180"
+                } 
+            }
+        }
+    },
+    "AUTO_TECHSUPPORT_VALID_DECIMAL_VALUE": {
+        "sonic-auto_techsupport:sonic-auto_techsupport": {
+            "sonic-auto_techsupport:AUTO_TECHSUPPORT": {
+                "sonic-auto_techsupport:GLOBAL": {
+                    "max_techsupport_size" : "11.23",
+                    "max_core_size" : "99.99"
+                } 
+            }
+        }
+    },
+    "AUTO_TECHSUPPORT_INVALID_FRACTION_DIGITS": {
+        "sonic-auto_techsupport:sonic-auto_techsupport": {
+            "sonic-auto_techsupport:AUTO_TECHSUPPORT": {
+                "sonic-auto_techsupport:GLOBAL": {
+                    "max_techsupport_size" : "11.111",
+                    "max_core_size" : "99.99"
+                } 
+            }
+        }
+    },
+    "AUTO_TECHSUPPORT_RATE_LIMIT_INTERVAL_TEST": {
+        "sonic-auto_techsupport:sonic-auto_techsupport": {
+            "sonic-auto_techsupport:AUTO_TECHSUPPORT_FEATURE": {
+                "AUTO_TECHSUPPORT_FEATURE_LIST": [
+                    {
+                        "feature_name" : "bgp",
+                        "state" : "enabled",
+                        "rate_limit_interval" : "600"
+                    },
+                    {
+                        "feature_name" : "swss",
+                        "state" : "disabled",
+                        "rate_limit_interval" : "400"
+                    }
+                ]
+            }
+        }
+    }
+}