Enhanced SupervisorD to exit immediately if one of its managed process get crashed which causes respective docker container to stop.Then container will be restarted gracefully.

dockers/docker-base/Dockerfile.j2

@@ -45,6 +45,7 @@ RUN mkdir -p /etc/supervisor
 RUN mkdir -p /var/log/supervisor
 
 COPY ["etc/supervisor/supervisord.conf", "/etc/supervisor/"]
+COPY ["etc/supervisor/kill_supervisor.py", "/usr/bin/"]
 
 RUN apt-get -y purge   \
     exim4              \

dockers/docker-base/etc/supervisor/kill_supervisor.py

@@ -0,0 +1,59 @@
+#!/usr/bin/python
+
+# Please follow the link for documentation: http://supervisord.org/events.html
+# SupervisorD exits immediately if one of its managed process get crashed as it subscribes to "EVENT Listener" process.
+
+import sys
+import os
+import signal
+import subprocess
+
+from supervisor.childutils import listener
+
+def write_stdout(s):
+    # only eventlistener protocol messages may be sent to stdout
+    sys.stdout.write(s)
+    sys.stdout.flush()
+
+def write_stderr(s):
+    sys.stderr.write(s)
+    sys.stderr.flush()
+
+def main():
+    while True:
+        all_service_list = []
+        proc = subprocess.Popen(["supervisorctl avail | cut -d' ' -f1"], shell=True, stdout=subprocess.PIPE)
+        (out, err) = proc.communicate()
+
+        all_service_list = out.split()
+
+        # "exception_service_list" contains all the program excluded  from event listener process.
+        exception_service_list = ["start.sh", "enable_counters", "swssconfig", "arp_update", "ledinit", "fancontrol", "lm-sensors", "ledd", "xcvrd", "configdb-load.sh", "snmpd-config-updater"]
+
+        service_list =  [x for x in all_service_list if x not in exception_service_list]
+        headers, body = listener.wait(sys.stdin, sys.stdout)
+        body = dict([pair.split(":") for pair in body.split(" ")])
+
+        write_stderr("Headers: %r\n" % repr(headers))
+        write_stderr("Body: %r\n" % repr(body))
+
+        process = body["processname"];
+        state = headers["eventname"].split('_')[2];
+        if process in service_list:
+            write_stderr("Process {} got {} !!! Time to kill Supervisord !!!\n".format(process,state))
+            try:
+                pidfile = open('/var/run/supervisord.pid','r')
+                pid = int(pidfile.readline());
+                os.kill(pid, signal.SIGQUIT)
+            except Exception as e:
+                write_stdout('Could not kill supervisor: ' + e.strerror + '\n')
+        else:
+            write_stderr("Process {} got {} !!! But no need to kill Supervisor !!!\n".format(process,state))
+
+        # # transition from READY to ACKNOWLEDGED
+        write_stdout("RESULT 2\nOK")
+
+
+if __name__ == "__main__":
+    main()
+

dockers/docker-database/supervisord.conf

@@ -27,3 +27,9 @@ autorestart=false
 startsecs=0
 stdout_logfile=syslog
 stderr_logfile=syslog
+
+[eventlistener:kill_supervisor]
+command=/usr/bin/kill_supervisor.py
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
+stdout_logfile=syslog
+stderr_logfile=syslog

dockers/docker-dhcp-relay/docker-dhcp-relay.supervisord.conf.j2

@@ -65,3 +65,9 @@ stderr_logfile=syslog
 {% endfor %}
 {% endif %}
 {% endif %}
+
+[eventlistener:kill_supervisor]
+command=/usr/bin/kill_supervisor.py
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
+stdout_logfile=syslog
+stderr_logfile=syslog

dockers/docker-fpm-quagga/supervisord.conf

@@ -56,3 +56,9 @@ autorestart=false
 startsecs=0
 stdout_logfile=syslog
 stderr_logfile=syslog
+
+[eventlistener:kill_supervisor]
+command=/usr/bin/kill_supervisor.py
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
+stdout_logfile=syslog
+stderr_logfile=syslog

dockers/docker-lldp-sv2/supervisord.conf

@@ -47,3 +47,9 @@ autostart=false
 autorestart=false
 stdout_logfile=syslog
 stderr_logfile=syslog
+
+[eventlistener:kill_supervisor]
+command=/usr/bin/kill_supervisor.py
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
+stdout_logfile=syslog
+stderr_logfile=syslog

dockers/docker-orchagent/supervisord.conf

@@ -116,3 +116,9 @@ autostart=false
 autorestart=false
 stdout_logfile=syslog
 stderr_logfile=syslog
+
+[eventlistener:kill_supervisor]
+command=/usr/bin/kill_supervisor.py
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
+stdout_logfile=syslog
+stderr_logfile=syslog

dockers/docker-platform-monitor/supervisord.conf

@@ -54,3 +54,9 @@ autorestart=false
 stdout_logfile=syslog
 stderr_logfile=syslog
 startsecs=0
+
+[eventlistener:kill_supervisor]
+command=/usr/bin/kill_supervisor.py
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
+stdout_logfile=syslog
+stderr_logfile=syslog

dockers/docker-router-advertiser/docker-router-advertiser.supervisord.conf

@@ -27,3 +27,9 @@ autostart=false
 autorestart=false
 stdout_logfile=syslog
 stderr_logfile=syslog
+
+[eventlistener:kill_supervisor]
+command=/usr/bin/kill_supervisor.py
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
+stdout_logfile=syslog
+stderr_logfile=syslog

dockers/docker-snmp-sv2/supervisord.conf

@@ -34,3 +34,9 @@ autostart=false
 autorestart=false
 stdout_logfile=syslog
 stderr_logfile=syslog
+
+[eventlistener:kill_supervisor]
+command=/usr/bin/kill_supervisor.py
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
+stdout_logfile=syslog
+stderr_logfile=syslog

dockers/docker-sonic-telemetry/supervisord.conf

@@ -34,3 +34,9 @@ autostart=false
 autorestart=true
 stdout_logfile=syslog
 stderr_logfile=syslog
+
+[eventlistener:kill_supervisor]
+command=/usr/bin/kill_supervisor.py
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
+stdout_logfile=syslog
+stderr_logfile=syslog

dockers/docker-teamd/supervisord.conf

@@ -34,3 +34,9 @@ autostart=false
 autorestart=false
 stdout_logfile=syslog
 stderr_logfile=syslog
+
+[eventlistener:kill_supervisor]
+command=/usr/bin/kill_supervisor.py
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
+stdout_logfile=syslog
+stderr_logfile=syslog

files/build_templates/database.service.j2

@@ -9,5 +9,7 @@ ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
 ExecStart=/usr/bin/{{docker_container_name}}.sh attach
 ExecStop=/usr/bin/{{docker_container_name}}.sh stop
 
+Restart=always
+
 [Install]
 WantedBy=multi-user.target

files/build_templates/dhcp_relay.service.j2

@@ -9,5 +9,7 @@ ExecStartPre=/usr/bin/{{ docker_container_name }}.sh start
 ExecStart=/usr/bin/{{ docker_container_name }}.sh attach
 ExecStop=/usr/bin/{{ docker_container_name }}.sh stop
 
+Restart=always
+
 [Install]
 WantedBy=multi-user.target teamd.service

files/build_templates/lldp.service.j2

@@ -9,5 +9,7 @@ ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
 ExecStart=/usr/bin/{{docker_container_name}}.sh attach
 ExecStop=/usr/bin/{{docker_container_name}}.sh stop
 
+Restart=always
+
 [Install]
 WantedBy=multi-user.target

files/build_templates/pmon.service.j2

@@ -9,5 +9,7 @@ ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
 ExecStart=/usr/bin/{{docker_container_name}}.sh attach
 ExecStop=/usr/bin/{{docker_container_name}}.sh stop
 
+Restart=always
+
 [Install]
 WantedBy=multi-user.target

files/build_templates/radv.service.j2

@@ -9,5 +9,7 @@ ExecStartPre=/usr/bin/{{ docker_container_name }}.sh start
 ExecStart=/usr/bin/{{ docker_container_name }}.sh attach
 ExecStop=/usr/bin/{{ docker_container_name }}.sh stop
 
+Restart=always
+
 [Install]
 WantedBy=multi-user.target

files/build_templates/snmp.service.j2

@@ -7,3 +7,5 @@ After=updategraph.service swss.service
 ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
 ExecStart=/usr/bin/{{docker_container_name}}.sh attach
 ExecStop=/usr/bin/{{docker_container_name}}.sh stop
+
+Restart=always

files/build_templates/swss.service.j2

@@ -15,5 +15,7 @@ Environment=sonic_asic_platform={{ sonic_asic_platform }}
 ExecStart=/usr/local/bin/swss.sh start
 ExecStop=/usr/local/bin/swss.sh stop
 
+Restart=always
+
 [Install]
 WantedBy=multi-user.target

files/build_templates/syncd.service.j2

@@ -20,5 +20,7 @@ Environment=sonic_asic_platform={{ sonic_asic_platform }}
 ExecStart=/usr/local/bin/syncd.sh start
 ExecStop=/usr/local/bin/syncd.sh stop
 
+Restart=always
+
 [Install]
 WantedBy=multi-user.target

files/build_templates/teamd.service.j2

@@ -9,5 +9,7 @@ ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
 ExecStart=/usr/bin/{{docker_container_name}}.sh attach
 ExecStop=/usr/bin/{{docker_container_name}}.sh stop
 
+Restart=always
+
 [Install]
 WantedBy=multi-user.target

files/build_templates/telemetry.service.j2

@@ -9,5 +9,7 @@ ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
 ExecStart=/usr/bin/{{docker_container_name}}.sh attach
 ExecStop=/usr/bin/{{docker_container_name}}.sh stop
 
+Restart=always
+
 [Install]
 WantedBy=multi-user.target

platform/barefoot/docker-syncd-bfn/supervisord.conf

@@ -27,3 +27,8 @@ autorestart=false
 stdout_logfile=syslog
 stderr_logfile=syslog
 
+[eventlistener:kill_supervisor]
+command=/usr/bin/kill_supervisor.py
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
+stdout_logfile=syslog
+stderr_logfile=syslog

platform/broadcom/docker-syncd-brcm/supervisord.conf

@@ -34,3 +34,9 @@ autostart=false
 autorestart=false
 stdout_logfile=syslog
 stderr_logfile=syslog
+
+[eventlistener:kill_supervisor]
+command=/usr/bin/kill_supervisor.py
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
+stdout_logfile=syslog
+stderr_logfile=syslog

platform/cavium/docker-syncd-cavm/supervisord.conf

@@ -26,3 +26,9 @@ autostart=false
 autorestart=false
 stdout_logfile=syslog
 stderr_logfile=syslog
+
+[eventlistener:kill_supervisor]
+command=/usr/bin/kill_supervisor.py
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
+stdout_logfile=syslog
+stderr_logfile=syslog

platform/centec/docker-syncd-centec/supervisord.conf

@@ -26,3 +26,9 @@ autostart=false
 autorestart=false
 stdout_logfile=syslog
 stderr_logfile=syslog
+
+[eventlistener:kill_supervisor]
+command=/usr/bin/kill_supervisor.py
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
+stdout_logfile=syslog
+stderr_logfile=syslog

platform/marvell/docker-syncd-mrvl/supervisord.conf

@@ -27,3 +27,8 @@ autorestart=false
 stdout_logfile=syslog
 stderr_logfile=syslog
 
+[eventlistener:kill_supervisor]
+command=/usr/bin/kill_supervisor.py
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
+stdout_logfile=syslog
+stderr_logfile=syslog

platform/mellanox/docker-syncd-mlnx/supervisord.conf

@@ -34,3 +34,9 @@ autostart=false
 autorestart=false
 stdout_logfile=syslog
 stderr_logfile=syslog
+
+[eventlistener:kill_supervisor]
+command=/usr/bin/kill_supervisor.py
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
+stdout_logfile=syslog
+stderr_logfile=syslog

platform/nephos/docker-syncd-nephos/supervisord.conf

@@ -26,3 +26,9 @@ autostart=false
 autorestart=false
 stdout_logfile=syslog
 stderr_logfile=syslog
+
+[eventlistener:kill_supervisor]
+command=/usr/bin/kill_supervisor.py
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
+stdout_logfile=syslog
+stderr_logfile=syslog

platform/vs/docker-sonic-vs/supervisord.conf

@@ -154,3 +154,9 @@ autostart=false
 autorestart=false
 stdout_logfile=syslog
 stderr_logfile=syslog
+
+[eventlistener:kill_supervisor]
+command=/usr/bin/kill_supervisor.py
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
+stdout_logfile=syslog
+stderr_logfile=syslog

src/sonic-config-engine/tests/sample_output/docker-dhcp-relay.supervisord.conf

@@ -31,3 +31,8 @@ stdout_logfile=syslog
 stderr_logfile=syslog
 
 
+[eventlistener:kill_supervisor]
+command=/usr/bin/kill_supervisor.py
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
+stdout_logfile=syslog
+stderr_logfile=syslog