diff --git a/files/scripts/swss.sh b/files/scripts/swss.sh index ba0437a25e9..39d134f7094 100755 --- a/files/scripts/swss.sh +++ b/files/scripts/swss.sh @@ -274,6 +274,8 @@ stop() { if [[ x"$WARM_BOOT" != x"true" ]] && [[ x"$FAST_BOOT" != x"true" ]]; then /usr/bin/${SERVICE}.sh stop $DEV debug "Stopped ${SERVICE}$DEV service..." + $SONIC_DB_CLI APPL_DB DEL PORT_TABLE:PortInitDone + debug "Cleared PortInitDone from APPL_DB..." else debug "Killing Docker swss..." /usr/bin/docker kill swss &> /dev/null || debug "Docker swss is not running ($?) ..." diff --git a/files/scripts/syncd.sh b/files/scripts/syncd.sh index 812cf30e400..a6eaaa5b6ff 100755 --- a/files/scripts/syncd.sh +++ b/files/scripts/syncd.sh @@ -2,6 +2,29 @@ . /usr/local/bin/syncd_common.sh +function collect_saisdkdump() { + TMP_DMP_DIR="/tmp/orch_abrt_sdkdump/" + HOST_SDKDUMP_LOC="/var/log/orch_abrt_sdkdump/" + local sai_dump_filename="sai_sdk_dump_$(date +"%m_%d_%Y_%I_%M_%p")" + local sai_dump_filename_epoch="sai_sdk_dump_$(date +%s)" + + /usr/bin/docker exec syncd$DEV rm -rf ${TMP_DMP_DIR} + /usr/bin/docker exec syncd$DEV rm -f /tmp/${sai_dump_filename_epoch}.tar + /usr/bin/docker exec syncd$DEV mkdir -p ${TMP_DMP_DIR} + timeout 30s bash -c "/usr/bin/docker exec syncd$DEV saisdkdump -f ${TMP_DMP_DIR}/${sai_dump_filename} > /dev/null" + /usr/bin/docker exec syncd$DEV tar -czf /tmp/${sai_dump_filename_epoch}.tar.gz -C ${TMP_DMP_DIR} . + + if [[ $? == 0 ]]; then + mkdir -p ${HOST_SDKDUMP_LOC} + /usr/bin/docker cp syncd$DEV:/tmp/${sai_dump_filename_epoch}.tar.gz ${HOST_SDKDUMP_LOC} + # Only retain the latest 3 files + ls -1td ${HOST_SDKDUMP_LOC}/sai_sdk_dump_* | tail -n +3 | xargs rm -rf + debug "${sai_dump_filename_epoch}.tar.gz collected before taking stopping syncd" + else + debug "Failed to collect saisdkdump before stopping syncd" + fi +} + function startplatform() { # platform specific tasks @@ -75,6 +98,17 @@ function waitplatform() { function stopplatform1() { + if [[ x"$(${SONIC_DB_CLI} STATE_DB GET ORCH_ABRT_STATUS)" == x"1" ]]; then + # Collecting saisdkdump before restarting syncd + # Runs when orchagent is aborted because of SAI failure. + # Only enabled for mellanox platform + if [[ x$sonic_asic_platform == x"mellanox" ]]; then + collect_saisdkdump + fi + # This is used to notify auto-techsupport process + touch /tmp/saidump_collection_notify_flag + fi + if [[ x$sonic_asic_platform == x"mellanox" ]] && [[ x$TYPE == x"cold" ]]; then debug "Stopping pmon service ahead of syncd..." /bin/systemctl stop pmon