From c935c31471586e350943fd765af52ef176c26174 Mon Sep 17 00:00:00 2001 From: Danny Allen Date: Mon, 26 Aug 2019 14:17:24 -0700 Subject: [PATCH 1/5] [fast-reboot] Check if ASIC config has changed before fast reboot * Adds script to compare checksums for config files between SONiC images * Adds pre-check to fast reboot script to confirm config files match before performing fast reboot Signed-off-by: Danny Allen daall@microsoft.com --- scripts/asic_config_check | 83 +++++++++++++++++++++++++++++++++++++++ scripts/fast-reboot | 11 ++++++ setup.py | 1 + 3 files changed, 95 insertions(+) create mode 100755 scripts/asic_config_check diff --git a/scripts/asic_config_check b/scripts/asic_config_check new file mode 100755 index 0000000000..ce1c719f83 --- /dev/null +++ b/scripts/asic_config_check @@ -0,0 +1,83 @@ +#!/bin/bash -e + +# Exit codes +ASIC_CONFIG_UNCHANGED=0 +ASIC_CONFIG_CHANGED=1 +SRC_ASIC_CONFIG_NOT_FOUND=2 +DST_ASIC_CONFIG_NOT_FOUND=3 + +# Logging utilities +function error() +{ + logger -p user.err "$@" +} + +function debug() +{ + logger -p user.info "$@" +} + +# Retrieve the source ASIC config checksum from the source image +# Exits with error SRC_ASIC_CONFIG_NOT_FOUND if the checksum is not found +function GetSourceASICConfigChecksum() { + if [[ ! -f "/etc/sonic/asic_config_checksum" ]]; then + error "ASIC config not found in src image, can't verify changes" + exit "${SRC_ASIC_CONFIG_NOT_FOUND}" + fi +} + +# Retrieve the destination ASIC config checksum from the destination image +# Exits with error DST_ASIC_CONFIG_NOT_FOUND if the checksum is not found +function GetDestinationASICConfigChecksum() { + DST_IMAGE_PATH="/host/image-${DST_SONIC_IMAGE#SONiC-OS-}" + FS_PATH="${DST_IMAGE_PATH}/fs.squashfs" + FS_MOUNTPOINT="/tmp/image-${DST_SONIC_IMAGE#SONiC-OS-}-fs" + + # Verify that the destination image exists + if [[ ! -d ${DST_IMAGE_PATH} ]]; then + error "ASIC config not found in dst image, can't verify changes" + exit "${DST_ASIC_CONFIG_NOT_FOUND}" + fi + + mkdir "${FS_MOUNTPOINT}" + mount -t squashfs "${FS_PATH}" "${FS_MOUNTPOINT}" + + if [[ ! -f "${FS_MOUNTPOINT}/etc/sonic/asic_config_checksum" ]]; then + error "ASIC config not found in dst image, can't verify changes" + umount "${FS_MOUNTPOINT}" + rm -rf "${FS_MOUNTPOINT}" + exit "${DST_ASIC_CONFIG_NOT_FOUND}" + fi + + cp "${FS_MOUNTPOINT}/etc/sonic/asic_config_checksum" /tmp/dst_asic_config_checksum + umount "${FS_MOUNTPOINT}" + rm -rf "${FS_MOUNTPOINT}" +} + +# Confirm that the src and dst ASIC config checksums match +# Exits with ASIC_CONFIG_CHANGED if the checksums differ +function ConfirmASICConfigChecksumsMatch() { + SRC_CONFIG_CHECKSUM=$(cat /etc/sonic/asic_config_checksum) + DST_CONFIG_CHECKSUM=$(cat /tmp/dst_asic_config_checksum) + if [[ "${SRC_CONFIG_CHECKSUM}" != "${DST_CONFIG_CHECKSUM}" ]]; then + error "ASIC config may have changed, checksum failed" + exit "${ASIC_CONFIG_CHANGED}" + fi +} + +# Main starts here +debug "Checking if ASIC config has changed" + +SRC_SONIC_IMAGE="$(sonic_installer list | grep "Current: " | cut -f2 -d' ')" +DST_SONIC_IMAGE="$(sonic_installer list | grep "Next: " | cut -f2 -d' ')" +if [[ "${SRC_SONIC_IMAGE}" == "${DST_SONIC_IMAGE}" ]]; then + debug "ASIC config unchanged, src and dst SONiC version are the same" + exit "${ASIC_CONFIG_UNCHANGED}" +fi + +GetSourceASICConfigChecksum +GetDestinationASICConfigChecksum +ConfirmASICConfigChecksumsMatch + +debug "ASIC config unchanged, checksum passed" +exit "${ASIC_CONFIG_UNCHANGED}" diff --git a/scripts/fast-reboot b/scripts/fast-reboot index 8b05650e56..2628e5338d 100755 --- a/scripts/fast-reboot +++ b/scripts/fast-reboot @@ -250,6 +250,17 @@ function reboot_pre_check() debug "Next image ${NEXT_SONIC_IMAGE} doesn't exist ..." exit ${EXIT_NEXT_IMAGE_NOT_EXISTS} fi + + # Make sure ASIC configuration has not changed between images + ASIC_CONFIG_CHECK_SCRIPT="/usr/bin/asic_config_check" + ASIC_CONFIG_CHECK_SUCCESS=0 + + debug "Checking that ASIC configuration has not changed" + ${ASIC_CONFIG_CHECK_SCRIPT} || ASIC_CONFIG_CHECK_EXIT_CODE=$? + if [[ "${ASIC_CONFIG_CHECK_EXIT_CODE}" != "${ASIC_CONFIG_CHECK_SUCCESS}" ]]; then + error "ASIC config may have changed: errno=${ASIC_CONFIG_CHECK_EXIT_CODE}" + exit "${EXIT_FAILURE}" + fi } function unload_kernel() diff --git a/setup.py b/setup.py index d12d9880b8..a4bdf7089e 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ }, scripts=[ 'scripts/aclshow', + 'scripts/asic_config_check', 'scripts/boot_part', 'scripts/coredump-compress', 'scripts/db_migrator.py', From eec5a0460d9132fe5715f0fc28a24b0c98199c9a Mon Sep 17 00:00:00 2001 From: Danny Allen Date: Mon, 26 Aug 2019 17:25:25 -0700 Subject: [PATCH 2/5] Fix formatting --- scripts/asic_config_check | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/asic_config_check b/scripts/asic_config_check index ce1c719f83..b9cc6f9d39 100755 --- a/scripts/asic_config_check +++ b/scripts/asic_config_check @@ -19,7 +19,8 @@ function debug() # Retrieve the source ASIC config checksum from the source image # Exits with error SRC_ASIC_CONFIG_NOT_FOUND if the checksum is not found -function GetSourceASICConfigChecksum() { +function GetSourceASICConfigChecksum() +{ if [[ ! -f "/etc/sonic/asic_config_checksum" ]]; then error "ASIC config not found in src image, can't verify changes" exit "${SRC_ASIC_CONFIG_NOT_FOUND}" @@ -28,7 +29,8 @@ function GetSourceASICConfigChecksum() { # Retrieve the destination ASIC config checksum from the destination image # Exits with error DST_ASIC_CONFIG_NOT_FOUND if the checksum is not found -function GetDestinationASICConfigChecksum() { +function GetDestinationASICConfigChecksum() +{ DST_IMAGE_PATH="/host/image-${DST_SONIC_IMAGE#SONiC-OS-}" FS_PATH="${DST_IMAGE_PATH}/fs.squashfs" FS_MOUNTPOINT="/tmp/image-${DST_SONIC_IMAGE#SONiC-OS-}-fs" @@ -56,7 +58,8 @@ function GetDestinationASICConfigChecksum() { # Confirm that the src and dst ASIC config checksums match # Exits with ASIC_CONFIG_CHANGED if the checksums differ -function ConfirmASICConfigChecksumsMatch() { +function ConfirmASICConfigChecksumsMatch() +{ SRC_CONFIG_CHECKSUM=$(cat /etc/sonic/asic_config_checksum) DST_CONFIG_CHECKSUM=$(cat /tmp/dst_asic_config_checksum) if [[ "${SRC_CONFIG_CHECKSUM}" != "${DST_CONFIG_CHECKSUM}" ]]; then From 9610d08f7417f5943565dda8356c18021f28faf3 Mon Sep 17 00:00:00 2001 From: Danny Allen Date: Tue, 27 Aug 2019 18:21:28 -0700 Subject: [PATCH 3/5] Incorporate feedback --- scripts/asic_config_check | 3 ++- scripts/fast-reboot | 9 +-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/scripts/asic_config_check b/scripts/asic_config_check index b9cc6f9d39..fb99e05bec 100755 --- a/scripts/asic_config_check +++ b/scripts/asic_config_check @@ -10,6 +10,7 @@ DST_ASIC_CONFIG_NOT_FOUND=3 function error() { logger -p user.err "$@" + echo $@ >&2 } function debug() @@ -69,7 +70,7 @@ function ConfirmASICConfigChecksumsMatch() } # Main starts here -debug "Checking if ASIC config has changed" +debug "Checking that ASIC configuration has not changed" SRC_SONIC_IMAGE="$(sonic_installer list | grep "Current: " | cut -f2 -d' ')" DST_SONIC_IMAGE="$(sonic_installer list | grep "Next: " | cut -f2 -d' ')" diff --git a/scripts/fast-reboot b/scripts/fast-reboot index 2628e5338d..94ea11819e 100755 --- a/scripts/fast-reboot +++ b/scripts/fast-reboot @@ -253,14 +253,7 @@ function reboot_pre_check() # Make sure ASIC configuration has not changed between images ASIC_CONFIG_CHECK_SCRIPT="/usr/bin/asic_config_check" - ASIC_CONFIG_CHECK_SUCCESS=0 - - debug "Checking that ASIC configuration has not changed" - ${ASIC_CONFIG_CHECK_SCRIPT} || ASIC_CONFIG_CHECK_EXIT_CODE=$? - if [[ "${ASIC_CONFIG_CHECK_EXIT_CODE}" != "${ASIC_CONFIG_CHECK_SUCCESS}" ]]; then - error "ASIC config may have changed: errno=${ASIC_CONFIG_CHECK_EXIT_CODE}" - exit "${EXIT_FAILURE}" - fi + ${ASIC_CONFIG_CHECK_SCRIPT} } function unload_kernel() From 69408c844d2456bd5d28365c5d5e38a3c5c57818 Mon Sep 17 00:00:00 2001 From: Danny Allen Date: Wed, 28 Aug 2019 15:32:26 -0700 Subject: [PATCH 4/5] Add check for reboot type and override ability --- scripts/fast-reboot | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/fast-reboot b/scripts/fast-reboot index 94ea11819e..c001140e6b 100755 --- a/scripts/fast-reboot +++ b/scripts/fast-reboot @@ -253,7 +253,22 @@ function reboot_pre_check() # Make sure ASIC configuration has not changed between images ASIC_CONFIG_CHECK_SCRIPT="/usr/bin/asic_config_check" - ${ASIC_CONFIG_CHECK_SCRIPT} + ASIC_CONFIG_CHECK_SUCCESS=0 + if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then + set +e + ${ASIC_CONFIG_CHECK_SCRIPT} + ASIC_CONFIG_CHECK_EXIT_CODE=$? + set -e + + if [[ "${ASIC_CONFIG_CHECK_EXIT_CODE}" != "${ASIC_CONFIG_CHECK_SUCCESS}" ]]; then + if [[ x"${FORCE}" == x"yes" ]]; then + debug "Ignoring ASIC config checksum failure..." + else + error "ASIC config may have changed: errno=${ASIC_CONFIG_CHECK_EXIT_CODE}" + exit "${EXIT_FAILURE}" + fi + fi + fi } function unload_kernel() From 9b6e0cb1b2ee4e6940bad27c9ff1499eb47a2720 Mon Sep 17 00:00:00 2001 From: Danny Allen Date: Thu, 29 Aug 2019 10:35:43 -0700 Subject: [PATCH 5/5] Clean up error check --- scripts/asic_config_check | 1 - scripts/fast-reboot | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/asic_config_check b/scripts/asic_config_check index fb99e05bec..23be6b9742 100755 --- a/scripts/asic_config_check +++ b/scripts/asic_config_check @@ -10,7 +10,6 @@ DST_ASIC_CONFIG_NOT_FOUND=3 function error() { logger -p user.err "$@" - echo $@ >&2 } function debug() diff --git a/scripts/fast-reboot b/scripts/fast-reboot index c001140e6b..d503c755ac 100755 --- a/scripts/fast-reboot +++ b/scripts/fast-reboot @@ -255,10 +255,8 @@ function reboot_pre_check() ASIC_CONFIG_CHECK_SCRIPT="/usr/bin/asic_config_check" ASIC_CONFIG_CHECK_SUCCESS=0 if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then - set +e - ${ASIC_CONFIG_CHECK_SCRIPT} - ASIC_CONFIG_CHECK_EXIT_CODE=$? - set -e + ASIC_CONFIG_CHECK_EXIT_CODE=0 + ${ASIC_CONFIG_CHECK_SCRIPT} || ASIC_CONFIG_CHECK_EXIT_CODE=$? if [[ "${ASIC_CONFIG_CHECK_EXIT_CODE}" != "${ASIC_CONFIG_CHECK_SUCCESS}" ]]; then if [[ x"${FORCE}" == x"yes" ]]; then