Skip to content

[fastboot] fastboot enhancement: Use warm-boot infrastructure for fast-boot #2286

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vaibhavhd merged 5 commits into from
Sep 20, 2022
Merged

[fastboot] fastboot enhancement: Use warm-boot infrastructure for fast-boot #2286

Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5271e5a
Use warm-boot infrastructure for fast-boot
arfeigin Jul 31, 2022
83770a5
remove redundant condition check
arfeigin Aug 11, 2022
5154f42
revert change - dump arp and fdb for reconciliation
arfeigin Aug 28, 2022
e1cc628
Revert "revert change - dump arp and fdb for reconciliation"
arfeigin Sep 8, 2022
b5f4629
Delete routes only after we are committed to reboot
arfeigin Sep 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 28 additions & 31 deletions scripts/fast-reboot
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,41 +130,36 @@ function parseOptions()
done
}

function common_clear()
function clear_boot()
{
# common_clear
debug "${REBOOT_TYPE} failure ($?) cleanup ..."

/sbin/kexec -u || /bin/true

teardown_control_plane_assistant
}

function clear_fast_boot()
{
common_clear

sonic-db-cli STATE_DB DEL "FAST_REBOOT|system" &>/dev/null || /bin/true
}

function clear_warm_boot()
{
common_clear

#clear_warm_boot
result=$(timeout 10s config warm_restart disable; res=$?; if [[ $res == 124 ]]; then echo timeout; else echo "code ($res)"; fi) || /bin/true
debug "Cancel warm-reboot: ${result}"

TIMESTAMP=$(date +%Y%m%d-%H%M%S)
if [[ -f ${WARM_DIR}/${REDIS_FILE} ]]; then
mv -f ${WARM_DIR}/${REDIS_FILE} ${WARM_DIR}/${REDIS_FILE}.${TIMESTAMP} || /bin/true
fi

#clear_fast_boot
if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then
sonic-db-cli STATE_DB DEL "FAST_REBOOT|system" &>/dev/null || /bin/true
fi
}

function init_warm_reboot_states()
{
# If the current running instance was booted up with warm reboot. Then
# the current DB contents will likely mark warm reboot is done.
# Clear these states so that the next boot up image won't get confused.
if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" || "$REBOOT_TYPE" = "fast-reboot" ]]; then
sonic-db-cli STATE_DB eval "
for _, key in ipairs(redis.call('keys', 'WARM_RESTART_TABLE|*')) do
redis.call('hdel', key, 'state')
Expand Down Expand Up @@ -271,7 +266,8 @@ function backup_database()
and not string.match(k, 'FG_ROUTE_TABLE|') \
and not string.match(k, 'WARM_RESTART_ENABLE_TABLE|') \
and not string.match(k, 'VXLAN_TUNNEL_TABLE|') \
and not string.match(k, 'BUFFER_MAX_PARAM_TABLE|') then
and not string.match(k, 'BUFFER_MAX_PARAM_TABLE|') \
and not string.match(k, 'FAST_REBOOT|') then
redis.call('del', k)
end
end
Expand Down Expand Up @@ -381,7 +377,7 @@ function check_docker_exec()

function check_db_integrity()
{
if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" || "$REBOOT_TYPE" = "fast-reboot" ]]; then
CHECK_DB_INTEGRITY=0
/usr/local/bin/check_db_integrity.py || CHECK_DB_INTEGRITY=$?
if [[ CHECK_DB_INTEGRITY -ne 0 ]]; then
Expand Down Expand Up @@ -464,7 +460,6 @@ function unload_kernel()
function save_counters_folder() {
if [[ "$REBOOT_TYPE" = "warm-reboot" ]]; then
debug "Saving counters folder before warmboot..."

counters_folder="/host/counters"
if [[ ! -d $counters_folder ]]; then
mkdir $counters_folder
Expand Down Expand Up @@ -533,9 +528,11 @@ sonic_asic_type=$(sonic-cfggen -y /etc/sonic/sonic_version.yml -v asic_type)
BOOT_TYPE_ARG="cold"
case "$REBOOT_TYPE" in
"fast-reboot")
check_warm_restart_in_progress
BOOT_TYPE_ARG=$REBOOT_TYPE
trap clear_fast_boot EXIT HUP INT QUIT TERM KILL ABRT ALRM
trap clear_boot EXIT HUP INT QUIT TERM KILL ABRT ALRM
sonic-db-cli STATE_DB SET "FAST_REBOOT|system" "1" "EX" "180" &>/dev/null
config warm_restart enable system
Copy link
Collaborator

@Junchao-Mellanox Junchao-Mellanox Aug 31, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For people who does not know the background, it is really confuse here, I would suggest to add comment here to explain why fastboot requires "enable warmboot".

Copy link
Contributor

@qiluo-msft qiluo-msft Oct 6, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agree, it confused me.

;;
"warm-reboot")
check_warm_restart_in_progress
Expand All @@ -548,7 +545,7 @@ case "$REBOOT_TYPE" in
else
BOOT_TYPE_ARG="warm"
fi
trap clear_warm_boot EXIT HUP INT QUIT TERM KILL ABRT ALRM
trap clear_boot EXIT HUP INT QUIT TERM KILL ABRT ALRM
config warm_restart enable system
;;
*)
Expand Down Expand Up @@ -618,7 +615,6 @@ if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then
unload_kernel
exit "${EXIT_FAST_REBOOT_DUMP_FAILURE}"
fi

FILTER_FDB_ENTRIES_RC=0
# Filter FDB entries using MAC addresses from ARP table
/usr/local/bin/filter_fdb_entries -f $DUMP_DIR/fdb.json -a $DUMP_DIR/arp.json -c $CONFIG_DB_FILE || FILTER_FDB_ENTRIES_RC=$?
Expand All @@ -633,7 +629,7 @@ init_warm_reboot_states

setup_control_plane_assistant

if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" || "$REBOOT_TYPE" = "fast-reboot" ]]; then
# Freeze orchagent for warm restart
# Ask orchagent_restart_check to try freeze 5 times with interval of 2 seconds,
# it is possible that the orchagent is in transient state and no opportunity to freeze
Expand Down Expand Up @@ -726,18 +722,19 @@ for service in ${SERVICES_TO_STOP}; do
if [[ "x$sonic_asic_type" == x"mellanox" ]]; then
check_issu_bank_file
fi
fi

# Warm reboot: dump state to host disk
if [[ "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
sonic-db-cli ASIC_DB FLUSHDB > /dev/null
sonic-db-cli COUNTERS_DB FLUSHDB > /dev/null
sonic-db-cli FLEX_COUNTER_DB FLUSHDB > /dev/null
fi

# TODO: backup_database preserves FDB_TABLE
# need to cleanup as well for fastfast boot case
backup_database
if [[ "$REBOOT_TYPE" = "fastfast-reboot" || "$REBOOT_TYPE" = "fast-reboot" ]]; then
# Advanced reboot: dump state to host disk
sonic-db-cli ASIC_DB FLUSHDB > /dev/null
sonic-db-cli COUNTERS_DB FLUSHDB > /dev/null
sonic-db-cli FLEX_COUNTER_DB FLUSHDB > /dev/null
fi

# TODO: backup_database preserves FDB_TABLE
# need to cleanup as well for fastfast boot case
backup_database

fi
done

Expand Down