From fd7e877032fd940f8e146869a5dbd674ad9eedb0 Mon Sep 17 00:00:00 2001 From: aviramd Date: Thu, 22 Aug 2024 13:46:01 +0000 Subject: [PATCH 1/6] add support for sai debug generate dump This update introduces support for the sai_dbg_gen_dump API, which is triggered by the show techsupport command within the generate_dump script. Currently, this API is only invoked in the vendor specific use case to avoid disrupting existing implementations for other vendors. It is recommended to adopt this new procedure for triggering sai_dbg_gen_dump and eventually phase out the standalone saisdkdump executable. By integrating this API directly within the process, we eliminate the need for a separate process, as not all SAI implementations will function correctly when the API is called from a different process other than syncd. Additionally, this change will reduce the system footprint by 70 MB, currently occupied by the saisdkdump executable. An auxiliary script "gen_sai_dbg_dump.sh" is also provided to generate the dump file directly without requiring the show techsupport command. --- scripts/gen_sai_dbg_dump.sh | 140 ++++++++++++++++++++++++++++++++ scripts/gen_sai_dbg_dump_lib.sh | 92 +++++++++++++++++++++ scripts/generate_dump | 81 ++++++++++++++++++ setup.py | 4 +- 4 files changed, 316 insertions(+), 1 deletion(-) create mode 100644 scripts/gen_sai_dbg_dump.sh create mode 100644 scripts/gen_sai_dbg_dump_lib.sh diff --git a/scripts/gen_sai_dbg_dump.sh b/scripts/gen_sai_dbg_dump.sh new file mode 100644 index 0000000000..96fcc7abdc --- /dev/null +++ b/scripts/gen_sai_dbg_dump.sh @@ -0,0 +1,140 @@ +#!/bin/sh + +. /usr/local/bin/gen_sai_dbg_dump_lib.sh + +############################################################################### +# Prints the usage information. +# Globals: +# None +# Arguments: +# None +# Returns: +# None +############################################################################### +usage() { + cat < + +Generate and retrieve the SAI debug dump file from the syncd Docker container. + +OPTIONS: + -f Specify the destination file path for the SAI debug dump. + -h Display this help and exit. + +EXAMPLES: + $0 -f /var/log/dbg_gen_dump.log +EOF +} + +############################################################################### +# Copies a given file from a specified Docker container to the given target location. +# Globals: +# TIMEOUT_MIN +# Arguments: +# docker: Docker container name +# filename: The filename to copy +# destination: Destination filename +# Returns: +# None +############################################################################### +copy_from_docker() { + TIMEOUT_MIN="1" + local docker=$1 + local filename=$2 + local dstpath=$3 + local timeout_cmd="timeout --foreground ${TIMEOUT_MIN}m" + + local touch_cmd="sudo docker exec ${docker} touch ${filename}" + local cp_cmd="sudo docker cp ${docker}:${filename} ${dstpath}" + + RC=0 + eval "${timeout_cmd} ${touch_cmd}" || RC=$? + if [ $RC -ne 0 ]; then + echo "Command: $touch_cmd timed out after ${TIMEOUT_MIN} minutes." + fi + eval "${timeout_cmd} ${cp_cmd}" || RC=$? + if [ $RC -ne 0 ]; then + echo "Command: $cp_cmd timed out after ${TIMEOUT_MIN} minutes." + fi +} + +############################################################################### +# Main script logic +# Description: +# This is the main entry point of the script, which handles the generation +# and retrieval of the SAI debug dump file. It parses command-line arguments, +# ensures necessary directories and the `syncd` container are available, and +# triggers the SAI debug dump process through Redis. The script waits for the +# dump file to be generated and then copies it from the Docker container to +# the specified location on the local system. +# +# Globals: +# None +# +# Arguments: +# -f : Specifies the output filename for the SAI debug dump file. +# -h : Displays usage information. +# +# Returns: +# 0 - On success +# 1 - On failure +############################################################################### +main() { + # Parse arguments + while getopts ":f:h" opt; do + case $opt in + f) + sai_dump_filename="$OPTARG" + ;; + h) + usage + exit 0 + ;; + /?) + echo "Invalid option: -$OPTARG" >&2 + usage + exit 1 + ;; + esac + done + + local syncd_sai_dump_filename="/var/log/dbg_gen_dump.log" + + # Ensure a filename was provided + if [ -z "$sai_dump_filename" ]; then + echo "Error: Missing filename." + usage + exit 1 + fi + + # Ensure the directory exists, create it if it doesn't + if [ ! -d "$(dirname "$sai_dump_filename")" ]; then + sudo mkdir -p "$(dirname "$sai_dump_filename")" + fi + + # Ensure the syncd container is running + if [ "$(docker container inspect -f '{{.State.Running}}' syncd)" != "true" ]; then + echo "Error: syncd container is not running." + exit 1 + fi + + generate_sai_dump "$syncd_sai_dump_filename" + if [ $? -ne 0 ]; then + echo "Failed to generate SAI debug dump." + exit 1 + fi + + # Copy the dump file from the Docker container + local + if ! copy_from_docker syncd $syncd_sai_dump_filename $sai_dump_filename; then + echo "Error: Failed to copy the SAI dump file from the container." + exit 1 + fi + + # Remove the dump file from the Docker container + docker exec syncd rm -rf $syncd_sai_dump_filename; + echo "$sai_dump_filename is ready!!!" + exit 0 +} + +main "$@" diff --git a/scripts/gen_sai_dbg_dump_lib.sh b/scripts/gen_sai_dbg_dump_lib.sh new file mode 100644 index 0000000000..f730ccb5b5 --- /dev/null +++ b/scripts/gen_sai_dbg_dump_lib.sh @@ -0,0 +1,92 @@ +#!/bin/sh + +############################################################################### +# generate_sai_dump +# Description: +# This function triggers the generation of a SAI debug dump file in the +# `syncd` Docker container through Redis and waits for the file to be ready. +# +# Arguments: +# $1 - Filename for the SAI debug dump file. +# +# Returns: +# 0 - On success +# 1 - On failure +############################################################################### +generate_sai_dump() { + local DB=4 + local KEY="DBG_GEN_DUMP_TABLE|DUMP" + local STATUS_KEY="DBG_GEN_DUMP_STATS_TABLE|DUMP" + local FIELD="file" + local STATUS_FIELD="status" + local STATUS="1" + + local TIMEOUT_FOR_GEN_DBG_DUMP_FILE_READYNESS=10 + local INTERVAL=1 + local TIME_PASSED=0 + local EXISTS + + local SYNCD_DUMP_FILE="$1" + if [ -z "$SYNCD_DUMP_FILE" ]; then + echo "Error: No filename provided for the SAI debug dump file." + return 1 + fi + + # Extract the directory from the SYNCD_DUMP_FILE path + local SYNCD_DUMP_DIR + SYNCD_DUMP_DIR=$(dirname "$SYNCD_DUMP_FILE") + + # Ensure the directory exists in the syncd container; if not, create it + if ! docker exec syncd test -d "$SYNCD_DUMP_DIR"; then + echo "Directory '$SYNCD_DUMP_DIR' does not exist in the syncd container. Creating it..." + if ! docker exec syncd mkdir -p "$SYNCD_DUMP_DIR"; then + echo "Error: Failed to create directory '$SYNCD_DUMP_DIR' inside the syncd container." + return 1 + fi + fi + + # Delete the tables from STATE_DB before triger the dump file + redis-cli -n $DB DEL $KEY > /dev/null 2>&1 + redis-cli -n $DB DEL $STATUS_KEY > /dev/null 2>&1 + + # Set the DBG_GEN_DUMP in the Redis DB to trigger the dump generation + if ! redis-cli -n $DB HSET $KEY $FIELD $SYNCD_DUMP_FILE > /dev/null 2>&1; then + echo "Error: Failed to set Redis key." + return 1 + fi + + # Timeout and interval for checking status of file readiness + + while [ $TIME_PASSED -lt $TIMEOUT_FOR_GEN_DBG_DUMP_FILE_READYNESS ]; do + EXISTS=$(redis-cli -n $DB EXISTS "$STATUS_KEY" 2>/dev/null | grep -o '^[0-9]*$') + if [ "$EXISTS" -eq 1 ]; then + STATUS=$(redis-cli -n $DB HGET "$STATUS_KEY" "$STATUS_FIELD" 2>/dev/null | grep -o '^[0-9]*$') + break + fi + + sleep $INTERVAL + TIME_PASSED=$((TIME_PASSED + INTERVAL)) + done + + # Delete the tables from STATE_DB after triger the dump file + redis-cli -n $DB DEL $KEY > /dev/null 2>&1 + redis-cli -n $DB DEL $STATUS_KEY > /dev/null 2>&1 + + if [ "$STATUS" -ne 0 ]; then + echo "Error: dump file operation failed, Status $STATUS" + return 1 + fi + + if [ $TIME_PASSED -ge $TIMEOUT_FOR_GEN_DBG_DUMP_FILE_READYNESS ]; then + echo "Timeout reached. Status was not ready in time." + return 1 + fi + + # Ensure the file exists in the Docker container + if ! docker exec syncd test -f $SYNCD_DUMP_FILE; then + echo "Error: SAI dump file does not exist in the syncd container." + return 1 + fi + + return 0 +} diff --git a/scripts/generate_dump b/scripts/generate_dump index 3d0ef3430d..48208d063c 100755 --- a/scripts/generate_dump +++ b/scripts/generate_dump @@ -1107,6 +1107,83 @@ save_file() { echo "[ save_file:$orig_path] : $(($end_t-$start_t)) msec" >> $TECHSUPPORT_TIME_INFO } +############################################################################### +# generate_sai_dbg_dump_file +# Description: +# This function triggers the generation of a SAI debug dump file in the +# `syncd` Docker container and saves it to the local system. The function +# communicates with the Redis database to initiate the dump process +# +# Globals: +# None +# +# Arguments: +# None +# +# Returns: +# 0 - On success +# 1 - On failure +############################################################################### +generate_sai_dbg_dump_file() { + trap 'handle_error $? $LINENO' ERR + local syncd_sai_dump_filename="/var/log/dbg_gen_dump.log" + + if $NOOP; then + echo "generate_sai_dump" + return 1 + fi + + # Validate the argument + local sai_dump_filename="$1" + if [ -z "$sai_dump_filename" ]; then + echo "Error: No filename provided for the SAI debug dump file." + return 1 + fi + + # Extract the directory from the filename + local sai_dump_dir + sai_dump_dir=$(dirname "$sai_dump_filename") + + # Check if the directory exists; if not, create it + if [ ! -d "$sai_dump_dir" ]; then + echo "Directory '$sai_dump_dir' does not exist. Creating it..." + mkdir -p "$sai_dump_dir" + if [ $? -ne 0 ]; then + echo "Error: Failed to create directory '$sai_dump_dir'." + return 1 + fi + fi + + # Ensure the syncd container is running + if [[ "$( docker container inspect -f '{{.State.Running}}' syncd )" != "true" ]]; then + echo "Error: syncd container is not running." + return 1 + fi + + # Generate the SAI dump file using the refactored function + source /usr/local/bin/gen_sai_dbg_dump_lib.sh + if ! generate_sai_dump "$syncd_sai_dump_filename"; then + echo "Error: Failed to generate SAI debug dump." + return 1 + fi + + # Copy the dump file from the Docker container + if ! copy_from_docker syncd $syncd_sai_dump_filename $sai_dump_filename; then + echo "Error: Failed to copy the SAI dump file from the container." + return 1 + fi + + # Save the file using the save_file function + if ! save_file $sai_dump_filename sai_sdk_dump false; then + echo "Error: Failed to save the SAI dump file." + return 1 + fi + + # remove the dump files after saving to show techsupport package + rm -rf $sai_dump_filename + docker exec syncd rm -rf $syncd_sai_dump_filename; + +} ############################################################################### # find_files routine # Globals: @@ -1341,6 +1418,10 @@ save_marvellcmd() { ############################################################################### collect_marvell() { trap 'handle_error $? $LINENO' ERR + + local sai_dump_folder="/tmp/saisdkdump" + local sai_dump_filename="${sai_dump_folder}/sai_sdk_dump_$(date +"%m_%d_%Y_%I_%M_%p")" + generate_sai_dbg_dump_file "$sai_dump_filename" save_marvellcmd "show version" "CPSS_version" save_marvellcmd "debug-mode;xps-api call xpsDataIntegrityDumpSerInfo all" "SER_table" diff --git a/setup.py b/setup.py index 6a66f012f9..3fcb7d9519 100644 --- a/setup.py +++ b/setup.py @@ -188,7 +188,9 @@ 'scripts/verify_image_sign.sh', 'scripts/verify_image_sign_common.sh', 'scripts/check_db_integrity.py', - 'scripts/sysreadyshow' + 'scripts/sysreadyshow', + 'scripts/gen_sai_dbg_dump.sh', + 'scripts/gen_sai_dbg_dump_lib.sh', ], entry_points={ 'console_scripts': [ From 0ecc5fb6233ffbdbc843989ded8b9f94480d0b02 Mon Sep 17 00:00:00 2001 From: aviramd Date: Wed, 30 Oct 2024 13:31:23 +0000 Subject: [PATCH 2/6] write debug dump to Appl DB instead of ConfigDb Signed-off-by: aviramd --- scripts/gen_sai_dbg_dump.sh | 20 +++++----------- scripts/gen_sai_dbg_dump_lib.sh | 40 ++++++++++++++++++++++++-------- scripts/generate_dump | 41 +++++++++++++-------------------- 3 files changed, 52 insertions(+), 49 deletions(-) diff --git a/scripts/gen_sai_dbg_dump.sh b/scripts/gen_sai_dbg_dump.sh index 96fcc7abdc..e63badfe0f 100644 --- a/scripts/gen_sai_dbg_dump.sh +++ b/scripts/gen_sai_dbg_dump.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash . /usr/local/bin/gen_sai_dbg_dump_lib.sh @@ -63,10 +63,9 @@ copy_from_docker() { # Description: # This is the main entry point of the script, which handles the generation # and retrieval of the SAI debug dump file. It parses command-line arguments, -# ensures necessary directories and the `syncd` container are available, and -# triggers the SAI debug dump process through Redis. The script waits for the -# dump file to be generated and then copies it from the Docker container to -# the specified location on the local system. +# ensures necessary directories are available, and triggers the SAI debug dump +# process through Redis. The script waits for the dump file to be generated and +# then copies it from the Docker container to the specified location on the local system. # # Globals: # None @@ -112,20 +111,13 @@ main() { sudo mkdir -p "$(dirname "$sai_dump_filename")" fi - # Ensure the syncd container is running - if [ "$(docker container inspect -f '{{.State.Running}}' syncd)" != "true" ]; then - echo "Error: syncd container is not running." - exit 1 - fi - generate_sai_dump "$syncd_sai_dump_filename" if [ $? -ne 0 ]; then echo "Failed to generate SAI debug dump." exit 1 fi - # Copy the dump file from the Docker container - local + # Copy the dump file from the Docker container if ! copy_from_docker syncd $syncd_sai_dump_filename $sai_dump_filename; then echo "Error: Failed to copy the SAI dump file from the container." exit 1 @@ -133,7 +125,7 @@ main() { # Remove the dump file from the Docker container docker exec syncd rm -rf $syncd_sai_dump_filename; - echo "$sai_dump_filename is ready!!!" + echo "file '$sai_dump_filename' is ready!!!" exit 0 } diff --git a/scripts/gen_sai_dbg_dump_lib.sh b/scripts/gen_sai_dbg_dump_lib.sh index f730ccb5b5..fc869473b8 100644 --- a/scripts/gen_sai_dbg_dump_lib.sh +++ b/scripts/gen_sai_dbg_dump_lib.sh @@ -1,10 +1,11 @@ -#!/bin/sh +#!/bin/bash ############################################################################### # generate_sai_dump # Description: # This function triggers the generation of a SAI debug dump file in the # `syncd` Docker container through Redis and waits for the file to be ready. +# it ensures that the `syncd` container is running before initiating the dump. # # Arguments: # $1 - Filename for the SAI debug dump file. @@ -14,9 +15,9 @@ # 1 - On failure ############################################################################### generate_sai_dump() { - local DB=4 - local KEY="DBG_GEN_DUMP_TABLE|DUMP" - local STATUS_KEY="DBG_GEN_DUMP_STATS_TABLE|DUMP" + local DB=0 + local KEY="DBG_GEN_DUMP_TABLE:DUMP" + local STATUS_KEY="DBG_GEN_DUMP_STATUS_TABLE:DUMP" local FIELD="file" local STATUS_FIELD="status" local STATUS="1" @@ -32,6 +33,12 @@ generate_sai_dump() { return 1 fi + # Ensure the syncd container is running + if [[ "$( docker container inspect -f '{{.State.Running}}' syncd )" != "true" ]]; then + echo "Error: syncd container is not running." + return 1 + fi + # Extract the directory from the SYNCD_DUMP_FILE path local SYNCD_DUMP_DIR SYNCD_DUMP_DIR=$(dirname "$SYNCD_DUMP_FILE") @@ -50,17 +57,30 @@ generate_sai_dump() { redis-cli -n $DB DEL $STATUS_KEY > /dev/null 2>&1 # Set the DBG_GEN_DUMP in the Redis DB to trigger the dump generation - if ! redis-cli -n $DB HSET $KEY $FIELD $SYNCD_DUMP_FILE > /dev/null 2>&1; then + if ! redis-cli SADD "DBG_GEN_DUMP_TABLE_KEY_SET" "DUMP" > /dev/null 2>&1; then + echo "Error: Failed to publish message to Redis DBG_GEN_DUMP_TABLE_CHANNEL." + return 1 + fi + + if ! redis-cli -n $DB HSET "_$KEY" $FIELD $SYNCD_DUMP_FILE > /dev/null 2>&1; then echo "Error: Failed to set Redis key." return 1 fi - # Timeout and interval for checking status of file readiness + + if ! redis-cli PUBLISH "DBG_GEN_DUMP_TABLE_CHANNEL@0" "G" > /dev/null 2>&1; then + echo "Error: Failed to publish message to Redis DBG_GEN_DUMP_TABLE_CHANNEL." + return 1 + fi + # Timeout and interval for checking status of file readiness while [ $TIME_PASSED -lt $TIMEOUT_FOR_GEN_DBG_DUMP_FILE_READYNESS ]; do - EXISTS=$(redis-cli -n $DB EXISTS "$STATUS_KEY" 2>/dev/null | grep -o '^[0-9]*$') - if [ "$EXISTS" -eq 1 ]; then - STATUS=$(redis-cli -n $DB HGET "$STATUS_KEY" "$STATUS_FIELD" 2>/dev/null | grep -o '^[0-9]*$') + # Get the status field value + STATUS=$(redis-cli -n $DB HGET "$STATUS_KEY" "$STATUS_FIELD" 2>/dev/null | grep -o '^[0-9]*$') + + # Check if STATUS is non-empty + if [ -n "$STATUS" ]; then + # STATUS field exists; you can use it as needed break fi @@ -72,7 +92,7 @@ generate_sai_dump() { redis-cli -n $DB DEL $KEY > /dev/null 2>&1 redis-cli -n $DB DEL $STATUS_KEY > /dev/null 2>&1 - if [ "$STATUS" -ne 0 ]; then + if [ -n "$STATUS" ] && [ "$STATUS" -ne 0 ]; then echo "Error: dump file operation failed, Status $STATUS" return 1 fi diff --git a/scripts/generate_dump b/scripts/generate_dump index 48208d063c..79df6d3ed3 100755 --- a/scripts/generate_dump +++ b/scripts/generate_dump @@ -1110,41 +1110,39 @@ save_file() { ############################################################################### # generate_sai_dbg_dump_file # Description: -# This function triggers the generation of a SAI debug dump file in the -# `syncd` Docker container and saves it to the local system. The function -# communicates with the Redis database to initiate the dump process +# This function triggers the generation of a SAI debug dump file and saves the +# dumped file in the show techsupport output directory. # # Globals: # None # # Arguments: -# None +# $1 - (required) The file name (without path) the SAI debug dump will be saved under +# this name in the show techsupport output directory. # # Returns: # 0 - On success # 1 - On failure ############################################################################### + generate_sai_dbg_dump_file() { trap 'handle_error $? $LINENO' ERR local syncd_sai_dump_filename="/var/log/dbg_gen_dump.log" if $NOOP; then echo "generate_sai_dump" - return 1 + return 0 fi - # Validate the argument - local sai_dump_filename="$1" - if [ -z "$sai_dump_filename" ]; then + # Validate the argument + if [ -z "$1" ]; then echo "Error: No filename provided for the SAI debug dump file." return 1 fi - # Extract the directory from the filename - local sai_dump_dir - sai_dump_dir=$(dirname "$sai_dump_filename") - - # Check if the directory exists; if not, create it + # set temp file path to generate the file, Check if the directory exists; if not, create it + local sai_dump_dir="/tmp/saisdkdump" + local sai_dump_filename="${sai_dump_dir}/$1" if [ ! -d "$sai_dump_dir" ]; then echo "Directory '$sai_dump_dir' does not exist. Creating it..." mkdir -p "$sai_dump_dir" @@ -1152,13 +1150,7 @@ generate_sai_dbg_dump_file() { echo "Error: Failed to create directory '$sai_dump_dir'." return 1 fi - fi - - # Ensure the syncd container is running - if [[ "$( docker container inspect -f '{{.State.Running}}' syncd )" != "true" ]]; then - echo "Error: syncd container is not running." - return 1 - fi + fi # Generate the SAI dump file using the refactored function source /usr/local/bin/gen_sai_dbg_dump_lib.sh @@ -1183,6 +1175,7 @@ generate_sai_dbg_dump_file() { rm -rf $sai_dump_filename docker exec syncd rm -rf $syncd_sai_dump_filename; + return 0 } ############################################################################### # find_files routine @@ -1417,11 +1410,9 @@ save_marvellcmd() { # None ############################################################################### collect_marvell() { - trap 'handle_error $? $LINENO' ERR - - local sai_dump_folder="/tmp/saisdkdump" - local sai_dump_filename="${sai_dump_folder}/sai_sdk_dump_$(date +"%m_%d_%Y_%I_%M_%p")" - generate_sai_dbg_dump_file "$sai_dump_filename" + trap 'handle_error $? $LINENO' ERR + + generate_sai_dbg_dump_file "sai_sdk_dump_$(date +"%m_%d_%Y_%I_%M_%p")" save_marvellcmd "show version" "CPSS_version" save_marvellcmd "debug-mode;xps-api call xpsDataIntegrityDumpSerInfo all" "SER_table" From 53c8e3f1c1c8ff98c12751e83bc1bf9955348726 Mon Sep 17 00:00:00 2001 From: aviramd Date: Sun, 3 Nov 2024 14:14:33 +0000 Subject: [PATCH 3/6] add file readyness timeout argument for gen_dump_file api (default 10sec) Signed-off-by: aviramd --- scripts/gen_sai_dbg_dump_lib.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/gen_sai_dbg_dump_lib.sh b/scripts/gen_sai_dbg_dump_lib.sh index fc869473b8..44f77bf68f 100644 --- a/scripts/gen_sai_dbg_dump_lib.sh +++ b/scripts/gen_sai_dbg_dump_lib.sh @@ -9,6 +9,7 @@ # # Arguments: # $1 - Filename for the SAI debug dump file. +# $2 - Optional timeout for file readiness (default: 10 seconds). # # Returns: # 0 - On success @@ -22,12 +23,12 @@ generate_sai_dump() { local STATUS_FIELD="status" local STATUS="1" - local TIMEOUT_FOR_GEN_DBG_DUMP_FILE_READYNESS=10 + local SYNCD_DUMP_FILE="$1" + local TIMEOUT_FOR_GEN_DBG_DUMP_FILE_READYNESS="${2:-10}" local INTERVAL=1 local TIME_PASSED=0 - local EXISTS - local SYNCD_DUMP_FILE="$1" + if [ -z "$SYNCD_DUMP_FILE" ]; then echo "Error: No filename provided for the SAI debug dump file." return 1 @@ -52,7 +53,7 @@ generate_sai_dump() { fi fi - # Delete the tables from STATE_DB before triger the dump file + # Delete the tables from STATE_DB before triggering the dump file redis-cli -n $DB DEL $KEY > /dev/null 2>&1 redis-cli -n $DB DEL $STATUS_KEY > /dev/null 2>&1 @@ -67,7 +68,6 @@ generate_sai_dump() { return 1 fi - if ! redis-cli PUBLISH "DBG_GEN_DUMP_TABLE_CHANNEL@0" "G" > /dev/null 2>&1; then echo "Error: Failed to publish message to Redis DBG_GEN_DUMP_TABLE_CHANNEL." return 1 @@ -88,7 +88,7 @@ generate_sai_dump() { TIME_PASSED=$((TIME_PASSED + INTERVAL)) done - # Delete the tables from STATE_DB after triger the dump file + # Delete the tables from STATE_DB after triggering the dump file redis-cli -n $DB DEL $KEY > /dev/null 2>&1 redis-cli -n $DB DEL $STATUS_KEY > /dev/null 2>&1 From 85555b05bba1b9d6e27f5747b63acb60a1b33222 Mon Sep 17 00:00:00 2001 From: aviramd Date: Sun, 3 Nov 2024 14:14:33 +0000 Subject: [PATCH 4/6] add file readyness timeout argument for gen_dump_file api (default 10sec) Signed-off-by: aviramd --- scripts/gen_sai_dbg_dump.sh | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/scripts/gen_sai_dbg_dump.sh b/scripts/gen_sai_dbg_dump.sh index e63badfe0f..5985eb0b27 100644 --- a/scripts/gen_sai_dbg_dump.sh +++ b/scripts/gen_sai_dbg_dump.sh @@ -72,6 +72,7 @@ copy_from_docker() { # # Arguments: # -f : Specifies the output filename for the SAI debug dump file. +# -t : Optional timeout for file readiness (in seconds). # -h : Displays usage information. # # Returns: @@ -79,12 +80,18 @@ copy_from_docker() { # 1 - On failure ############################################################################### main() { + local sai_dump_filename="" + local timeout_for_file_readiness="" + # Parse arguments - while getopts ":f:h" opt; do + while getopts ":f:t:h" opt; do case $opt in f) sai_dump_filename="$OPTARG" ;; + t) + timeout_for_file_readiness="$OPTARG" + ;; h) usage exit 0 @@ -111,7 +118,13 @@ main() { sudo mkdir -p "$(dirname "$sai_dump_filename")" fi - generate_sai_dump "$syncd_sai_dump_filename" + # Call generate_sai_dump with or without the timeout argument + if [ -n "$timeout_for_file_readiness" ]; then + generate_sai_dump "$syncd_sai_dump_filename" "$timeout_for_file_readiness" + else + generate_sai_dump "$syncd_sai_dump_filename" + fi + if [ $? -ne 0 ]; then echo "Failed to generate SAI debug dump." exit 1 @@ -124,7 +137,7 @@ main() { fi # Remove the dump file from the Docker container - docker exec syncd rm -rf $syncd_sai_dump_filename; + docker exec syncd rm -rf $syncd_sai_dump_filename echo "file '$sai_dump_filename' is ready!!!" exit 0 } From 3e858d3e5390dee548d5a48898ebedc8f26f5d24 Mon Sep 17 00:00:00 2001 From: aviramd Date: Sun, 3 Nov 2024 14:14:33 +0000 Subject: [PATCH 5/6] add file readyness timeout argument for gen_dump_file api (default 10sec) Signed-off-by: aviramd --- scripts/generate_dump | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/generate_dump b/scripts/generate_dump index 79df6d3ed3..9e9b78fdd9 100755 --- a/scripts/generate_dump +++ b/scripts/generate_dump @@ -1410,9 +1410,7 @@ save_marvellcmd() { # None ############################################################################### collect_marvell() { - trap 'handle_error $? $LINENO' ERR - - generate_sai_dbg_dump_file "sai_sdk_dump_$(date +"%m_%d_%Y_%I_%M_%p")" + trap 'handle_error $? $LINENO' ERR save_marvellcmd "show version" "CPSS_version" save_marvellcmd "debug-mode;xps-api call xpsDataIntegrityDumpSerInfo all" "SER_table" @@ -2129,6 +2127,8 @@ main() { save_saidump_by_route_size + generate_sai_dbg_dump_file "sai_sdk_dump_$(date +"%m_%d_%Y_%I_%M_%p")" + if [ "$asic" = "barefoot" ]; then collect_barefoot fi From 9893ebf50527817a5ce7e20f3000637a86fcd9b4 Mon Sep 17 00:00:00 2001 From: aviramd Date: Mon, 11 Nov 2024 16:38:01 +0000 Subject: [PATCH 6/6] Update gen_sai_dbg_dump scripts and remove gen_sai_dbg_dump_lib.sh --- scripts/gen_sai_dbg_dump.sh | 148 +++++++++++++++++++++++++++----- scripts/gen_sai_dbg_dump_lib.sh | 112 ------------------------ scripts/generate_dump | 3 +- setup.py | 1 - 4 files changed, 129 insertions(+), 135 deletions(-) delete mode 100644 scripts/gen_sai_dbg_dump_lib.sh diff --git a/scripts/gen_sai_dbg_dump.sh b/scripts/gen_sai_dbg_dump.sh index 5985eb0b27..2c36be48a3 100644 --- a/scripts/gen_sai_dbg_dump.sh +++ b/scripts/gen_sai_dbg_dump.sh @@ -1,7 +1,5 @@ #!/bin/bash -. /usr/local/bin/gen_sai_dbg_dump_lib.sh - ############################################################################### # Prints the usage information. # Globals: @@ -58,12 +56,128 @@ copy_from_docker() { fi } +############################################################################### +# generate_sai_dump +# Description: +# This function triggers the generation of a SAI debug dump file in the +# `syncd` Docker container through Redis and waits for the file to be ready. +# Arguments: +# $1 - Filename for the SAI debug dump file. +# $2 - Optional timeout for file readiness (default: 10 seconds). +# Returns: +# 0 - On success +# 1 - On failure +############################################################################### +generate_sai_dump() { + local DB=0 + local KEY="DBG_GEN_DUMP_TABLE:DUMP" + local STATUS_KEY="DBG_GEN_DUMP_STATUS_TABLE:DUMP" + local FIELD="file" + local STATUS_FIELD="status" + local STATUS="1" + + local SYNCD_DUMP_FILE="$1" + local TIMEOUT_FOR_GEN_DBG_DUMP_FILE_READYNESS="${2:-60}" + local INTERVAL=1 + local TIME_PASSED=0 + + if [ -z "$SYNCD_DUMP_FILE" ]; then + echo "Error: No filename provided for the SAI debug dump file." + return 1 + fi + + # Ensure the syncd container is running + if [[ "$( docker container inspect -f '{{.State.Running}}' syncd )" != "true" ]]; then + echo "Error: syncd container is not running." + return 1 + fi + + # Extract the directory from the SYNCD_DUMP_FILE path + local SYNCD_DUMP_DIR + SYNCD_DUMP_DIR=$(dirname "$SYNCD_DUMP_FILE") + + # Ensure the directory exists in the syncd container; if not, create it + if ! docker exec syncd test -d "$SYNCD_DUMP_DIR"; then + #echo "Creating directory '$SYNCD_DUMP_DIR' inside the syncd container..." + if ! docker exec syncd mkdir -p "$SYNCD_DUMP_DIR"; then + echo "Error: Failed to create directory inside the syncd container." + return 1 + fi + fi + + # Delete the tables from STATE_DB before triggering the dump file + redis-cli -n $DB DEL $KEY > /dev/null 2>&1 + redis-cli -n $DB DEL $STATUS_KEY > /dev/null 2>&1 + + # Set the DBG_GEN_DUMP in the Redis DB to trigger the dump generation + if ! redis-cli SADD "DBG_GEN_DUMP_TABLE_KEY_SET" "DUMP" > /dev/null 2>&1; then + echo "Error: Failed to publish message to Redis DBG_GEN_DUMP_TABLE_CHANNEL." + return 1 + fi + + if ! redis-cli -n $DB HSET "_$KEY" $FIELD $SYNCD_DUMP_FILE > /dev/null 2>&1; then + echo "Error: Failed to set Redis key." + return 1 + fi + + if ! redis-cli PUBLISH "DBG_GEN_DUMP_TABLE_CHANNEL@0" "G" > /dev/null 2>&1; then + echo "Error: Failed to publish message to Redis DBG_GEN_DUMP_TABLE_CHANNEL." + return 1 + fi + + # Timeout and interval for checking status of file readiness + while [ $TIME_PASSED -lt $TIMEOUT_FOR_GEN_DBG_DUMP_FILE_READYNESS ]; do + # Get the status field value + STATUS=$(redis-cli -n $DB HGET "$STATUS_KEY" "$STATUS_FIELD" 2>/dev/null | grep -o '^[0-9]*$') + + # Check if STATUS is non-empty + if [ -n "$STATUS" ]; then + # STATUS field exists; you can use it as needed + break + fi + sleep $INTERVAL + TIME_PASSED=$((TIME_PASSED + INTERVAL)) + done + + # Delete the tables from STATE_DB after triggering the dump file + redis-cli -n $DB DEL $KEY > /dev/null 2>&1 + redis-cli -n $DB DEL $STATUS_KEY > /dev/null 2>&1 + + if [ -n "$STATUS" ] && [ "$STATUS" -ne 0 ]; then + echo "Error: dump file operation failed, Status $STATUS" + return 1 + fi + + if [ $TIME_PASSED -ge $TIMEOUT_FOR_GEN_DBG_DUMP_FILE_READYNESS ]; then + echo "Timeout reached. Status was not ready in time." + return 1 + fi + + # Poll for file existence in the syncd container with a timeout + TIME_PASSED=0 + while [ $TIME_PASSED -lt $TIMEOUT_FOR_GEN_DBG_DUMP_FILE_READYNESS ]; do + if docker exec syncd test -f "$SYNCD_DUMP_FILE"; then + #echo "SAI dump file successfully generated in the syncd container." + break + fi + sleep $INTERVAL + TIME_PASSED=$((TIME_PASSED + INTERVAL)) + done + + if [ $TIME_PASSED -ge $TIMEOUT_FOR_GEN_DBG_DUMP_FILE_READYNESS ]; then + echo "Error: SAI dump file does not exist in the syncd container after waiting ${TIME_PASSED} seconds." + return 1 + fi + + return 0 +} + ############################################################################### # Main script logic # Description: # This is the main entry point of the script, which handles the generation # and retrieval of the SAI debug dump file. It parses command-line arguments, -# ensures necessary directories are available, and triggers the SAI debug dump +# ensures necessary directories are available, and triggers the SAI debug dump # process through Redis. The script waits for the dump file to be generated and # then copies it from the Docker container to the specified location on the local system. # @@ -86,21 +200,10 @@ main() { # Parse arguments while getopts ":f:t:h" opt; do case $opt in - f) - sai_dump_filename="$OPTARG" - ;; - t) - timeout_for_file_readiness="$OPTARG" - ;; - h) - usage - exit 0 - ;; - /?) - echo "Invalid option: -$OPTARG" >&2 - usage - exit 1 - ;; + f) sai_dump_filename="$OPTARG" ;; + t) timeout_for_file_readiness="$OPTARG" ;; + h) usage; exit 0 ;; + ?) echo "Invalid option: -$OPTARG" >&2; usage; exit 1 ;; esac done @@ -117,11 +220,13 @@ main() { if [ ! -d "$(dirname "$sai_dump_filename")" ]; then sudo mkdir -p "$(dirname "$sai_dump_filename")" fi - + # Call generate_sai_dump with or without the timeout argument if [ -n "$timeout_for_file_readiness" ]; then + #echo generate_sai_dump "$syncd_sai_dump_filename" "$timeout_for_file_readiness" generate_sai_dump "$syncd_sai_dump_filename" "$timeout_for_file_readiness" else + #echo generate_sai_dump "$syncd_sai_dump_filename" generate_sai_dump "$syncd_sai_dump_filename" fi @@ -142,4 +247,7 @@ main() { exit 0 } -main "$@" +# Only call main if script is executed directly +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/scripts/gen_sai_dbg_dump_lib.sh b/scripts/gen_sai_dbg_dump_lib.sh deleted file mode 100644 index 44f77bf68f..0000000000 --- a/scripts/gen_sai_dbg_dump_lib.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash - -############################################################################### -# generate_sai_dump -# Description: -# This function triggers the generation of a SAI debug dump file in the -# `syncd` Docker container through Redis and waits for the file to be ready. -# it ensures that the `syncd` container is running before initiating the dump. -# -# Arguments: -# $1 - Filename for the SAI debug dump file. -# $2 - Optional timeout for file readiness (default: 10 seconds). -# -# Returns: -# 0 - On success -# 1 - On failure -############################################################################### -generate_sai_dump() { - local DB=0 - local KEY="DBG_GEN_DUMP_TABLE:DUMP" - local STATUS_KEY="DBG_GEN_DUMP_STATUS_TABLE:DUMP" - local FIELD="file" - local STATUS_FIELD="status" - local STATUS="1" - - local SYNCD_DUMP_FILE="$1" - local TIMEOUT_FOR_GEN_DBG_DUMP_FILE_READYNESS="${2:-10}" - local INTERVAL=1 - local TIME_PASSED=0 - - - if [ -z "$SYNCD_DUMP_FILE" ]; then - echo "Error: No filename provided for the SAI debug dump file." - return 1 - fi - - # Ensure the syncd container is running - if [[ "$( docker container inspect -f '{{.State.Running}}' syncd )" != "true" ]]; then - echo "Error: syncd container is not running." - return 1 - fi - - # Extract the directory from the SYNCD_DUMP_FILE path - local SYNCD_DUMP_DIR - SYNCD_DUMP_DIR=$(dirname "$SYNCD_DUMP_FILE") - - # Ensure the directory exists in the syncd container; if not, create it - if ! docker exec syncd test -d "$SYNCD_DUMP_DIR"; then - echo "Directory '$SYNCD_DUMP_DIR' does not exist in the syncd container. Creating it..." - if ! docker exec syncd mkdir -p "$SYNCD_DUMP_DIR"; then - echo "Error: Failed to create directory '$SYNCD_DUMP_DIR' inside the syncd container." - return 1 - fi - fi - - # Delete the tables from STATE_DB before triggering the dump file - redis-cli -n $DB DEL $KEY > /dev/null 2>&1 - redis-cli -n $DB DEL $STATUS_KEY > /dev/null 2>&1 - - # Set the DBG_GEN_DUMP in the Redis DB to trigger the dump generation - if ! redis-cli SADD "DBG_GEN_DUMP_TABLE_KEY_SET" "DUMP" > /dev/null 2>&1; then - echo "Error: Failed to publish message to Redis DBG_GEN_DUMP_TABLE_CHANNEL." - return 1 - fi - - if ! redis-cli -n $DB HSET "_$KEY" $FIELD $SYNCD_DUMP_FILE > /dev/null 2>&1; then - echo "Error: Failed to set Redis key." - return 1 - fi - - if ! redis-cli PUBLISH "DBG_GEN_DUMP_TABLE_CHANNEL@0" "G" > /dev/null 2>&1; then - echo "Error: Failed to publish message to Redis DBG_GEN_DUMP_TABLE_CHANNEL." - return 1 - fi - - # Timeout and interval for checking status of file readiness - while [ $TIME_PASSED -lt $TIMEOUT_FOR_GEN_DBG_DUMP_FILE_READYNESS ]; do - # Get the status field value - STATUS=$(redis-cli -n $DB HGET "$STATUS_KEY" "$STATUS_FIELD" 2>/dev/null | grep -o '^[0-9]*$') - - # Check if STATUS is non-empty - if [ -n "$STATUS" ]; then - # STATUS field exists; you can use it as needed - break - fi - - sleep $INTERVAL - TIME_PASSED=$((TIME_PASSED + INTERVAL)) - done - - # Delete the tables from STATE_DB after triggering the dump file - redis-cli -n $DB DEL $KEY > /dev/null 2>&1 - redis-cli -n $DB DEL $STATUS_KEY > /dev/null 2>&1 - - if [ -n "$STATUS" ] && [ "$STATUS" -ne 0 ]; then - echo "Error: dump file operation failed, Status $STATUS" - return 1 - fi - - if [ $TIME_PASSED -ge $TIMEOUT_FOR_GEN_DBG_DUMP_FILE_READYNESS ]; then - echo "Timeout reached. Status was not ready in time." - return 1 - fi - - # Ensure the file exists in the Docker container - if ! docker exec syncd test -f $SYNCD_DUMP_FILE; then - echo "Error: SAI dump file does not exist in the syncd container." - return 1 - fi - - return 0 -} diff --git a/scripts/generate_dump b/scripts/generate_dump index 9e9b78fdd9..7dadcc0de1 100755 --- a/scripts/generate_dump +++ b/scripts/generate_dump @@ -1153,7 +1153,7 @@ generate_sai_dbg_dump_file() { fi # Generate the SAI dump file using the refactored function - source /usr/local/bin/gen_sai_dbg_dump_lib.sh + source /usr/local/bin/gen_sai_dbg_dump.sh if ! generate_sai_dump "$syncd_sai_dump_filename"; then echo "Error: Failed to generate SAI debug dump." return 1 @@ -2157,7 +2157,6 @@ main() { collect_marvell fi - # 2nd counter snapshot late. Need 2 snapshots to make sense of counters trend. save_counter_snapshot $asic 2 diff --git a/setup.py b/setup.py index 3fcb7d9519..c25cf9bd68 100644 --- a/setup.py +++ b/setup.py @@ -190,7 +190,6 @@ 'scripts/check_db_integrity.py', 'scripts/sysreadyshow', 'scripts/gen_sai_dbg_dump.sh', - 'scripts/gen_sai_dbg_dump_lib.sh', ], entry_points={ 'console_scripts': [