[generate_dump] remove secrets from dump files (#1886)

Remove secrets from dump files.

What I did
    Add bash functions to remove secrets from dump files.
How I did it
    For tacacs key, radius key, snmp community srring, use sed command with regex to remove user secrets from dump files.
    For certs, update tar command exclude list to remove those certs from dump file.
How to verify it
    Run 'show techsupport' command and check secrets removed from dump files.
Previous command output (if the output of a command-line utility has changed)
New command output (if the output of a command-line utility has changed)

Author: liuh-80

scripts/generate_dump

@@ -42,7 +42,6 @@ SAVE_STDERR=true
 RETURN_CODE=0
 DEBUG_DUMP=false
 
-
 handle_signal()
 {
     echo "Generate Dump received interrupt" >&2
@@ -155,6 +154,7 @@ save_bcmcmd_all_ns() {
 #  cmd: The command to run. Make sure that arguments with spaces have quotes
 #  filename: the filename to save the output as in $BASE/dump
 #  do_gzip: (OPTIONAL) true or false. Should the output be gzipped
+#  cleanup_method: (OPTIONAL) the cleanup method to procress dump file after it generated.
 # Returns:
 #  None
 ###############################################################################
@@ -168,6 +168,7 @@ save_cmd() {
     local do_gzip=${3:-false}
     local tarpath="${BASE}/dump/$filename"
     local timeout_cmd="timeout --foreground ${TIMEOUT_MIN}m"
+    local cleanup_method=${4:-dummy_cleanup_method}
     local redirect='&>'
     local redirect_eval='2>&1'
     if [ ! -d $LOGDIR ]; then
@@ -187,7 +188,9 @@ save_cmd() {
     if $do_gzip; then
         tarpath="${tarpath}.gz"
         filepath="${filepath}.gz"
-        local cmds="$cmd $redirect_eval | gzip -c > '${filepath}'"
+        # cleanup_method will run in a sub-shell, need declare it first
+        local cleanup_method_declration=$(declare -f $cleanup_method)
+        local cmds="$cleanup_method_declration; $cmd $redirect_eval | $cleanup_method | gzip -c > '${filepath}'"
         if $NOOP; then
             echo "${timeout_cmd} bash -c \"${cmds}\""
         else
@@ -199,22 +202,36 @@ save_cmd() {
         fi
     else
         if $NOOP; then
-            echo "${timeout_cmd} $cmd $redirect '$filepath'"
+            echo "${timeout_cmd} $cmd | $cleanup_method $redirect '$filepath'"
         else
             RC=0
-            eval "${timeout_cmd} $cmd" "$redirect" "$filepath" || RC=$?
+            eval "${timeout_cmd} $cmd | $cleanup_method" "$redirect" "$filepath" || RC=$?
             if [ $RC -ne 0 ]; then
                 echo "Command: $cmd timedout after ${TIMEOUT_MIN} minutes."
             fi
         fi
     fi
+
     ($TAR $V -rhf $TARFILE -C $DUMPDIR "$tarpath" \
         || abort "${ERROR_TAR_FAILED}" "tar append operation failed. Aborting to prevent data loss.") \
         && $RM $V -rf "$filepath"
     end_t=$(date +%s%3N)
     echo "[ save_cmd:$cmd ] : $(($end_t-$start_t)) msec" >> $TECHSUPPORT_TIME_INFO
 }
 
+###############################################################################
+# Dummy cleanup method.
+# Globals:
+#  None
+# Arguments:
+#  None
+# Returns:
+#  None
+###############################################################################
+dummy_cleanup_method() {
+    cat
+}
+
 ###############################################################################
 # Runs a given command in all namesapces in case of multi ASIC platform, in
 # default (host) namespace in single ASIC platform
@@ -224,22 +241,24 @@ save_cmd() {
 #  cmd: The command to run. Make sure that arguments with spaces have quotes
 #  filename: the filename to save the output as in $BASE/dump
 #  do_gzip: (OPTIONAL) true or false. Should the output be gzipped
+#  cleanup_method: (OPTIONAL) the cleanup method to procress dump file after it generated.
 # Returns:
 #  None
 ###############################################################################
 save_cmd_all_ns() {
     trap 'handle_error $? $LINENO' ERR
     local do_zip=${3:-false}
+    local cleanup_method=${4:-dummy_cleanup_method}
 
     # host or default namespace
-    save_cmd "$1" "$2" "$do_zip"
+    save_cmd "$1" "$2" "$do_zip" $cleanup_method
 
     if [[ ( "$NUM_ASICS" > 1 ) ]] ; then
       for (( i=0; i<$NUM_ASICS; i++ ))
       do
           local cmd="sonic-netns-exec asic$i $1"
           local file="$2.$i"
-          save_cmd "$cmd" "$file" "$do_zip"
+          save_cmd "$cmd" "$file" "$do_zip" $cleanup_method
       done
     fi
 }
@@ -591,7 +610,8 @@ save_redis_info() {
     save_redis "APPL_DB"
     save_redis "ASIC_DB"
     save_redis "COUNTERS_DB"
-    save_redis "CONFIG_DB"
+    # There are secrets in CONFIG_DB need to be cleanup.
+    save_redis "CONFIG_DB" "CONFIG_DB" remove_secret_from_config_db_dump
     save_redis "FLEX_COUNTER_DB"
     save_redis "STATE_DB"
 }
@@ -637,18 +657,20 @@ save_proc() {
 # Arguments:
 #  DB name: DB name
 #  Filename: Destination filename, if not given then filename would be DB name
+#  cleanup_method: (OPTIONAL) the cleanup method to procress dump file after it generated.
 # Returns:
 #  None
 ###############################################################################
 save_redis() {
+    local cleanup_method=${3:-dummy_cleanup_method}
     trap 'handle_error $? $LINENO' ERR
     local db_name=$1
     if [ $# -ge 2 ] && [ -n "$2" ]; then
         local dest_file_name=$2
     else
         local dest_file_name="$db_name"
     fi
-    save_cmd_all_ns "sonic-db-dump -n '$db_name' -y" "$dest_file_name.json"
+    save_cmd_all_ns "sonic-db-dump -n '$db_name' -y" "$dest_file_name.json" false $cleanup_method
 }
 
 ###############################################################################
@@ -1259,6 +1281,9 @@ main() {
         rm $rm_list
     fi
 
+    # Remove secret from /etc files before tar
+    remove_secret_from_etc_files $TARDIR
+
     start_t=$(date +%s%3N)
     ($TAR $V --warning=no-file-removed -rhf $TARFILE -C $DUMPDIR --mode=+rw \
         --exclude="etc/alternatives" \
@@ -1271,6 +1296,13 @@ main() {
         --exclude="*snmpd.conf*" \
         --exclude="/etc/mlnx" \
         --exclude="/etc/mft" \
+        --exclude="*/etc/sonic/*.cer" \
+        --exclude="*/etc/sonic/*.crt" \
+        --exclude="*/etc/sonic/*.pem" \
+        --exclude="*/etc/sonic/*.key" \
+        --exclude="*/etc/ssl/*.pem" \
+        --exclude="*/etc/ssl/certs/*" \
+        --exclude="*/etc/ssl/private/*" \
         $BASE/etc \
         || abort "${ERROR_TAR_FAILED}" "Tar append operation failed. Aborting for safety.") \
         && $RM $V -rf $TARDIR
@@ -1310,6 +1342,57 @@ main() {
     exit $RETURN_CODE
 }
 
+###############################################################################
+# Remove secret from pipeline inout and output result to pipeline.
+# Globals:
+#  None
+# Arguments:
+#  None
+# Returns:
+#  None
+###############################################################################
+remove_secret_from_config_db_dump() {
+    # Remove tacacs & radius passkey and snmp community from config DB
+    sed -E 's/\"passkey\"\s*:\s*\"([^\"]*)\"/\"passkey\":\"****\"/g; /SNMP_COMMUNITY/,/\s{2,4}\},/d'
+}
+
+###############################################################################
+# Remove secret from dump files.
+# Globals:
+# Arguments:
+#  dumppath: the dump file path.
+# Returns:
+#  None
+###############################################################################
+remove_secret_from_etc_files() {
+    local dumppath=$1
+    echo "Remove secret from etc files."
+    # Remove tacacs passkey from tacplus_nss.conf
+    local secret_regex='s/(secret=)([^,|\S]*)(.*)/\1****\3/g'
+    sed -i -E $secret_regex $dumppath/etc/tacplus_nss.conf
+
+    # Remove radius passkey from radius_nss.conf
+    sed -i -E $secret_regex $dumppath/etc/radius_nss.conf
+
+    # Remove tacacs passkey from common-auth-sonic
+    sed -i -E 's/(secret=)(\S*)/\1****/g' $dumppath/etc/pam.d/common-auth-sonic
+
+    # Remove tacacs passkey from pam_radius_auth.conf
+    sed -i -E 's/^([^#]\S*\s*)(\S*)/\1****/g' $dumppath/etc/pam_radius_auth.conf
+
+    # Remove radius passkey from per-server conf file /etc/pam_radius_auth.d/{ip}_{port}.conf
+    for filename in $dumppath/etc/pam_radius_auth.d/*.conf; do
+        sed -i -E 's/^([^#]\S*\s*)(\S*)/\1****/g' $filename
+    done
+
+    # Remove snmp community string from snmp.yml
+    sed -i -E 's/(\s*snmp_\S*community\s*:\s*)(\S*)/\1****/g' $dumppath/etc/sonic/snmp.yml
+    
+    # Remove secret from /etc/sonic/config_db.json
+    cat $dumppath/etc/sonic/config_db.json | remove_secret_from_config_db_dump > $dumppath/etc/sonic/config_db.json.temp
+    mv $dumppath/etc/sonic/config_db.json.temp $dumppath/etc/sonic/config_db.json
+}
+
 ###############################################################################
 # Terminates generate_dump early just in case we have issues.
 # Globals: