feat: enable-debug option (sonic-net#18009)

Description of PR
Summary:
This PR adds remote debugging functionality using https://github.com/microsoft/debugpy support with vs-code for sonic-mgmt container. This will need the follow PR to be mereged as well:

feat: add debugpy dependency sonic-buildimage#22341
In summary, one can setup a remote debug container when provide option --enable-debug. For example:

./setup-containers.sh sonic-mgmt-user_master --enable-debug
Which will automatically scan for available ports and establish a tunnel between docker and host for debugpy purpose. The user then can execute normal run_test command with --enable-debug added to the end to connect with their vscode client. For example:

./run_tests.sh -n vms-kvm-t0 -d vlab-01 -c bgp/test_bgp_fact.py -f vtestbed.yaml -i ../ansible/veos_vtb --enable-debug
Fixes # (issue) 32317137

Approach
What is the motivation for this PR?
To provide a native-like debug experience, improve our development efficiency.

How did you do it?
Described above

How did you verify/test it?
Verified on dev server

Signed-off-by: Austin Pham <[email protected]>
Signed-off-by: Guy Shemesh <[email protected]>

Author: auspham

setup-container.sh

@@ -49,6 +49,11 @@ FORCE_REMOVAL="${NO_PARAM}"
 VERBOSE_LEVEL="${VERBOSE_MIN}"
 SILENT_HOOK="&> /dev/null"
 
+# Sonic-mgmt remote debug feature
+DEBUG_PORT_START_RANGE=50000
+DEBUG_PORT_END_RANGE=60000
+DEFAULT_LOCK_FOLDER="/tmp/sonic-mgmt-locks/"
+
 #
 # Functions -----------------------------------------------------------------------------------------------------------
 #
@@ -110,13 +115,15 @@ function show_help_and_exit() {
     echo "  -v                   explain what is being done"
     echo "  -x                   show execution details"
     echo "  -h                   display this help and exit"
+    echo "  --enable-debug       enable debug mode"
     echo
     echo "Examples:"
     echo "  ./${SCRIPT_NAME} -n sonic-mgmt-${USER}_master"
     echo "  ./${SCRIPT_NAME} -n sonic-mgmt-${USER}_master -i sonicdev-microsoft.azurecr.io:443/docker-sonic-mgmt:latest"
     echo "  ./${SCRIPT_NAME} -n sonic-mgmt-${USER}_master -d /var/src"
     echo "  ./${SCRIPT_NAME} -n sonic-mgmt-${USER}_master -m /my/working/dir"
     echo "  ./${SCRIPT_NAME} -n sonic-mgmt-${USER}_master -p 192.0.2.1:8080:80/tcp"
+    echo "  ./${SCRIPT_NAME} -n sonic-mgmt-${USER}_master --enable-debug"
     echo "  ./${SCRIPT_NAME} -h"
     echo
     exit ${1}
@@ -133,6 +140,16 @@ function show_local_container_login() {
     echo "EXEC: docker exec --user ${USER} -ti ${CONTAINER_NAME} bash"
     echo "SSH:  ssh -i ~/.ssh/id_rsa_docker_sonic_mgmt ${USER}@${CONTAINER_IPV4}"
     echo "******************************************************************************"
+
+    if [[ -n ${SELECTED_DEBUG_PORT} ]]; then
+        echo
+        echo "*********************************[IMPORTANT]*********************************"
+        echo "DEBUG PORT: $SELECTED_DEBUG_PORT"
+        echo "Please use the above debug port in your vscode extensions"
+        echo "When running the test, add --enable-debug to the end of your ./run_tests.sh to use"
+        echo "You can check which port was assigned to you again by running 'docker ps' and search for your container"
+        echo "*********************************[IMPORTANT]*********************************"
+    fi
 }
 
 function pull_sonic_mgmt_docker_image() {
@@ -259,6 +276,10 @@ RUN if ! pip3 list | grep -c pytest >/dev/null && \
 /bin/bash -c '${HOME}/env-python3/bin/pip install $(/var/AzDevOps/env-python3/bin/pip freeze | grep -vE "distro|PyGObject|python-apt|unattended-upgrades|dbus-python")'; \
 fi
 
+# Remote debug port setup
+{% if SONIC_MGMT_DEBUG_PORT %}
+ENV SONIC_MGMT_DEBUG_PORT={{ SONIC_MGMT_DEBUG_PORT }}
+{% endif %}
 EOF
 
     log_info "prepare an environment file: ${TMP_DIR}/data.env"
@@ -272,6 +293,7 @@ GROUP_NAME=${USER}
 USER_NAME=${USER}
 USER_PASS=${USER_PASS}
 ROOT_PASS=${ROOT_PASS}
+SONIC_MGMT_DEBUG_PORT=${SELECTED_DEBUG_PORT}
 EOF
 
     log_info "generate a Dockerfile: ${TMP_DIR}/Dockerfile"
@@ -387,6 +409,29 @@ function parse_arguments() {
     fi
 }
 
+function find_debug_port() {
+    mkdir -p "$DEFAULT_LOCK_FOLDER"
+    for port in $(seq $DEBUG_PORT_START_RANGE $DEBUG_PORT_END_RANGE); do
+        if ! ss -tuln | grep -q ":$port\b" && mkdir $DEFAULT_LOCK_FOLDER/$port.lock 2>/dev/null; then
+            trap "rm -rf $DEFAULT_LOCK_FOLDER/$port.lock" EXIT # Remove the port.lock file when done
+            SELECTED_DEBUG_PORT=$port
+            return 0
+        fi
+    done
+    return 1
+}
+
+
+ARGS=()
+for arg in "$@"; do
+    if [[ "$arg" == "--enable-debug" ]]; then
+        ENABLE_DEBUG=1
+    else
+        ARGS+=("$arg")
+    fi
+done
+
+set -- "${ARGS[@]}"
 #
 # Script --------------------------------------------------------------------------------------------------------------
 #
@@ -431,6 +476,17 @@ while getopts "n:i:d:m:p:fvxh" opt; do
     esac
 done
 
+if [[ "$ENABLE_DEBUG" -eq 1 ]]; then
+    find_debug_port
+    if [[ -n "$SELECTED_DEBUG_PORT" ]]; then
+        PUBLISH_PORTS+=" -p \"$SELECTED_DEBUG_PORT:$SELECTED_DEBUG_PORT\""
+    else
+        echo "FAILURE: Cannot find an eligible debug port within the range [$DEBUG_PORT_START_RANGE, $DEBUG_PORT_END_RANGE]"
+        echo "Please re-run without --enable-debug option."
+        exit 1
+    fi
+fi
+
 parse_arguments
 
 if [[ "$(id -u)" = "${ROOT_UID}" ]]; then

tests/run_tests.sh

@@ -96,6 +96,7 @@ function validate_parameters()
 function setup_environment()
 {
     SCRIPT=$0
+    PYTEST_EXEC="python3 -m pytest"
     FULL_PATH=$(realpath ${SCRIPT})
     SCRIPT_PATH=$(dirname ${FULL_PATH})
     BASE_PATH=$(dirname ${SCRIPT_PATH})
@@ -301,22 +302,22 @@ function pre_post_extra_params()
 function prepare_dut()
 {
     echo "=== Preparing DUT for subsequent tests ==="
-    echo Running: python3 -m pytest ${PYTEST_UTIL_OPTS} ${PRET_LOGGING_OPTIONS} ${UTIL_TOPOLOGY_OPTIONS} $(pre_post_extra_params) -m pretest
-    python3 -m pytest ${PYTEST_UTIL_OPTS} ${PRET_LOGGING_OPTIONS} ${UTIL_TOPOLOGY_OPTIONS} $(pre_post_extra_params) -m pretest
+    echo Running: ${PYTEST_EXEC} ${PYTEST_UTIL_OPTS} ${PRET_LOGGING_OPTIONS} ${UTIL_TOPOLOGY_OPTIONS} $(pre_post_extra_params) -m pretest
+    ${PYTEST_EXEC} ${PYTEST_UTIL_OPTS} ${PRET_LOGGING_OPTIONS} ${UTIL_TOPOLOGY_OPTIONS} $(pre_post_extra_params) -m pretest
 }
 
 function cleanup_dut()
 {
     echo "=== Cleaning up DUT after tests ==="
-    echo Running: python3 -m pytest ${PYTEST_UTIL_OPTS} ${POST_LOGGING_OPTIONS} ${UTIL_TOPOLOGY_OPTIONS} $(pre_post_extra_params) -m posttest
-    python3 -m pytest ${PYTEST_UTIL_OPTS} ${POST_LOGGING_OPTIONS} ${UTIL_TOPOLOGY_OPTIONS} $(pre_post_extra_params) -m posttest
+    echo Running: ${PYTEST_EXEC} ${PYTEST_UTIL_OPTS} ${POST_LOGGING_OPTIONS} ${UTIL_TOPOLOGY_OPTIONS} $(pre_post_extra_params) -m posttest
+    ${PYTEST_EXEC} ${PYTEST_UTIL_OPTS} ${POST_LOGGING_OPTIONS} ${UTIL_TOPOLOGY_OPTIONS} $(pre_post_extra_params) -m posttest
 }
 
 function run_group_tests()
 {
     echo "=== Running tests in groups ==="
-    echo Running: python3 -m pytest ${TEST_CASES} ${PYTEST_COMMON_OPTS} ${TEST_LOGGING_OPTIONS} ${TEST_TOPOLOGY_OPTIONS} ${EXTRA_PARAMETERS}
-    python3 -m pytest ${TEST_CASES} ${PYTEST_COMMON_OPTS} ${TEST_LOGGING_OPTIONS} ${TEST_TOPOLOGY_OPTIONS} ${EXTRA_PARAMETERS} --cache-clear
+    echo Running: ${PYTEST_EXEC} ${TEST_CASES} ${PYTEST_COMMON_OPTS} ${TEST_LOGGING_OPTIONS} ${TEST_TOPOLOGY_OPTIONS} ${EXTRA_PARAMETERS}
+    ${PYTEST_EXEC} ${TEST_CASES} ${PYTEST_COMMON_OPTS} ${TEST_LOGGING_OPTIONS} ${TEST_TOPOLOGY_OPTIONS} ${EXTRA_PARAMETERS} --cache-clear
 }
 
 function run_individual_tests()
@@ -383,6 +384,26 @@ function run_bsl_tests()
 
 setup_environment
 
+for arg in "$@"; do
+    if [[ "$arg" == "--enable-debug" ]]; then
+        if [[ -z $SONIC_MGMT_DEBUG_PORT ]]; then
+            echo "*********************************[WARNING]*********************************"
+            echo "This container was not setup with --enable-debug option. Please re-setup this container with --enable-debug option"
+            echo "Please re-run without '--enable-debug' option to continue."
+            echo "*********************************[WARNING]*********************************"
+            exit 1
+        fi
+
+        if [[ "$arg" != "${@: -1}" ]]; then
+            echo "Please put '--enable-debug' as the last option of your './run_tests.sh' command to avoid conflicts with pytest and run_tests options"
+            echo "Example: ./run_tests.sh ... --enable-debug"
+            exit 1
+        fi
+
+        PYTEST_EXEC="python3 -m debugpy --listen 0.0.0.0:$SONIC_MGMT_DEBUG_PORT --wait-for-client -m pytest"
+        set -- "${@/$arg/}" # remove this option so getopts can process the rest
+    fi
+done
 
 while getopts "h?a:b:Bc:C:d:e:Ef:F:H:i:I:k:l:m:n:oOp:q:rs:S:t:ux" opt; do
     case ${opt} in