Skip to content

Support recovering Nexus devices via console #11264

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wangxin merged 6 commits into from
Jan 18, 2024
Merged

Support recovering Nexus devices via console #11264

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a93402c
Add nexus testbed recovery
yutongzhang-microsoft Jan 12, 2024
1e4abfa
Cover different ssh connection fail
yutongzhang-microsoft Jan 17, 2024
41c1e8c
Add constants
yutongzhang-microsoft Jan 17, 2024
5a84c8f
Add constants in ONIE
yutongzhang-microsoft Jan 17, 2024
23136c0
Change type to device_type to avoid conflict
yutongzhang-microsoft Jan 17, 2024
97ac5ad
Remove a TODO
yutongzhang-microsoft Jan 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 12 additions & 7 deletions .azure-pipelines/recover_testbed/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,9 @@ def get_pdu_managers(sonichosts, conn_graph_facts):
return pdu_managers


def posix_shell_onie(dut_console, mgmt_ip, image_url):
def posix_shell_onie(dut_console, mgmt_ip, image_url, is_nexus=False):
oldtty = termios.tcgetattr(sys.stdin)
enter_onie_flag = 0
enter_onie_flag = True
gw_ip = list(ipaddress.ip_interface(mgmt_ip).network.hosts())[0]
try:
tty.setraw(sys.stdin.fileno())
Expand All @@ -66,20 +66,25 @@ def posix_shell_onie(dut_console, mgmt_ip, image_url):

x = x.decode('ISO-8859-9')

if "GNU GRUB" in x:
enter_onie_flag += 1
if is_nexus and "loader" in x and ">" in x:
dut_console.remote_conn.send('reboot\n')
continue

if "SONiC-OS-" in x and enter_onie_flag == 1:
# if "GNU GRUB" in x:
# enter_onie_flag += 1
# continue

if "-OS-" in x and enter_onie_flag is True:
Copy link
Copy Markdown
Collaborator

@yejianquan yejianquan Jan 17, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if "-OS-" in x and enter_onie_flag:

Copy link
Copy Markdown
Contributor Author

@yutongzhang-microsoft yutongzhang-microsoft Jan 17, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I do some tests, and because of the rule of Python, we can use bool(enter_onie_flag) or use enter_onie_flag is True here. Simply enter_onie_flag will not be considered as a boolean.

# Send arrow key "down" here.
dut_console.remote_conn.send(b'\x1b[B')
continue

if "*ONIE" in x and "Install OS" not in x:
dut_console.remote_conn.send("\n")
enter_onie_flag += 1
enter_onie_flag = False
Copy link
Copy Markdown
Collaborator

@yejianquan yejianquan Jan 17, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you create constant for the strings like "-OS-", "*ONIE", "Install OS" and others, so that it would be clear to speculate the usage from the constant name

Copy link
Copy Markdown
Contributor Author

@yutongzhang-microsoft yutongzhang-microsoft Jan 17, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done


if "ONIE: Starting ONIE Service Discovery" in x:
# "ONIE: Starting ONIE Service Discovery"
if "Discovery" in x:
# TODO: Define a function to send command here
for i in range(5):
dut_console.remote_conn.send('onie-discovery-stop\n')
Expand Down
3 changes: 3 additions & 0 deletions .azure-pipelines/recover_testbed/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
RC_SSH_SUCCESS = 0
RC_SSH_FAILED = 1
RC_PASSWORD_FAILED = 2
11 changes: 4 additions & 7 deletions .azure-pipelines/recover_testbed/dut_connection.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@
import os
import sys
import paramiko
import socket
import glob
import re
import yaml
import jinja2
from tests.common.connections.console_host import ConsoleHost
from paramiko.ssh_exception import AuthenticationException
from constants import RC_SSH_FAILED, RC_PASSWORD_FAILED

_self_dir = os.path.dirname(os.path.abspath(__file__))
base_path = os.path.realpath(os.path.join(_self_dir, "../.."))
Expand All @@ -22,10 +22,6 @@

logger = logging.getLogger(__name__)

RC_SSH_SUCCESS = 0
RC_SOCKET_TIMEOUT = 1
RC_PASSWORD_FAILED = 2


def creds_on_dut(sonichost):
groups = sonichost.im.get_host(sonichost.hostname).get_vars()['group_names']
Expand Down Expand Up @@ -137,7 +133,8 @@ def duthost_ssh(sonichost):
return sonic_username, password, sonic_ip
except AuthenticationException:
continue
except socket.timeout as e:
# Errors such like timeout, connection fails
except Exception as e:
logger.info("Cannot access DUT {} via ssh, error: {}".format(sonichost.hostname, e))
return RC_SOCKET_TIMEOUT
return RC_SSH_FAILED
return RC_PASSWORD_FAILED
9 changes: 5 additions & 4 deletions .azure-pipelines/recover_testbed/recover_testbed.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import os
import sys
from common import do_power_cycle, check_sonic_installer, posix_shell_aboot, posix_shell_onie
from constants import RC_SSH_FAILED

_self_dir = os.path.dirname(os.path.abspath(__file__))
base_path = os.path.realpath(os.path.join(_self_dir, "../.."))
Expand Down Expand Up @@ -40,9 +41,9 @@ def recover_via_console(sonichost, conn_graph_facts, localhost, mgmt_ip, image_u

if type in ["arista"]:
posix_shell_aboot(dut_console, mgmt_ip, image_url)
# elif type in ["Cisco"]:
# return
elif type in ["mellanox", "nexus", "acs"]:
elif type in ["nexus"]:
Copy link
Copy Markdown
Collaborator

@wangxin wangxin Jan 17, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you rename variable type to something like device_type? The reason is that it conflicts with python's builtin type.

Copy link
Copy Markdown
Contributor Author

@yutongzhang-microsoft yutongzhang-microsoft Jan 17, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

posix_shell_onie(dut_console, mgmt_ip, image_url, is_nexus=True)
elif type in ["mellanox", "cisco", "acs"]:
Copy link
Copy Markdown
Collaborator

@wangxin wangxin Jan 17, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We also have some Nokia devices. Are they covered?

posix_shell_onie(dut_console, mgmt_ip, image_url)
else:
return
Expand Down Expand Up @@ -74,7 +75,7 @@ def recover_testbed(sonichosts, conn_graph_facts, localhost, image_url, hwsku):
logger.info("Exception caught while executing cmd. Error message: {}".format(e))
need_to_recover = True
# TODO: Define the return message like RC_SOCKET_TIMEOUT in common file
elif dut_ssh == 1:
elif dut_ssh == RC_SSH_FAILED:
# Do power cycle
need_to_recover = True
else:
Expand Down