Skip to content

[action] [PR:17722] [dualtor_io] Collect syslog to debug #17741

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mssonicbld merged 1 commit into from
Apr 16, 2025
Merged

[action] [PR:17722] [dualtor_io] Collect syslog to debug #17741

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions tests/dualtor_io/test_link_failure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -421,17 +421,17 @@ def test_active_link_admin_down_config_reload_downstream(
upper_tor_host.shell("config save -y")


@pytest.mark.disable_loganalyzer
@pytest.mark.enable_active_active
@pytest.mark.skip_active_standby
def test_active_link_admin_down_config_reload_link_up_upstream(
upper_tor_host, lower_tor_host, send_server_to_t1_with_action, # noqa F811
cable_type, active_active_ports # noqa F811
cable_type, active_active_ports, setup_loganalyzer # noqa F811
):
"""
Send traffic from server to T1 and unshut the active-active mux ports.
Verify switchover and disruption.
"""
setup_loganalyzer(upper_tor_host, collect_only=True)
if cable_type == CableType.active_active:
try:
config_interface_admin_status(upper_tor_host, active_active_ports, "down")
Expand Down Expand Up @@ -474,17 +474,17 @@ def test_active_link_admin_down_config_reload_link_up_upstream(
upper_tor_host.shell("config save -y")


@pytest.mark.disable_loganalyzer
@pytest.mark.enable_active_active
@pytest.mark.skip_active_standby
def test_active_link_admin_down_config_reload_link_up_downstream_standby(
upper_tor_host, lower_tor_host, send_t1_to_server_with_action, # noqa F811
cable_type, active_active_ports # noqa F811
cable_type, active_active_ports, setup_loganalyzer # noqa F811
):
"""
Send traffic from T1 to standby ToR and unshut the active-active mux ports.
Verify switchover and disruption.
"""
setup_loganalyzer(upper_tor_host, collect_only=True)
if cable_type == CableType.active_active:
try:
config_interface_admin_status(upper_tor_host, active_active_ports, "down")
Expand Down
128 changes: 60 additions & 68 deletions tests/dualtor_io/test_tor_failure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,117 +56,109 @@ def toggle_lower_tor_pdu(lower_tor_host, get_pdu_controller): # noqa F811


@pytest.mark.enable_active_active
@pytest.mark.disable_loganalyzer
def test_active_tor_reboot_upstream(
upper_tor_host, lower_tor_host, send_server_to_t1_with_action, # noqa F811
toggle_all_simulator_ports_to_upper_tor, toggle_upper_tor_pdu, # noqa F811
wait_for_device_reachable, wait_for_mux_container, cable_type, # noqa F811
wait_for_pmon_container # noqa F811
wait_for_pmon_container, setup_loganalyzer # noqa F811
):
"""
Send upstream traffic and reboot the active ToR. Confirm switchover
occurred and disruption lasts < 1 second
"""
with LogAnalyzer(ansible_host=lower_tor_host,
marker_prefix="test_active_tor_reboot_upstream"):
send_server_to_t1_with_action(
upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC,
action=toggle_upper_tor_pdu, stop_after=60
)
wait_for_device_reachable(upper_tor_host)
wait_for_mux_container(upper_tor_host)
wait_for_pmon_container(upper_tor_host)
setup_loganalyzer(upper_tor_host, collect_only=True)
send_server_to_t1_with_action(
upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC,
action=toggle_upper_tor_pdu, stop_after=60
)
wait_for_device_reachable(upper_tor_host)
wait_for_mux_container(upper_tor_host)
wait_for_pmon_container(upper_tor_host)

if cable_type == CableType.active_standby:
verify_tor_states(
expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host
)
elif cable_type == CableType.active_active:
verify_tor_states(
expected_active_host=[upper_tor_host, lower_tor_host],
expected_standby_host=None,
cable_type=cable_type,
verify_db_timeout=60
)
if cable_type == CableType.active_standby:
verify_tor_states(
expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host
)
elif cable_type == CableType.active_active:
verify_tor_states(
expected_active_host=[upper_tor_host, lower_tor_host],
expected_standby_host=None,
cable_type=cable_type,
verify_db_timeout=60
)


@pytest.mark.disable_loganalyzer
def test_active_tor_reboot_downstream_standby(
upper_tor_host, lower_tor_host, send_t1_to_server_with_action, # noqa F811
toggle_all_simulator_ports_to_upper_tor, toggle_upper_tor_pdu, # noqa F811
wait_for_device_reachable, wait_for_mux_container, # noqa F811
wait_for_pmon_container # noqa F811
wait_for_pmon_container, setup_loganalyzer # noqa F811
):
"""
Send downstream traffic to the standby ToR and reboot the active ToR.
Confirm switchover occurred and disruption lasts < 1 second
"""
with LogAnalyzer(ansible_host=lower_tor_host,
marker_prefix="test_active_tor_reboot_downstream_standby"):
send_t1_to_server_with_action(
lower_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC,
action=toggle_upper_tor_pdu, stop_after=60
)
wait_for_device_reachable(upper_tor_host)
wait_for_mux_container(upper_tor_host)
wait_for_pmon_container(upper_tor_host)
verify_tor_states(
expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host
)
setup_loganalyzer(upper_tor_host, collect_only=True)
send_t1_to_server_with_action(
lower_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC,
action=toggle_upper_tor_pdu, stop_after=60
)
wait_for_device_reachable(upper_tor_host)
wait_for_mux_container(upper_tor_host)
wait_for_pmon_container(upper_tor_host)
verify_tor_states(
expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host
)


@pytest.mark.disable_loganalyzer
def test_standby_tor_reboot_upstream(
upper_tor_host, lower_tor_host, send_server_to_t1_with_action, # noqa F811
toggle_all_simulator_ports_to_upper_tor, toggle_lower_tor_pdu, # noqa F811
wait_for_device_reachable, wait_for_mux_container, # noqa F811
wait_for_pmon_container # noqa F811
wait_for_pmon_container, setup_loganalyzer # noqa F811
):
"""
Send upstream traffic and reboot the standby ToR. Confirm no switchover
occurred and no disruption
"""
with LogAnalyzer(ansible_host=upper_tor_host,
marker_prefix="test_standby_tor_reboot_upstream"):
send_server_to_t1_with_action(
upper_tor_host, verify=True,
action=toggle_lower_tor_pdu, stop_after=60
)
wait_for_device_reachable(lower_tor_host)
wait_for_mux_container(lower_tor_host)
wait_for_pmon_container(lower_tor_host)
verify_tor_states(
expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host
)
setup_loganalyzer(lower_tor_host, collect_only=True)
send_server_to_t1_with_action(
upper_tor_host, verify=True,
action=toggle_lower_tor_pdu, stop_after=60
)
wait_for_device_reachable(lower_tor_host)
wait_for_mux_container(lower_tor_host)
wait_for_pmon_container(lower_tor_host)
verify_tor_states(
expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host
)


@pytest.mark.disable_loganalyzer
def test_standby_tor_reboot_downstream_active(
upper_tor_host, lower_tor_host, send_t1_to_server_with_action, # noqa F811
toggle_all_simulator_ports_to_upper_tor, toggle_lower_tor_pdu, # noqa F811
wait_for_device_reachable, wait_for_mux_container, # noqa F811
wait_for_pmon_container # noqa F811
wait_for_pmon_container, setup_loganalyzer # noqa F811
):
"""
Send downstream traffic to the active ToR and reboot the standby ToR.
Confirm no switchover occurred and no disruption
"""
with LogAnalyzer(ansible_host=upper_tor_host,
marker_prefix="test_standby_tor_reboot_downstream_active"):
send_t1_to_server_with_action(
upper_tor_host, verify=True,
action=toggle_lower_tor_pdu, stop_after=60
)
wait_for_device_reachable(lower_tor_host)
wait_for_mux_container(lower_tor_host)
wait_for_pmon_container(lower_tor_host)
verify_tor_states(
expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host
)
setup_loganalyzer(lower_tor_host, collect_only=True)
send_t1_to_server_with_action(
upper_tor_host, verify=True,
action=toggle_lower_tor_pdu, stop_after=60
)
wait_for_device_reachable(lower_tor_host)
wait_for_mux_container(lower_tor_host)
wait_for_pmon_container(lower_tor_host)
verify_tor_states(
expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host
)


@pytest.mark.enable_active_active
Expand Down
Loading