Skip to content

feat: support stress routes test on T2 topo #13341

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yejianquan merged 3 commits into from
Jun 21, 2024
Merged

feat: support stress routes test on T2 topo #13341

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f3dda59
feat: support stress routes test on T2 topo
cyw233 Jun 18, 2024
64578cf
feat: optimize imports
cyw233 Jun 18, 2024
03d15df
feat: update fixtures
cyw233 Jun 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 19 additions & 10 deletions tests/stress/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import logging
import pytest
import time
from tests.common.utilities import wait_until
from utils import get_crm_resources, check_queue_status, sleep_to_wait

import pytest

from tests.common import config_reload
from tests.common.utilities import wait_until
from utils import get_crm_resource_status, check_queue_status, sleep_to_wait

CRM_POLLING_INTERVAL = 1
CRM_DEFAULT_POLL_INTERVAL = 300
Expand All @@ -18,7 +20,8 @@ def get_function_conpleteness_level(pytestconfig):


@pytest.fixture(scope="module", autouse=True)
def set_polling_interval(duthost):
def set_polling_interval(duthosts, enum_rand_one_per_hwsku_frontend_hostname):
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
wait_time = 2
duthost.command("crm config polling interval {}".format(CRM_POLLING_INTERVAL))
logger.info("Waiting {} sec for CRM counters to become updated".format(wait_time))
Expand All @@ -32,7 +35,12 @@ def set_polling_interval(duthost):


@pytest.fixture(scope='module')
def withdraw_and_announce_existing_routes(duthost, localhost, tbinfo):
def withdraw_and_announce_existing_routes(duthosts, localhost, tbinfo, enum_rand_one_per_hwsku_frontend_hostname,
enum_rand_one_frontend_asic_index):
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
asichost = duthost.asic_instance(enum_rand_one_frontend_asic_index)
namespace = asichost.namespace

ptf_ip = tbinfo["ptf_ip"]
topo_name = tbinfo["topo"]["name"]

Expand All @@ -41,8 +49,8 @@ def withdraw_and_announce_existing_routes(duthost, localhost, tbinfo):

wait_until(MAX_WAIT_TIME, CRM_POLLING_INTERVAL, 0, lambda: check_queue_status(duthost, "inq") is True)
sleep_to_wait(CRM_POLLING_INTERVAL * 100)
ipv4_route_used_before = get_crm_resources(duthost, "ipv4_route", "used")
ipv6_route_used_before = get_crm_resources(duthost, "ipv6_route", "used")
ipv4_route_used_before = get_crm_resource_status(duthost, "ipv4_route", "used", namespace)
ipv6_route_used_before = get_crm_resource_status(duthost, "ipv6_route", "used", namespace)
logger.info("ipv4 route used {}".format(ipv4_route_used_before))
logger.info("ipv6 route used {}".format(ipv6_route_used_before))

Expand All @@ -53,12 +61,13 @@ def withdraw_and_announce_existing_routes(duthost, localhost, tbinfo):

wait_until(MAX_WAIT_TIME, CRM_POLLING_INTERVAL, 0, lambda: check_queue_status(duthost, "outq") is True)
sleep_to_wait(CRM_POLLING_INTERVAL * 5)
logger.info("ipv4 route used {}".format(get_crm_resources(duthost, "ipv4_route", "used")))
logger.info("ipv6 route used {}".format(get_crm_resources(duthost, "ipv6_route", "used")))
logger.info("ipv4 route used {}".format(get_crm_resource_status(duthost, "ipv4_route", "used", namespace)))
logger.info("ipv6 route used {}".format(get_crm_resource_status(duthost, "ipv6_route", "used", namespace)))


@pytest.fixture(scope="module", autouse=True)
def check_system_memmory(duthost):
def check_system_memmory(duthosts, enum_rand_one_per_hwsku_frontend_hostname):
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
for index in range(1, 4):
cmd = 'echo {} > /proc/sys/vm/drop_caches'.format(index)
duthost.shell(cmd, module_ignore_errors=True)
Expand Down
25 changes: 14 additions & 11 deletions tests/stress/test_stress_routes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
#!/usr/bin/env python

import logging

import pytest

from tests.common.helpers.assertions import pytest_assert
from tests.common.utilities import wait_until
from utils import get_crm_resources, check_queue_status, sleep_to_wait, LOOP_TIMES_LEVEL_MAP
from utils import get_crm_resource_status, check_queue_status, sleep_to_wait, LOOP_TIMES_LEVEL_MAP

ALLOW_ROUTES_CHANGE_NUMS = 5
CRM_POLLING_INTERVAL = 1
Expand All @@ -14,35 +15,37 @@
logger = logging.getLogger(__name__)

pytestmark = [
pytest.mark.topology('t0', 't1', 'm0', 'mx')
pytest.mark.topology('t0', 't1', 'm0', 'mx', 't2')
]


def announce_withdraw_routes(duthost, localhost, ptf_ip, topo_name):
def announce_withdraw_routes(duthost, namespace, localhost, ptf_ip, topo_name):
logger.info("announce ipv4 and ipv6 routes")
localhost.announce_routes(topo_name=topo_name, ptf_ip=ptf_ip, action="announce", path="../ansible/")

wait_until(MAX_WAIT_TIME, CRM_POLLING_INTERVAL, 0, lambda: check_queue_status(duthost, "outq") is True)

logger.info("ipv4 route used {}".format(get_crm_resources(duthost, "ipv4_route", "used")))
logger.info("ipv6 route used {}".format(get_crm_resources(duthost, "ipv6_route", "used")))
logger.info("ipv4 route used {}".format(get_crm_resource_status(duthost, "ipv4_route", "used", namespace)))
logger.info("ipv6 route used {}".format(get_crm_resource_status(duthost, "ipv6_route", "used", namespace)))
sleep_to_wait(CRM_POLLING_INTERVAL * 5)

logger.info("withdraw ipv4 and ipv6 routes")
localhost.announce_routes(topo_name=topo_name, ptf_ip=ptf_ip, action="withdraw", path="../ansible/")

wait_until(MAX_WAIT_TIME, CRM_POLLING_INTERVAL, 0, lambda: check_queue_status(duthost, "inq") is True)
sleep_to_wait(CRM_POLLING_INTERVAL * 5)
logger.info("ipv4 route used {}".format(get_crm_resources(duthost, "ipv4_route", "used")))
logger.info("ipv6 route used {}".format(get_crm_resources(duthost, "ipv6_route", "used")))
logger.info("ipv4 route used {}".format(get_crm_resource_status(duthost, "ipv4_route", "used", namespace)))
logger.info("ipv6 route used {}".format(get_crm_resource_status(duthost, "ipv6_route", "used", namespace)))


def test_announce_withdraw_route(duthosts, localhost, tbinfo, get_function_conpleteness_level,
withdraw_and_announce_existing_routes, loganalyzer,
enum_rand_one_per_hwsku_frontend_hostname):
enum_rand_one_per_hwsku_frontend_hostname, enum_rand_one_frontend_asic_index):
ptf_ip = tbinfo["ptf_ip"]
topo_name = tbinfo["topo"]["name"]
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
asichost = duthost.asic_instance(enum_rand_one_frontend_asic_index)
namespace = asichost.namespace

if loganalyzer:
ignoreRegex = [
Expand Down Expand Up @@ -71,13 +74,13 @@ def test_announce_withdraw_route(duthosts, localhost, tbinfo, get_function_conpl
loop_times = LOOP_TIMES_LEVEL_MAP[normalized_level]

while loop_times > 0:
announce_withdraw_routes(duthost, localhost, ptf_ip, topo_name)
announce_withdraw_routes(duthost, namespace, localhost, ptf_ip, topo_name)
loop_times -= 1

sleep_to_wait(CRM_POLLING_INTERVAL * 120)

ipv4_route_used_after = get_crm_resources(duthost, "ipv4_route", "used")
ipv6_route_used_after = get_crm_resources(duthost, "ipv6_route", "used")
ipv4_route_used_after = get_crm_resource_status(duthost, "ipv4_route", "used", namespace)
ipv6_route_used_after = get_crm_resource_status(duthost, "ipv6_route", "used", namespace)

pytest_assert(abs(ipv4_route_used_after - ipv4_route_used_before) < ALLOW_ROUTES_CHANGE_NUMS,
"ipv4 route used after is not equal to it used before")
Expand Down
6 changes: 4 additions & 2 deletions tests/stress/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import re
import time

from tests.common.helpers.constants import DEFAULT_NAMESPACE

TOPO_FILENAME_TEMPLATE = 'topo_{}.yml'
SHOW_BGP_SUMMARY_CMD = "show ip bgp summary"
LOOP_TIMES_LEVEL_MAP = {
Expand All @@ -12,8 +14,8 @@
}


def get_crm_resources(duthost, resource, status):
return duthost.get_crm_resources().get("main_resources").get(resource).get(status)
def get_crm_resource_status(duthost, resource, status, namespace=DEFAULT_NAMESPACE):
Copy link
Contributor

@wenyiz2021 wenyiz2021 Jun 18, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

since you changed this funciton name, can you make sure all other places that were using get_crm_resources are fine?

Copy link
Contributor Author

@cyw233 cyw233 Jun 19, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi @wenyiz2021, I did some search in the repo and found the following tests are using function get_crm_resources() (same function name from various places):

  • arp/test_stress_arp.py
  • crm/test_crm.py
  • fdb/test_fdb_mac_move.py
  • vxlan/test_vxlan_crm.py
  • vxlan/test_vxlan_ecmp.py

Then I ran these tests and can confirm they are still working as expected.

Copy link
Contributor

@yutongzhang-microsoft yutongzhang-microsoft Jun 19, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi, @cyw233 , should we change other test scripts to support T2 as well?

Copy link
Contributor Author

@cyw233 cyw233 Jun 19, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hey @yutongzhang-microsoft, we have checked all the skipped tests on T2 and documented the ones that indeed need to be supported on T2. For example, this test_stress_routes should run on T2, while test_stress_arp should always be skipped on T2 because it's supposed to be run on T0 only.
This is the very beginning of the T2 test gap and we will make more test cases supported on T2, for example, supporting test_bfd.py on T2 is my very next ticket.

return duthost.get_crm_resources(namespace).get("main_resources").get(resource).get(status)


def check_queue_status(duthost, queue):
Expand Down