Skip to content

[cherry pick] #9523 Fix PR #9312 for KeyError and AttributeError in 202012 #9590

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yutongzhang-microsoft merged 1 commit into from
Aug 23, 2023
Merged

[cherry pick] #9523 Fix PR #9312 for KeyError and AttributeError in 202012 #9590

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
147 changes: 76 additions & 71 deletions tests/cacl/test_cacl_function.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import logging
from tests.common.helpers.assertions import pytest_assert
from tests.common.helpers.snmp_helpers import get_snmp_facts
from tests.common.utilities import get_data_acl, recover_acl_rule

try:
import ntplib
Expand All @@ -25,6 +26,7 @@ def test_cacl_function(duthosts, rand_one_dut_hostname, localhost, creds):
"""Test control plane ACL functionality on a SONiC device"""

duthost = duthosts[rand_one_dut_hostname]
data_acl = get_data_acl(duthost)
dut_mgmt_ip = duthost.mgmt_ip

# Start an NTP client
Expand All @@ -48,76 +50,79 @@ def test_cacl_function(duthosts, rand_one_dut_hostname, localhost, creds):
ntp_client.request(dut_mgmt_ip)
except ntplib.NTPException:
pytest.fail("NTP did timed out when expected to succeed!")

# Copy config_service_acls.sh to the DuT (this also implicitly verifies we can successfully SSH to the DuT)
duthost.copy(src="scripts/config_service_acls.sh", dest="/tmp/config_service_acls.sh", mode="0755")

# We run the config_service_acls.sh script in the background because it
# will install ACL rules which will only allow control plane traffic
# to an unused IP range. Thus, if it works properly, it will sever our
# SSH session, but we don't want the script itself to get killed,
# because it is also responsible for resetting the control plane ACLs
# back to their previous, working state
duthost.shell("nohup /tmp/config_service_acls.sh < /dev/null > /dev/null 2>&1 &")

# Wait until we are unable to SSH into the DuT
res = localhost.wait_for(host=dut_mgmt_ip,
port=SONIC_SSH_PORT,
state='stopped',
search_regex=SONIC_SSH_REGEX,
delay=30,
timeout=40,
module_ignore_errors=True)

pytest_assert(not res.is_failed, "SSH port did not stop. {}".format(res.get('msg', '')))

# Try to SSH back into the DuT, it should time out
res = localhost.wait_for(host=dut_mgmt_ip,
port=SONIC_SSH_PORT,
state='started',
search_regex=SONIC_SSH_REGEX,
delay=0,
timeout=10,
module_ignore_errors=True)

pytest_assert(res.is_failed, "SSH did not timeout when expected. {}".format(res.get('msg', '')))

# Ensure we CANNOT gather basic SNMP facts from the device
res = get_snmp_facts(localhost, host=dut_mgmt_ip, version='v2c', community=creds['snmp_rocommunity'],
module_ignore_errors=True)

pytest_assert('ansible_facts' not in res and "No SNMP response received before timeout" in res.get('msg', ''))

# Ensure we cannot send an NTP request to the DUT
if NTPLIB_INSTALLED:
try:
ntp_client.request(dut_mgmt_ip)
pytest.fail("NTP did not time out when expected")
except ntplib.NTPException:
pass

# Wait until the original service ACLs are reinstated and the SSH port on the
# DUT is open to us once again. Note that the timeout here should be set sufficiently
# long enough to allow config_service_acls.sh to reset the ACLs to their original
# configuration.
res = localhost.wait_for(host=dut_mgmt_ip,
port=SONIC_SSH_PORT,
state='started',
search_regex=SONIC_SSH_REGEX,
delay=0,
timeout=90,
try:
# Copy config_service_acls.sh to the DuT (this also implicitly verifies we can successfully SSH to the DuT)
duthost.copy(src="scripts/config_service_acls.sh", dest="/tmp/config_service_acls.sh", mode="0755")

# We run the config_service_acls.sh script in the background because it
# will install ACL rules which will only allow control plane traffic
# to an unused IP range. Thus, if it works properly, it will sever our
# SSH session, but we don't want the script itself to get killed,
# because it is also responsible for resetting the control plane ACLs
# back to their previous, working state
duthost.shell("nohup /tmp/config_service_acls.sh < /dev/null > /dev/null 2>&1 &")

# Wait until we are unable to SSH into the DuT
res = localhost.wait_for(host=dut_mgmt_ip,
port=SONIC_SSH_PORT,
state='stopped',
search_regex=SONIC_SSH_REGEX,
delay=30,
timeout=40,
module_ignore_errors=True)

pytest_assert(not res.is_failed, "SSH port did not stop. {}".format(res.get('msg', '')))

# Try to SSH back into the DuT, it should time out
res = localhost.wait_for(host=dut_mgmt_ip,
port=SONIC_SSH_PORT,
state='started',
search_regex=SONIC_SSH_REGEX,
delay=0,
timeout=10,
module_ignore_errors=True)

pytest_assert(res.is_failed, "SSH did not timeout when expected. {}".format(res.get('msg', '')))

# Ensure we CANNOT gather basic SNMP facts from the device
res = get_snmp_facts(localhost, host=dut_mgmt_ip, version='v2c', community=creds['snmp_rocommunity'],
module_ignore_errors=True)

pytest_assert(not res.is_failed, "SSH did not start working when expected. {}".format(res.get('msg', '')))

# Delete config_service_acls.sh from the DuT
duthost.file(path="/tmp/config_service_acls.sh", state="absent")

# Ensure we can gather basic SNMP facts from the device once again. Should fail on timeout
get_snmp_facts(localhost,
host=dut_mgmt_ip,
version="v2c",
community=creds['snmp_rocommunity'],
wait=True,
timeout = 20,
interval=20)
pytest_assert('ansible_facts' not in res and "No SNMP response received before timeout" in res.get('msg', ''))

# Ensure we cannot send an NTP request to the DUT
if NTPLIB_INSTALLED:
try:
ntp_client.request(dut_mgmt_ip)
pytest.fail("NTP did not time out when expected")
except ntplib.NTPException:
pass

# Wait until the original service ACLs are reinstated and the SSH port on the
# DUT is open to us once again. Note that the timeout here should be set sufficiently
# long enough to allow config_service_acls.sh to reset the ACLs to their original
# configuration.
res = localhost.wait_for(host=dut_mgmt_ip,
port=SONIC_SSH_PORT,
state='started',
search_regex=SONIC_SSH_REGEX,
delay=0,
timeout=90,
module_ignore_errors=True)

pytest_assert(not res.is_failed, "SSH did not start working when expected. {}".format(res.get('msg', '')))

# Delete config_service_acls.sh from the DuT
duthost.file(path="/tmp/config_service_acls.sh", state="absent")

# Ensure we can gather basic SNMP facts from the device once again. Should fail on timeout
get_snmp_facts(localhost,
host=dut_mgmt_ip,
version="v2c",
community=creds['snmp_rocommunity'],
wait=True,
timeout = 20,
interval=20)
finally:
if data_acl:
recover_acl_rule(duthost, data_acl)
38 changes: 38 additions & 0 deletions tests/common/templates/default_acl_rules.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{
"acl": {
"acl-sets": {
"acl-set": {
"dataacl": {
"acl-entries": {
"acl-entry": {
"1": {
"actions": {
"config": {
"forwarding-action": "ACCEPT"
}
},
"config": {
"sequence-id": 1
},
"l2": {
"config": {
"ethertype": "2048",
"vlan_id": "1000"
}
},
"input_interface": {
"interface_ref":
{
"config": {
"interface": "Ethernet12,Ethernet16,Ethernet20,Ethernet24,Ethernet28,Ethernet32,Ethernet36,Ethernet4,Ethernet40,Ethernet44,Ethernet48,Ethernet52,Ethernet56,Ethernet60,Ethernet64,Ethernet68,Ethernet72,Ethernet76,Ethernet8"
}
}
}
}
}
}
}
}
}
}
}
39 changes: 39 additions & 0 deletions tests/common/utilities.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
import time
import traceback
import json
import copy
import tempfile
import os
from io import BytesIO

import pytest
Expand Down Expand Up @@ -638,3 +641,39 @@ def delete_running_config(config_entry, duthost, is_json=True):
duthost.copy(src=config_entry, dest="/tmp/del_config_entry.json")
duthost.shell("configlet -d -j {}".format("/tmp/del_config_entry.json"))
duthost.shell("rm -f {}".format("/tmp/del_config_entry.json"))

def get_data_acl(duthost):
acl_facts = duthost.acl_facts()["ansible_facts"]["ansible_acl_facts"]
pre_acl_rules = acl_facts.get("DATAACL", {}).get("rules", None)
return pre_acl_rules


def recover_acl_rule(duthost, data_acl):
base_dir = os.path.dirname(os.path.realpath(__file__))
template_dir = os.path.join(base_dir, "templates")
acl_rules_template = "default_acl_rules.json"
dut_tmp_dir = "/tmp"
dut_conf_file_path = os.path.join(dut_tmp_dir, acl_rules_template)

for key, value in data_acl.items():
if key != "DEFAULT_RULE":
seq_id = key.split('_')[1]
acl_config = json.loads(open(os.path.join(template_dir, acl_rules_template)).read())
acl_entry_template = \
acl_config["acl"]["acl-sets"]["acl-set"]["dataacl"]["acl-entries"]["acl-entry"]["1"]
acl_entry_config = acl_config["acl"]["acl-sets"]["acl-set"]["dataacl"]["acl-entries"]["acl-entry"]

acl_entry_config[seq_id] = copy.deepcopy(acl_entry_template)
acl_entry_config[seq_id]["config"]["sequence-id"] = seq_id
acl_entry_config[seq_id]["l2"]["config"]["ethertype"] = value["ETHER_TYPE"]
acl_entry_config[seq_id]["l2"]["config"]["vlan_id"] = value["VLAN_ID"]
acl_entry_config[seq_id]["input_interface"]["interface_ref"]["config"]["interface"] = value["IN_PORTS"]

with tempfile.NamedTemporaryFile(suffix=".json", prefix="acl_config", mode="w") as fp:
json.dump(acl_config, fp)
fp.flush()
logger.info("Generating config for ACL rule, ACL table - DATAACL")
duthost.template(src=fp.name, dest=dut_conf_file_path, force=True)

logger.info("Applying {}".format(dut_conf_file_path))
duthost.command("acl-loader update full {}".format(dut_conf_file_path))
6 changes: 5 additions & 1 deletion tests/crm/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

from test_crm import RESTORE_CMDS, CRM_POLLING_INTERVAL
from tests.common.errors import RunAnsibleModuleFail
from tests.common.utilities import recover_acl_rule

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -52,7 +53,10 @@ def pytest_runtest_teardown(item, nextitem):
for cmd in RESTORE_CMDS[test_name]:
logger.info(cmd)
try:
dut.shell(cmd)
if isinstance(cmd, dict):
recover_acl_rule(dut, cmd["data_acl"])
else:
dut.shell(cmd)
except RunAnsibleModuleFail as err:
failures.append("Failure during command execution '{command}':\n{error}".format(command=cmd,
error=str(err)))
Expand Down
86 changes: 45 additions & 41 deletions tests/crm/test_crm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from tests.common.fixtures.duthost_utils import disable_route_checker
from tests.common.fixtures.duthost_utils import disable_fdb_aging
from tests.common.utilities import wait_until
from tests.common.utilities import wait_until, get_data_acl


pytestmark = [
Expand Down Expand Up @@ -851,62 +852,65 @@ def recover_acl_rule(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_f
duthost.command("acl-loader update full {}".format(dut_conf_file_path))


def test_acl_entry(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_frontend_asic_index, collector, recover_acl_rule):
def test_acl_entry(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_frontend_asic_index, collector):
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
data_acl = get_data_acl(duthost)
asichost = duthost.asic_instance(enum_frontend_asic_index)
asic_collector = collector[asichost.asic_index]
try:
apply_acl_config(duthost, asichost, "test_acl_entry", asic_collector)
acl_tbl_key = asic_collector["acl_tbl_key"]
get_acl_entry_stats = "{db_cli} COUNTERS_DB HMGET {acl_tbl_key} \
crm_stats_acl_entry_used \
crm_stats_acl_entry_available"\
.format(db_cli=asichost.sonic_db_cli,
acl_tbl_key=acl_tbl_key)

apply_acl_config(duthost, asichost, "test_acl_entry", asic_collector)
acl_tbl_key = asic_collector["acl_tbl_key"]
get_acl_entry_stats = "{db_cli} COUNTERS_DB HMGET {acl_tbl_key} \
crm_stats_acl_entry_used \
crm_stats_acl_entry_available"\
.format(db_cli=asichost.sonic_db_cli,
acl_tbl_key=acl_tbl_key)
base_dir = os.path.dirname(os.path.realpath(__file__))
template_dir = os.path.join(base_dir, "templates")
acl_rules_template = "acl.json"
dut_tmp_dir = "/tmp"

base_dir = os.path.dirname(os.path.realpath(__file__))
template_dir = os.path.join(base_dir, "templates")
acl_rules_template = "acl.json"
dut_tmp_dir = "/tmp"
RESTORE_CMDS["crm_threshold_name"] = "acl_entry"

RESTORE_CMDS["crm_threshold_name"] = "acl_entry"
crm_stats_acl_entry_used = 0
crm_stats_acl_entry_available = 0

crm_stats_acl_entry_used = 0
crm_stats_acl_entry_available = 0
# Get new "crm_stats_acl_entry" used and available counter value
new_crm_stats_acl_entry_used, new_crm_stats_acl_entry_available = get_crm_stats(get_acl_entry_stats, duthost)

# Get new "crm_stats_acl_entry" used and available counter value
new_crm_stats_acl_entry_used, new_crm_stats_acl_entry_available = get_crm_stats(get_acl_entry_stats, duthost)
# Verify "crm_stats_acl_entry_used" counter was incremented
pytest_assert(new_crm_stats_acl_entry_used - crm_stats_acl_entry_used == 2, \
"\"crm_stats_acl_entry_used\" counter was not incremented")

# Verify "crm_stats_acl_entry_used" counter was incremented
pytest_assert(new_crm_stats_acl_entry_used - crm_stats_acl_entry_used == 2, \
"\"crm_stats_acl_entry_used\" counter was not incremented")
crm_stats_acl_entry_available = new_crm_stats_acl_entry_available + new_crm_stats_acl_entry_used

crm_stats_acl_entry_available = new_crm_stats_acl_entry_available + new_crm_stats_acl_entry_used
used_percent = get_used_percent(new_crm_stats_acl_entry_used, new_crm_stats_acl_entry_available)
if used_percent < 1:
# Preconfiguration needed for used percentage verification
nexthop_group_num = get_entries_num(new_crm_stats_acl_entry_used, new_crm_stats_acl_entry_available)

used_percent = get_used_percent(new_crm_stats_acl_entry_used, new_crm_stats_acl_entry_available)
if used_percent < 1:
# Preconfiguration needed for used percentage verification
nexthop_group_num = get_entries_num(new_crm_stats_acl_entry_used, new_crm_stats_acl_entry_available)
apply_acl_config(duthost, asichost, "test_acl_entry", asic_collector, nexthop_group_num)

apply_acl_config(duthost, asichost, "test_acl_entry", asic_collector, nexthop_group_num)
logger.info("Waiting {} seconds for SONiC to update resources...".format(SONIC_RES_UPDATE_TIME))
# Make sure SONIC configure expected entries
time.sleep(SONIC_RES_UPDATE_TIME)

logger.info("Waiting {} seconds for SONiC to update resources...".format(SONIC_RES_UPDATE_TIME))
# Make sure SONIC configure expected entries
time.sleep(SONIC_RES_UPDATE_TIME)

# Verify thresholds for "ACL entry" CRM resource
verify_thresholds(duthost, asichost, crm_cli_res="acl group entry", crm_cmd=get_acl_entry_stats)
# Verify thresholds for "ACL entry" CRM resource
verify_thresholds(duthost, asichost, crm_cli_res="acl group entry", crm_cmd=get_acl_entry_stats)

# Remove ACL
duthost.command("acl-loader delete")

crm_stats_checker = wait_until(30, 5, 0, check_crm_stats, get_acl_entry_stats, duthost,
crm_stats_acl_entry_used,
crm_stats_acl_entry_available)
pytest_assert(crm_stats_checker,
"\"crm_stats_acl_entry_used\" counter was not decremented or "
"\"crm_stats_acl_entry_available\" counter was not incremented")
# Remove ACL
duthost.command("acl-loader delete")

crm_stats_checker = wait_until(30, 5, 0, check_crm_stats, get_acl_entry_stats, duthost,
crm_stats_acl_entry_used,
crm_stats_acl_entry_available)
pytest_assert(crm_stats_checker,
"\"crm_stats_acl_entry_used\" counter was not decremented or "
"\"crm_stats_acl_entry_available\" counter was not incremented")
finally:
if data_acl:
RESTORE_CMDS["test_acl_entry"].append({"data_acl": data_acl})

def test_acl_counter(duthosts, enum_rand_one_per_hwsku_frontend_hostname,enum_frontend_asic_index, collector):
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
Expand Down