Skip to content

Enhancing core_dump_and_config_check to be multi-asic aware #6527

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
SuvarnaMeenakshi merged 6 commits into from
Nov 21, 2022
Merged

Enhancing core_dump_and_config_check to be multi-asic aware #6527

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
bdd1a86
Enhancing check_dut_health_status to be multi-asic aware
sanmalho-git Oct 7, 2022
9d2e1f1
Include safe_reload and check_ifs_are_up as part of config_reload
sanmalho-git Oct 27, 2022
9c679e2
Changes to support golden_running_config as part of core_dump_and_con…
sanmalho-git Nov 8, 2022
4704b61
Review comments on PR#6257 - core_dump_and_config_check fixture
sanmalho-git Nov 17, 2022
cdcb0aa
Fix for failing test cases
sanmalho-git Nov 18, 2022
9eb2766
Fixing flake8 issues
sanmalho-git Nov 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions ansible/config_sonic_basedon_testbed.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -597,6 +597,13 @@
file: path=/etc/sonic/running_golden_config.json
state=absent

- name: remove running golden config files for all the asics if exists
become: True
file: path=/etc/sonic/running_golden_config{{ item }}.json
state=absent
with_sequence: start=0 end={{ num_asics - 1 }}
when: num_asics > 1

- name: cleanup all cached facts
shell: python ../tests/common/cache/facts_cache.py
delegate_to: localhost
Expand Down
164 changes: 102 additions & 62 deletions tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1743,8 +1743,18 @@ def __dut_reload(duts_data, node=None, results=None):
logger.error('Missing kwarg "node" or "results"')
return
logger.info("dut reload called on {}".format(node.hostname))
node.copy(content=json.dumps(duts_data[node.hostname]["pre_running_config"], indent=4),
node.copy(content=json.dumps(duts_data[node.hostname]["pre_running_config"][None], indent=4),
dest='/etc/sonic/config_db.json', verbose=False)

if node.is_multi_asic:
for asic_index in range(0, node.facts.get('num_asic')):
asic_ns = "asic{}".format(asic_index)
asic_cfg_file = "/tmp/{}_config_db{}.json".format(node.hostname, asic_index)
with open(asic_cfg_file, "w") as outfile:
outfile.write(json.dumps(duts_data[node.hostname]['pre_running_config'][asic_ns], indent=4))
node.copy(src=asic_cfg_file, dest='/etc/sonic/config_db{}.json'.format(asic_index), verbose=False)
os.remove(asic_cfg_file)

config_reload(node)


Expand Down Expand Up @@ -1784,12 +1794,25 @@ def core_dump_and_config_check(duthosts, request):
pre_existing_core_dumps = duthost.shell('ls /var/core/')['stdout'].split()
duts_data[duthost.hostname]["pre_core_dumps"] = pre_existing_core_dumps

logger.info("Collecting running config before test on {}".format(duthost.hostname))
duts_data[duthost.hostname]["pre_running_config"] = {}
if not duthost.stat(path="/etc/sonic/running_golden_config.json")['stat']['exists']:
logger.info("Collecting running golden config before test on {}".format(duthost.hostname))
duthost.shell("sonic-cfggen -d --print-data > /etc/sonic/running_golden_config.json")
duts_data[duthost.hostname]["pre_running_config"] = \
duts_data[duthost.hostname]["pre_running_config"][None] = \
json.loads(duthost.shell("cat /etc/sonic/running_golden_config.json", verbose=False)['stdout'])

if duthost.is_multi_asic:
for asic_index in range(0, duthost.facts.get('num_asic')):
asic_ns = "asic{}".format(asic_index)
if not duthost.stat(
path="/etc/sonic/running_golden_config{}.json".format(asic_index))['stat']['exists']:
duthost.shell("sonic-cfggen -n {} -d --print-data > /etc/sonic/running_golden_config{}.json".
format(asic_ns, asic_index))
duts_data[duthost.hostname]['pre_running_config'][asic_ns] = \
json.loads(duthost.shell("cat /etc/sonic/running_golden_config{}.json".format(asic_index),
verbose=False)['stdout'])

yield

if check_flag:
Expand All @@ -1813,8 +1836,16 @@ def core_dump_and_config_check(duthosts, request):
core_dump_check_pass = False

logger.info("Collecting running config after test on {}".format(duthost.hostname))
duts_data[duthost.hostname]["cur_running_config"] = \
# get running config after running
duts_data[duthost.hostname]["cur_running_config"] = {}
duts_data[duthost.hostname]["cur_running_config"][None] = \
json.loads(duthost.shell("sonic-cfggen -d --print-data", verbose=False)['stdout'])
if duthost.is_multi_asic:
for asic_index in range(0, duthost.facts.get('num_asic')):
asic_ns = "asic{}".format(asic_index)
duts_data[duthost.hostname]["cur_running_config"][asic_ns] = \
json.loads(duthost.shell("sonic-cfggen -n {} -d --print-data".format(asic_ns),
verbose=False)['stdout'])

# The tables that we don't care
EXCLUDE_CONFIG_TABLE_NAMES = set([])
Expand All @@ -1833,72 +1864,81 @@ def _remove_entry(table_name, key_name, config):
if len(config[table_name]) == 0:
config.pop(table_name)

# Remove ignored keys from base config
for exclude_key in EXCLUDE_CONFIG_KEY_NAMES:
fields = exclude_key.split('|')
if len(fields) != 2:
continue
_remove_entry(fields[0], fields[1], duts_data[duthost.hostname]["pre_running_config"])
_remove_entry(fields[0], fields[1], duts_data[duthost.hostname]["cur_running_config"])

# Get common keys in pre running config and cur running config

pre_running_config_keys = set(duts_data[duthost.hostname]["pre_running_config"].keys())
cur_running_config_keys = set(duts_data[duthost.hostname]["cur_running_config"].keys())

# Check if there are extra keys in pre running config
pre_config_extra_keys = list(pre_running_config_keys - cur_running_config_keys - EXCLUDE_CONFIG_TABLE_NAMES)
for key in pre_config_extra_keys:
pre_only_config[duthost.hostname].update({key: duts_data[duthost.hostname]["pre_running_config"][key]})

# Check if there are extra keys in cur running config
cur_config_extra_keys = list(cur_running_config_keys - pre_running_config_keys - EXCLUDE_CONFIG_TABLE_NAMES)
for key in cur_config_extra_keys:
cur_only_config[duthost.hostname].update({key: duts_data[duthost.hostname]["cur_running_config"][key]})

common_config_keys = list(pre_running_config_keys & cur_running_config_keys - EXCLUDE_CONFIG_TABLE_NAMES)
# Check if the running config is modified after module running
for key in common_config_keys:
# TODO: remove these code when solve the problem of "FLEX_COUNTER_DELAY_STATUS"
if key == "FLEX_COUNTER_TABLE":
for sub_key, sub_value in duts_data[duthost.hostname]["pre_running_config"][key].items():
try:
pre_value = duts_data[duthost.hostname]["pre_running_config"][key][sub_key]
cur_value = duts_data[duthost.hostname]["cur_running_config"][key][sub_key]
if pre_value["FLEX_COUNTER_STATUS"] != cur_value["FLEX_COUNTER_STATUS"]:
inconsistent_config[duthost.hostname].update(
for cfg_context in duts_data[duthost.hostname]['pre_running_config']:
pre_only_config[duthost.hostname][cfg_context] = {}
cur_only_config[duthost.hostname][cfg_context] = {}
inconsistent_config[duthost.hostname][cfg_context] = {}

pre_running_config = duts_data[duthost.hostname]["pre_running_config"][cfg_context]
cur_running_config = duts_data[duthost.hostname]["cur_running_config"][cfg_context]

pre_running_config_keys = set(pre_running_config.keys())
cur_running_config_keys = set(cur_running_config.keys())

# Remove ignored keys from base config
for exclude_key in EXCLUDE_CONFIG_KEY_NAMES:
fields = exclude_key.split('|')
if len(fields) != 2:
continue
_remove_entry(fields[0], fields[1], pre_running_config)
_remove_entry(fields[0], fields[1], cur_running_config)

# Check if there are extra keys in pre running config
pre_config_extra_keys = list(
pre_running_config_keys - cur_running_config_keys - EXCLUDE_CONFIG_TABLE_NAMES)
for key in pre_config_extra_keys:
pre_only_config[duthost.hostname].update({key: pre_running_config[key]})

# Check if there are extra keys in cur running config
cur_config_extra_keys = list(
cur_running_config_keys - pre_running_config_keys - EXCLUDE_CONFIG_TABLE_NAMES)
for key in cur_config_extra_keys:
cur_only_config[duthost.hostname].update({key: cur_running_config[key]})

# Get common keys in pre running config and cur running config
common_config_keys = list(pre_running_config_keys & cur_running_config_keys -
EXCLUDE_CONFIG_TABLE_NAMES)

# Check if the running config is modified after module running
for key in common_config_keys:
# TODO: remove these code when solve the problem of "FLEX_COUNTER_DELAY_STATUS"
if key == "FLEX_COUNTER_TABLE":
for sub_key, sub_value in pre_running_config[key].items():
try:
pre_value = pre_running_config[key][sub_key]
cur_value = cur_running_config[key][sub_key]
if pre_value["FLEX_COUNTER_STATUS"] != cur_value["FLEX_COUNTER_STATUS"]:
inconsistent_config[duthost.hostname][cfg_context].update(
{
key: {
"pre_value": pre_running_config[key],
"cur_value": cur_running_config[key]
}
}
)
except KeyError:
inconsistent_config[duthost.hostname][cfg_context].update(
{
key: {
"pre_value": duts_data[duthost.hostname]["pre_running_config"][key],
"cur_value": duts_data[duthost.hostname]["cur_running_config"][key]
"pre_value": pre_running_config[key],
"cur_value": cur_running_config[key]
}
}
)
except KeyError:
inconsistent_config[duthost.hostname].update(
{
key: {
"pre_value": duts_data[duthost.hostname]["pre_running_config"][key],
"cur_value": duts_data[duthost.hostname]["cur_running_config"][key]
}
}
)
elif duts_data[duthost.hostname]["pre_running_config"][key] != \
duts_data[duthost.hostname]["cur_running_config"][key]:
inconsistent_config[duthost.hostname].update(
{
key: {
"pre_value": duts_data[duthost.hostname]["pre_running_config"][key],
"cur_value": duts_data[duthost.hostname]["cur_running_config"][key]
elif pre_running_config[key] != cur_running_config[key]:
inconsistent_config[duthost.hostname][cfg_context].update(
{
key: {
"pre_value": pre_running_config[key],
"cur_value": cur_running_config[key]
}
}
)

if pre_only_config[duthost.hostname] or \
cur_only_config[duthost.hostname] or \
inconsistent_config[duthost.hostname]:
config_db_check_pass = False
}
)

if pre_only_config[duthost.hostname][cfg_context] or \
cur_only_config[duthost.hostname][cfg_context] or \
inconsistent_config[duthost.hostname][cfg_context]:
config_db_check_pass = False
if not (core_dump_check_pass and config_db_check_pass):
check_result = {
"core_dump_check": {
Expand Down
5 changes: 5 additions & 0 deletions tests/test_pretest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,3 +294,8 @@ def test_generate_running_golden_config(duthosts):
"""
for duthost in duthosts:
duthost.shell("sonic-cfggen -d --print-data > /etc/sonic/running_golden_config.json")
if duthost.is_multi_asic:
for asic_index in range(0, duthost.facts.get('num_asic')):
asic_ns = 'asic{}'.format(asic_index)
duthost.shell("sonic-cfggen -n {} -d --print-data > /etc/sonic/running_golden_config{}.json".
format(asic_ns, asic_index))