diff --git a/tests/autorestart/test_container_autorestart.py b/tests/autorestart/test_container_autorestart.py index 2174a3f8cb5..c2a8406d60e 100755 --- a/tests/autorestart/test_container_autorestart.py +++ b/tests/autorestart/test_container_autorestart.py @@ -28,7 +28,7 @@ def config_reload_after_tests(duthost): config_reload(duthost) @pytest.fixture(autouse=True) -def ignore_expected_loganalyzer_exception(loganalyzer, enum_dut_feature): +def ignore_expected_loganalyzer_exception(duthost, loganalyzer, enum_dut_feature): """ Ignore expected failure/error messages during testing the autorestart feature. @@ -95,9 +95,9 @@ def ignore_expected_loganalyzer_exception(loganalyzer, enum_dut_feature): _, feature = decode_dut_port_name(enum_dut_feature) if loganalyzer: - loganalyzer.ignore_regex.extend(ignore_regex_dict['common']) + loganalyzer[duthost.hostname].ignore_regex.extend(ignore_regex_dict['common']) if feature in ignore_regex_dict: - loganalyzer.ignore_regex.extend(ignore_regex_dict[feature]) + loganalyzer[duthost.hostname].ignore_regex.extend(ignore_regex_dict[feature]) def get_group_program_info(duthost, container_name, group_name): diff --git a/tests/common/plugins/loganalyzer/__init__.py b/tests/common/plugins/loganalyzer/__init__.py index 9df3879a01d..0725ef860b8 100644 --- a/tests/common/plugins/loganalyzer/__init__.py +++ b/tests/common/plugins/loganalyzer/__init__.py @@ -3,6 +3,7 @@ from loganalyzer import LogAnalyzer from tests.common.errors import RunAnsibleModuleFail +import re def pytest_addoption(parser): @@ -11,33 +12,40 @@ def pytest_addoption(parser): @pytest.fixture(autouse=True) -def loganalyzer(duthosts, rand_one_dut_hostname, request): - duthost = duthosts[rand_one_dut_hostname] +def loganalyzer(duthosts, request): if request.config.getoption("--disable_loganalyzer") or "disable_loganalyzer" in request.keywords: logging.info("Log analyzer is disabled") yield return - # Force rotate logs - try: - duthost.shell( - "/usr/sbin/logrotate -f /etc/logrotate.conf > /dev/null 2>&1" - ) - except RunAnsibleModuleFail as e: - logging.warning("logrotate is failed. Command returned:\n" - "Stdout: {}\n" - "Stderr: {}\n" - "Return code: {}".format(e.results["stdout"], e.results["stderr"], e.results["rc"])) - - loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix=request.node.name) - logging.info("Add start marker into DUT syslog") - marker = loganalyzer.init() - logging.info("Load config and analyze log") - # Read existed common regular expressions located with legacy loganalyzer module - loganalyzer.load_common_config() - - yield loganalyzer + analyzers = {} + markers = {} + # Analyze all the duts + for duthost in duthosts: + # Force rotate logs + try: + duthost.shell( + "/usr/sbin/logrotate -f /etc/logrotate.conf > /dev/null 2>&1" + ) + except RunAnsibleModuleFail as e: + logging.warning("logrotate is failed. Command returned:\n" + "Stdout: {}\n" + "Stderr: {}\n" + "Return code: {}".format(e.results["stdout"], e.results["stderr"], e.results["rc"])) + + loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix=request.node.name) + logging.info("Add start marker into DUT syslog") + marker = loganalyzer.init() + logging.info("Load config and analyze log") + # Read existed common regular expressions located with legacy loganalyzer module + loganalyzer.load_common_config() + analyzers[duthost.hostname] = loganalyzer + markers[duthost.hostname] = marker + + yield analyzers + # Skip LogAnalyzer if case is skipped if "rep_call" in request.node.__dict__ and request.node.rep_call.skipped: return - loganalyzer.analyze(marker) + for dut_hostname, dut_analyzer in analyzers.items(): + dut_analyzer.analyze(markers[dut_hostname]) diff --git a/tests/copp/test_copp.py b/tests/copp/test_copp.py index 7cb475065b7..10ec576af42 100644 --- a/tests/copp/test_copp.py +++ b/tests/copp/test_copp.py @@ -131,7 +131,7 @@ def copp_testbed( _teardown_testbed(duthost, creds, ptfhost, test_params) @pytest.fixture(autouse=True) -def ignore_expected_loganalyzer_exceptions(loganalyzer): +def ignore_expected_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer): """ Ignore expected failures logs during test execution. @@ -152,7 +152,7 @@ def ignore_expected_loganalyzer_exceptions(loganalyzer): ] if loganalyzer: # Skip if loganalyzer is disabled - loganalyzer.ignore_regex.extend(ignoreRegex) + loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex) def _copp_runner(dut, ptf, protocol, test_params, dut_type): """ diff --git a/tests/dhcp_relay/test_dhcp_relay.py b/tests/dhcp_relay/test_dhcp_relay.py index 51818ab0e0c..46c8943a7f2 100644 --- a/tests/dhcp_relay/test_dhcp_relay.py +++ b/tests/dhcp_relay/test_dhcp_relay.py @@ -16,13 +16,13 @@ DEFAULT_DHCP_CLIENT_PORT = 68 @pytest.fixture(autouse=True) -def ignore_expected_loganalyzer_exceptions(loganalyzer): +def ignore_expected_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer): """Ignore expected failures logs during test execution.""" if loganalyzer: ignoreRegex = [ ".*ERR snmp#snmp-subagent.*", ] - loganalyzer.ignore_regex.extend(ignoreRegex) + loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex) yield diff --git a/tests/drop_packets/drop_packets.py b/tests/drop_packets/drop_packets.py index fa4c5ccd326..b1079e698f2 100644 --- a/tests/drop_packets/drop_packets.py +++ b/tests/drop_packets/drop_packets.py @@ -170,16 +170,16 @@ def rif_port_down(duthosts, rand_one_dut_hostname, setup, fanouthosts, loganalyz fanout_neighbor, fanout_intf = fanout_switch_port_lookup(fanouthosts, duthost.hostname, rif_member_iface) - loganalyzer.expect_regex = [LOG_EXPECT_PORT_OPER_DOWN_RE.format(rif_member_iface)] - with loganalyzer as _: + loganalyzer[rand_one_dut_hostname].expect_regex = [LOG_EXPECT_PORT_OPER_DOWN_RE.format(rif_member_iface)] + with loganalyzer[rand_one_dut_hostname] as _: fanout_neighbor.shutdown(fanout_intf) time.sleep(1) yield ip_dst - loganalyzer.expect_regex = [LOG_EXPECT_PORT_OPER_UP_RE.format(rif_member_iface)] - with loganalyzer as _: + loganalyzer[rand_one_dut_hostname].expect_regex = [LOG_EXPECT_PORT_OPER_UP_RE.format(rif_member_iface)] + with loganalyzer[rand_one_dut_hostname] as _: fanout_neighbor.no_shutdown(fanout_intf) time.sleep(wait_after_ports_up) diff --git a/tests/drop_packets/test_drop_counters.py b/tests/drop_packets/test_drop_counters.py index 8e1eb1c679a..a8edf9a85ef 100755 --- a/tests/drop_packets/test_drop_counters.py +++ b/tests/drop_packets/test_drop_counters.py @@ -74,14 +74,14 @@ def acl_setup(duthosts, rand_one_dut_hostname, loganalyzer): logger.info("Applying {}".format(dut_conf_file_path)) - loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE] - with loganalyzer as analyzer: + loganalyzer[rand_one_dut_hostname].expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE] + with loganalyzer[rand_one_dut_hostname] as analyzer: duthost.command("config acl update full {}".format(dut_conf_file_path)) yield - loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE] - with loganalyzer as analyzer: + loganalyzer[rand_one_dut_hostname].expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE] + with loganalyzer[rand_one_dut_hostname] as analyzer: logger.info("Applying {}".format(dut_clear_conf_file_path)) duthost.command("config acl update full {}".format(dut_clear_conf_file_path)) logger.info("Removing {}".format(dut_tmp_dir)) diff --git a/tests/k8s/conftest.py b/tests/k8s/conftest.py index 1d41dd992a4..a57910e5442 100644 --- a/tests/k8s/conftest.py +++ b/tests/k8s/conftest.py @@ -19,7 +19,7 @@ def check_k8s_vms(k8scluster): @pytest.fixture(autouse=True) -def ignore_expected_loganalyzer_exceptions(loganalyzer): +def ignore_expected_loganalyzer_exceptions(duthost, loganalyzer): """ Ignore expected failures logs during test execution @@ -35,5 +35,5 @@ def ignore_expected_loganalyzer_exceptions(loganalyzer): ".*for troubleshooting tips.*", ".*kubeproxy.*", ] - loganalyzer.ignore_regex.extend(ignoreRegex) + loganalyzer[duthost.hostname].ignore_regex.extend(ignoreRegex) yield diff --git a/tests/lldp/test_lldp.py b/tests/lldp/test_lldp.py index 4bc7747f06c..2ddd3d1e1a3 100644 --- a/tests/lldp/test_lldp.py +++ b/tests/lldp/test_lldp.py @@ -29,7 +29,7 @@ def test_lldp(duthosts, rand_one_dut_hostname, localhost, collect_techsupport): assert v['chassis']['name'] == config_facts['DEVICE_NEIGHBOR'][k]['name'] # Compare the LLDP neighbor interface with minigraph neigbhor interface (exclude the management port) assert v['port']['ifname'] == config_facts['DEVICE_NEIGHBOR'][k]['port'] - + def test_lldp_neighbor(duthosts, rand_one_dut_hostname, localhost, eos, collect_techsupport, loganalyzer): @@ -37,7 +37,7 @@ def test_lldp_neighbor(duthosts, rand_one_dut_hostname, localhost, eos, duthost = duthosts[rand_one_dut_hostname] if loganalyzer: - loganalyzer.ignore_regex.extend([ + loganalyzer[rand_one_dut_hostname].ignore_regex.extend([ ".*ERR syncd#syncd: :- check_fdb_event_notification_data.*", ".*ERR syncd#syncd: :- process_on_fdb_event: invalid OIDs in fdb \ notifications, NOT translating and NOT storing in ASIC DB.*", diff --git a/tests/pc/test_po_cleanup.py b/tests/pc/test_po_cleanup.py index ea970e35bf8..8d1b6fd60c5 100644 --- a/tests/pc/test_po_cleanup.py +++ b/tests/pc/test_po_cleanup.py @@ -9,7 +9,7 @@ ] @pytest.fixture(autouse=True) -def ignore_expected_loganalyzer_exceptions(loganalyzer): +def ignore_expected_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer): """ Ignore expected failures logs during test execution. @@ -25,12 +25,12 @@ def ignore_expected_loganalyzer_exceptions(loganalyzer): ignoreRegex = [ ".*", ] - loganalyzer.ignore_regex.extend(ignoreRegex) + loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex) expectRegex = [ ".*teamd#teammgrd: :- cleanTeamProcesses.*", ".*teamd#teamsyncd: :- cleanTeamSync.*" ] - loganalyzer.expect_regex.extend(expectRegex) + loganalyzer[rand_one_dut_hostname].expect_regex.extend(expectRegex) def check_kernel_po_interface_cleaned(duthost): diff --git a/tests/pc/test_po_update.py b/tests/pc/test_po_update.py index 673909f15e5..a702f2bd3d5 100644 --- a/tests/pc/test_po_update.py +++ b/tests/pc/test_po_update.py @@ -11,7 +11,7 @@ ] @pytest.fixture(autouse=True) -def ignore_expected_loganalyzer_exceptions(loganalyzer): +def ignore_expected_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer): """ Ignore expected failures logs during test execution. @@ -29,7 +29,7 @@ def ignore_expected_loganalyzer_exceptions(loganalyzer): ".*ERR syncd#syncd: :- process_on_fdb_event: FDB notification was not sent since it contain invalid OIDs, bug.*", ".*ERR syncd#syncd: :- translate_vid_to_rid: unable to get RID for VID.*", ] - loganalyzer.ignore_regex.extend(ignoreRegex) + loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex) yield diff --git a/tests/pfcwd/test_pfcwd_timer_accuracy.py b/tests/pfcwd/test_pfcwd_timer_accuracy.py index ffb398676ba..92a62e33910 100644 --- a/tests/pfcwd/test_pfcwd_timer_accuracy.py +++ b/tests/pfcwd/test_pfcwd_timer_accuracy.py @@ -39,7 +39,7 @@ def ignore_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer): ".*ERR syncd#syncd: :- process_on_fdb_event: invalid OIDs in fdb notifications, NOT translating and NOT storing in ASIC DB.*", ".*ERR syncd#syncd: :- process_on_fdb_event: FDB notification was not sent since it contain invalid OIDs, bug.*" ] - loganalyzer.ignore_regex.extend(ignoreRegex) + loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex) yield diff --git a/tests/qos/qos_sai_base.py b/tests/qos/qos_sai_base.py index 1549b33bcdc..e969cd317d5 100644 --- a/tests/qos/qos_sai_base.py +++ b/tests/qos/qos_sai_base.py @@ -500,7 +500,7 @@ def updateDockerService(host, docker="", action="", service=""): updateDockerService(duthost, action="start", **service) @pytest.fixture(autouse=True) - def updateLoganalyzerExceptions(self, loganalyzer): + def updateLoganalyzerExceptions(self, rand_one_dut_hostname, loganalyzer): """ Update loganalyzer ignore regex list @@ -530,7 +530,7 @@ def updateLoganalyzerExceptions(self, loganalyzer): ".*ERR monit.*'bgp\|bgpmon' status failed.*'/usr/bin/python.* /usr/local/bin/bgpmon' is not running.*", ".*ERR monit.*bgp\|fpmsyncd.*status failed.*NoSuchProcess process no longer exists.*", ] - loganalyzer.ignore_regex.extend(ignoreRegex) + loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex) yield diff --git a/tests/route/test_route_perf.py b/tests/route/test_route_perf.py index 966756f1688..d247dc34e0a 100644 --- a/tests/route/test_route_perf.py +++ b/tests/route/test_route_perf.py @@ -20,7 +20,7 @@ ROUTE_TABLE_NAME = 'ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY' @pytest.fixture(autouse=True) -def ignore_expected_loganalyzer_exceptions(loganalyzer): +def ignore_expected_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer): """ Ignore expected failures logs during test execution. @@ -38,7 +38,7 @@ def ignore_expected_loganalyzer_exceptions(loganalyzer): ] if loganalyzer: # Skip if loganalyzer is disabled - loganalyzer.ignore_regex.extend(ignoreRegex) + loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex) @pytest.fixture(params=[4, 6]) def ip_versions(request):