diff --git a/ansible/roles/test/files/ptftests/pfc_pause_test.py b/ansible/roles/test/files/ptftests/pfc_pause_test.py index 7fdf5a156d2..ea41dfb22d3 100755 --- a/ansible/roles/test/files/ptftests/pfc_pause_test.py +++ b/ansible/roles/test/files/ptftests/pfc_pause_test.py @@ -1,5 +1,8 @@ +import datetime +import glob import ipaddress import logging +import os import random import socket import sys @@ -10,6 +13,7 @@ import ptf import ptf.packet as scapy import ptf.dataplane as dataplane +import scapy as sc from ptf import config from ptf.base_tests import BaseTest @@ -31,7 +35,7 @@ def capture_matched_packets(test, exp_packet, port, device_number=0, timeout=1): """ if timeout <= 0: raise Exception("%s() requires positive timeout value." % sys._getframe().f_code.co_name) - + pkts = list() while True: result = dp_poll(test, device_number=device_number, port_number=port, timeout=timeout) @@ -40,9 +44,9 @@ def capture_matched_packets(test, exp_packet, port, device_number=0, timeout=1): pkts.append(result.packet) else: break - - return pkts - + + return pkts + class PfcPauseTest(BaseTest): def __init__(self): BaseTest.__init__(self) @@ -54,7 +58,7 @@ def setUp(self): self.mac_src = self.test_params['mac_src'] self.mac_dst = self.test_params['mac_dst'] self.pkt_count = int(self.test_params['pkt_count']) - self.pkt_intvl = float(self.test_params['pkt_intvl']) + self.pkt_intvl = float(self.test_params['pkt_intvl']) self.port_src = int(self.test_params['port_src']) self.port_dst = self.test_params['port_dst'] self.ip_src = self.test_params['ip_src'] @@ -65,34 +69,42 @@ def setUp(self): self.queue_paused = self.test_params['queue_paused'] """ if DUT has MAC information """ self.dut_has_mac = self.test_params['dut_has_mac'] - + self.debug = self.test_params.get('debug', False) + def runTest(self): pass_cnt = 0 tos = self.dscp<<2 tos_bg = self.dscp_bg<<2 - + if self.debug: + # remove previous debug files + files = glob.glob("/tmp/pfc_pause_{}*".format(self.dscp)) + for file in files: + os.remove(file) + current_time = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + log_file = open("/tmp/pfc_pause_{}_{}".format(self.dscp, current_time), "w") + """ If DUT needs to learn MAC addresses """ - if not self.dut_has_mac: + if not self.dut_has_mac: pkt = simple_udp_packet( eth_dst=self.mac_dst, eth_src=self.mac_src, ip_src=self.ip_src, ip_dst=self.ip_dst) - + send_packet(self, self.port_src, pkt, 5) - + pkt = simple_udp_packet( eth_dst=self.mac_src, eth_src=self.mac_dst, ip_src=self.ip_dst, ip_dst=self.ip_src) - + send_packet(self, self.port_dst, pkt, 5) - + for x in range(self.pkt_count): sport = random.randint(0, 65535) dport = random.randint(0, 65535) - + pkt = simple_udp_packet( eth_dst=self.mac_dst, eth_src=self.mac_src, @@ -102,7 +114,7 @@ def runTest(self): udp_sport=sport, udp_dport=dport, ip_ttl=64) - + pkt_bg = simple_udp_packet( eth_dst=self.mac_dst, eth_src=self.mac_src, @@ -112,7 +124,7 @@ def runTest(self): udp_sport=sport, udp_dport=dport, ip_ttl=64) - + exp_pkt = simple_udp_packet( ip_src=self.ip_src, ip_dst=self.ip_dst, @@ -120,30 +132,37 @@ def runTest(self): udp_sport=sport, udp_dport=dport, ip_ttl=63) - + masked_exp_pkt = Mask(exp_pkt) masked_exp_pkt.set_do_not_care_scapy(scapy.Ether, "src") masked_exp_pkt.set_do_not_care_scapy(scapy.Ether, "dst") masked_exp_pkt.set_do_not_care_scapy(scapy.IP, "ttl") masked_exp_pkt.set_do_not_care_scapy(scapy.IP, "chksum") masked_exp_pkt.set_do_not_care_scapy(scapy.IP, "tos") - + send_packet(self, self.port_src, pkt, 1) send_packet(self, self.port_src, pkt_bg, 1) - + pkts = capture_matched_packets(self, masked_exp_pkt, self.port_dst) - + + if self.debug: + for i, pkt in enumerate(pkts): + dump_msg = "Iteration {}:\n Pkt num {}:\n Hex dump: {}\n\n".format(x, i, sc.utils.hexstr(pkt)) + log_file.write(dump_msg) + time.sleep(self.pkt_intvl) - + """ If the queue is paused, we should only receive the background packet """ if self.queue_paused: pass_cnt += int(len(pkts) == 1 and scapy.Ether(pkts[0])[scapy.IP].tos == tos_bg) else: pass_cnt += int(len(pkts) == 2) - + + if self.debug: + log_file.close() print "Passes: %d / %d" % (pass_cnt, self.pkt_count) - - def tearDown(self): + + def tearDown(self): reset_filters() BaseTest.tearDown(self) diff --git a/tests/qos/test_pfc_pause.py b/tests/qos/test_pfc_pause.py index 2a607024172..719cd99f027 100644 --- a/tests/qos/test_pfc_pause.py +++ b/tests/qos/test_pfc_pause.py @@ -1,21 +1,23 @@ - -import pytest +import logging import os -import time -import re -import struct +import pytest import random -from tests.common.fixtures.conn_graph_facts import conn_graph_facts -from qos_fixtures import lossless_prio_dscp_map, leaf_fanouts -from qos_helpers import ansible_stdout_to_str, eos_to_linux_intf, start_pause, stop_pause, setup_testbed, gen_testbed_t0, PFC_GEN_FILE, PFC_GEN_REMOTE_PATH +import time +from qos_fixtures import lossless_prio_dscp_map +from qos_helpers import ansible_stdout_to_str, get_phy_intfs, get_addrs_in_subnet, get_active_vlan_members, get_vlan_subnet, natural_keys +from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # lgtm[py/unused-import] from tests.common.fixtures.ptfhost_utils import change_mac_addresses # lgtm[py/unused-import] +from tests.common.helpers.assertions import pytest_assert +from tests.common.helpers.pfc_storm import PFCStorm pytestmark = [ pytest.mark.topology('t0') ] +logger = logging.getLogger(__name__) + PFC_PKT_COUNT = 1000000000 PTF_FILE_REMOTE_PATH = '~/ptftests/pfc_pause_test.py' @@ -26,45 +28,90 @@ """ Maximum number of interfaces to test on a DUT """ MAX_TEST_INTFS_COUNT = 4 -def run_test_t0(fanouthosts, - duthost, - ptfhost, - conn_graph_facts, - leaf_fanouts, - dscp, - dscp_bg, - queue_paused, - send_pause, - pfc_pause, - pause_prio, - pause_time=65535, - max_test_intfs_count=128): +@pytest.fixture(scope="module", autouse=True) +def pfc_test_setup(duthosts, rand_one_dut_hostname): """ - @Summary: Run a series of tests on a T0 topology. - For the T0 topology, we only test Vlan (server-faced) interfaces. - @param conn_graph_facts: Testbed topology - @param leaf_fanouts: Leaf fanout switches - @param dscp: DSCP value of test data packets - @param dscp_bg: DSCP value of background data packets - @param queue_paused: if the queue is expected to be paused - @param send_pause: send pause frames or not - @param pfc_pause: send PFC pause frames or not - @param pause_prio: priority of PFC franme - @param pause_time: pause time quanta. It is 65535 (maximum pause time quanta) by default. - @param max_test_intfs_count: maximum count of interfaces to test. By default, it is a very large value to cover all the interfaces. - return: Return # of iterations and # of passed iterations for each tested interface. + Generate configurations for the tests + + Args: + duthosts(AnsibleHost) : multi dut instance + rand_one_dut_hostname(string) : one of the dut instances from the multi dut + + Yields: + setup(dict): DUT interfaces, PTF interfaces, PTF IP addresses, and PTF MAC addresses """ - """ Clear DUT's PFC counters """ - duthost.sonic_pfc_counters(method="clear") + """ Get all the active physical interfaces enslaved to the Vlan """ + """ These interfaces are actually server-faced interfaces at T0 """ + duthost = duthosts[rand_one_dut_hostname] + vlan_members = get_active_vlan_members(duthost) + + """ Get Vlan subnet """ + vlan_subnet = get_vlan_subnet(duthost) + + """ Generate IP addresses for servers in the Vlan """ + vlan_ip_addrs = get_addrs_in_subnet(vlan_subnet, len(vlan_members)) + + """ Generate MAC addresses 00:00:00:00:00:XX for servers in the Vlan """ + vlan_mac_addrs = [5 * '00:' + format(k, '02x') for k in random.sample(range(1, 256), len(vlan_members))] + + """ Find correspoinding interfaces on PTF """ + phy_intfs = get_phy_intfs(duthost) + phy_intfs.sort(key=natural_keys) + vlan_members.sort(key=natural_keys) + vlan_members_index = [phy_intfs.index(intf) for intf in vlan_members] + ptf_intfs = ['eth' + str(i) for i in vlan_members_index] """ Disable DUT's PFC wd """ duthost.shell('sudo pfcwd stop') - """ Generate a T0 testbed configuration """ - dut_intfs, ptf_intfs, ptf_ip_addrs, ptf_mac_addrs = gen_testbed_t0(duthost) + yield { + 'vlan_members': vlan_members, + 'ptf_intfs': ptf_intfs, + 'vlan_ip_addrs': vlan_ip_addrs, + 'vlan_mac_addrs': vlan_mac_addrs + } + + """ Enable DUT's PFC wd """ + duthost.shell('sudo pfcwd start_default') + +def run_test(pfc_test_setup, fanouthosts, duthost, ptfhost, conn_graph_facts, + fanout_info, traffic_params, pause_prio=None, queue_paused=True, + send_pause=True, pfc_pause=True, max_test_intfs_count=128): + """ + Run the test + + Args: + pfc_test_setup(fixture) : setup fixture + fanouthosts(AnsibleHost) : fanout instance + duthost(AnsibleHost) : dut instance + ptfhost(AnsibleHost) : ptf instance + conn_graph_facts(fixture) : Testbed topology + fanout_info(fixture) : fanout graph info + traffic_params(dict) : dict containing the dscp of test dscp and background dscp + pause_prio(string) : priority of PFC franme + queue_paused(bool) : if the queue is expected to be paused + send_pause(bool) : send pause frames or not + pfc_pause(bool) : send PFC pause frames or not + max_test_intfs_count(int) : maximum count of interfaces to test. By default, it is a very large value to cover all the interfaces + + Return: + Number of iterations and number of passed iterations for each tested interface. + """ + + setup = pfc_test_setup + dut_intfs = setup['vlan_members'] + ptf_intfs = setup['ptf_intfs'] + ptf_ip_addrs = setup['vlan_ip_addrs'] + ptf_mac_addrs = setup['vlan_mac_addrs'] + """ Clear DUT's PFC counters """ + duthost.sonic_pfc_counters(method="clear") + + results = dict() + all_peer_dev = set() + storm_handle = None for i in range(min(max_test_intfs_count, len(ptf_intfs))): src_index = i dst_index = (i + 1) % len(ptf_intfs) @@ -88,31 +135,37 @@ def run_test_t0(fanouthosts, if send_pause: peer_device = conn_graph_facts['device_conn'][duthost.hostname][dut_intf_paused]['peerdevice'] peer_port = conn_graph_facts['device_conn'][duthost.hostname][dut_intf_paused]['peerport'] - peer_port_name = eos_to_linux_intf(peer_port) - peerdev_ans = fanouthosts[peer_device] + peer_info = { 'peerdevice': peer_device, + 'pfc_fanout_interface': peer_port + } if not pfc_pause: pause_prio = None - start_pause(host_ans=peerdev_ans, - pkt_gen_path=PFC_GEN_REMOTE_PATH, - intf=peer_port_name, - pkt_count=PFC_PKT_COUNT, - pause_duration=pause_time, - pause_priority=pause_prio) - + if not storm_handle: + storm_handle = PFCStorm(duthost, fanout_info, fanouthosts, + pfc_queue_idx=pause_prio, + pfc_frames_number=PFC_PKT_COUNT, + peer_info=peer_info) + + storm_handle.update_peer_info(peer_info) + if not all_peer_dev or peer_device not in all_peer_dev: + storm_handle.deploy_pfc_gen() + all_peer_dev.add(peer_device) + storm_handle.start_storm() """ Wait for PFC pause frame generation """ time.sleep(1) """ Run PTF test """ + logger.info("Running test: src intf: {} dest intf: {}".format(dut_intfs[src_index], dut_intfs[dst_index])) intf_info = '--interface %d@%s --interface %d@%s' % (src_index, src_intf, dst_index, dst_intf) test_params = ("mac_src=\'%s\';" % src_mac + "mac_dst=\'%s\';" % dst_mac + "ip_src=\'%s\';" % src_ip + "ip_dst=\'%s\';" % dst_ip - + "dscp=%d;" % dscp - + "dscp_bg=%d;" % dscp_bg + + "dscp=%d;" % traffic_params['dscp'] + + "dscp_bg=%d;" % traffic_params['dscp_bg'] + "pkt_count=%d;" % PTF_PKT_COUNT + "pkt_intvl=%f;" % PTF_PKT_INTVL_SEC + "port_src=%d;" % src_index @@ -139,197 +192,151 @@ def run_test_t0(fanouthosts, if send_pause: """ Stop PFC / FC storm """ - stop_pause(peerdev_ans, PFC_GEN_FILE) + storm_handle.stop_storm() time.sleep(1) return results +def test_pfc_pause_lossless(pfc_test_setup, fanouthosts, duthost, ptfhost, + conn_graph_facts, fanout_graph_facts, + lossless_prio_dscp_map, enum_dut_lossless_prio): -def run_test(fanouthosts, - duthost, - ptfhost, - tbinfo, - conn_graph_facts, - leaf_fanouts, - dscp, - dscp_bg, - queue_paused, - send_pause, - pfc_pause, - pause_prio, - pause_time=65535, - max_test_intfs_count=128): """ - @Summary: Run a series of tests (only support T0 topology) - @param tbinfo: Testbed information - @param conn_graph_facts: Testbed topology - @param leaf_fanouts: Leaf fanout switches - @param dscp: DSCP value of test data packets - @param dscp_bg: DSCP value of background data packets - @param queue_paused: if the queue is expected to be paused - @param send_pause: send pause frames or not - @param pfc_pause: send PFC pause frames or not - @param pause_prio: priority of PFC franme - @param pause_time: pause time quanta. It is 65535 (maximum pause time quanta) by default. - @param max_test_intfs_count: maximum count of interfaces to test. By default, it is a very large value to cover all the interfaces. - return: Return # of iterations and # of passed iterations for each tested interface. + Test if PFC pause frames can pause a lossless priority without affecting the other priorities + + Args: + pfc_test_setup(fixture) : setup fixture + fanouthosts(AnsibleHost) : fanout instance + duthost(AnsibleHost) : dut instance + ptfhost(AnsibleHost) : ptf instance + conn_graph_facts(fixture) : Testbed topology + fanout_graph_facts(fixture) : fanout graph info + lossless_prio_dscp_map(dict) : lossless priorities and their DSCP values + enum_dut_lossless_prio (str): name of lossless priority to test """ - print tbinfo - if tbinfo['topo']['name'].startswith('t0'): - return run_test_t0(fanouthosts=fanouthosts, - duthost=duthost, - ptfhost=ptfhost, - conn_graph_facts=conn_graph_facts, leaf_fanouts=leaf_fanouts, - dscp=dscp, - dscp_bg=dscp_bg, - queue_paused=queue_paused, - send_pause=send_pause, - pfc_pause=pfc_pause, - pause_prio=pause_prio, - pause_time=pause_time, - max_test_intfs_count=max_test_intfs_count) - - else: - return None - -def test_pfc_pause_lossless(fanouthosts, - duthost, - ptfhost, - tbinfo, - conn_graph_facts, - leaf_fanouts, - lossless_prio_dscp_map): - - """ - @Summary: Test if PFC pause frames can pause a lossless priority without affecting the other priorities - @param tbinfo: Testbed information - @param conn_graph_facts: Testbed topology - @param lossless_prio_dscp_map: lossless priorities and their DSCP values - """ - setup_testbed(fanouthosts=fanouthosts, - ptfhost=ptfhost, - leaf_fanouts=leaf_fanouts) - + test_errors = "" errors = [] + prio = int(enum_dut_lossless_prio.split('|')[-1]) + dscp = lossless_prio_dscp_map[prio] + other_lossless_prio = 4 if prio == 3 else 3 + - """ DSCP vlaues for lossless priorities """ - lossless_dscps = [int(dscp) for prio in lossless_prio_dscp_map for dscp in lossless_prio_dscp_map[prio]] + """ DSCP values for other lossless priority """ + other_lossless_dscps = lossless_prio_dscp_map[other_lossless_prio] """ DSCP values for lossy priorities """ - lossy_dscps = list(set(range(64)) - set(lossless_dscps)) - - for prio in lossless_prio_dscp_map: - """ DSCP values of the other lossless priorities """ - other_lossless_dscps = list(set(lossless_dscps) - set(lossless_prio_dscp_map[prio])) - """ We also need to test some DSCP values for lossy priorities """ - other_dscps = other_lossless_dscps + lossy_dscps[0:2] - - for dscp in lossless_prio_dscp_map[prio]: - for dscp_bg in other_dscps: - results = run_test(fanouthosts=fanouthosts, - duthost=duthost, - ptfhost=ptfhost, - tbinfo=tbinfo, - conn_graph_facts=conn_graph_facts, - leaf_fanouts=leaf_fanouts, - dscp=dscp, - dscp_bg=dscp_bg, - queue_paused=True, - send_pause=True, - pfc_pause=True, - pause_prio=prio, - pause_time=65535, - max_test_intfs_count=MAX_TEST_INTFS_COUNT) - - """ results should not be none """ - if results is None: - assert 0 - - errors = dict() - for intf in results: - if len(results[intf]) != 2: - continue - - pass_count = results[intf][0] - total_count = results[intf][1] - - if total_count == 0: - continue - - if pass_count < total_count * PTF_PASS_RATIO_THRESH: - errors[intf] = results[intf] - - if len(errors) > 0: - print "errors occured:\n{}".format("\n".join(errors)) - assert 0 - -def test_no_pfc(fanouthosts, - duthost, - ptfhost, - tbinfo, - conn_graph_facts, - leaf_fanouts, - lossless_prio_dscp_map): + lossy_dscps = list(set(range(64)) - set(other_lossless_dscps) - set(dscp)) + + """ We also need to test some DSCP values for lossy priorities """ + other_dscps = other_lossless_dscps + lossy_dscps[0:2] + + for dscp_bg in other_dscps: + logger.info("Testing dscp: {} and background dscp: {}".format(dscp, dscp_bg)) + traffic_params = {'dscp': dscp[0], 'dscp_bg': dscp_bg} + results = run_test(pfc_test_setup, + fanouthosts, + duthost, + ptfhost, + conn_graph_facts, + fanout_graph_facts, + traffic_params, + queue_paused=True, + send_pause=True, + pfc_pause=True, + pause_prio=prio, + max_test_intfs_count=MAX_TEST_INTFS_COUNT) + + """ results should not be none """ + if results is None: + test_errors += "Dscp: {}, Background Dscp: {}, Result is empty\n".format(dscp, dscp_bg) + + errors = dict() + for intf in results: + if len(results[intf]) != 2: + continue + + pass_count = results[intf][0] + total_count = results[intf][1] + + if total_count == 0: + continue + + if pass_count < total_count * PTF_PASS_RATIO_THRESH: + errors[intf] = results[intf] + + if len(errors) > 0: + test_errors += "Dscp: {}, Background Dscp: {}, errors occured: {}\n"\ + .format(dscp, dscp_bg, " ".join(["{}:{}".format(k,v) for k, v in errors.items()])) + + pytest_assert(len(test_errors) == 0, test_errors) + +def test_no_pfc(pfc_test_setup, fanouthosts, duthost, ptfhost, conn_graph_facts, + fanout_graph_facts, lossless_prio_dscp_map, enum_dut_lossless_prio): """ - @Summary: Test if lossless and lossy priorities can forward packets in the absence of PFC pause frames - @param fanouthosts: Fixture for fanout hosts - @param tbinfo: Testbed information - @param conn_graph_facts: Testbed topology - @param lossless_prio_dscp_map: lossless priorities and their DSCP values + Test if lossless and lossy priorities can forward packets in the absence of PFC pause frames + + Args: + pfc_test_setup(fixture) : setup fixture + fanouthosts(AnsibleHost) : fanout instance + duthost(AnsibleHost) : dut instance + ptfhost(AnsibleHost) : ptf instance + conn_graph_facts(fixture) : Testbed topology + fanout_graph_facts(fixture) : fanout graph info + lossless_prio_dscp_map(dict) : lossless priorities and their DSCP values + enum_dut_lossless_prio (str): name of lossless priority to test """ - setup_testbed(fanouthosts=fanouthosts, - ptfhost=ptfhost, - leaf_fanouts=leaf_fanouts) + test_errors = "" errors = [] + prio = int(enum_dut_lossless_prio.split('|')[-1]) + dscp = lossless_prio_dscp_map[prio] + other_lossless_prio = 4 if prio == 3 else 3 - """ DSCP vlaues for lossless priorities """ - lossless_dscps = [int(dscp) for prio in lossless_prio_dscp_map for dscp in lossless_prio_dscp_map[prio]] + """ DSCP values for other lossless priority """ + other_lossless_dscps = lossless_prio_dscp_map[other_lossless_prio] """ DSCP values for lossy priorities """ - lossy_dscps = list(set(range(64)) - set(lossless_dscps)) - - for prio in lossless_prio_dscp_map: - """ DSCP values of the other lossless priorities """ - other_lossless_dscps = list(set(lossless_dscps) - set(lossless_prio_dscp_map[prio])) - """ We also need to test some DSCP values for lossy priorities """ - other_dscps = other_lossless_dscps + lossy_dscps[0:2] - - for dscp in lossless_prio_dscp_map[prio]: - for dscp_bg in other_dscps: - results = run_test(fanouthosts=fanouthosts, - duthost=duthost, - ptfhost=ptfhost, - tbinfo=tbinfo, - conn_graph_facts=conn_graph_facts, - leaf_fanouts=leaf_fanouts, - dscp=dscp, - dscp_bg=dscp_bg, - queue_paused=False, - send_pause=False, - pfc_pause=None, - pause_prio=None, - pause_time=None, - max_test_intfs_count=MAX_TEST_INTFS_COUNT) - - """ results should not be none """ - if results is None: - assert 0 - - errors = dict() - for intf in results: - if len(results[intf]) != 2: - continue - - pass_count = results[intf][0] - total_count = results[intf][1] - - if total_count == 0: - continue - - if pass_count < total_count * PTF_PASS_RATIO_THRESH: - errors[intf] = results[intf] - - if len(errors) > 0: - print "errors occured:\n{}".format("\n".join(errors)) - assert 0 + lossy_dscps = list(set(range(64)) - set(other_lossless_dscps) - set(dscp)) + + """ We also need to test some DSCP values for lossy priorities """ + other_dscps = other_lossless_dscps + lossy_dscps[0:2] + + for dscp_bg in other_dscps: + logger.info("Testing dscp: {} and background dscp: {}".format(dscp, dscp_bg)) + traffic_params = {'dscp': dscp[0], 'dscp_bg': dscp_bg} + results = run_test(pfc_test_setup, + fanouthosts, + duthost, + ptfhost, + conn_graph_facts, + fanout_graph_facts, + traffic_params, + queue_paused=False, + send_pause=False, + pfc_pause=None, + pause_prio=None, + max_test_intfs_count=MAX_TEST_INTFS_COUNT) + + """ results should not be none """ + if results is None: + test_errors += "Dscp: {}, Background Dscp: {}, Result is empty\n".format(dscp, dscp_bg) + + errors = dict() + for intf in results: + if len(results[intf]) != 2: + continue + + pass_count = results[intf][0] + total_count = results[intf][1] + + if total_count == 0: + continue + + if pass_count < total_count * PTF_PASS_RATIO_THRESH: + errors[intf] = results[intf] + + if len(errors) > 0: + test_errors += "Dscp: {}, Background Dscp: {}, errors occured: {}\n"\ + .format(dscp, dscp_bg, " ".join(["{}:{}".format(k,v) for k, v in errors.items()])) + + pytest_assert(len(test_errors) == 0, test_errors)