From 7314f3ca13b2c2d887c296cd806c9599efdf45a1 Mon Sep 17 00:00:00 2001 From: Shu0T1an ChenG Date: Tue, 15 Oct 2019 11:42:30 -0700 Subject: [PATCH 1/2] [test]: Add mirror session warm reboot test This test validates the post baking stage that recovers the monitor port picked before the warm reboot. Signed-off-by: Shu0T1an ChenG --- tests/conftest.py | 16 +++++++ tests/test_warm_reboot.py | 87 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 8fba25d8d8d..53983f882bb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -781,6 +781,22 @@ def remove_neighbor(self, interface, ip): tbl._del(interface + ":" + ip) time.sleep(1) + def add_route(self, prefix, nexthop): + self.runcmd("ip route add " + prefix + " via " + nexthop) + time.sleep(1) + + def add_route_ecmp(self, prefix, nexthops): + cmd = "" + for nexthop in nexthops: + cmd += " nexthop via " + nexthop + + self.runcmd("ip route add " + prefix + cmd) + time.sleep(1) + + def remove_route(self, prefix): + self.runcmd("ip route del " + prefix) + time.sleep(1) + def setup_db(self): self.pdb = swsscommon.DBConnector(0, self.redis_sock, 0) self.adb = swsscommon.DBConnector(1, self.redis_sock, 0) diff --git a/tests/test_warm_reboot.py b/tests/test_warm_reboot.py index fc8ca3debd2..dcf52374579 100644 --- a/tests/test_warm_reboot.py +++ b/tests/test_warm_reboot.py @@ -234,7 +234,94 @@ def ping_new_ips(dvs): dvs.runcmd(['sh', '-c', "ping -c 1 -W 0 -q {}.0.0.{} > /dev/null 2>&1".format(i*4,j+NUM_NEIGH_PER_INTF+2)]) dvs.runcmd(['sh', '-c', "ping6 -c 1 -W 0 -q {}00::{} > /dev/null 2>&1".format(i*4,j+NUM_NEIGH_PER_INTF+2)]) +def create_mirror_session(dvs, name, src, dst, gre, dscp, ttl, queue): + tbl = swsscommon.Table(dvs.cdb, "MIRROR_SESSION") + fvs = swsscommon.FieldValuePairs([("src_ip", src), + ("dst_ip", dst), + ("gre_type", gre), + ("dscp", dscp), + ("ttl", ttl), + ("queue", queue)]) + tbl.set(name, fvs) + time.sleep(1) + +def remove_mirror_session(dvs, name): + tbl = swsscommon.Table(dvs.cdb, "MIRROR_SESSION") + tbl._del(name) + time.sleep(1) + +def get_mirror_session_state(dvs, name): + tbl = swsscommon.Table(dvs.sdb, "MIRROR_SESSION_TABLE") + (status, fvs) = tbl.get(name) + assert status == True + assert len(fvs) > 0 + return { fv[0]: fv[1] for fv in fvs } + + class TestWarmReboot(object): + def test_swss_mirror_post_baking(self, dvs, testlog): + dvs.setup_db() + adb = swsscommon.DBConnector(swsscommon.APPL_DB, dvs.redis_sock, 0) + cdb = swsscommon.DBConnector(swsscommon.CONFIG_DB, dvs.redis_sock, 0) + sdb = swsscommon.DBConnector(swsscommon.STATE_DB, dvs.redis_sock, 0) + + dvs.runcmd("config warm_restart enable swss") + + dvs.set_interface_status("Ethernet12", "up") + dvs.set_interface_status("Ethernet16", "up") + dvs.set_interface_status("Ethernet20", "up") + + dvs.add_ip_address("Ethernet12", "10.0.0.0/31") + dvs.add_ip_address("Ethernet16", "11.0.0.0/31") + dvs.add_ip_address("Ethernet20", "12.0.0.0/31") + + dvs.add_neighbor("Ethernet12", "10.0.0.1", "02:04:06:08:10:12") + dvs.add_neighbor("Ethernet16", "11.0.0.1", "03:04:06:08:10:12") + dvs.add_neighbor("Ethernet20", "12.0.0.1", "04:04:06:08:10:12") + + # set test1 monitor port as Ethernet20 + dvs.runcmd("ip route add 2.2.2.2 via 12.0.0.1") + create_mirror_session(dvs, "test1", "1.1.1.1", "2.2.2.2", "0x6558", "8", "100", "0") + + # set test2 monitor port as Ethernet16 + dvs.runcmd("ip route change 2.2.2.2 nexthop via 11.0.0.1 nexthop via 12.0.0.1") + create_mirror_session(dvs, "test2", "1.1.1.1", "2.2.2.2", "0x6558", "8", "100", "0") + + # set test3 monitor port as Ethernet12 + dvs.runcmd("ip route change 2.2.2.2 nexthop via 10.0.0.1 nexthop via 11.0.0.1 nexthop via 12.0.0.1") + create_mirror_session(dvs, "test3", "1.1.1.1", "2.2.2.2", "0x6558", "8", "100", "0") + + test1_mp = get_mirror_session_state(dvs, "test1")["monitor_port"] + test2_mp = get_mirror_session_state(dvs, "test2")["monitor_port"] + test3_mp = get_mirror_session_state(dvs, "test3")["monitor_port"] + + dvs.stop_swss() + dvs.start_swss() + time.sleep(5) + + # after warm reboot, all monitor ports are not changed + assert test1_mp == get_mirror_session_state(dvs, "test1")["monitor_port"] + assert test2_mp == get_mirror_session_state(dvs, "test2")["monitor_port"] + assert test3_mp == get_mirror_session_state(dvs, "test3")["monitor_port"] + + remove_mirror_session(dvs, "test1") + remove_mirror_session(dvs, "test2") + remove_mirror_session(dvs, "test3") + + dvs.remove_route("2.2.2.2") + + dvs.remove_neighbor("Ethernet12", "10.0.0.1") + dvs.remove_neighbor("Ethernet16", "11.0.0.1") + dvs.remove_neighbor("Ethernet20", "12.0.0.1") + + dvs.remove_ip_address("Ethernet12", "10.0.0.0/31") + dvs.remove_ip_address("Ethernet16", "11.0.0.0/31") + dvs.remove_ip_address("Ethernet20", "12.0.0.0/31") + + dvs.set_interface_status("Ethernet12", "down") + dvs.set_interface_status("Ethernet16", "down") + dvs.set_interface_status("Ethernet20", "down") + def test_PortSyncdWarmRestart(self, dvs, testlog): conf_db = swsscommon.DBConnector(swsscommon.CONFIG_DB, dvs.redis_sock, 0) From 62aac6609b32237bf2b83ae1f71243abd7a72ad9 Mon Sep 17 00:00:00 2001 From: Danny Allen Date: Wed, 13 Nov 2019 11:33:37 -0800 Subject: [PATCH 2/2] Respond to PR feedback --- tests/conftest.py | 38 ++++++++++++++++++ tests/test_warm_reboot.py | 81 +++++++++++++++++++++++++-------------- 2 files changed, 91 insertions(+), 28 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 53983f882bb..d3288b66152 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -777,15 +777,36 @@ def add_neighbor(self, interface, ip, mac): time.sleep(1) def remove_neighbor(self, interface, ip): + """ + Removes a neighbor from the device. This method will wait 1 second + to allow redis updates to propagate. + """ tbl = swsscommon.ProducerStateTable(self.pdb, "NEIGH_TABLE") tbl._del(interface + ":" + ip) time.sleep(1) def add_route(self, prefix, nexthop): + """ + Adds a route from nexthop to prefix. This method will wait 1 second + to allow redis updates to propagate. + """ self.runcmd("ip route add " + prefix + " via " + nexthop) time.sleep(1) + def update_route(self, prefix, nexthop): + """ + Adds a route from nexthop to prefix, overwriting an existing route + to prefix. This method will wait 1 second to allow redis updates + to propagate. + """ + self.runcmd("ip route change " + prefix + " via " + nexthop) + time.sleep(1) + def add_route_ecmp(self, prefix, nexthops): + """ + Adds an ecmp route from any of the provided nexthops to prefix. + This method will wait 1 second to allow redis updates to propagate. + """ cmd = "" for nexthop in nexthops: cmd += " nexthop via " + nexthop @@ -793,7 +814,24 @@ def add_route_ecmp(self, prefix, nexthops): self.runcmd("ip route add " + prefix + cmd) time.sleep(1) + def update_route_ecmp(self, prefix, nexthops): + """ + Adds an ecmp route from any of the provided nexthops to prefix, + overwriting an existing ecmp route to prefix. This method will wait + 1 second to allow redis updates to propagate. + """ + cmd = "" + for nexthop in nexthops: + cmd += " nexthop via " + nexthop + + self.runcmd("ip route change " + prefix + cmd) + time.sleep(1) + def remove_route(self, prefix): + """ + Removes a route to prefix. This method will wait 1 second to allow + redis updates to propagate. + """ self.runcmd("ip route del " + prefix) time.sleep(1) diff --git a/tests/test_warm_reboot.py b/tests/test_warm_reboot.py index dcf52374579..1e40062e33b 100644 --- a/tests/test_warm_reboot.py +++ b/tests/test_warm_reboot.py @@ -234,7 +234,12 @@ def ping_new_ips(dvs): dvs.runcmd(['sh', '-c', "ping -c 1 -W 0 -q {}.0.0.{} > /dev/null 2>&1".format(i*4,j+NUM_NEIGH_PER_INTF+2)]) dvs.runcmd(['sh', '-c', "ping6 -c 1 -W 0 -q {}00::{} > /dev/null 2>&1".format(i*4,j+NUM_NEIGH_PER_INTF+2)]) + def create_mirror_session(dvs, name, src, dst, gre, dscp, ttl, queue): + """ + Creates a mirror session. This method will wait 1 second to allow + redis updates to propagate. + """ tbl = swsscommon.Table(dvs.cdb, "MIRROR_SESSION") fvs = swsscommon.FieldValuePairs([("src_ip", src), ("dst_ip", dst), @@ -245,28 +250,48 @@ def create_mirror_session(dvs, name, src, dst, gre, dscp, ttl, queue): tbl.set(name, fvs) time.sleep(1) + def remove_mirror_session(dvs, name): + """ + Deletes a mirror session. This method will wait 1 second to allow + redis updates to propagate. + """ tbl = swsscommon.Table(dvs.cdb, "MIRROR_SESSION") tbl._del(name) time.sleep(1) + def get_mirror_session_state(dvs, name): + """ + Gets the current state of a mirror session. This method will cause + your test to fail if the given mirror session does not exist. + + Example output: + { + "src_ip": "10.0.0.0", + "dst_ip": "192.168.1.1", + ... + } + """ tbl = swsscommon.Table(dvs.sdb, "MIRROR_SESSION_TABLE") (status, fvs) = tbl.get(name) - assert status == True + assert status assert len(fvs) > 0 - return { fv[0]: fv[1] for fv in fvs } + return {fv[0]: fv[1] for fv in fvs} class TestWarmReboot(object): - def test_swss_mirror_post_baking(self, dvs, testlog): - dvs.setup_db() - adb = swsscommon.DBConnector(swsscommon.APPL_DB, dvs.redis_sock, 0) - cdb = swsscommon.DBConnector(swsscommon.CONFIG_DB, dvs.redis_sock, 0) - sdb = swsscommon.DBConnector(swsscommon.STATE_DB, dvs.redis_sock, 0) + def test_MirrorSessionWarmReboot(self, dvs, testlog): + """ + This test verifies that mirror session monitor ports are not + changed after warm reboot. + """ + # Setup the virtual switch + dvs.setup_db() dvs.runcmd("config warm_restart enable swss") + # Setup neighbors for the test dvs.set_interface_status("Ethernet12", "up") dvs.set_interface_status("Ethernet16", "up") dvs.set_interface_status("Ethernet20", "up") @@ -279,34 +304,36 @@ def test_swss_mirror_post_baking(self, dvs, testlog): dvs.add_neighbor("Ethernet16", "11.0.0.1", "03:04:06:08:10:12") dvs.add_neighbor("Ethernet20", "12.0.0.1", "04:04:06:08:10:12") - # set test1 monitor port as Ethernet20 - dvs.runcmd("ip route add 2.2.2.2 via 12.0.0.1") - create_mirror_session(dvs, "test1", "1.1.1.1", "2.2.2.2", "0x6558", "8", "100", "0") + # Set test_session1 monitor port as Ethernet12 + dvs.add_route("2.2.2.2", "10.0.0.1") + create_mirror_session(dvs, "test_session1", "1.1.1.1", "2.2.2.2", "0x6558", "8", "100", "0") - # set test2 monitor port as Ethernet16 - dvs.runcmd("ip route change 2.2.2.2 nexthop via 11.0.0.1 nexthop via 12.0.0.1") - create_mirror_session(dvs, "test2", "1.1.1.1", "2.2.2.2", "0x6558", "8", "100", "0") + # Set test_session2 monitor port as Ethernet16 + dvs.update_route_ecmp("2.2.2.2", ["11.0.0.1", "10.0.0.1"]) + create_mirror_session(dvs, "test_session2", "1.1.1.1", "2.2.2.2", "0x6558", "8", "100", "0") - # set test3 monitor port as Ethernet12 - dvs.runcmd("ip route change 2.2.2.2 nexthop via 10.0.0.1 nexthop via 11.0.0.1 nexthop via 12.0.0.1") - create_mirror_session(dvs, "test3", "1.1.1.1", "2.2.2.2", "0x6558", "8", "100", "0") + # Set test_session3 monitor port as Ethernet20 + dvs.update_route_ecmp("2.2.2.2", ["12.0.0.1", "11.0.0.1", "10.0.0.1"]) + create_mirror_session(dvs, "test_session3", "1.1.1.1", "2.2.2.2", "0x6558", "8", "100", "0") - test1_mp = get_mirror_session_state(dvs, "test1")["monitor_port"] - test2_mp = get_mirror_session_state(dvs, "test2")["monitor_port"] - test3_mp = get_mirror_session_state(dvs, "test3")["monitor_port"] + test_session1_mp = get_mirror_session_state(dvs, "test_session1")["monitor_port"] + test_session2_mp = get_mirror_session_state(dvs, "test_session2")["monitor_port"] + test_session3_mp = get_mirror_session_state(dvs, "test_session3")["monitor_port"] + # Perform the warm reboot dvs.stop_swss() dvs.start_swss() time.sleep(5) - # after warm reboot, all monitor ports are not changed - assert test1_mp == get_mirror_session_state(dvs, "test1")["monitor_port"] - assert test2_mp == get_mirror_session_state(dvs, "test2")["monitor_port"] - assert test3_mp == get_mirror_session_state(dvs, "test3")["monitor_port"] + # Verify that all monitor ports are still the same after warm reboot + assert test_session1_mp == get_mirror_session_state(dvs, "test_session1")["monitor_port"] + assert test_session2_mp == get_mirror_session_state(dvs, "test_session2")["monitor_port"] + assert test_session3_mp == get_mirror_session_state(dvs, "test_session3")["monitor_port"] - remove_mirror_session(dvs, "test1") - remove_mirror_session(dvs, "test2") - remove_mirror_session(dvs, "test3") + # Teardown the mirror sessions and routes + remove_mirror_session(dvs, "test_session1") + remove_mirror_session(dvs, "test_session2") + remove_mirror_session(dvs, "test_session3") dvs.remove_route("2.2.2.2") @@ -412,7 +439,6 @@ def test_PortSyncdWarmRestart(self, dvs, testlog): intf_tbl._del("Ethernet20") time.sleep(2) - def test_VlanMgrdWarmRestart(self, dvs, testlog): conf_db = swsscommon.DBConnector(swsscommon.CONFIG_DB, dvs.redis_sock, 0) @@ -902,7 +928,6 @@ def test_swss_neighbor_syncup(self, dvs, testlog): intf_tbl._del("{}".format(intfs[1])) time.sleep(2) - # TODO: The condition of warm restart readiness check is still under discussion. def test_OrchagentWarmRestartReadyCheck(self, dvs, testlog):