From 5276a903b110cce5a9713562adf036f2f697bd5d Mon Sep 17 00:00:00 2001 From: Neetha John Date: Thu, 23 Apr 2020 10:59:02 -0700 Subject: [PATCH 1/3] Pytest organization proposal Signed-off-by: Neetha John --- tests/pytest.org.md | 122 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 tests/pytest.org.md diff --git a/tests/pytest.org.md b/tests/pytest.org.md new file mode 100644 index 00000000000..63241e0f1d1 --- /dev/null +++ b/tests/pytest.org.md @@ -0,0 +1,122 @@ +# Pytest organization proposal + +This proposal intends to achieve the following + - Have a standard way of categorizing tests + - Have some guidelines around test file organization + - Have a master wrapper for test execution + - Follow common documentation style + - Test result collection + +## Test categorization +Leverage pytest custom markers to group tests based on topology, platform, features and connection type. + +``` +pytest.ini +[pytest] +markers: + topology(topo_name): The topologies this particular testcase can run against. topo_name can be individual topology names like 't0', 't1', 'ptf', 'any' or a comma separated like ('t0', 't1') if supported on multiple topologies + platform(platform_name): used for platform specific test(broadcom, mellanox, vs etc) + feature(feature_name): feature this test is written for. eg. acl, nat + connection_type(name): names can be 'fabric' (which indicates the presence of a fanout switch) or 'direct' if a testcase uses directly connected links + +``` +conftest.py + +``` +def pytest_addoption(parser): + parser.addoption("--topology", action="store", metavar="NAME", + help="only run tests matching the topology NAME") + +def pytest_runtest_setup(item): + toponames = [mark.args for mark in item.iter_markers(name="topology")] + if toponames: + cfg_topos = item.config.getoption("--topology").split(',') + if all(topo not in toponames[0] for topo in cfg_topos): + pytest.skip("test requires topology in {!r}".format(toponames)) + else: + if item.config.getoption("--topology"): + pytest.skip("test does not match topology") + +``` + +Sample test file: test_topo.py + +``` +@pytest.mark.topology('t0', 't1') +def test_all(): + assert 1 == 1 + +@pytest.mark.topology('t0') +def test_t0(): + assert 1 == 1 + + +@pytest.mark.topology('any') +def test_any(): + assert 1 == 1 + +``` + +Sample test file: test_notopo.py + +``` +def test_notopo(): + assert 1 == 1 + +``` + +Test run + +``` +py.test --inventory inv --host-pattern dut1 --module-path ../ansible/library/ --testbed tb --testbed_file tb.csv --topology t1 test_topo.py test_notopo.py -rA + +platform linux2 -- Python 2.7.12, pytest-4.6.9, py-1.8.1, pluggy-0.13.1 +ansible: 2.8.7 +rootdir: /var/nejo/Networking-acs-sonic-mgmt/tests, inifile: pytest.ini +plugins: ansible-2.2.2 +collected 4 items + +test_topo.py .ss [ 75%] +test_notopo.py s + +.... + +.... +PASSED test_topo.py::test_all +SKIPPED [1] /var/nejo/Networking-acs-sonic-mgmt/tests/conftest.py:293: test requires topology in [('t0',)] +SKIPPED [1] /var/nejo/Networking-acs-sonic-mgmt/tests/conftest.py:293: test requires topology in [('any',)] +SKIPPED [1] /var/nejo/Networking-acs-sonic-mgmt/tests/conftest.py:295: test does not match topology + +``` + +## Test file organization +- Have 2 broad categories (platform and feature). Feature specific tests and their helpers go into specific feature folders. + +``` +tests + |_ common + |_ platform + |_ ptftests + |_ nat + |_ test_nat_bindings.py + |_ files + |_ all helpers for the nat feature + |_ acl + +``` + +- Any reusable code needs to go under tests/common + +- File naming convention + + +## Master wrapper +Make it easier to run a nightly test against a feature/platform/topology from the command line. Have something similar to the 'ansible/testbed-cli.sh' script which can be invoked with just the basic parameters (testbed name, what flavor of test to run) + + +## Documentation style +Follow a common style of documentation for test methods which can be used by some tool to generate html content + + +## Test result collection +Use the --junitxml attribute to collect test results. Can leverage the existing format used in sonic-utilities/sonic-swss repo for reporting test results. From 69232dcdb6c505ff3a5242c39e1b21f419093537 Mon Sep 17 00:00:00 2001 From: Neetha John Date: Tue, 28 Apr 2020 14:39:22 -0700 Subject: [PATCH 2/3] Address review comments Signed-off-by: Neetha John --- tests/pytest.org.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/pytest.org.md b/tests/pytest.org.md index 63241e0f1d1..0b13d7e8d0b 100644 --- a/tests/pytest.org.md +++ b/tests/pytest.org.md @@ -8,16 +8,19 @@ This proposal intends to achieve the following - Test result collection ## Test categorization -Leverage pytest custom markers to group tests based on topology, platform, features and connection type. +Leverage pytest custom markers to group tests based on topology, asic, features, device type and connection type. +Every testcase needs to have a topology marker. Feature markers are recommended for any feature test that are getting added. +'Device_type' is optional but needs to be specified if there is a specific requirement that the test needs a physical DUT as opposed to a VS. The same criteria applies for 'connection_type' ``` pytest.ini [pytest] markers: topology(topo_name): The topologies this particular testcase can run against. topo_name can be individual topology names like 't0', 't1', 'ptf', 'any' or a comma separated like ('t0', 't1') if supported on multiple topologies - platform(platform_name): used for platform specific test(broadcom, mellanox, vs etc) + asic(vendor_name): used for asic specific test(broadcom, mellanox etc) feature(feature_name): feature this test is written for. eg. acl, nat connection_type(name): names can be 'fabric' (which indicates the presence of a fanout switch) or 'direct' if a testcase uses directly connected links + device_type(name): name can 'physical' (if this test requires a physical dut) or 'vs' (if this test can be run on a virtual switch) ``` conftest.py @@ -76,8 +79,10 @@ rootdir: /var/nejo/Networking-acs-sonic-mgmt/tests, inifile: pytest.ini plugins: ansible-2.2.2 collected 4 items -test_topo.py .ss [ 75%] -test_notopo.py s +test_topo.py::test_all PASSED [ 25%] +test_topo.py::test_t0 PASSED [ 50%] +test_topo.py::test_any SKIPPED [ 75%] +test_notopo.py::test_notopo SKIPPED [100%] .... @@ -108,6 +113,7 @@ tests - Any reusable code needs to go under tests/common - File naming convention + The objective here is to provide meaningful names for helper files/testcase files so that the user gets a general idea of the file contents. ## Master wrapper From 798676d7589b99f0ad38bef897915127e2d7aba0 Mon Sep 17 00:00:00 2001 From: Neetha John Date: Tue, 28 Apr 2020 14:43:48 -0700 Subject: [PATCH 3/3] Move to docs folder Signed-off-by: Neetha John --- tests/docs/pytest.org.md | 128 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 tests/docs/pytest.org.md diff --git a/tests/docs/pytest.org.md b/tests/docs/pytest.org.md new file mode 100644 index 00000000000..0b13d7e8d0b --- /dev/null +++ b/tests/docs/pytest.org.md @@ -0,0 +1,128 @@ +# Pytest organization proposal + +This proposal intends to achieve the following + - Have a standard way of categorizing tests + - Have some guidelines around test file organization + - Have a master wrapper for test execution + - Follow common documentation style + - Test result collection + +## Test categorization +Leverage pytest custom markers to group tests based on topology, asic, features, device type and connection type. +Every testcase needs to have a topology marker. Feature markers are recommended for any feature test that are getting added. +'Device_type' is optional but needs to be specified if there is a specific requirement that the test needs a physical DUT as opposed to a VS. The same criteria applies for 'connection_type' + +``` +pytest.ini +[pytest] +markers: + topology(topo_name): The topologies this particular testcase can run against. topo_name can be individual topology names like 't0', 't1', 'ptf', 'any' or a comma separated like ('t0', 't1') if supported on multiple topologies + asic(vendor_name): used for asic specific test(broadcom, mellanox etc) + feature(feature_name): feature this test is written for. eg. acl, nat + connection_type(name): names can be 'fabric' (which indicates the presence of a fanout switch) or 'direct' if a testcase uses directly connected links + device_type(name): name can 'physical' (if this test requires a physical dut) or 'vs' (if this test can be run on a virtual switch) + +``` +conftest.py + +``` +def pytest_addoption(parser): + parser.addoption("--topology", action="store", metavar="NAME", + help="only run tests matching the topology NAME") + +def pytest_runtest_setup(item): + toponames = [mark.args for mark in item.iter_markers(name="topology")] + if toponames: + cfg_topos = item.config.getoption("--topology").split(',') + if all(topo not in toponames[0] for topo in cfg_topos): + pytest.skip("test requires topology in {!r}".format(toponames)) + else: + if item.config.getoption("--topology"): + pytest.skip("test does not match topology") + +``` + +Sample test file: test_topo.py + +``` +@pytest.mark.topology('t0', 't1') +def test_all(): + assert 1 == 1 + +@pytest.mark.topology('t0') +def test_t0(): + assert 1 == 1 + + +@pytest.mark.topology('any') +def test_any(): + assert 1 == 1 + +``` + +Sample test file: test_notopo.py + +``` +def test_notopo(): + assert 1 == 1 + +``` + +Test run + +``` +py.test --inventory inv --host-pattern dut1 --module-path ../ansible/library/ --testbed tb --testbed_file tb.csv --topology t1 test_topo.py test_notopo.py -rA + +platform linux2 -- Python 2.7.12, pytest-4.6.9, py-1.8.1, pluggy-0.13.1 +ansible: 2.8.7 +rootdir: /var/nejo/Networking-acs-sonic-mgmt/tests, inifile: pytest.ini +plugins: ansible-2.2.2 +collected 4 items + +test_topo.py::test_all PASSED [ 25%] +test_topo.py::test_t0 PASSED [ 50%] +test_topo.py::test_any SKIPPED [ 75%] +test_notopo.py::test_notopo SKIPPED [100%] + +.... + +.... +PASSED test_topo.py::test_all +SKIPPED [1] /var/nejo/Networking-acs-sonic-mgmt/tests/conftest.py:293: test requires topology in [('t0',)] +SKIPPED [1] /var/nejo/Networking-acs-sonic-mgmt/tests/conftest.py:293: test requires topology in [('any',)] +SKIPPED [1] /var/nejo/Networking-acs-sonic-mgmt/tests/conftest.py:295: test does not match topology + +``` + +## Test file organization +- Have 2 broad categories (platform and feature). Feature specific tests and their helpers go into specific feature folders. + +``` +tests + |_ common + |_ platform + |_ ptftests + |_ nat + |_ test_nat_bindings.py + |_ files + |_ all helpers for the nat feature + |_ acl + +``` + +- Any reusable code needs to go under tests/common + +- File naming convention + The objective here is to provide meaningful names for helper files/testcase files so that the user gets a general idea of the file contents. + + +## Master wrapper +Make it easier to run a nightly test against a feature/platform/topology from the command line. Have something similar to the 'ansible/testbed-cli.sh' script which can be invoked with just the basic parameters (testbed name, what flavor of test to run) + + +## Documentation style +Follow a common style of documentation for test methods which can be used by some tool to generate html content + + +## Test result collection +Use the --junitxml attribute to collect test results. Can leverage the existing format used in sonic-utilities/sonic-swss repo for reporting test results.