Merge pull request #7902 from opensourcerouting/topotest-marker

tests: Standardized Topotest PyTest Markers

doc/developer/topotests-markers.rst

@@ -0,0 +1,115 @@
+.. _topotests-markers:
+
+Markers
+--------
+
+To allow for automated selective testing on large scale continuous integration
+systems, all tests must be marked with at least one of the following markers:
+
+* babeld
+* bfdd
+* bgpd
+* eigrpd
+* isisd
+* ldpd
+* nhrpd
+* ospf6d
+* ospfd
+* pathd
+* pbrd
+* pimd
+* ripd
+* ripngd
+* sharpd
+* staticd
+* vrrpd
+
+The markers corespond to the daemon subdirectories in FRR's source code and have
+to be added to tests on a module level depending on which daemons are used
+during the test.
+
+The goal is to have continuous integration systems scan code submissions, detect
+changes to files in a daemons subdirectory and select only tests using that
+daemon to run to shorten developers waiting times for test results and save test
+infrastructure resources.
+
+Newly written modules and code changes on tests, which do not contain any or
+incorrect markers will be rejected by reviewers.
+
+
+Registering markers
+^^^^^^^^^^^^^^^^^^^
+The Registration of new markers takes place in the file
+``tests/topotests/pytest.ini`` and should be discussed with members of the TSC
+beforehand.
+
+.. code:: python3
+
+    # tests/topotests/pytest.ini
+    [pytest]
+    ...
+    markers =
+        babeld: Tests that run against BABELD
+        bfdd: Tests that run against BFDD
+        ...
+        vrrpd: Tests that run against VRRPD
+
+
+Adding markers to tests
+^^^^^^^^^^^^^^^^^^^^^^^
+Markers are added to a test by placing a global variable in the test module.
+
+Adding a single marker:
+
+.. code:: python3
+
+    import pytest
+    
+    ...
+    
+    pytestmark = pytest.mark.bfdd
+    
+    ...
+    
+    def test_using_bfdd():
+
+
+Adding multiple markers:
+
+.. code:: python3
+
+    import pytest
+    
+    ...
+    
+    pytestmark = [
+        pytest.mark.bgpd,
+        pytest.mark.ospfd,
+        pytest.mark.ospf6d
+    ]
+    
+    ...
+    
+    def test_using_bgpd_ospfd_ospf6d():
+
+
+Selecting marked modules fort testing
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Selecting by a single marker:
+
+.. code:: bash
+
+    pytest -v -m isisd
+
+Selecting by multiple markers:
+
+.. code:: bash
+
+    pytest -v -m "isisd or ldpd or nhrpd"
+
+
+Further Information
+^^^^^^^^^^^^^^^^^^^
+The `online pytest documentation <https://docs.pytest.org/en/stable/example/markers.html>`_
+provides further information and usage examples for pytest markers.
+

doc/developer/topotests.rst

@@ -792,6 +792,8 @@ Requirements:
   conforms with this, run it without the :option:`-s` parameter.
 - Use `black <https://github.com/psf/black>`_ code formatter before creating
   a pull request. This ensures we have a unified code style.
+- Mark test modules with pytest markers depending on the daemons used during the
+  tests (s. Markers)
 
 Tips:
 
@@ -950,6 +952,8 @@ does what you need. If nothing is similar, then you may create a new topology,
 preferably, using the newest template
 (:file:`tests/topotests/example-test/test_template.py`).
 
+.. include:: topotests-markers.rst
+
 .. include:: topotests-snippets.rst
 
 License

tests/topotests/example-test/test_template.py

@@ -44,6 +44,18 @@ from lib.topolog import logger
 from mininet.topo import Topo
 
 
+#TODO: select markers based on daemons used during test
+# pytest module level markers
+"""
+pytestmark = pytest.mark.bfdd # single marker
+pytestmark = [
+	pytest.mark.bgpd,
+	pytest.mark.ospfd,
+	pytest.mark.ospf6d
+] # multiple markers
+"""
+
+
 class TemplateTopo(Topo):
     "Test topology builder"
 

tests/topotests/example-topojson-test/test_topo_json_multiple_links/test_example_topojson_multiple_links.py

@@ -53,6 +53,19 @@ from lib.topolog import logger
 from lib.bgp import verify_bgp_convergence
 from lib.topojson import build_topo_from_json, build_config_from_json
 
+
+#TODO: select markers based on daemons used during test
+# pytest module level markers
+"""
+pytestmark = pytest.mark.bfdd # single marker
+pytestmark = [
+	pytest.mark.bgpd,
+	pytest.mark.ospfd,
+	pytest.mark.ospf6d
+] # multiple markers
+"""
+
+
 # Reading the data from JSON File for topology and configuration creation
 jsonFile = "{}/example_topojson_multiple_links.json".format(CWD)
 try:

tests/topotests/example-topojson-test/test_topo_json_single_link/test_example_topojson.py

@@ -52,6 +52,19 @@ from lib.topolog import logger
 from lib.bgp import verify_bgp_convergence
 from lib.topojson import build_topo_from_json, build_config_from_json
 
+
+#TODO: select markers based on daemons used during test
+# pytest module level markers
+"""
+pytestmark = pytest.mark.bfdd # single marker
+pytestmark = [
+	pytest.mark.bgpd,
+	pytest.mark.ospfd,
+	pytest.mark.ospf6d
+] # multiple markers
+"""
+
+
 # Reading the data from JSON File for topology and configuration creation
 jsonFile = "{}/example_topojson.json".format(CWD)
 

tests/topotests/example-topojson-test/test_topo_json_single_link_loopback/test_example_topojson.py

@@ -54,6 +54,19 @@ from lib.topolog import logger
 from lib.bgp import verify_bgp_convergence
 from lib.topojson import build_topo_from_json, build_config_from_json
 
+
+#TODO: select markers based on daemons used during test
+# pytest module level markers
+"""
+pytestmark = pytest.mark.bfdd # single marker
+pytestmark = [
+	pytest.mark.bgpd,
+	pytest.mark.ospfd,
+	pytest.mark.ospf6d
+] # multiple markers
+"""
+
+
 # Reading the data from JSON File for topology and configuration creation
 jsonFile = "{}/example_topojson.json".format(CWD)
 

tests/topotests/pytest.ini

@@ -1,16 +1,29 @@
 # Skip pytests example directory
 [pytest]
 norecursedirs = .git example-test example-topojson-test lib docker
+
+# Markers
+#
+# Please consult the documentation and discuss with TSC members before applying
+# any changes to this list.
 markers =
-	babel: Tests that run against BABEL
-	bfd: Tests that run against BFDD
-	eigrp: Tests that run against EIGRPD
-	isis: Tests that run against ISISD
-	ldp: Tests that run against LDPD
-	ospf: Tests that run against OSPF( v2 and v3 )
-	pbr: Tests that run against PBRD
-	pim: Tests that run against pim
-	rip: Tests that run against RIP, both v4 and v6
+	babeld: Tests that run against BABELD
+	bfdd: Tests that run against BFDD
+	bgpd: Tests that run against BGPD
+	eigrpd: Tests that run against EIGRPD
+	isisd: Tests that run against ISISD
+	ldpd: Tests that run against LDPD
+	nhrpd: Tests that run against NHRPD
+	ospf6d: Tests that run against OSPF6D
+	ospfd: Tests that run against OSPFD
+	pathd: Tests that run against PATHD
+	pbrd: Tests that run against PBRD
+	pimd: Tests that run against PIMD
+	ripd: Tests that run against RIPD
+	ripngd: Tests that run against RIPNGD
+	sharpd: Tests that run against SHARPD
+	staticd: Tests that run against STATICD
+	vrrpd: Tests that run against VRRPD
 
 [topogen]
 # Default configuration values