tests: update the test template and doc

- Update the template and documentation to use newer pytest fixutres for
setup and teardown, as well as skipping tests when the suite fails.

Signed-off-by: Christian Hopps <chopps@labn.net>

doc/developer/topotests.rst

@@ -983,22 +983,20 @@ Writing Tests
 """""""""""""
 
 Test topologies should always be bootstrapped from
-:file:`tests/topotests/example-test/test_template.py` because it contains
+:file:`tests/topotests/example_test/test_template.py` because it contains
 important boilerplate code that can't be avoided, like:
 
 Example:
 
 .. code:: py
 
-   # For all registered routers, load the zebra configuration file
+       # For all routers arrange for:
-   CWD = os.path.dirname(os.path.realpath(__file__))
+       # - starting zebra using config file from <rtrname>/zebra.conf
+       # - starting ospfd using an empty config file.
        for rname, router in router_list.items():
-       router.load_config(
+           router.load_config(TopoRouter.RD_ZEBRA, "zebra.conf")
-           TopoRouter.RD_ZEBRA,
+           router.load_config(TopoRouter.RD_OSPF)
-           os.path.join(CWD, '{}/zebra.conf'.format(rname))
+
-       )
-       # os.path.join() joins the CWD string with arguments adding the necessary
-       # slashes ('/'). Arguments must not begin with '/'.
 
 - The topology definition or build function
 
@@ -1013,27 +1011,31 @@ Example:
        # topology build code
        ...
 
-- pytest ``setup_module()`` and ``teardown_module()`` to start the topology
+- pytest setup/teardown fixture to start the topology and supply `tgen` argument
+  to tests.
 
 .. code:: py
 
-   def setup_module(module):
+
+   @pytest.fixture(scope="module")
+   def tgen(request):
+       "Setup/Teardown the environment and provide tgen argument to tests"
+
        tgen = Topogen(topodef, module.__name__)
        # or
        tgen = Topogen(build_topo, module.__name__)
 
-       tgen.start_topology('debug')
+       ...
 
-   def teardown_module(_m):
+       # Start and configure the router daemons
-       tgen = get_topogen()
+       tgen.start_router()
+
+       # Provide tgen as argument to each test function
+       yield tgen
+
+       # Teardown after last test runs
        tgen.stop_topology()
 
-- ``__main__`` initialization code (to support running the script directly)
-
-.. code:: py
-
-   if __name__ == '__main__':
-       sys.exit(pytest.main(["-s"]))
 
 Requirements:
 

tests/topotests/example_test/r1/zebra.conf

@@ -0,0 +1,8 @@
+interface r1-eth0
+  ip address 192.168.1.1/24
+
+interface r1-eth1
+  ip address 192.168.2.1/24
+
+interface r1-eth2
+  ip address 192.168.3.1/24

tests/topotests/example_test/r2/zebra.conf

@@ -0,0 +1,4 @@
+interface r2-eth0
+  ip address 192.168.1.2/24
+interface r2-eth1
+  ip address 192.168.3.2/24

tests/topotests/example_test/test_template.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-
+# -*- coding: utf-8 eval: (blacken-mode 1) -*-
 #
 # <template>.py
 # Part of NetDEF Topology Tests
@@ -29,54 +29,69 @@
 import sys
 import pytest
 
-# Import topogen and topotest helpers
+from lib.topogen import Topogen, TopoRouter
-from lib.topogen import Topogen, TopoRouter, get_topogen
+from lib.topolog import logger
-
 
 # TODO: select markers based on daemons used during test
 # pytest module level markers
-"""
-pytestmark = pytest.mark.bfdd # single marker
 pytestmark = [
-	pytest.mark.bgpd,
+    # pytest.mark.babeld,
+    # pytest.mark.bfdd,
+    # pytest.mark.bgpd,
+    # pytest.mark.eigrpd,
+    # pytest.mark.isisd,
+    # pytest.mark.ldpd,
+    # pytest.mark.nhrpd,
+    # pytest.mark.ospf6d,
     pytest.mark.ospfd,
-	pytest.mark.ospf6d
+    # pytest.mark.pathd,
-] # multiple markers
+    # pytest.mark.pbrd,
-"""
+    # pytest.mark.pimd,
-
+    # pytest.mark.ripd,
+    # pytest.mark.ripngd,
+    # pytest.mark.sharpd,
+    # pytest.mark.staticd,
+    # pytest.mark.vrrpd,
+]
 
+# Function we pass to Topogen to create the topology
 def build_topo(tgen):
     "Build function"
 
     # Create 2 routers
-    for routern in range(1, 3):
+    r1 = tgen.add_router("r1")
-        tgen.add_router("r{}".format(routern))
+    r2 = tgen.add_router("r2")
 
-    # Create a switch with just one router connected to it to simulate a
+    # Create a p2p connection between r1 and r2
-    # empty network.
+    tgen.add_link(r1, r2)
+
+    # Create a switch with one router connected to it to simulate a empty network.
     switch = tgen.add_switch("s1")
-    switch.add_link(tgen.gears["r1"])
+    switch.add_link(r1)
 
-    # Create a connection between r1 and r2
+    # Create a p2p connection between r1 and r2
     switch = tgen.add_switch("s2")
-    switch.add_link(tgen.gears["r1"])
+    switch.add_link(r1)
-    switch.add_link(tgen.gears["r2"])
+    switch.add_link(r2)
 
 
-def setup_module(mod):
+# New form of setup/teardown using pytest fixture
-    "Sets up the pytest environment"
+@pytest.fixture(scope="module")
+def tgen(request):
+    "Setup/Teardown the environment and provide tgen argument to tests"
 
     # This function initiates the topology build with Topogen...
-    tgen = Topogen(build_topo, mod.__name__)
+    tgen = Topogen(build_topo, request.module.__name__)
 
-    # The basic topology above could also have be more easily specified using a
+    # A basic topology similar to the above could also have be more easily specified
-    # dictionary, remove the build_topo function and use the following instead:
+    # using a # dictionary, remove the build_topo function and use the following
+    # instead:
     #
     # topodef = {
     #     "s1": "r1"
     #     "s2": ("r1", "r2")
     # }
-    # tgen = Topogen(topodef, mod.__name__)
+    # tgen = Topogen(topodef, request.module.__name__)
 
     # ... and here it calls initialization functions.
     tgen.start_topology()
@@ -84,42 +99,69 @@ def setup_module(mod):
     # This is a sample of configuration loading.
     router_list = tgen.routers()
 
-    # For all registred routers, load the zebra configuration file
+    # For all routers arrange for:
-    # CWD = os.path.dirname(os.path.realpath(__file__))
+    # - starting zebra using config file from <rtrname>/zebra.conf
+    # - starting ospfd using an empty config file.
     for rname, router in router_list.items():
-        router.load_config(
+        router.load_config(TopoRouter.RD_ZEBRA, "zebra.conf")
-            TopoRouter.RD_ZEBRA,
+        router.load_config(TopoRouter.RD_OSPF)
-            # Uncomment next line to load configuration from ./router/zebra.conf
-            # os.path.join(CWD, '{}/zebra.conf'.format(rname))
-        )
 
-    # After loading the configurations, this function loads configured daemons.
+    # Start and configure the router daemons
     tgen.start_router()
 
+    # Provide tgen as argument to each test function
+    yield tgen
 
-def teardown_module(mod):
+    # Teardown after last test runs
-    "Teardown the pytest environment"
-    tgen = get_topogen()
-
-    # This function tears down the whole topology.
     tgen.stop_topology()
 
 
-def test_call_cli():
+# Fixture that executes before each test
-    "Dummy test that just calls tgen.cli() so we can interact with the build."
+@pytest.fixture(autouse=True)
-    tgen = get_topogen()
+def skip_on_failure(tgen):
-    # Don't run this test if we have any failure.
     if tgen.routers_have_failure():
-        pytest.skip(tgen.errors)
+        pytest.skip("skipped because of previous test failure")
 
-    # logger.info("calling CLI")
+
-    # tgen.cli()
+# ===================
+# The tests functions
+# ===================
+
+
+def test_get_version(tgen):
+    "Test the logs the FRR version"
+
+    r1 = tgen.gears["r1"]
+    version = r1.vtysh_cmd("show version")
+    logger.info("FRR version is: " + version)
+
+
+def test_connectivity(tgen):
+    "Test the logs the FRR version"
+
+    r1 = tgen.gears["r1"]
+    r2 = tgen.gears["r2"]
+    output = r1.cmd_raises("ping -c1 192.168.1.2")
+    output = r2.cmd_raises("ping -c1 192.168.3.1")
+
+
+@pytest.mark.xfail
+def test_expect_failure(tgen):
+    "A test that is current expected to fail but should be fixed"
+
+    assert False, "Example of temporary expected failure that will eventually be fixed"
+
+
+@pytest.mark.skip
+def test_will_be_skipped(tgen):
+    "A test that will be skipped"
+    assert False
 
 
 # Memory leak test template
-def test_memory_leak():
+def test_memory_leak(tgen):
     "Run the memory leak test and report results."
-    tgen = get_topogen()
+
     if not tgen.is_memleak_enabled():
         pytest.skip("Memory leak test/report is disabled")