(all tests): Add extra check to make sure daemons are still running after each essential stage.

If daemon crashed at a later stage (not startup), then the test scripts didn't properly detect it and report unpredictable errors. This will properly log the daemon crashes

Signed-off-by: Martin Winter <mwinter@opensourcerouting.org>

tests/topotests/all-protocol-startup/test_all_protocol_startup.py

@@ -205,6 +205,11 @@ def test_error_messages_vtysh():
 
         assert vtystderr == '', "Vtysh StdErr Output check failed for router r%s:\n%s" % (i, vtystderr)
 
+    # Make sure that all daemons are running
+    for i in range(1, 2):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     # CLI(net)
 
@@ -291,6 +296,11 @@ def test_converge_protocols():
     # Not really implemented yet - just sleep 60 secs for now
     sleep(60)
 
+    # Make sure that all daemons are running
+    for i in range(1, 2):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     ## CLI(net)
 
@@ -339,6 +349,11 @@ def test_rip_status():
 
             assert failures == 0, "IP RIP status failed for router r%s:\n%s" % (i, diff)
 
+    # Make sure that all daemons are running
+    for i in range(1, 2):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     # CLI(net)
 
@@ -389,6 +404,11 @@ def test_ripng_status():
 
             assert failures == 0, "IPv6 RIPng status failed for router r%s:\n%s" % (i, diff)
 
+    # Make sure that all daemons are running
+    for i in range(1, 2):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     # CLI(net)
 
@@ -443,6 +463,11 @@ def test_ospfv2_interfaces():
 
             assert failures == 0, "SHOW IP OSPF INTERFACE failed for router r%s:\n%s" % (i, diff)
 
+    # Make sure that all daemons are running
+    for i in range(1, 2):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     # CLI(net)
 
@@ -491,6 +516,11 @@ def test_isis_interfaces():
 
             assert failures == 0, "SHOW ISIS INTERFACE DETAIL failed for router r%s:\n%s" % (i, diff)
 
+    # Make sure that all daemons are running
+    for i in range(1, 2):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     # CLI(net)
 
@@ -567,6 +597,11 @@ def test_bgp_summary():
 
             assert failures == 0, "SHOW IP BGP SUMMARY failed for router r%s:\n%s" % (i, diff)
 
+    # Make sure that all daemons are running
+    for i in range(1, 2):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     # CLI(net)
 
@@ -644,6 +679,11 @@ def test_bgp_ipv6_summary():
 
             assert failures == 0, "SHOW BGP IPv6 SUMMARY failed for router r%s:\n%s" % (i, diff)
 
+    # Make sure that all daemons are running
+    for i in range(1, 2):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     # CLI(net)
 
@@ -692,6 +732,11 @@ def test_bgp_ipv4():
 
             assert failures == 0, "SHOW BGP IPv4 failed for router r%s:\n%s" % (i, diff)
 
+    # Make sure that all daemons are running
+    for i in range(1, 2):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     # CLI(net)
 
@@ -740,6 +785,11 @@ def test_bgp_ipv6():
 
             assert failures == 0, "SHOW BGP IPv6 failed for router r%s:\n%s" % (i, diff)
 
+    # Make sure that all daemons are running
+    for i in range(1, 2):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     # CLI(net)
 
@@ -794,6 +844,11 @@ def test_mpls_interfaces():
 
             assert failures == 0, "MPLS LDP Interface status failed for router r%s:\n%s" % (i, diff)
 
+    # Make sure that all daemons are running
+    for i in range(1, 2):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     # CLI(net)
 

tests/topotests/bgp_multiview_topo1/test_bgp_multiview_topo1.py

@@ -251,6 +251,11 @@ def test_bgp_converge():
     #     print("\nwaiting 15s for routes to populate")
     #     sleep(15)
 
+    # Make sure that all daemons are running
+    for i in range(1, 2):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting Quagga/FRR daemons, uncomment the next line
     # CLI(net)
 
@@ -307,6 +312,11 @@ def test_bgp_routingTable():
 
             assert failures == 0, "Routing Table verification failed for router r%s, view %s:\n%s" % (i, view, diff)
 
+    # Make sure that all daemons are running
+    for i in range(1, 2):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     # CLI(net)
 

tests/topotests/ldp-topo1/test_ldp_topo1.py

@@ -252,6 +252,11 @@ def test_mpls_interfaces():
 
             assert failures == 0, "MPLS LDP Interface status failed for router r%s:\n%s" % (i, diff)
 
+    # Make sure that all daemons are running
+    for i in range(1, 5):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     # CLI(net)
 
@@ -313,6 +318,11 @@ def test_mpls_ldp_neighbor_establish():
         print("\nwaiting 15s for LDP sessions to establish")
         sleep(15)
   
+    # Make sure that all daemons are running
+    for i in range(1, 5):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
 
 def test_mpls_ldp_discovery():
     global fatal_error
@@ -360,6 +370,11 @@ def test_mpls_ldp_discovery():
 
             assert failures == 0, "MPLS LDP Interface discovery output for router r%s:\n%s" % (i, diff)
 
+    # Make sure that all daemons are running
+    for i in range(1, 5):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     # CLI(net)
 
@@ -420,6 +435,11 @@ def test_mpls_ldp_neighbor():
 
             assert failures == 0, "MPLS LDP Interface neighbor output for router r%s:\n%s" % (i, diff)
 
+    # Make sure that all daemons are running
+    for i in range(1, 5):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     #CLI(net)
 
@@ -496,6 +516,11 @@ def test_mpls_ldp_binding():
 
             assert failures == 0, "MPLS LDP Interface binding output for router r%s:\n%s" % (i, diff)
 
+    # Make sure that all daemons are running
+    for i in range(1, 5):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     #CLI(net)
 
@@ -549,6 +574,11 @@ def test_zebra_ipv4_routingTable():
 
             assert failures == 0, "IPv4 Zebra Routing Table verification failed for router r%s:\n%s" % (i, diff)
 
+    # Make sure that all daemons are running
+    for i in range(1, 5):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     # CLI(net)
 
@@ -614,6 +644,11 @@ def test_mpls_table():
 
             assert failures == 0, "MPLS table output for router r%s:\n%s" % (i, diff)
 
+    # Make sure that all daemons are running
+    for i in range(1, 5):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     # CLI(net)
 
@@ -713,6 +748,11 @@ def test_linux_mpls_routes():
 
             assert failures == 0, "Linux Kernel MPLS route output for router r%s:\n%s" % (i, diff)
 
+    # Make sure that all daemons are running
+    for i in range(1, 5):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     # CLI(net)
 

tests/topotests/ospf6-topo1/test_ospf6_topo1.py

@@ -208,8 +208,7 @@ def test_router_running():
     print("******************************************\n")
     sleep(5)
 
-    # CLI(net)
-    failedRunning = ""
+    # Make sure that all daemons are running
     for i in range(1, 5):
         fatal_error = net['r%s' % i].checkRouterRunning()
         assert fatal_error == "", fatal_error
@@ -259,6 +258,11 @@ def test_ospf6_converged():
         print("\nwaiting 15s for routes to populate")
         sleep(15)
 
+    # Make sure that all daemons are still running
+    for i in range(1, 5):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
 def test_ospfv3_routingTable():
     global fatal_error
     global net
@@ -304,6 +308,11 @@ def test_ospfv3_routingTable():
 
             assert failures == 0, "OSPFv3 (IPv6) Routing Table verification failed for router r%s:\n%s" % (i, diff)
 
+    # Make sure that all daemons are still running
+    for i in range(1, 5):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     # CLI(net)
 

tests/topotests/ripng-topo1/test_ripng_topo1.py

@@ -173,6 +173,11 @@ def test_converge_protocols():
     # Not really implemented yet - just sleep 60 secs for now
     sleep(60)
 
+    # Make sure that all daemons are running
+    for i in range(1, 4):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     #CLI(net)
 
@@ -224,6 +229,11 @@ def test_ripng_status():
 
             assert failures == 0, "IPv6 RIPng status failed for router r%s:\n%s" % (i, diff)
 
+    # Make sure that all daemons are running
+    for i in range(1, 4):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     # CLI(net)
 
@@ -276,6 +286,11 @@ def test_ripng_routes():
 
             assert failures == 0, "SHOW IPv6 RIPng failed for router r%s:\n%s" % (i, diff)
 
+    # Make sure that all daemons are running
+    for i in range(1, 4):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     # CLI(net)
 
@@ -325,6 +340,11 @@ def test_zebra_ipv6_routingTable():
 
             assert failures == 0, "Zebra IPv6 Routing Table verification failed for router r%s:\n%s" % (i, diff)
 
+    # Make sure that all daemons are running
+    for i in range(1, 4):
+        fatal_error = net['r%s' % i].checkRouterRunning()
+        assert fatal_error == "", fatal_error
+
     # For debugging after starting FRR/Quagga daemons, uncomment the next line
     # CLI(net)