tests: Adding BGP convergence verification before starting PIM tests

Issue: Sometimes BGP neighbors are not up before doing any PIM
operation, that is causing some tests failures.
https://github.com/FRRouting/frr/issues/14441

Fix: Added BGP convergence for all tests where BGP is used to make
sure all BGP neigbhors

Signed-off-by: Kuldeep Kashyap <kashyapk@vmware.com>

tests/topotests/lib/bgp.py

@@ -1325,7 +1325,7 @@ def verify_router_id(tgen, topo, input_dict, expected=True):
 
 
 @retry(retry_timeout=150)
-def verify_bgp_convergence(tgen, topo=None, dut=None, expected=True):
+def verify_bgp_convergence(tgen, topo=None, dut=None, expected=True, addr_type=None):
     """
     API will verify if BGP is converged with in the given time frame.
     Running "show bgp summary json" command and verify bgp neighbor
@@ -1336,6 +1336,7 @@ def verify_bgp_convergence(tgen, topo=None, dut=None, expected=True):
     * `tgen`: topogen object
     * `topo`: input json file data
     * `dut`: device under test
+    * `addr_type` : address type for which verification to be done, by-default both v4 and v6
 
     Usage
     -----
@@ -1439,20 +1440,27 @@ def verify_bgp_convergence(tgen, topo=None, dut=None, expected=True):
                     return errormsg
             else:
                 total_peer = 0
-                for addr_type in bgp_addr_type.keys():
+                for _addr_type in bgp_addr_type.keys():
-                    if not check_address_types(addr_type):
+                    if not check_address_types(_addr_type):
                         continue
 
-                    bgp_neighbors = bgp_addr_type[addr_type]["unicast"]["neighbor"]
+                    if addr_type and addr_type != _addr_type:
+                        continue
+
+                    bgp_neighbors = bgp_addr_type[_addr_type]["unicast"]["neighbor"]
 
                     for bgp_neighbor in bgp_neighbors:
                         total_peer += len(bgp_neighbors[bgp_neighbor]["dest_link"])
 
                 no_of_peer = 0
-                for addr_type in bgp_addr_type.keys():
+                for _addr_type in bgp_addr_type.keys():
                     if not check_address_types(addr_type):
                         continue
-                    bgp_neighbors = bgp_addr_type[addr_type]["unicast"]["neighbor"]
+
+                    if addr_type and addr_type != _addr_type:
+                        continue
+
+                    bgp_neighbors = bgp_addr_type[_addr_type]["unicast"]["neighbor"]
 
                     for bgp_neighbor, peer_data in bgp_neighbors.items():
                         for dest_link in peer_data["dest_link"].keys():
@@ -1473,7 +1481,7 @@ def verify_bgp_convergence(tgen, topo=None, dut=None, expected=True):
                                 elif "source_link" in peer_details:
                                     neighbor_ip = topo["routers"][bgp_neighbor][
                                         "links"
-                                    ][peer_details["source_link"]][addr_type].split(
+                                    ][peer_details["source_link"]][_addr_type].split(
                                         "/"
                                     )[
                                         0
@@ -1484,12 +1492,12 @@ def verify_bgp_convergence(tgen, topo=None, dut=None, expected=True):
                                 ):
                                     neighbor_ip = data[dest_link]["peer-interface"]
                                 else:
-                                    neighbor_ip = data[dest_link][addr_type].split("/")[
+                                    neighbor_ip = data[dest_link][_addr_type].split(
-                                        0
+                                        "/"
-                                    ]
+                                    )[0]
                                 nh_state = None
                                 neighbor_ip = neighbor_ip.lower()
-                                if addr_type == "ipv4":
+                                if _addr_type == "ipv4":
                                     ipv4_data = show_bgp_json[vrf]["ipv4Unicast"][
                                         "peers"
                                     ]

tests/topotests/multicast_mld_join_topo1/test_multicast_mld_local_join.py

@@ -42,6 +42,10 @@ from lib.pim import (
     verify_pim_rp_info,
     verify_upstream_iif,
 )
+from lib.bgp import (
+    verify_bgp_convergence,
+)
+
 from lib.topogen import Topogen, get_topogen
 from lib.topojson import build_config_from_json
 from lib.topolog import logger
@@ -126,6 +130,13 @@ def setup_module(mod):
 
     # Creating configuration from JSON
     build_config_from_json(tgen, topo)
+
+    # Verify BGP convergence
+    BGP_CONVERGENCE = verify_bgp_convergence(tgen, topo, addr_type="ipv6")
+    assert BGP_CONVERGENCE is True, "setup_module : Failed \n Error:" " {}".format(
+        BGP_CONVERGENCE
+    )
+
     # Verify PIM neighbors
     result = verify_pim_neighbors(tgen, topo)
     assert result is True, " Verify PIM neighbor: Failed Error: {}".format(result)
@@ -177,6 +188,10 @@ def test_mld_local_joins_p0(request):
 
     reset_config_on_routers(tgen)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo, addr_type="ipv6")
+    assert result is True, "Testcase {} : Failed \n Error: {}".format(tc_name, result)
+
     step("configure BGP on R1, R2, R3, R4 and enable redistribute static/connected")
     step("Enable the MLD on R11 interfac of R1 and configure local mld groups")
     intf_r1_i1 = topo["routers"]["r1"]["links"]["i1"]["interface"]
@@ -249,6 +264,10 @@ def test_mroute_with_mld_local_joins_p0(request):
 
     reset_config_on_routers(tgen)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo, addr_type="ipv6")
+    assert result is True, "Testcase {} : Failed \n Error: {}".format(tc_name, result)
+
     app_helper.stop_all_hosts()
 
     step("Enable the PIM on all the interfaces of R1, R2, R3, R4")
@@ -442,6 +461,10 @@ def test_remove_add_mld_local_joins_p1(request):
 
     reset_config_on_routers(tgen)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo, addr_type="ipv6")
+    assert result is True, "Testcase {} : Failed \n Error: {}".format(tc_name, result)
+
     app_helper.stop_all_hosts()
 
     step("Enable the PIM on all the interfaces of R1, R2, R3, R4")
@@ -694,6 +717,10 @@ def test_remove_add_mld_config_with_local_joins_p1(request):
 
     reset_config_on_routers(tgen)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo, addr_type="ipv6")
+    assert result is True, "Testcase {} : Failed \n Error: {}".format(tc_name, result)
+
     app_helper.stop_all_hosts()
 
     step("Enable the PIM on all the interfaces of R1, R2, R3, R4")

tests/topotests/multicast_pim6_sm_topo1/test_multicast_pim6_sm1.py

@@ -61,6 +61,9 @@ from lib.pim import (
     verify_sg_traffic,
     verify_upstream_iif,
 )
+from lib.bgp import (
+    verify_bgp_convergence,
+)
 from lib.topogen import Topogen, get_topogen
 from lib.topojson import build_config_from_json
 from lib.topolog import logger
@@ -140,6 +143,12 @@ def setup_module(mod):
     global app_helper
     app_helper = McastTesterHelper(tgen)
 
+    # Verify BGP convergence
+    BGP_CONVERGENCE = verify_bgp_convergence(tgen, topo, addr_type="ipv6")
+    assert BGP_CONVERGENCE is True, "setup_module : Failed \n Error:" " {}".format(
+        BGP_CONVERGENCE
+    )
+
     logger.info("Running setup_module() done")
 
 
@@ -276,6 +285,10 @@ def test_multicast_data_traffic_static_RP_send_traffic_then_join_p0(request):
     # Creating configuration from JSON
     reset_config_on_routers(tgen)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo, addr_type="ipv6")
+    assert result is True, "Testcase {} : Failed \n Error {}".format(tc_name, result)
+
     app_helper.stop_all_hosts()
 
     # Don"t run this test if we have any failure.
@@ -480,6 +493,10 @@ def test_verify_mroute_when_receiver_is_outside_frr_p0(request):
     # Creating configuration from JSON
     reset_config_on_routers(tgen)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo, addr_type="ipv6")
+    assert result is True, "Testcase {} : Failed \n Error {}".format(tc_name, result)
+
     # Don"t run this test if we have any failure.
     if tgen.routers_have_failure():
         pytest.skip(tgen.errors)
@@ -648,6 +665,10 @@ def test_verify_mroute_when_frr_is_transit_router_p2(request):
     # Creating configuration from JSON
     reset_config_on_routers(tgen)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo, addr_type="ipv6")
+    assert result is True, "Testcase {} : Failed \n Error {}".format(tc_name, result)
+
     app_helper.stop_all_hosts()
 
     # Don"t run this test if we have any failure.
@@ -803,6 +824,10 @@ def test_verify_mroute_when_RP_unreachable_p1(request):
     # Creating configuration from JSON
     reset_config_on_routers(tgen)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo, addr_type="ipv6")
+    assert result is True, "Testcase {} : Failed \n Error {}".format(tc_name, result)
+
     app_helper.stop_all_hosts()
 
     # Don"t run this test if we have any failure.
@@ -929,6 +954,10 @@ def test_modify_mld_query_timer_p0(request):
     # Creating configuration from JSON
     reset_config_on_routers(tgen)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo, addr_type="ipv6")
+    assert result is True, "Testcase {} : Failed \n Error {}".format(tc_name, result)
+
     app_helper.stop_all_hosts()
 
     # Don"t run this test if we have any failure.
@@ -1108,6 +1137,10 @@ def test_modify_mld_max_query_response_timer_p0(request):
     # Creating configuration from JSON
     reset_config_on_routers(tgen)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo, addr_type="ipv6")
+    assert result is True, "Testcase {} : Failed \n Error {}".format(tc_name, result)
+
     app_helper.stop_all_hosts()
 
     # Don"t run this test if we have any failure.
@@ -1377,6 +1410,10 @@ def test_verify_impact_on_multicast_traffic_when_RP_removed_p0(request):
     # Creating configuration from JSON
     reset_config_on_routers(tgen)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo, addr_type="ipv6")
+    assert result is True, "Testcase {} : Failed \n Error {}".format(tc_name, result)
+
     app_helper.stop_all_hosts()
 
     # Don"t run this test if we have any failure.

tests/topotests/multicast_pim6_sm_topo1/test_multicast_pim6_sm2.py

@@ -43,6 +43,9 @@ from lib.pim import (
     verify_sg_traffic,
     verify_upstream_iif,
 )
+from lib.bgp import (
+    verify_bgp_convergence,
+)
 from lib.topogen import Topogen, get_topogen
 from lib.topojson import build_config_from_json
 from lib.topolog import logger
@@ -129,6 +132,12 @@ def setup_module(mod):
     # Creating configuration from JSON
     build_config_from_json(tgen, tgen.json_topo)
 
+    # Verify BGP convergence
+    BGP_CONVERGENCE = verify_bgp_convergence(tgen, topo, addr_type="ipv6")
+    assert BGP_CONVERGENCE is True, "setup_module : Failed \n Error:" " {}".format(
+        BGP_CONVERGENCE
+    )
+
     logger.info("Running setup_module() done")
 
 
@@ -213,6 +222,10 @@ def test_clear_mroute_and_verify_multicast_data_p0(request, app_helper):
     # Creating configuration from JSON
     reset_config_on_routers(tgen)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo, addr_type="ipv6")
+    assert result is True, "Testcase {} : Failed \n Error {}".format(tc_name, result)
+
     app_helper.stop_all_hosts()
 
     # Don"t run this test if we have any failure.
@@ -444,6 +457,10 @@ def test_verify_SPT_switchover_when_RPT_and_SPT_path_is_different_p0(
     # Creating configuration from JSON
     reset_config_on_routers(tgen)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo, addr_type="ipv6")
+    assert result is True, "Testcase {} : Failed \n Error {}".format(tc_name, result)
+
     # Don"t run this test if we have any failure.
     if tgen.routers_have_failure():
         pytest.skip(tgen.errors)

tests/topotests/multicast_pim_uplink_topo1/test_multicast_pim_uplink_topo1.py

@@ -52,7 +52,10 @@ from lib.common_config import (
     create_static_routes,
     required_linux_kernel_version,
 )
-from lib.bgp import create_router_bgp
+from lib.bgp import (
+    create_router_bgp,
+    verify_bgp_convergence,
+)
 from lib.pim import (
     create_pim_config,
     create_igmp_config,
@@ -152,6 +155,12 @@ def setup_module(mod):
     global app_helper
     app_helper = McastTesterHelper(tgen)
 
+    # Verify BGP convergence
+    BGP_CONVERGENCE = verify_bgp_convergence(tgen, topo)
+    assert BGP_CONVERGENCE is True, "setup_module : Failed \n Error:" " {}".format(
+        BGP_CONVERGENCE
+    )
+
     logger.info("Running setup_module() done")
 
 
@@ -191,7 +200,6 @@ def get_interfaces_names(topo):
     """
 
     for link in range(1, 5):
-
         intf = topo["routers"]["r1"]["links"]["r2-link{}".format(link)]["interface"]
         r1_r2_links.append(intf)
 
@@ -401,6 +409,10 @@ def test_mroutes_updated_with_correct_oil_iif_when_receiver_is_in_and_outside_DU
     reset_config_on_routers(tgen)
     clear_pim_interface_traffic(tgen, topo)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo)
+    assert result is True, "Testcase {} : Failed \n Error: {}".format(tc_name, result)
+
     # Don"t run this test if we have any failure.
     if tgen.routers_have_failure():
         pytest.skip(tgen.errors)
@@ -795,6 +807,10 @@ def test_mroutes_updated_with_correct_oil_iif_when_source_is_in_and_outside_DUT_
     reset_config_on_routers(tgen)
     clear_pim_interface_traffic(tgen, topo)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo)
+    assert result is True, "Testcase {} : Failed \n Error: {}".format(tc_name, result)
+
     # Don"t run this test if we have any failure.
     if tgen.routers_have_failure():
         pytest.skip(tgen.errors)
@@ -1164,6 +1180,10 @@ def test_verify_mroutes_forwarding_p0(request):
     reset_config_on_routers(tgen)
     clear_pim_interface_traffic(tgen, topo)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo)
+    assert result is True, "Testcase {} : Failed \n Error: {}".format(tc_name, result)
+
     # Don"t run this test if we have any failure.
     if tgen.routers_have_failure():
         pytest.skip(tgen.errors)
@@ -1478,6 +1498,10 @@ def test_mroutes_updated_correctly_after_source_interface_shut_noshut_p1(request
     reset_config_on_routers(tgen)
     clear_pim_interface_traffic(tgen, topo)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo)
+    assert result is True, "Testcase {} : Failed \n Error: {}".format(tc_name, result)
+
     # Don"t run this test if we have any failure.
     if tgen.routers_have_failure():
         pytest.skip(tgen.errors)
@@ -1838,6 +1862,10 @@ def test_mroutes_updated_correctly_after_receiver_interface_shut_noshut_p1(reque
     reset_config_on_routers(tgen)
     clear_pim_interface_traffic(tgen, topo)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo)
+    assert result is True, "Testcase {} : Failed \n Error: {}".format(tc_name, result)
+
     # Don"t run this test if we have any failure.
     if tgen.routers_have_failure():
         pytest.skip(tgen.errors)
@@ -2153,6 +2181,10 @@ def test_mroutes_updated_after_sending_IGMP_prune_and_join_p1(request):
     reset_config_on_routers(tgen)
     clear_pim_interface_traffic(tgen, topo)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo)
+    assert result is True, "Testcase {} : Failed \n Error: {}".format(tc_name, result)
+
     # Don"t run this test if we have any failure.
     if tgen.routers_have_failure():
         pytest.skip(tgen.errors)
@@ -2412,6 +2444,10 @@ def test_mroutes_updated_after_after_clear_mroute_p1(request):
     reset_config_on_routers(tgen)
     clear_pim_interface_traffic(tgen, topo)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo)
+    assert result is True, "Testcase {} : Failed \n Error: {}".format(tc_name, result)
+
     # Don"t run this test if we have any failure.
     if tgen.routers_have_failure():
         pytest.skip(tgen.errors)
@@ -2601,6 +2637,10 @@ def test_mroutes_updated_after_changing_rp_config_p1(request):
     reset_config_on_routers(tgen)
     clear_pim_interface_traffic(tgen, topo)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo)
+    assert result is True, "Testcase {} : Failed \n Error: {}".format(tc_name, result)
+
     # Don"t run this test if we have any failure.
     if tgen.routers_have_failure():
         pytest.skip(tgen.errors)
@@ -3067,6 +3107,10 @@ def test_mroutes_after_restart_frr_services_p2(request):
     reset_config_on_routers(tgen)
     clear_pim_interface_traffic(tgen, topo)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo)
+    assert result is True, "Testcase {} : Failed \n Error: {}".format(tc_name, result)
+
     # Don"t run this test if we have any failure.
     if tgen.routers_have_failure():
         pytest.skip(tgen.errors)

tests/topotests/multicast_pim_uplink_topo2/test_multicast_pim_uplink_topo2.py

@@ -56,6 +56,9 @@ from lib.pim import (
     verify_pim_interface_traffic,
     McastTesterHelper,
 )
+from lib.bgp import (
+    verify_bgp_convergence,
+)
 from lib.topolog import logger
 from lib.topojson import build_config_from_json
 
@@ -131,6 +134,12 @@ def setup_module(mod):
     global app_helper
     app_helper = McastTesterHelper(tgen)
 
+    # Verify BGP convergence
+    BGP_CONVERGENCE = verify_bgp_convergence(tgen, topo)
+    assert BGP_CONVERGENCE is True, "setup_module : Failed \n Error:" " {}".format(
+        BGP_CONVERGENCE
+    )
+
     logger.info("Running setup_module() done")
 
 
@@ -170,7 +179,6 @@ def get_interfaces_names(topo):
     """
 
     for link in range(1, 5):
-
         intf = topo["routers"]["r1"]["links"]["r2-link{}".format(link)]["interface"]
         r1_r2_links.append(intf)
 
@@ -255,6 +263,10 @@ def test_iif_oil_when_RP_address_changes_from_static_to_BSR_p1(request):
     reset_config_on_routers(tgen)
     clear_pim_interface_traffic(tgen, topo)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo)
+    assert result is True, "Testcase {} : Failed \n Error: {}".format(tc_name, result)
+
     # Don"t run this test if we have any failure.
     if tgen.routers_have_failure():
         pytest.skip(tgen.errors)
@@ -516,6 +528,10 @@ def test_mroute_when_RPT_and_SPT_path_is_different_p1(request):
     reset_config_on_routers(tgen)
     clear_pim_interface_traffic(tgen, topo)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo)
+    assert result is True, "Testcase {} : Failed \n Error: {}".format(tc_name, result)
+
     # Don"t run this test if we have any failure.
     if tgen.routers_have_failure():
         pytest.skip(tgen.errors)
@@ -724,6 +740,10 @@ def test_mroutes_updated_with_correct_oil_iif_after_shut_noshut_upstream_interfa
     reset_config_on_routers(tgen)
     clear_pim_interface_traffic(tgen, topo)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo)
+    assert result is True, "Testcase {} : Failed \n Error: {}".format(tc_name, result)
+
     # Don"t run this test if we have any failure.
     if tgen.routers_have_failure():
         pytest.skip(tgen.errors)
@@ -1094,6 +1114,10 @@ def test_mroutes_updated_with_correct_oil_iif_after_shut_noshut_downstream_inter
     reset_config_on_routers(tgen)
     clear_pim_interface_traffic(tgen, topo)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo)
+    assert result is True, "Testcase {} : Failed \n Error: {}".format(tc_name, result)
+
     # Don"t run this test if we have any failure.
     if tgen.routers_have_failure():
         pytest.skip(tgen.errors)

tests/topotests/multicast_pim_uplink_topo3/test_multicast_pim_uplink_topo3.py

@@ -53,6 +53,9 @@ from lib.pim import (
     verify_local_igmp_groups,
     McastTesterHelper,
 )
+from lib.bgp import (
+    verify_bgp_convergence,
+)
 from lib.topolog import logger
 from lib.topojson import build_config_from_json
 
@@ -181,6 +184,12 @@ def setup_module(mod):
     global app_helper
     app_helper = McastTesterHelper(tgen)
 
+    # Verify BGP convergence
+    BGP_CONVERGENCE = verify_bgp_convergence(tgen, topo)
+    assert BGP_CONVERGENCE is True, "setup_module : Failed \n Error:" " {}".format(
+        BGP_CONVERGENCE
+    )
+
     logger.info("Running setup_module() done")
 
 
@@ -220,7 +229,6 @@ def get_interfaces_names(topo):
     """
 
     for link in range(1, 5):
-
         intf = topo["routers"]["r1"]["links"]["r2-link{}".format(link)]["interface"]
         r1_r2_links.append(intf)
 
@@ -331,6 +339,10 @@ def test_ip_igmp_local_joins_p0(request):
     reset_config_on_routers(tgen)
     clear_pim_interface_traffic(tgen, topo)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo)
+    assert result is True, "Testcase {} : Failed \n Error: {}".format(tc_name, result)
+
     # Don"t run this test if we have any failure.
     if tgen.routers_have_failure():
         pytest.skip(tgen.errors)
@@ -412,6 +424,10 @@ def test_mroute_with_igmp_local_joins_p0(request):
     reset_config_on_routers(tgen)
     clear_pim_interface_traffic(tgen, topo)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo)
+    assert result is True, "Testcase {} : Failed \n Error {}".format(tc_name, result)
+
     # Don"t run this test if we have any failure.
     if tgen.routers_have_failure():
         pytest.skip(tgen.errors)
@@ -606,6 +622,10 @@ def test_igmp_local_join_with_reserved_address_p0(request):
     reset_config_on_routers(tgen)
     clear_pim_interface_traffic(tgen, topo)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo)
+    assert result is True, "Testcase {} : Failed \n Error {}".format(tc_name, result)
+
     # Don"t run this test if we have any failure.
     if tgen.routers_have_failure():
         pytest.skip(tgen.errors)
@@ -671,6 +691,10 @@ def test_remove_add_igmp_local_joins_p1(request):
     reset_config_on_routers(tgen)
     clear_pim_interface_traffic(tgen, topo)
 
+    # Verify BGP convergence
+    result = verify_bgp_convergence(tgen, topo)
+    assert result is True, "Testcase {} : Failed \n Error {}".format(tc_name, result)
+
     # Don"t run this test if we have any failure.
     if tgen.routers_have_failure():
         pytest.skip(tgen.errors)