From a5722d5a78c40286bbb68d3b6b611f05e3807683 Mon Sep 17 00:00:00 2001 From: Donatas Abraitis Date: Fri, 18 Nov 2022 17:22:46 +0200 Subject: [PATCH 1/2] tests: Fail tests immediately if they use too low wait/count values This is for run_and_expect_type and run_and_expect topotests method. Some contributions unintentionally get merged with very low values, that leads to CI failures, let's guard this a bit. Signed-off-by: Donatas Abraitis --- tests/topotests/lib/topotest.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/topotests/lib/topotest.py b/tests/topotests/lib/topotest.py index 5a3f586f8..61cf16944 100644 --- a/tests/topotests/lib/topotest.py +++ b/tests/topotests/lib/topotest.py @@ -355,6 +355,16 @@ def run_and_expect(func, what, count=20, wait=3): else: func_name = func.__name__ + # Just a safety-check to avoid running topotests with very + # small wait/count arguments. + wait_time = wait * count + if wait_time < 5: + assert ( + wait_time >= 5 + ), "Waiting time is too small (count={}, wait={}), adjust timer values".format( + count, wait + ) + logger.info( "'{}' polling started (interval {} secs, maximum {} tries)".format( func_name, wait, count @@ -402,6 +412,16 @@ def run_and_expect_type(func, etype, count=20, wait=3, avalue=None): else: func_name = func.__name__ + # Just a safety-check to avoid running topotests with very + # small wait/count arguments. + wait_time = wait * count + if wait_time < 5: + assert ( + wait_time >= 5 + ), "Waiting time is too small (count={}, wait={}), adjust timer values".format( + count, wait + ) + logger.info( "'{}' polling started (interval {} secs, maximum wait {} secs)".format( func_name, wait, int(wait * count) From d3a6af081ed18ce245ccaa89d6096ac034bb09cf Mon Sep 17 00:00:00 2001 From: Donatas Abraitis Date: Tue, 22 Nov 2022 11:17:27 +0200 Subject: [PATCH 2/2] tests: Set minimum wait time for tests to 5 seconds Signed-off-by: Donatas Abraitis --- .../topotests/isis_lfa_topo1/test_isis_lfa_topo1.py | 2 +- .../isis_tilfa_topo1/test_isis_tilfa_topo1.py | 12 ++++++------ .../topotests/ospf6_gr_topo1/test_ospf6_gr_topo1.py | 4 ++-- tests/topotests/ospf6_topo2/test_ospf6_topo2.py | 2 +- tests/topotests/ospf_gr_topo1/test_ospf_gr_topo1.py | 4 ++-- tests/topotests/pim_basic/test_pim.py | 2 +- .../pim_basic_topo2/test_pim_basic_topo2.py | 4 ++-- tests/topotests/srv6_locator/test_srv6_locator.py | 4 ++-- .../test_srv6_locator.py | 4 ++-- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/topotests/isis_lfa_topo1/test_isis_lfa_topo1.py b/tests/topotests/isis_lfa_topo1/test_isis_lfa_topo1.py index 61caf257e..012814919 100755 --- a/tests/topotests/isis_lfa_topo1/test_isis_lfa_topo1.py +++ b/tests/topotests/isis_lfa_topo1/test_isis_lfa_topo1.py @@ -1054,7 +1054,7 @@ def test_rib_ipv6_step24(): rname, "show ipv6 route isis json", outputs[rname][15]["show_ipv6_route.ref"], - count=5, + count=10, ) diff --git a/tests/topotests/isis_tilfa_topo1/test_isis_tilfa_topo1.py b/tests/topotests/isis_tilfa_topo1/test_isis_tilfa_topo1.py index fa76072b4..213439d3f 100755 --- a/tests/topotests/isis_tilfa_topo1/test_isis_tilfa_topo1.py +++ b/tests/topotests/isis_tilfa_topo1/test_isis_tilfa_topo1.py @@ -831,19 +831,19 @@ def test_rt6_step11(): rname, "show ip route isis json", outputs[rname][11]["show_ip_route.ref"], - count=5, + count=10, ) router_compare_json_output( rname, "show ipv6 route isis json", outputs[rname][11]["show_ipv6_route.ref"], - count=5, + count=10, ) router_compare_json_output( rname, "show mpls table json", outputs[rname][11]["show_mpls_table.ref"], - count=5, + count=10, ) @@ -1028,19 +1028,19 @@ def test_rt6_step14(): rname, "show ip route isis json", outputs[rname][11]["show_ip_route.ref"], - count=5, + count=10, ) router_compare_json_output( rname, "show ipv6 route isis json", outputs[rname][11]["show_ipv6_route.ref"], - count=5, + count=10, ) router_compare_json_output( rname, "show mpls table json", outputs[rname][11]["show_mpls_table.ref"], - count=5, + count=10, ) diff --git a/tests/topotests/ospf6_gr_topo1/test_ospf6_gr_topo1.py b/tests/topotests/ospf6_gr_topo1/test_ospf6_gr_topo1.py index e59333ebd..1876dabed 100755 --- a/tests/topotests/ospf6_gr_topo1/test_ospf6_gr_topo1.py +++ b/tests/topotests/ospf6_gr_topo1/test_ospf6_gr_topo1.py @@ -224,7 +224,7 @@ def check_routers(initial_convergence=False, exiting=None, restarting=None): if restarting != None: tries = 40 else: - tries = 1 + tries = 10 router_compare_json_output( rname, "show ipv6 route ospf json", "show_ipv6_route.json", tries ) @@ -246,7 +246,7 @@ def check_routers(initial_convergence=False, exiting=None, restarting=None): if initial_convergence == True or restarting == rname: tries = 240 else: - tries = 1 + tries = 10 router_compare_json_output( rname, "show ipv6 ospf database json", diff --git a/tests/topotests/ospf6_topo2/test_ospf6_topo2.py b/tests/topotests/ospf6_topo2/test_ospf6_topo2.py index d17aeda3e..f16e8f396 100644 --- a/tests/topotests/ospf6_topo2/test_ospf6_topo2.py +++ b/tests/topotests/ospf6_topo2/test_ospf6_topo2.py @@ -243,7 +243,7 @@ def test_ospf6_default_route(): "show ipv6 route json", {route: [{"metric": metric}]}, ) - _, result = topotest.run_and_expect(test_func, None, count=4, wait=1) + _, result = topotest.run_and_expect(test_func, None, count=5, wait=1) assertmsg = '"{}" convergence failure'.format(router) assert result is None, assertmsg diff --git a/tests/topotests/ospf_gr_topo1/test_ospf_gr_topo1.py b/tests/topotests/ospf_gr_topo1/test_ospf_gr_topo1.py index debf7ad76..429b7dc96 100755 --- a/tests/topotests/ospf_gr_topo1/test_ospf_gr_topo1.py +++ b/tests/topotests/ospf_gr_topo1/test_ospf_gr_topo1.py @@ -233,7 +233,7 @@ def check_routers(initial_convergence=False, exiting=None, restarting=None): if restarting != None: tries = 60 else: - tries = 1 + tries = 10 router_compare_json_output( rname, "show ip route ospf json", "show_ip_route.json", tries ) @@ -252,7 +252,7 @@ def check_routers(initial_convergence=False, exiting=None, restarting=None): if initial_convergence == True or restarting == rname: tries = 240 else: - tries = 1 + tries = 10 router_compare_json_output( rname, "show ip ospf database json", "show_ip_ospf_database.json", tries ) diff --git a/tests/topotests/pim_basic/test_pim.py b/tests/topotests/pim_basic/test_pim.py index 6cea521aa..2c1bc52d0 100644 --- a/tests/topotests/pim_basic/test_pim.py +++ b/tests/topotests/pim_basic/test_pim.py @@ -225,7 +225,7 @@ def test_pim_igmp_report(): test_func = partial( topotest.router_json_cmp, r1, "show ip pim upstream json", expected ) - _, result = topotest.run_and_expect(test_func, None, count=5, wait=0.5) + _, result = topotest.run_and_expect(test_func, None, count=10, wait=0.5) assertmsg = '"{}" JSON output mismatches'.format(r1.name) assert result is None, assertmsg finally: diff --git a/tests/topotests/pim_basic_topo2/test_pim_basic_topo2.py b/tests/topotests/pim_basic_topo2/test_pim_basic_topo2.py index 9506c3c6d..5aa313137 100644 --- a/tests/topotests/pim_basic_topo2/test_pim_basic_topo2.py +++ b/tests/topotests/pim_basic_topo2/test_pim_basic_topo2.py @@ -175,7 +175,7 @@ def test_pim_reconvergence(): "show ip pim neighbor json", {interface: {peer: None}}, ) - _, result = topotest.run_and_expect(test_func, None, count=4, wait=1) + _, result = topotest.run_and_expect(test_func, None, count=5, wait=1) assertmsg = '"{}" PIM convergence failure'.format(router) assert result is None, assertmsg @@ -201,7 +201,7 @@ def test_pim_bfd_profile(): "show bfd peers json", [settings], ) - _, result = topotest.run_and_expect(test_func, None, count=4, wait=1) + _, result = topotest.run_and_expect(test_func, None, count=5, wait=1) assertmsg = '"{}" BFD convergence failure'.format(router) assert result is None, assertmsg diff --git a/tests/topotests/srv6_locator/test_srv6_locator.py b/tests/topotests/srv6_locator/test_srv6_locator.py index bc5fa409d..b918da065 100755 --- a/tests/topotests/srv6_locator/test_srv6_locator.py +++ b/tests/topotests/srv6_locator/test_srv6_locator.py @@ -94,12 +94,12 @@ def test_srv6(): def check_srv6_locator(router, expected_file): func = functools.partial(_check_srv6_locator, router, expected_file) - success, result = topotest.run_and_expect(func, None, count=5, wait=0.5) + success, result = topotest.run_and_expect(func, None, count=10, wait=0.5) assert result is None, "Failed" def check_sharpd_chunk(router, expected_file): func = functools.partial(_check_sharpd_chunk, router, expected_file) - success, result = topotest.run_and_expect(func, None, count=5, wait=0.5) + success, result = topotest.run_and_expect(func, None, count=10, wait=0.5) assert result is None, "Failed" # FOR DEVELOPER: diff --git a/tests/topotests/srv6_locator_custom_bits_length/test_srv6_locator.py b/tests/topotests/srv6_locator_custom_bits_length/test_srv6_locator.py index ecc085637..cc250d5fc 100755 --- a/tests/topotests/srv6_locator_custom_bits_length/test_srv6_locator.py +++ b/tests/topotests/srv6_locator_custom_bits_length/test_srv6_locator.py @@ -90,12 +90,12 @@ def test_srv6(): def check_srv6_locator(router, expected_file): func = functools.partial(_check_srv6_locator, router, expected_file) - success, result = topotest.run_and_expect(func, None, count=5, wait=0.5) + success, result = topotest.run_and_expect(func, None, count=10, wait=0.5) assert result is None, "Failed" def check_sharpd_chunk(router, expected_file): func = functools.partial(_check_sharpd_chunk, router, expected_file) - success, result = topotest.run_and_expect(func, None, count=5, wait=0.5) + success, result = topotest.run_and_expect(func, None, count=10, wait=0.5) assert result is None, "Failed" # FOR DEVELOPER: