From 26732abf54ce72cc67139b349950bd970006f3b2 Mon Sep 17 00:00:00 2001 From: Lou Berger Date: Wed, 26 Jun 2019 17:53:46 +0000 Subject: [PATCH 1/3] bgp_l3vpn_to_bgp_vrf: refactor to try to avoid periodic failure, also collect more info Signed-off-by: Lou Berger --- .../topotests/bgp_l3vpn_to_bgp_vrf/scripts/scale_down.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/topotests/bgp_l3vpn_to_bgp_vrf/scripts/scale_down.py b/tests/topotests/bgp_l3vpn_to_bgp_vrf/scripts/scale_down.py index c17bf1168c..db53c1aacd 100644 --- a/tests/topotests/bgp_l3vpn_to_bgp_vrf/scripts/scale_down.py +++ b/tests/topotests/bgp_l3vpn_to_bgp_vrf/scripts/scale_down.py @@ -6,14 +6,19 @@ if ret != False and found != None: luCommand('ce3', 'vtysh -c "show bgp sum"', '.', 'pass', 'See %s sharp routes' % num) if num > 0: + rtrs = ['ce1', 'ce2', 'ce3'] + for rtr in rtrs: + luCommand(rtr, 'vtysh -c "show bgp ipv4 uni" | grep Display','.', 'none', 'BGP routes pre remove') + luCommand(rtr, 'ip route show | cat -n | tail','.', 'none', 'Linux routes pre remove') wait = 2*num/500 luCommand('ce1', 'vtysh -c "sharp remove routes 10.0.0.0 {}"'.format(num),'.','none','Removing {} routes'.format(num)) luCommand('ce2', 'vtysh -c "sharp remove routes 10.0.0.0 {}"'.format(num),'.','none','Removing {} routes'.format(num)) - rtrs = ['ce1', 'ce2', 'ce3'] for rtr in rtrs: - luCommand(rtr, 'vtysh -c "show bgp ipv4 uni" | grep -c 10\\.\\*/32','^0$', 'wait', 'BGP routes removed', wait) + luCommand(rtr, 'vtysh -c "show bgp ipv4 uni" | grep Display',' 10 route', 'wait', 'BGP routes removed', wait) + luCommand(rtr, 'vtysh -c "show bgp ipv4 uni"','.', 'none', 'BGP routes post remove') for rtr in rtrs: luCommand(rtr, 'ip route show | grep -c \\^10\\.','^0$', 'wait', 'Linux routes removed', wait) + luCommand(rtr, 'ip route show','.', 'none', 'Linux routes post remove') rtrs = ['r1', 'r3', 'r4'] for rtr in rtrs: luCommand(rtr, 'ip route show vrf {}-cust1 | grep -c \\^10\\.'.format(rtr),'^0$','wait','VRF route removed',wait) From 487921759cdfd168b3c15cec17326c184df60401 Mon Sep 17 00:00:00 2001 From: Lou Berger Date: Wed, 26 Jun 2019 17:54:20 +0000 Subject: [PATCH 2/3] bgp_l3vpn_to_bgp_vrf: report zebra memory stats Signed-off-by: Lou Berger --- .../bgp_l3vpn_to_bgp_vrf/scripts/scale_up.py | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/tests/topotests/bgp_l3vpn_to_bgp_vrf/scripts/scale_up.py b/tests/topotests/bgp_l3vpn_to_bgp_vrf/scripts/scale_up.py index 7f114d10df..3619cdf84f 100644 --- a/tests/topotests/bgp_l3vpn_to_bgp_vrf/scripts/scale_up.py +++ b/tests/topotests/bgp_l3vpn_to_bgp_vrf/scripts/scale_up.py @@ -11,14 +11,17 @@ if c > 0: else: d = r wait = 2*num/1000 -mem = {} +mem_z = {} +mem_b = {} rtrs = ['ce1', 'ce2', 'ce3', 'r1', 'r2', 'r3', 'r4'] for rtr in rtrs: - mem[rtr] = {'value': 0, 'units': 'unknown'} - ret = luCommand(rtr, 'vtysh -c "show memory"', 'bgpd: System allocator statistics: Total heap allocated: *(\d*) ([A-Za-z]*)', 'none', 'collect bgpd memory stats') + mem_z[rtr] = {'value': 0, 'units': 'unknown'} + mem_b[rtr] = {'value': 0, 'units': 'unknown'} + ret = luCommand(rtr, 'vtysh -c "show memory"', 'zebra: System allocator statistics: Total heap allocated: *(\d*) ([A-Za-z]*) .*bgpd: System allocator statistics: Total heap allocated: *(\d*) ([A-Za-z]*)', 'none', 'collect bgpd memory stats') found = luLast() if ret != False and found != None: - mem[rtr] = {'value': int(found.group(1)), 'units': found.group(2)} + mem_z[rtr] = {'value': int(found.group(1)), 'units': found.group(2)} + mem_b[rtr] = {'value': int(found.group(3)), 'units': found.group(4)} luCommand('ce1', 'vtysh -c "sharp data nexthop"', 'sharpd is not running', 'none','check if sharpd running') doSharp = True @@ -54,13 +57,20 @@ if doSharp == True: luCommand(rtr, 'ip route show vrf {}-cust1 | grep -c \\^10\\.'.format(rtr), str(num), 'wait','See {} linux routes'.format(num), wait) rtrs = ['ce1', 'ce2', 'ce3', 'r1', 'r2', 'r3', 'r4'] for rtr in rtrs: - ret = luCommand(rtr, 'vtysh -c "show memory"', 'bgpd: System allocator statistics: Total heap allocated: *(\d*) ([A-Za-z]*)', 'none', 'collect bgpd memory stats') + ret = luCommand(rtr, 'vtysh -c "show memory"', 'zebra: System allocator statistics: Total heap allocated: *(\d*) ([A-Za-z]*) .*bgpd: System allocator statistics: Total heap allocated: *(\d*) ([A-Za-z]*)', 'none', 'collect bgpd memory stats') found = luLast() if ret != False and found != None: - val = int(found.group(1)) - if mem[rtr]['units'] != found.group(2): - val *= 1000 - delta = val - int(mem[rtr]['value']) - ave = float(delta)/float(num) - luCommand(rtr, 'vtysh -c "show thread cpu"', '.', 'pass', 'BGPd heap: {0} {1} --> {2} {3} ({4} {1}/route)'.format(mem[rtr]['value'], mem[rtr]['units'], found.group(1), found.group(2), round(ave,4))) + val_z = int(found.group(1)) + if mem_z[rtr]['units'] != found.group(2): + val_z *= 1000 + delta_z = val_z - int(mem_z[rtr]['value']) + ave_z = float(delta_z)/float(num) + + val_b = int(found.group(3)) + if mem_b[rtr]['units'] != found.group(4): + val_b *= 1000 + delta_b = val_b - int(mem_b[rtr]['value']) + ave_b = float(delta_b)/float(num) + luCommand(rtr, 'vtysh -c "show thread cpu"', '.', 'pass', 'BGPd heap: {0} {1} --> {2} {3} ({4} {1}/vpn route)'.format(mem_b[rtr]['value'], mem_b[rtr]['units'], found.group(3), found.group(4), round(ave_b,4))) + luCommand(rtr, 'vtysh -c "show thread cpu"', '.', 'pass', 'Zebra heap: {0} {1} --> {2} {3} ({4} {1}/vpn route)'.format(mem_z[rtr]['value'], mem_z[rtr]['units'], found.group(1), found.group(2), round(ave_z,4))) #done From d850e66733b767b314affbd0e1e04ae1dcd2087b Mon Sep 17 00:00:00 2001 From: Lou Berger Date: Wed, 26 Jun 2019 20:30:59 +0000 Subject: [PATCH 3/3] topotest: reduce chattiness of lutil output Signed-off-by: Lou Berger --- tests/topotests/lib/lutil.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/topotests/lib/lutil.py b/tests/topotests/lib/lutil.py index 3ae18018bf..e58b4725eb 100755 --- a/tests/topotests/lib/lutil.py +++ b/tests/topotests/lib/lutil.py @@ -37,7 +37,7 @@ class lUtil: base_log_dir = '.' fout_name = 'output.log' fsum_name = 'summary.txt' - l_level = 9 + l_level = 6 CallOnFail = False l_total = 0 @@ -53,12 +53,12 @@ class lUtil: fsum = '' net = '' - def log(self, str): + def log(self, str, level=6): if self.l_level > 0: if self.fout == '': self.fout = open(self.fout_name, 'w', 0) self.fout.write(str+'\n') - if self.l_level > 5: + if level <= self.l_level: print(str) def summary(self, str): @@ -226,14 +226,16 @@ Total %-4d %-4d %d\n\ ret = success else: ret = search.group() - self.log('found:%s:' % ret) if op != 'fail': success = True + level = 7 else: success = False + level = 5 + self.log('found:%s:' % ret, level) # Experiment: compare matched strings obtained each way if self.l_dotall_experiment and (group_nl_converted != ret): - self.log('DOTALL experiment: strings differ dotall=[%s] orig=[%s]' % (group_nl_converted, ret)) + self.log('DOTALL experiment: strings differ dotall=[%s] orig=[%s]' % (group_nl_converted, ret), 9) if op == 'pass' or op == 'fail': self.result(target, success, result) if js != None: @@ -265,7 +267,7 @@ LUtil=None #entry calls def luStart(baseScriptDir='.', baseLogDir='.', net='', - fout='output.log', fsum='summary.txt', level=9): + fout='output.log', fsum='summary.txt', level=None): global LUtil #init class LUtil=lUtil() @@ -276,7 +278,8 @@ def luStart(baseScriptDir='.', baseLogDir='.', net='', LUtil.fout_name = baseLogDir + '/' + fout if fsum != None: LUtil.fsum_name = baseLogDir + '/' + fsum - LUtil.l_level = level + if level != None: + LUtil.l_level = level LUtil.l_dotall_experiment = False LUtil.l_dotall_experiment = True @@ -289,11 +292,11 @@ def luCommand(target, command, regexp='.', op='none', result='', time=10, return def luLast(usenl=False): if usenl: if LUtil.l_last_nl != None: - LUtil.log('luLast:%s:' % LUtil.l_last_nl.group()) + LUtil.log('luLast:%s:' % LUtil.l_last_nl.group(), 7) return LUtil.l_last_nl else: if LUtil.l_last != None: - LUtil.log('luLast:%s:' % LUtil.l_last.group()) + LUtil.log('luLast:%s:' % LUtil.l_last.group(), 7) return LUtil.l_last def luInclude(filename, CallOnFail=None):