diff --git a/tests/topotests/lib/common_config.py b/tests/topotests/lib/common_config.py index 1f2acb7f88..21ed47fc4b 100644 --- a/tests/topotests/lib/common_config.py +++ b/tests/topotests/lib/common_config.py @@ -39,10 +39,9 @@ import socket import ipaddr from lib.topolog import logger, logger_config -from lib.topogen import TopoRouter +from lib.topogen import TopoRouter, get_topogen from lib.topotest import interface_set_status - FRRCFG_FILE = "frr_json.conf" FRRCFG_BKUP_FILE = "frr_json_initial.conf" @@ -475,25 +474,19 @@ def reset_config_on_routers(tgen, routerName=None): for line in t_delta.split("\n"): line = line.strip() - if ( - line == "Lines To Delete" - or line == "===============" - or not line - ): + if line == "Lines To Delete" or line == "===============" or not line: continue - if (line == "Lines To Add"): + if line == "Lines To Add": check_debug = False continue - if (line == "============" - or not line - ): + if line == "============" or not line: continue # Leave debugs and log output alone if check_debug: - if ('debug' in line or 'log file' in line): + if "debug" in line or "log file" in line: continue delta.write(line) @@ -648,6 +641,33 @@ def get_frr_ipv6_linklocal(tgen, router, intf=None, vrf=None): return errormsg +def generate_support_bundle(): + """ + API to generate support bundle on any verification ste failure. + it runs a python utility, /usr/lib/frr/generate_support_bundle.py, + which basically runs defined CLIs and dumps the data to specified location + """ + + tgen = get_topogen() + router_list = tgen.routers() + test_name = sys._getframe(2).f_code.co_name + TMPDIR = os.path.join(LOGDIR, tgen.modname) + + for rname, rnode in router_list.iteritems(): + logger.info("Generating support bundle for {}".format(rname)) + rnode.run("mkdir -p /var/log/frr") + bundle_log = rnode.run("python2 /usr/lib/frr/generate_support_bundle.py") + logger.info(bundle_log) + + dst_bundle = "{}/{}/support_bundles/{}".format(TMPDIR, rname, test_name) + src_bundle = "/var/log/frr" + rnode.run("rm -rf {}".format(dst_bundle)) + rnode.run("mkdir -p {}".format(dst_bundle)) + rnode.run("mv -f {}/* {}".format(src_bundle, dst_bundle)) + + return True + + def start_topology(tgen): """ Starting topology, create tmp files which are loaded to routers @@ -1202,9 +1222,11 @@ def retry(attempts=3, wait=2, return_is_str=True, initial_wait=0, return_is_dict return ret if _attempts == i: + generate_support_bundle() return ret except Exception as err: if _attempts == i: + generate_support_bundle() logger.info("Max number of attempts (%r) reached", _attempts) raise else: @@ -1278,9 +1300,11 @@ def create_interfaces_cfg(tgen, topo, build=False): # Include vrf if present if "vrf" in data: - interface_data.append("interface {} vrf {}".format( - str(interface_name), - str(data["vrf"]))) + interface_data.append( + "interface {} vrf {}".format( + str(interface_name), str(data["vrf"]) + ) + ) else: interface_data.append("interface {}".format(str(interface_name))) diff --git a/tests/topotests/lib/topotest.py b/tests/topotests/lib/topotest.py index b65acad23e..22ed4b4d0f 100644 --- a/tests/topotests/lib/topotest.py +++ b/tests/topotests/lib/topotest.py @@ -1168,7 +1168,6 @@ class Router(Node): return self.startRouterDaemons() - def getStdErr(self, daemon): return self.getLog("err", daemon) @@ -1181,6 +1180,13 @@ class Router(Node): def startRouterDaemons(self, daemons=None): "Starts all FRR daemons for this router." + bundle_data = subprocess.check_output( + ["cat /etc/frr/support_bundle_commands.conf"], shell=True + ) + self.cmd( + "echo '{}' > /etc/frr/support_bundle_commands.conf".format(bundle_data) + ) + # Starts actual daemons without init (ie restart) # cd to per node directory self.cmd("cd {}/{}".format(self.logdir, self.name)) @@ -1192,7 +1198,7 @@ class Router(Node): # XXX: glue code forward ported from removed function. if self.version == None: self.version = self.cmd( - os.path.join(self.daemondir, 'bgpd') + ' -v' + os.path.join(self.daemondir, "bgpd") + " -v" ).split()[2] logger.info("{}: running version: {}".format(self.name, self.version)) @@ -1206,7 +1212,7 @@ class Router(Node): daemons_list.append(daemon) # Start Zebra first - if 'zebra' in daemons_list: + if "zebra" in daemons_list: zebra_path = os.path.join(self.daemondir, "zebra") zebra_option = self.daemons_options["zebra"] self.cmd( @@ -1217,11 +1223,11 @@ class Router(Node): logger.debug("{}: {} zebra started".format(self, self.routertype)) # Remove `zebra` so we don't attempt to start it again. - while 'zebra' in daemons_list: - daemons_list.remove('zebra') + while "zebra" in daemons_list: + daemons_list.remove("zebra") # Start staticd next if required - if 'staticd' in daemons_list: + if "staticd" in daemons_list: staticd_path = os.path.join(self.daemondir, "staticd") staticd_option = self.daemons_options["staticd"] self.cmd( @@ -1232,8 +1238,8 @@ class Router(Node): logger.debug("{}: {} staticd started".format(self, self.routertype)) # Remove `staticd` so we don't attempt to start it again. - while 'staticd' in daemons_list: - daemons_list.remove('staticd') + while "staticd" in daemons_list: + daemons_list.remove("staticd") # Fix Link-Local Addresses # Somehow (on Mininet only), Zebra removes the IPv6 Link-Local addresses on start. Fix this @@ -1256,18 +1262,18 @@ class Router(Node): logger.debug("{}: {} {} started".format(self, self.routertype, daemon)) # Check if daemons are running. - rundaemons = self.cmd('ls -1 /var/run/%s/*.pid' % self.routertype) + rundaemons = self.cmd("ls -1 /var/run/%s/*.pid" % self.routertype) if re.search(r"No such file or directory", rundaemons): return "Daemons are not running" return "" - - def killRouterDaemons(self, daemons, wait=True, assertOnError=True, - minErrorVersion='5.1'): + def killRouterDaemons( + self, daemons, wait=True, assertOnError=True, minErrorVersion="5.1" + ): # Kill Running Quagga or FRR specific # Daemons(user specified daemon only) using SIGKILL - rundaemons = self.cmd('ls -1 /var/run/%s/*.pid' % self.routertype) + rundaemons = self.cmd("ls -1 /var/run/%s/*.pid" % self.routertype) errors = "" daemonsNotRunning = [] if re.search(r"No such file or directory", rundaemons): @@ -1277,45 +1283,54 @@ class Router(Node): numRunning = 0 for d in StringIO.StringIO(rundaemons): if re.search(r"%s" % daemon, d): - daemonpid = self.cmd('cat %s' % d.rstrip()).rstrip() - if (daemonpid.isdigit() and pid_exists(int(daemonpid))): - logger.info('{}: killing {}'.format( - self.name, - os.path.basename(d.rstrip().rsplit(".", 1)[0]) - )) - self.cmd('kill -9 %s' % daemonpid) + daemonpid = self.cmd("cat %s" % d.rstrip()).rstrip() + if daemonpid.isdigit() and pid_exists(int(daemonpid)): + logger.info( + "{}: killing {}".format( + self.name, + os.path.basename(d.rstrip().rsplit(".", 1)[0]), + ) + ) + self.cmd("kill -9 %s" % daemonpid) self.waitOutput() if pid_exists(int(daemonpid)): numRunning += 1 if wait and numRunning > 0: - sleep(2, '{}: waiting for {} daemon to be stopped'.\ - format(self.name, daemon)) + sleep( + 2, + "{}: waiting for {} daemon to be stopped".format( + self.name, daemon + ), + ) # 2nd round of kill if daemons didn't exit for d in StringIO.StringIO(rundaemons): if re.search(r"%s" % daemon, d): - daemonpid = \ - self.cmd('cat %s' % d.rstrip()).rstrip() - if (daemonpid.isdigit() and pid_exists( - int(daemonpid))): - logger.info('{}: killing {}'.format( - self.name, - os.path.basename(d.rstrip().\ - rsplit(".", 1)[0]) - )) - self.cmd('kill -9 %s' % daemonpid) + daemonpid = self.cmd("cat %s" % d.rstrip()).rstrip() + if daemonpid.isdigit() and pid_exists( + int(daemonpid) + ): + logger.info( + "{}: killing {}".format( + self.name, + os.path.basename( + d.rstrip().rsplit(".", 1)[0] + ), + ) + ) + self.cmd("kill -9 %s" % daemonpid) self.waitOutput() - self.cmd('rm -- {}'.format(d.rstrip())) + self.cmd("rm -- {}".format(d.rstrip())) if wait: errors = self.checkRouterCores(reportOnce=True) - if self.checkRouterVersion('<', minErrorVersion): - #ignore errors in old versions + if self.checkRouterVersion("<", minErrorVersion): + # ignore errors in old versions errors = "" if assertOnError and len(errors) > 0: assert "Errors found - details follow:" == 0, errors else: daemonsNotRunning.append(daemon) if len(daemonsNotRunning) > 0: - errors = errors+"Daemons are not running", daemonsNotRunning + errors = errors + "Daemons are not running", daemonsNotRunning return errors