proxmox-mirrors/mirror_frr
1
0
Fork 0
mirror of https://git.proxmox.com/git/mirror_frr synced 2025-08-04 11:33:48 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

tests: enable code coverage reporting with topotests

Browse Source 
Signed-off-by: Christian Hopps <chopps@labn.net>
This commit is contained in:
Christian Hopps 2024-03-17 04:27:31 -04:00
parent 387a035b66
commit 71368f9e06
4 changed files with 144 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
doc/developer/topotests.rst
View File

@ -704,6 +704,44 @@ Here's an example of collecting ``rr`` execution state from ``mgmtd`` on router
To specify additional arguments for ``rr record``, one can use the
``--rr-options``.
.. _code_coverage:
Code coverage
"""""""""""""
Code coverage reporting requires installation of the ``gcov`` and ``lcov``
packages.
Code coverage can automatically be gathered for any topotest run. To support
this FRR must first be compiled with the ``--enable-gcov`` configure option.
This will cause *.gnco files to be created during the build. When topotests are
run the statistics are generated and stored in *.gcda files. Topotest
infrastructure will gather these files, capture the information into a
``coverage.info`` ``lcov`` file and also report the coverage summary.
To enable code coverage support pass the ``--cov-topotest`` argument to pytest.
If you build your FRR in a directory outside of the FRR source directory you
will also need to pass the ``--cov-frr-build-dir`` argument specifying the build
directory location.
During the topotest run the *.gcda files are generated into a ``gcda``
sub-directory of the top-level run directory (i.e., normally
``/tmp/topotests/gcda``). These files will then be copied at the end of the
topotest run into the FRR build directory where the ``gcov`` and ``lcov``
utilities expect to find them. This is done to deal with the various different
file ownership and permissions.
At the end of the run ``lcov`` will be run to capture all of the coverage data
into a ``coverage.info`` file. This file will be located in the top-level run
directory (i.e., normally ``/tmp/topotests/coverage.info``).
The ``coverage.info`` file can then be used to generate coverage reports or file
markup (e.g., using the ``genhtml`` utility) or enable markup within your
IDE/editor if supported (e.g., the emacs ``cov-mode`` package)
NOTE: the *.gcda files in ``/tmp/topotests/gcda`` are cumulative so if you do
not remove them they will aggregate data across multiple topotest runs.
.. _topotests_docker:
Running Tests with Docker

7
lib/libfrr.c
View File

@ -319,7 +319,12 @@ void frr_preinit(struct frr_daemon_info *daemon, int argc, char **argv)
char *p = strrchr(argv[0], '/');
di->progname = p ? p + 1 : argv[0];
umask(0027);
if (!getenv("GCOV_PREFIX"))
umask(0027);
else {
/* If we are profiling use a more generous umask */
umask(0002);
}
log_args_init(daemon->early_logging);

104
tests/topotests/conftest.py
View File

@ -68,6 +68,10 @@ def log_handler(basename, logpath):
topolog.logfinish(basename, logpath)
def is_main_runner():
return "PYTEST_XDIST_WORKER" not in os.environ
def pytest_addoption(parser):
"""
Add topology-only option to the topology tester. This option makes pytest
@ -85,6 +89,17 @@ def pytest_addoption(parser):
help="Mininet cli on test failure",
)
parser.addoption(
"--cov-topotest",
action="store_true",
help="Enable reporting of coverage",
)
parser.addoption(
"--cov-frr-build-dir",
help="Dir of coverage-enable build being run, default is the source dir",
)
parser.addoption(
"--gdb-breakpoints",
metavar="SYMBOL[,SYMBOL...]",
@ -456,6 +471,37 @@ def pytest_assertrepr_compare(op, left, right):
return json_result.gen_report()
def setup_coverage(config):
commander = Commander("pytest")
if config.option.cov_frr_build_dir:
bdir = Path(config.option.cov_frr_build_dir).resolve()
output = commander.cmd_raises(f"find {bdir} -name zebra_nb.gcno").strip()
else:
# Support build sub-directory of main source dir
bdir = Path(__file__).resolve().parent.parent.parent
output = commander.cmd_raises(f"find {bdir} -name zebra_nb.gcno").strip()
m = re.match(f"({bdir}.*)/zebra/zebra_nb.gcno", output)
if not m:
logger.warning(
"No coverage data files (*.gcno) found, try specifying --cov-frr-build-dir"
)
return
bdir = Path(m.group(1))
# Save so we can get later from g_pytest_config
rundir = Path(config.option.rundir).resolve()
gcdadir = rundir / "gcda"
os.environ["FRR_BUILD_DIR"] = str(bdir)
os.environ["GCOV_PREFIX_STRIP"] = str(len(bdir.parts) - 1)
os.environ["GCOV_PREFIX"] = str(gcdadir)
if is_main_runner():
commander.cmd_raises(f"find {bdir} -name '*.gc??' -exec chmod o+rw {{}} +")
commander.cmd_raises(f"mkdir -p {gcdadir}")
commander.cmd_raises(f"chown -R root:frr {gcdadir}")
commander.cmd_raises(f"chmod 2775 {gcdadir}")
def pytest_configure(config):
"""
Assert that the environment is correctly configured, and get extra config.
@ -556,8 +602,6 @@ def pytest_configure(config):
if config.option.topology_only and is_xdist:
pytest.exit("Cannot use --topology-only with distributed test mode")
pytest.exit("Cannot use --topology-only with distributed test mode")
# Check environment now that we have config
if not diagnose_env(rundir):
pytest.exit("environment has errors, please read the logs in %s" % rundir)
@ -572,27 +616,25 @@ def pytest_configure(config):
if "TOPOTESTS_CHECK_STDERR" in os.environ:
del os.environ["TOPOTESTS_CHECK_STDERR"]
if config.option.cov_topotest:
setup_coverage(config)
@pytest.fixture(autouse=True, scope="session")
def setup_session_auto():
def session_autouse():
# Aligns logs nicely
logging.addLevelName(logging.WARNING, " WARN")
logging.addLevelName(logging.INFO, " INFO")
if "PYTEST_TOPOTEST_WORKER" not in os.environ:
is_worker = False
elif not os.environ["PYTEST_TOPOTEST_WORKER"]:
is_worker = False
else:
is_worker = True
is_main = is_main_runner()
logger.debug("Before the run (is_worker: %s)", is_worker)
if not is_worker:
logger.debug("Before the run (is_main: %s)", is_main)
if is_main:
cleanup_previous()
yield
if not is_worker:
if is_main:
cleanup_current()
logger.debug("After the run (is_worker: %s)", is_worker)
logger.debug("After the run (is_main: %s)", is_main)
def pytest_runtest_setup(item):
@ -719,6 +761,42 @@ def pytest_runtest_makereport(item, call):
pause_test()
def coverage_finish(terminalreporter, config):
commander = Commander("pytest")
rundir = Path(config.option.rundir).resolve()
bdir = Path(os.environ["FRR_BUILD_DIR"])
gcdadir = Path(os.environ["GCOV_PREFIX"])
# Get the data files into the build directory
logger.info("Copying gcda files from '%s' to '%s'", gcdadir, bdir)
user = os.environ.get("SUDO_USER", os.environ["USER"])
commander.cmd_raises(f"chmod -R ugo+r {gcdadir}")
commander.cmd_raises(
f"tar -C {gcdadir} -cf - . | su {user} -c 'tar -C {bdir} -xf -'"
)
# Get the results into a summary file
data_file = rundir / "coverage.info"
logger.info("Gathering coverage data into: %s", data_file)
commander.cmd_raises(f"lcov --directory {bdir} --capture --output-file {data_file}")
# Get coverage info filtered to a specific set of files
report_file = rundir / "coverage.info"
logger.debug("Generating coverage summary from: %s\n%s", report_file)
output = commander.cmd_raises(f"lcov --summary {data_file}")
logger.info("\nCOVERAGE-SUMMARY-START\n%s\nCOVERAGE-SUMMARY-END", output)
terminalreporter.write(
f"\nCOVERAGE-SUMMARY-START\n{output}\nCOVERAGE-SUMMARY-END\n"
)
def pytest_terminal_summary(terminalreporter, exitstatus, config):
# Only run if we are the top level test runner
is_xdist_worker = "PYTEST_XDIST_WORKER" in os.environ
if config.option.cov_topotest and not is_xdist_worker:
coverage_finish(terminalreporter, config)
#
# Add common fixtures available to all tests as parameters
#

26
tests/topotests/lib/topotest.py
View File

@ -27,6 +27,7 @@ import time
import logging
from collections.abc import Mapping
from copy import deepcopy
from pathlib import Path
import lib.topolog as topolog
from lib.micronet_compat import Node
@ -1523,7 +1524,7 @@ class Router(Node):
pass
return ret
def stopRouter(self, assertOnError=True, minErrorVersion="5.1"):
def stopRouter(self, assertOnError=True):
# Stop Running FRR Daemons
running = self.listDaemons()
if not running:
@ -1570,9 +1571,6 @@ class Router(Node):
)
errors = self.checkRouterCores(reportOnce=True)
if self.checkRouterVersion("<", minErrorVersion):
# ignore errors in old versions
errors = ""
if assertOnError and (errors is not None) and len(errors) > 0:
assert "Errors found - details follow:" == 0, errors
return errors
@ -1803,6 +1801,8 @@ class Router(Node):
"Starts FRR daemons for this router."
asan_abort = bool(g_pytest_config.option.asan_abort)
cov_option = bool(g_pytest_config.option.cov_topotest)
cov_dir = Path(g_pytest_config.option.rundir) / "gcda"
gdb_breakpoints = g_pytest_config.get_option_list("--gdb-breakpoints")
gdb_daemons = g_pytest_config.get_option_list("--gdb-daemons")
gdb_routers = g_pytest_config.get_option_list("--gdb-routers")
@ -1836,13 +1836,6 @@ class Router(Node):
# Re-enable to allow for report per run
self.reportCores = True
# XXX: glue code forward ported from removed function.
if self.version is None:
self.version = self.cmd(
os.path.join(self.daemondir, "bgpd") + " -v"
).split()[2]
logger.info("{}: running version: {}".format(self.name, self.version))
perfds = {}
perf_options = g_pytest_config.get_option("--perf-options", "-g")
for perf in g_pytest_config.get_option("--perf", []):
@ -1928,6 +1921,10 @@ class Router(Node):
self.logdir, self.name, daemon
)
if cov_option:
scount = os.environ["GCOV_PREFIX_STRIP"]
cmdenv += f"GCOV_PREFIX_STRIP={scount} GCOV_PREFIX={cov_dir}"
if valgrind_memleaks:
this_dir = os.path.dirname(
os.path.abspath(os.path.realpath(__file__))
@ -2277,9 +2274,7 @@ class Router(Node):
rc, o, e = self.cmd_status("kill -0 " + str(pid), warn=False)
return rc == 0 or "No such process" not in e
def killRouterDaemons(
self, daemons, wait=True, assertOnError=True, minErrorVersion="5.1"
):
def killRouterDaemons(self, daemons, wait=True, assertOnError=True):
# Kill Running FRR
# Daemons(user specified daemon only) using SIGKILL
rundaemons = self.cmd("ls -1 /var/run/%s/*.pid" % self.routertype)
@ -2339,9 +2334,6 @@ class Router(Node):
self.cmd("rm -- {}".format(daemonpidfile))
if wait:
errors = self.checkRouterCores(reportOnce=True)
if self.checkRouterVersion("<", minErrorVersion):
# ignore errors in old versions
errors = ""
if assertOnError and len(errors) > 0:
assert "Errors found - details follow:" == 0, errors
else: