Merge pull request #13365 from LabNConsulting/chopps/add-perf-option

Add ability to profile daemons in topotest with --perf

doc/developer/topotests.rst

@@ -296,14 +296,14 @@ Execute single test
 .. code:: shell
 
    cd test_to_be_run
-   ./test_to_be_run.py
+   sudo -E pytest ./test_to_be_run.py
 
 For example, and assuming you are inside the frr directory:
 
 .. code:: shell
 
    cd tests/topotests/bgp_l3vpn_to_bgp_vrf
-   ./test_bgp_l3vpn_to_bgp_vrf.py
+   sudo -E pytest ./test_bgp_l3vpn_to_bgp_vrf.py
 
 For further options, refer to pytest documentation.
 
@@ -576,6 +576,27 @@ memleak detection is enabled.
 
    sudo -E pytest --valgrind-memleaks all-protocol-startup
 
+Collecting Performance Data using perf(1)
+"""""""""""""""""""""""""""""""""""""""""
+
+Topotest can automatically launch any daemon under ``perf(1)`` to collect
+performance data. The daemon is run in non-daemon mode with ``perf record -g``.
+The ``perf.data`` file will be saved in the router specific directory under the
+tests run directoy.
+
+Here's an example of collecting performance data from ``mgmtd`` on router ``r1``
+during the config_timing test.
+
+.. code:: console
+
+   $ sudo -E pytest --perf=mgmtd,r1 config_timing
+   ...
+   $ find /tmp/topotests/ -name '*perf.data*'
+   /tmp/topotests/config_timing.test_config_timing/r1/perf.data
+
+To specify different arguments for ``perf record``, one can use the
+``--perf-options`` this will replace the ``-g`` used by default.
+
 .. _topotests_docker:
 
 Running Tests with Docker

tests/topotests/conftest.py

@@ -80,8 +80,8 @@ def pytest_addoption(parser):
         action="append",
         metavar="DAEMON[,ROUTER[,...]",
         help=(
-            "Tail-F of DAEMON log file. Specify routers in comma-separated list after "
+            "Tail-F the DAEMON log file on all or a subset of ROUTERs."
-            "daemon to limit to a subset of routers"
+            " Option can be given multiple times."
         ),
     )
 
@@ -117,6 +117,23 @@ def pytest_addoption(parser):
         help="Comma-separated list of networks to capture packets on, or 'all'",
     )
 
+    parser.addoption(
+        "--perf",
+        action="append",
+        metavar="DAEMON[,ROUTER[,...]",
+        help=(
+            "Collect performance data from given DAEMON on all or a subset of ROUTERs."
+            " Option can be given multiple times."
+        ),
+    )
+
+    parser.addoption(
+        "--perf-options",
+        metavar="OPTS",
+        default="-g",
+        help="Options to pass to `perf record`.",
+    )
+
     rundir_help = "directory for running in and log files"
     parser.addini("rundir", rundir_help, default="/tmp/topotests")
     parser.addoption("--rundir", metavar="DIR", help=rundir_help)

tests/topotests/lib/topotest.py

@@ -1184,10 +1184,9 @@ def rlimit_atleast(rname, min_value, raises=False):
 
 
 def fix_netns_limits(ns):
-
     # Maximum read and write socket buffer sizes
-    sysctl_atleast(ns, "net.ipv4.tcp_rmem", [10 * 1024, 87380, 16 * 2 ** 20])
+    sysctl_atleast(ns, "net.ipv4.tcp_rmem", [10 * 1024, 87380, 16 * 2**20])
-    sysctl_atleast(ns, "net.ipv4.tcp_wmem", [10 * 1024, 87380, 16 * 2 ** 20])
+    sysctl_atleast(ns, "net.ipv4.tcp_wmem", [10 * 1024, 87380, 16 * 2**20])
 
     sysctl_assure(ns, "net.ipv4.conf.all.rp_filter", 0)
     sysctl_assure(ns, "net.ipv4.conf.default.rp_filter", 0)
@@ -1246,8 +1245,8 @@ def fix_host_limits():
     sysctl_atleast(None, "net.core.netdev_max_backlog", 4 * 1024)
 
     # Maximum read and write socket buffer sizes
-    sysctl_atleast(None, "net.core.rmem_max", 16 * 2 ** 20)
+    sysctl_atleast(None, "net.core.rmem_max", 16 * 2**20)
-    sysctl_atleast(None, "net.core.wmem_max", 16 * 2 ** 20)
+    sysctl_atleast(None, "net.core.wmem_max", 16 * 2**20)
 
     # Garbage Collection Settings for ARP and Neighbors
     sysctl_atleast(None, "net.ipv4.neigh.default.gc_thresh2", 4 * 1024)
@@ -1289,7 +1288,6 @@ class Router(Node):
     "A Node with IPv4/IPv6 forwarding enabled"
 
     def __init__(self, name, *posargs, **params):
-
         # Backward compatibility:
         #   Load configuration defaults like topogen.
         self.config_defaults = configparser.ConfigParser(
@@ -1305,6 +1303,8 @@ class Router(Node):
             os.path.join(os.path.dirname(os.path.realpath(__file__)), "../pytest.ini")
         )
 
+        self.perf_daemons = {}
+
         # If this topology is using old API and doesn't have logdir
         # specified, then attempt to generate an unique logdir.
         self.logdir = params.get("logdir")
@@ -1724,6 +1724,16 @@ class Router(Node):
             ).split()[2]
             logger.info("{}: running version: {}".format(self.name, self.version))
 
+        perfds = {}
+        perf_options = g_pytest_config.get_option("--perf-options", "-g")
+        for perf in g_pytest_config.get_option("--perf", []):
+            if "," in perf:
+                daemon, routers = perf.split(",", 1)
+                perfds[daemon] = routers.split(",")
+            else:
+                daemon = perf
+                perfds[daemon] = ["all"]
+
         logd_options = {}
         for logd in g_pytest_config.get_option("--logd", []):
             if "," in logd:
@@ -1805,7 +1815,6 @@ class Router(Node):
                         tail_log_files.append(
                             "{}/{}/{}.log".format(self.logdir, self.name, daemon)
                         )
-
             if extra_opts:
                 cmdopt += " " + extra_opts
 
@@ -1832,6 +1841,23 @@ class Router(Node):
                 logger.info(
                     "%s: %s %s launched in gdb window", self, self.routertype, daemon
                 )
+            elif daemon in perfds and (self.name in perfds[daemon] or "all" in perfds[daemon]):
+                cmdopt += rediropt
+                cmd = " ".join(["perf record {} --".format(perf_options), binary, cmdopt])
+                p = self.popen(cmd)
+                self.perf_daemons[daemon] = p
+                if p.poll() and p.returncode:
+                    self.logger.error(
+                        '%s: Failed to launch "%s" (%s) with perf using: %s',
+                        self,
+                        daemon,
+                        p.returncode,
+                        cmd,
+                    )
+                else:
+                    logger.debug(
+                        "%s: %s %s started with perf", self, self.routertype, daemon
+                    )
             else:
                 if daemon != "snmpd":
                     cmdopt += " -d "