zebra: infastructure for the new dataplane plugin

Signed-off-by: Anuradha Karuppiah <anuradhak@nvidia.com>

configure.ac

@@ -1793,7 +1793,6 @@ AS_IF([test "$enable_pathd" != "no"], [
   AC_DEFINE([HAVE_PATHD], [1], [pathd])
 ])
 
-
 if test "$ac_cv_lib_json_c_json_object_get" = "no" -a "$BFDD" = "bfdd"; then
   AC_MSG_ERROR(["you must use json-c library to use bfdd"])
 fi
@@ -2707,6 +2706,7 @@ AM_CONDITIONAL([FABRICD], [test "$enable_fabricd" != "no"])
 AM_CONDITIONAL([VRRPD], [test "$enable_vrrpd" != "no"])
 AM_CONDITIONAL([PATHD], [test "$enable_pathd" != "no"])
 AM_CONDITIONAL([PATHD_PCEP], [test "$enable_pathd" != "no"])
+AM_CONDITIONAL([DP_DPDK], [test "$enable_dp_dpdk" = "yes"])
 
 AC_CONFIG_FILES([Makefile],[
 	test "$enable_dev_build" = "yes" && makefile_devbuild="--dev-build"

vtysh/vtysh.h

@@ -66,6 +66,7 @@ extern struct thread_master *master;
 /* Daemons who can process nexthop-group configs */
 #define VTYSH_NH_GROUP    VTYSH_PBRD|VTYSH_SHARPD
 #define VTYSH_SR          VTYSH_ZEBRA|VTYSH_PATHD
+#define VTYSH_DPDK VTYSH_ZEBRA
 
 enum vtysh_write_integrated {
 	WRITE_INTEGRATED_UNSPECIFIED,

zebra/debug.c

@@ -38,6 +38,7 @@ unsigned long zebra_debug_mpls;
 unsigned long zebra_debug_vxlan;
 unsigned long zebra_debug_pw;
 unsigned long zebra_debug_dplane;
+unsigned long zebra_debug_dplane_dpdk;
 unsigned long zebra_debug_mlag;
 unsigned long zebra_debug_nexthop;
 unsigned long zebra_debug_evpn_mh;
@@ -108,6 +109,11 @@ DEFUN_NOSH (show_debugging_zebra,
 		vty_out(vty, "  Zebra detailed dataplane debugging is on\n");
 	else if (IS_ZEBRA_DEBUG_DPLANE)
 		vty_out(vty, "  Zebra dataplane debugging is on\n");
+	if (IS_ZEBRA_DEBUG_DPLANE_DPDK_DETAIL)
+		vty_out(vty,
+			"  Zebra detailed dpdk dataplane debugging is on\n");
+	else if (IS_ZEBRA_DEBUG_DPLANE_DPDK)
+		vty_out(vty, "  Zebra dataplane dpdk debugging is on\n");
 	if (IS_ZEBRA_DEBUG_MLAG)
 		vty_out(vty, "  Zebra mlag debugging is on\n");
 	if (IS_ZEBRA_DEBUG_NHG_DETAIL)
@@ -317,6 +323,29 @@ DEFUN (debug_zebra_dplane,
 	return CMD_SUCCESS;
 }
 
+DEFPY(debug_zebra_dplane_dpdk, debug_zebra_dplane_dpdk_cmd,
+      "[no$no] debug zebra dplane dpdk [detailed$detail]",
+      NO_STR DEBUG_STR
+      "Zebra configuration\n"
+      "Debug zebra dataplane events\n"
+      "Debug zebra DPDK offload events\n"
+      "Detailed debug information\n")
+{
+	if (no) {
+		UNSET_FLAG(zebra_debug_dplane_dpdk, ZEBRA_DEBUG_DPLANE_DPDK);
+		UNSET_FLAG(zebra_debug_dplane_dpdk,
+			   ZEBRA_DEBUG_DPLANE_DPDK_DETAIL);
+	} else {
+		SET_FLAG(zebra_debug_dplane_dpdk, ZEBRA_DEBUG_DPLANE_DPDK);
+
+		if (detail)
+			SET_FLAG(zebra_debug_dplane,
+				 ZEBRA_DEBUG_DPLANE_DPDK_DETAIL);
+	}
+
+	return CMD_SUCCESS;
+}
+
 DEFUN (debug_zebra_pbr,
        debug_zebra_pbr_cmd,
        "debug zebra pbr",
@@ -698,6 +727,14 @@ static int config_write_debug(struct vty *vty)
 		write++;
 	}
 
+	if (CHECK_FLAG(zebra_debug_dplane, ZEBRA_DEBUG_DPLANE_DPDK_DETAIL)) {
+		vty_out(vty, "debug zebra dplane dpdk detailed\n");
+		write++;
+	} else if (CHECK_FLAG(zebra_debug_dplane, ZEBRA_DEBUG_DPLANE_DPDK)) {
+		vty_out(vty, "debug zebra dplane dpdk\n");
+		write++;
+	}
+
 	if (CHECK_FLAG(zebra_debug_nexthop, ZEBRA_DEBUG_NHG_DETAILED)) {
 		vty_out(vty, "debug zebra nexthop detail\n");
 		write++;
@@ -730,6 +767,7 @@ void zebra_debug_init(void)
 	zebra_debug_vxlan = 0;
 	zebra_debug_pw = 0;
 	zebra_debug_dplane = 0;
+	zebra_debug_dplane_dpdk = 0;
 	zebra_debug_mlag = 0;
 	zebra_debug_evpn_mh = 0;
 	zebra_debug_nht = 0;
@@ -758,6 +796,7 @@ void zebra_debug_init(void)
 	install_element(ENABLE_NODE, &debug_zebra_nexthop_cmd);
 	install_element(ENABLE_NODE, &debug_zebra_pbr_cmd);
 	install_element(ENABLE_NODE, &debug_zebra_neigh_cmd);
+	install_element(ENABLE_NODE, &debug_zebra_dplane_dpdk_cmd);
 	install_element(ENABLE_NODE, &no_debug_zebra_events_cmd);
 	install_element(ENABLE_NODE, &no_debug_zebra_nht_cmd);
 	install_element(ENABLE_NODE, &no_debug_zebra_mpls_cmd);
@@ -786,6 +825,7 @@ void zebra_debug_init(void)
 	install_element(CONFIG_NODE, &debug_zebra_rib_cmd);
 	install_element(CONFIG_NODE, &debug_zebra_fpm_cmd);
 	install_element(CONFIG_NODE, &debug_zebra_dplane_cmd);
+	install_element(CONFIG_NODE, &debug_zebra_dplane_dpdk_cmd);
 	install_element(CONFIG_NODE, &debug_zebra_nexthop_cmd);
 	install_element(CONFIG_NODE, &debug_zebra_pbr_cmd);
 	install_element(CONFIG_NODE, &debug_zebra_neigh_cmd);

zebra/debug.h

@@ -58,6 +58,9 @@ extern "C" {
 #define ZEBRA_DEBUG_DPLANE           0x01
 #define ZEBRA_DEBUG_DPLANE_DETAILED  0x02
 
+#define ZEBRA_DEBUG_DPLANE_DPDK 0x01
+#define ZEBRA_DEBUG_DPLANE_DPDK_DETAIL 0x02
+
 #define ZEBRA_DEBUG_MLAG    0x01
 
 #define ZEBRA_DEBUG_NHG             0x01
@@ -105,6 +108,11 @@ extern "C" {
 #define IS_ZEBRA_DEBUG_DPLANE_DETAIL \
 	(zebra_debug_dplane & ZEBRA_DEBUG_DPLANE_DETAILED)
 
+#define IS_ZEBRA_DEBUG_DPLANE_DPDK                                             \
+	(zebra_debug_dplane & ZEBRA_DEBUG_DPLANE_DPDK)
+#define IS_ZEBRA_DEBUG_DPLANE_DPDK_DETAIL                                      \
+	(zebra_debug_dplane & ZEBRA_DEBUG_DPLANE_DPDK_DETAIL)
+
 #define IS_ZEBRA_DEBUG_MLAG (zebra_debug_mlag & ZEBRA_DEBUG_MLAG)
 
 #define IS_ZEBRA_DEBUG_NHG (zebra_debug_nexthop & ZEBRA_DEBUG_NHG)
@@ -135,6 +143,7 @@ extern unsigned long zebra_debug_mpls;
 extern unsigned long zebra_debug_vxlan;
 extern unsigned long zebra_debug_pw;
 extern unsigned long zebra_debug_dplane;
+extern unsigned long zebra_debug_dplane_dpdk;
 extern unsigned long zebra_debug_mlag;
 extern unsigned long zebra_debug_nexthop;
 extern unsigned long zebra_debug_evpn_mh;

zebra/dpdk/zebra_dplane_dpdk.c

@@ -0,0 +1,215 @@
+/*
+ * Zebra dataplane plugin for DPDK based hw offload
+ *
+ * Copyright (C) 2021 Nvidia
+ * Anuradha Karuppiah
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; see the file COPYING; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h" /* Include this explicitly */
+#endif
+
+#include "lib/libfrr.h"
+
+#include "zebra/debug.h"
+#include "zebra/interface.h"
+#include "zebra/zebra_dplane.h"
+#include "zebra/debug.h"
+#include "zebra/zebra_pbr.h"
+
+#include "zebra/dpdk/zebra_dplane_dpdk_private.h"
+
+static const char *plugin_name = "zebra_dplane_dpdk";
+
+static struct zd_dpdk_ctx dpdk_ctx_buf, *dpdk_ctx = &dpdk_ctx_buf;
+#define dpdk_stat (&dpdk_ctx->stats)
+
+
+void zd_dpdk_stat_show(struct vty *vty)
+{
+	uint32_t tmp_cnt;
+
+	vty_out(vty, "%30s\n%30s\n", "Dataplane DPDK counters",
+		"=======================");
+
+#define ZD_DPDK_SHOW_COUNTER(label, counter)                                   \
+	do {                                                                   \
+		tmp_cnt =                                                      \
+			atomic_load_explicit(&counter, memory_order_relaxed);  \
+		vty_out(vty, "%28s: %u\n", (label), (tmp_cnt));                \
+	} while (0);
+
+	ZD_DPDK_SHOW_COUNTER("PBR rule adds", dpdk_stat->rule_adds);
+	ZD_DPDK_SHOW_COUNTER("PBR rule dels", dpdk_stat->rule_dels);
+	ZD_DPDK_SHOW_COUNTER("Ignored updates", dpdk_stat->ignored_updates);
+}
+
+
+static void zd_dpdk_rule_add(struct zebra_dplane_ctx *ctx)
+{
+	/* XXX - place holder */
+}
+
+
+static void zd_dpdk_rule_del(const char *ifname, int in_ifindex,
+			     intptr_t dp_flow_ptr)
+{
+
+	/* XXX - place holder */
+}
+
+
+static void zd_dpdk_rule_update(struct zebra_dplane_ctx *ctx)
+{
+	enum dplane_op_e op;
+	int in_ifindex;
+	intptr_t dp_flow_ptr;
+
+	if (IS_ZEBRA_DEBUG_DPLANE_DPDK_DETAIL) {
+		zlog_debug("Dplane %s", dplane_op2str(dplane_ctx_get_op(ctx)));
+	}
+
+	op = dplane_ctx_get_op(ctx);
+	switch (op) {
+	case DPLANE_OP_RULE_ADD:
+		atomic_fetch_add_explicit(&dpdk_stat->rule_adds, 1,
+					  memory_order_relaxed);
+		zd_dpdk_rule_add(ctx);
+		break;
+
+	case DPLANE_OP_RULE_UPDATE:
+		/* delete old rule and install new one */
+		atomic_fetch_add_explicit(&dpdk_stat->rule_adds, 1,
+					  memory_order_relaxed);
+		in_ifindex = dplane_ctx_get_ifindex(ctx);
+		dp_flow_ptr = dplane_ctx_rule_get_old_dp_flow_ptr(ctx);
+		zd_dpdk_rule_del(dplane_ctx_rule_get_ifname(ctx), in_ifindex,
+				 dp_flow_ptr);
+		zd_dpdk_rule_add(ctx);
+		break;
+
+	case DPLANE_OP_RULE_DELETE:
+		atomic_fetch_add_explicit(&dpdk_stat->rule_dels, 1,
+					  memory_order_relaxed);
+		in_ifindex = dplane_ctx_get_ifindex(ctx);
+		dp_flow_ptr = dplane_ctx_rule_get_dp_flow_ptr(ctx);
+		zd_dpdk_rule_del(dplane_ctx_rule_get_ifname(ctx), in_ifindex,
+				 dp_flow_ptr);
+		break;
+
+	default:;
+	}
+}
+
+
+/* DPDK provider callback.
+ */
+static void zd_dpdk_process_update(struct zebra_dplane_ctx *ctx)
+{
+	switch (dplane_ctx_get_op(ctx)) {
+
+	case DPLANE_OP_RULE_ADD:
+	case DPLANE_OP_RULE_UPDATE:
+	case DPLANE_OP_RULE_DELETE:
+		zd_dpdk_rule_update(ctx);
+		break;
+
+	default:
+		atomic_fetch_add_explicit(&dpdk_stat->ignored_updates, 1,
+					  memory_order_relaxed);
+
+		break;
+	}
+}
+
+
+static int zd_dpdk_process(struct zebra_dplane_provider *prov)
+{
+	struct zebra_dplane_ctx *ctx;
+	int counter, limit;
+
+	if (IS_ZEBRA_DEBUG_DPLANE_DPDK_DETAIL)
+		zlog_debug("processing %s", dplane_provider_get_name(prov));
+
+	limit = dplane_provider_get_work_limit(prov);
+	for (counter = 0; counter < limit; counter++) {
+		ctx = dplane_provider_dequeue_in_ctx(prov);
+		if (!ctx)
+			break;
+
+		zd_dpdk_process_update(ctx);
+		dplane_ctx_set_status(ctx, ZEBRA_DPLANE_REQUEST_SUCCESS);
+		dplane_provider_enqueue_out_ctx(prov, ctx);
+	}
+
+	return 0;
+}
+
+
+static int zd_dpdk_start(struct zebra_dplane_provider *prov)
+{
+	if (IS_ZEBRA_DEBUG_DPLANE_DPDK)
+		zlog_debug("%s start", dplane_provider_get_name(prov));
+
+	/* XXX - place holder */
+	return 0;
+}
+
+
+static int zd_dpdk_finish(struct zebra_dplane_provider *prov, bool early)
+{
+	if (early) {
+		if (IS_ZEBRA_DEBUG_DPLANE_DPDK)
+			zlog_debug("%s early finish",
+				   dplane_provider_get_name(prov));
+
+		return 0;
+	}
+
+	if (IS_ZEBRA_DEBUG_DPLANE_DPDK)
+		zlog_debug("%s finish", dplane_provider_get_name(prov));
+
+
+	/* XXX - place holder */
+	return 0;
+}
+
+
+static int zd_dpdk_plugin_init(struct thread_master *tm)
+{
+	int ret;
+
+	ret = dplane_provider_register(
+		plugin_name, DPLANE_PRIO_KERNEL, DPLANE_PROV_FLAGS_DEFAULT,
+		zd_dpdk_start, zd_dpdk_process, zd_dpdk_finish, dpdk_ctx, NULL);
+
+	if (IS_ZEBRA_DEBUG_DPLANE_DPDK)
+		zlog_debug("%s register status %d", plugin_name, ret);
+
+	return 0;
+}
+
+
+static int zd_dpdk_module_init(void)
+{
+	hook_register(frr_late_init, zd_dpdk_plugin_init);
+	return 0;
+}
+
+FRR_MODULE_SETUP(.name = "dplane_dpdk", .version = "0.0.1",
+		 .description = "Data plane plugin using dpdk for hw offload",
+		 .init = zd_dpdk_module_init, );

zebra/dpdk/zebra_dplane_dpdk.h

@@ -0,0 +1,30 @@
+/*
+ * Zebra dataplane plugin for DPDK based hw offload
+ *
+ * Copyright (C) 2021 Nvidia
+ * Anuradha Karuppiah
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; see the file COPYING; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef _ZEBRA_DPLANE_DPDK_H
+#define _ZEBRA_DPLANE_DPDK_H
+
+#include <zebra.h>
+
+extern void zd_dpdk_stat_show(struct vty *vty);
+extern void zd_dpdk_vty_init(void);
+
+#endif

zebra/dpdk/zebra_dplane_dpdk_private.h

@@ -0,0 +1,43 @@
+/*
+ * Zebra dataplane plugin for DPDK based hw offload
+ *
+ * Copyright (C) 2021 Nvidia
+ * Anuradha Karuppiah
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; see the file COPYING; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef _ZEBRA_DPLANE_DPDK_PRIVATE_H
+#define _ZEBRA_DPLANE_DPDK_PRIVATE_H
+
+#include <zebra.h>
+
+#include "zebra_dplane_dpdk.h"
+
+
+struct zd_dpdk_stat {
+	_Atomic uint32_t ignored_updates;
+
+	_Atomic uint32_t rule_adds;
+	_Atomic uint32_t rule_dels;
+};
+
+struct zd_dpdk_ctx {
+	/* Stats */
+	struct zd_dpdk_stat stats;
+	int dpdk_logtype;
+};
+
+#endif

zebra/dpdk/zebra_dplane_dpdk_vty.c

@@ -0,0 +1,45 @@
+/*
+ * Zebra dataplane plugin for DPDK based hw offload
+ *
+ * Copyright (C) 2021 Nvidia
+ * Donald Sharp
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; see the file COPYING; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+#include <zebra.h>
+
+#include "lib/json.h"
+#include "zebra/dpdk/zebra_dplane_dpdk.h"
+
+#ifndef VTYSH_EXTRACT_PL
+#include "zebra/dpdk/zebra_dplane_dpdk_vty_clippy.c"
+#endif
+
+#define ZD_STR "Zebra dataplane information\n"
+#define ZD_DPDK_STR "DPDK offload information\n"
+
+DEFPY(zd_dpdk_show_counters, zd_dpdk_show_counters_cmd,
+      "show dplane dpdk counters",
+      SHOW_STR ZD_STR ZD_DPDK_STR "show counters\n")
+{
+	zd_dpdk_stat_show(vty);
+
+	return CMD_SUCCESS;
+}
+
+void zd_dpdk_vty_init(void)
+{
+	install_element(VIEW_NODE, &zd_dpdk_show_counters_cmd);
+}

zebra/subdir.am

@@ -20,6 +20,7 @@ vtysh_scan += \
 	zebra/zebra_vty.c \
 	zebra/zserv.c \
 	zebra/zebra_vrf.c \
+	zebra/dpdk/zebra_dplane_dpdk_vty.c \
 	# end
 
 # can be loaded as DSO - always include for vtysh
@@ -139,6 +140,7 @@ clippy_scan += \
 	zebra/zebra_vty.c \
 	zebra/zebra_srv6_vty.c \
 	zebra/zebra_vrf.c \
+	zebra/dpdk/zebra_dplane_dpdk_vty.c \
 	# end
 
 noinst_HEADERS += \
@@ -200,6 +202,8 @@ noinst_HEADERS += \
 	zebra/zebra_evpn_mh.h \
 	zebra/zebra_neigh.h \
 	zebra/zserv.h \
+	zebra/dpdk/zebra_dplane_dpdk.h \
+	zebra/dpdk/zebra_dplane_dpdk_private.h \
 	# end
 
 zebra_zebra_irdp_la_SOURCES = \
@@ -257,3 +261,10 @@ zebra_zebra_SOURCES += \
 	zebra/debug_nl.c \
 	# end
 endif
+
+if DP_DPDK
+module_LTLIBRARIES += zebra/zebra_dplane_dpdk.la
+endif
+
+zebra_zebra_dplane_dpdk_la_SOURCES = zebra/dpdk/zebra_dplane_dpdk.c zebra/dpdk/zebra_dplane_dpdk_vty.c
+zebra_zebra_dplane_dpdk_la_LDFLAGS = -avoid-version -module -shared -export-dynamic -L/usr/local/lib -v