diff --git a/configure.ac b/configure.ac index ff53e1224f..b901d7a423 100644 --- a/configure.ac +++ b/configure.ac @@ -1793,7 +1793,6 @@ AS_IF([test "$enable_pathd" != "no"], [ AC_DEFINE([HAVE_PATHD], [1], [pathd]) ]) - if test "$ac_cv_lib_json_c_json_object_get" = "no" -a "$BFDD" = "bfdd"; then AC_MSG_ERROR(["you must use json-c library to use bfdd"]) fi @@ -2707,6 +2706,7 @@ AM_CONDITIONAL([FABRICD], [test "$enable_fabricd" != "no"]) AM_CONDITIONAL([VRRPD], [test "$enable_vrrpd" != "no"]) AM_CONDITIONAL([PATHD], [test "$enable_pathd" != "no"]) AM_CONDITIONAL([PATHD_PCEP], [test "$enable_pathd" != "no"]) +AM_CONDITIONAL([DP_DPDK], [test "$enable_dp_dpdk" = "yes"]) AC_CONFIG_FILES([Makefile],[ test "$enable_dev_build" = "yes" && makefile_devbuild="--dev-build" diff --git a/vtysh/vtysh.h b/vtysh/vtysh.h index 6053955be9..1f02efee20 100644 --- a/vtysh/vtysh.h +++ b/vtysh/vtysh.h @@ -66,6 +66,7 @@ extern struct thread_master *master; /* Daemons who can process nexthop-group configs */ #define VTYSH_NH_GROUP VTYSH_PBRD|VTYSH_SHARPD #define VTYSH_SR VTYSH_ZEBRA|VTYSH_PATHD +#define VTYSH_DPDK VTYSH_ZEBRA enum vtysh_write_integrated { WRITE_INTEGRATED_UNSPECIFIED, diff --git a/zebra/debug.c b/zebra/debug.c index 98e25af857..69aaed33ac 100644 --- a/zebra/debug.c +++ b/zebra/debug.c @@ -38,6 +38,7 @@ unsigned long zebra_debug_mpls; unsigned long zebra_debug_vxlan; unsigned long zebra_debug_pw; unsigned long zebra_debug_dplane; +unsigned long zebra_debug_dplane_dpdk; unsigned long zebra_debug_mlag; unsigned long zebra_debug_nexthop; unsigned long zebra_debug_evpn_mh; @@ -108,6 +109,11 @@ DEFUN_NOSH (show_debugging_zebra, vty_out(vty, " Zebra detailed dataplane debugging is on\n"); else if (IS_ZEBRA_DEBUG_DPLANE) vty_out(vty, " Zebra dataplane debugging is on\n"); + if (IS_ZEBRA_DEBUG_DPLANE_DPDK_DETAIL) + vty_out(vty, + " Zebra detailed dpdk dataplane debugging is on\n"); + else if (IS_ZEBRA_DEBUG_DPLANE_DPDK) + vty_out(vty, " Zebra dataplane dpdk debugging is on\n"); if (IS_ZEBRA_DEBUG_MLAG) vty_out(vty, " Zebra mlag debugging is on\n"); if (IS_ZEBRA_DEBUG_NHG_DETAIL) @@ -317,6 +323,29 @@ DEFUN (debug_zebra_dplane, return CMD_SUCCESS; } +DEFPY(debug_zebra_dplane_dpdk, debug_zebra_dplane_dpdk_cmd, + "[no$no] debug zebra dplane dpdk [detailed$detail]", + NO_STR DEBUG_STR + "Zebra configuration\n" + "Debug zebra dataplane events\n" + "Debug zebra DPDK offload events\n" + "Detailed debug information\n") +{ + if (no) { + UNSET_FLAG(zebra_debug_dplane_dpdk, ZEBRA_DEBUG_DPLANE_DPDK); + UNSET_FLAG(zebra_debug_dplane_dpdk, + ZEBRA_DEBUG_DPLANE_DPDK_DETAIL); + } else { + SET_FLAG(zebra_debug_dplane_dpdk, ZEBRA_DEBUG_DPLANE_DPDK); + + if (detail) + SET_FLAG(zebra_debug_dplane, + ZEBRA_DEBUG_DPLANE_DPDK_DETAIL); + } + + return CMD_SUCCESS; +} + DEFUN (debug_zebra_pbr, debug_zebra_pbr_cmd, "debug zebra pbr", @@ -698,6 +727,14 @@ static int config_write_debug(struct vty *vty) write++; } + if (CHECK_FLAG(zebra_debug_dplane, ZEBRA_DEBUG_DPLANE_DPDK_DETAIL)) { + vty_out(vty, "debug zebra dplane dpdk detailed\n"); + write++; + } else if (CHECK_FLAG(zebra_debug_dplane, ZEBRA_DEBUG_DPLANE_DPDK)) { + vty_out(vty, "debug zebra dplane dpdk\n"); + write++; + } + if (CHECK_FLAG(zebra_debug_nexthop, ZEBRA_DEBUG_NHG_DETAILED)) { vty_out(vty, "debug zebra nexthop detail\n"); write++; @@ -730,6 +767,7 @@ void zebra_debug_init(void) zebra_debug_vxlan = 0; zebra_debug_pw = 0; zebra_debug_dplane = 0; + zebra_debug_dplane_dpdk = 0; zebra_debug_mlag = 0; zebra_debug_evpn_mh = 0; zebra_debug_nht = 0; @@ -758,6 +796,7 @@ void zebra_debug_init(void) install_element(ENABLE_NODE, &debug_zebra_nexthop_cmd); install_element(ENABLE_NODE, &debug_zebra_pbr_cmd); install_element(ENABLE_NODE, &debug_zebra_neigh_cmd); + install_element(ENABLE_NODE, &debug_zebra_dplane_dpdk_cmd); install_element(ENABLE_NODE, &no_debug_zebra_events_cmd); install_element(ENABLE_NODE, &no_debug_zebra_nht_cmd); install_element(ENABLE_NODE, &no_debug_zebra_mpls_cmd); @@ -786,6 +825,7 @@ void zebra_debug_init(void) install_element(CONFIG_NODE, &debug_zebra_rib_cmd); install_element(CONFIG_NODE, &debug_zebra_fpm_cmd); install_element(CONFIG_NODE, &debug_zebra_dplane_cmd); + install_element(CONFIG_NODE, &debug_zebra_dplane_dpdk_cmd); install_element(CONFIG_NODE, &debug_zebra_nexthop_cmd); install_element(CONFIG_NODE, &debug_zebra_pbr_cmd); install_element(CONFIG_NODE, &debug_zebra_neigh_cmd); diff --git a/zebra/debug.h b/zebra/debug.h index e761e5e3e3..73546de632 100644 --- a/zebra/debug.h +++ b/zebra/debug.h @@ -58,6 +58,9 @@ extern "C" { #define ZEBRA_DEBUG_DPLANE 0x01 #define ZEBRA_DEBUG_DPLANE_DETAILED 0x02 +#define ZEBRA_DEBUG_DPLANE_DPDK 0x01 +#define ZEBRA_DEBUG_DPLANE_DPDK_DETAIL 0x02 + #define ZEBRA_DEBUG_MLAG 0x01 #define ZEBRA_DEBUG_NHG 0x01 @@ -105,6 +108,11 @@ extern "C" { #define IS_ZEBRA_DEBUG_DPLANE_DETAIL \ (zebra_debug_dplane & ZEBRA_DEBUG_DPLANE_DETAILED) +#define IS_ZEBRA_DEBUG_DPLANE_DPDK \ + (zebra_debug_dplane & ZEBRA_DEBUG_DPLANE_DPDK) +#define IS_ZEBRA_DEBUG_DPLANE_DPDK_DETAIL \ + (zebra_debug_dplane & ZEBRA_DEBUG_DPLANE_DPDK_DETAIL) + #define IS_ZEBRA_DEBUG_MLAG (zebra_debug_mlag & ZEBRA_DEBUG_MLAG) #define IS_ZEBRA_DEBUG_NHG (zebra_debug_nexthop & ZEBRA_DEBUG_NHG) @@ -135,6 +143,7 @@ extern unsigned long zebra_debug_mpls; extern unsigned long zebra_debug_vxlan; extern unsigned long zebra_debug_pw; extern unsigned long zebra_debug_dplane; +extern unsigned long zebra_debug_dplane_dpdk; extern unsigned long zebra_debug_mlag; extern unsigned long zebra_debug_nexthop; extern unsigned long zebra_debug_evpn_mh; diff --git a/zebra/dpdk/zebra_dplane_dpdk.c b/zebra/dpdk/zebra_dplane_dpdk.c new file mode 100644 index 0000000000..a8dfdf86d5 --- /dev/null +++ b/zebra/dpdk/zebra_dplane_dpdk.c @@ -0,0 +1,215 @@ +/* + * Zebra dataplane plugin for DPDK based hw offload + * + * Copyright (C) 2021 Nvidia + * Anuradha Karuppiah + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; see the file COPYING; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" /* Include this explicitly */ +#endif + +#include "lib/libfrr.h" + +#include "zebra/debug.h" +#include "zebra/interface.h" +#include "zebra/zebra_dplane.h" +#include "zebra/debug.h" +#include "zebra/zebra_pbr.h" + +#include "zebra/dpdk/zebra_dplane_dpdk_private.h" + +static const char *plugin_name = "zebra_dplane_dpdk"; + +static struct zd_dpdk_ctx dpdk_ctx_buf, *dpdk_ctx = &dpdk_ctx_buf; +#define dpdk_stat (&dpdk_ctx->stats) + + +void zd_dpdk_stat_show(struct vty *vty) +{ + uint32_t tmp_cnt; + + vty_out(vty, "%30s\n%30s\n", "Dataplane DPDK counters", + "======================="); + +#define ZD_DPDK_SHOW_COUNTER(label, counter) \ + do { \ + tmp_cnt = \ + atomic_load_explicit(&counter, memory_order_relaxed); \ + vty_out(vty, "%28s: %u\n", (label), (tmp_cnt)); \ + } while (0); + + ZD_DPDK_SHOW_COUNTER("PBR rule adds", dpdk_stat->rule_adds); + ZD_DPDK_SHOW_COUNTER("PBR rule dels", dpdk_stat->rule_dels); + ZD_DPDK_SHOW_COUNTER("Ignored updates", dpdk_stat->ignored_updates); +} + + +static void zd_dpdk_rule_add(struct zebra_dplane_ctx *ctx) +{ + /* XXX - place holder */ +} + + +static void zd_dpdk_rule_del(const char *ifname, int in_ifindex, + intptr_t dp_flow_ptr) +{ + + /* XXX - place holder */ +} + + +static void zd_dpdk_rule_update(struct zebra_dplane_ctx *ctx) +{ + enum dplane_op_e op; + int in_ifindex; + intptr_t dp_flow_ptr; + + if (IS_ZEBRA_DEBUG_DPLANE_DPDK_DETAIL) { + zlog_debug("Dplane %s", dplane_op2str(dplane_ctx_get_op(ctx))); + } + + op = dplane_ctx_get_op(ctx); + switch (op) { + case DPLANE_OP_RULE_ADD: + atomic_fetch_add_explicit(&dpdk_stat->rule_adds, 1, + memory_order_relaxed); + zd_dpdk_rule_add(ctx); + break; + + case DPLANE_OP_RULE_UPDATE: + /* delete old rule and install new one */ + atomic_fetch_add_explicit(&dpdk_stat->rule_adds, 1, + memory_order_relaxed); + in_ifindex = dplane_ctx_get_ifindex(ctx); + dp_flow_ptr = dplane_ctx_rule_get_old_dp_flow_ptr(ctx); + zd_dpdk_rule_del(dplane_ctx_rule_get_ifname(ctx), in_ifindex, + dp_flow_ptr); + zd_dpdk_rule_add(ctx); + break; + + case DPLANE_OP_RULE_DELETE: + atomic_fetch_add_explicit(&dpdk_stat->rule_dels, 1, + memory_order_relaxed); + in_ifindex = dplane_ctx_get_ifindex(ctx); + dp_flow_ptr = dplane_ctx_rule_get_dp_flow_ptr(ctx); + zd_dpdk_rule_del(dplane_ctx_rule_get_ifname(ctx), in_ifindex, + dp_flow_ptr); + break; + + default:; + } +} + + +/* DPDK provider callback. + */ +static void zd_dpdk_process_update(struct zebra_dplane_ctx *ctx) +{ + switch (dplane_ctx_get_op(ctx)) { + + case DPLANE_OP_RULE_ADD: + case DPLANE_OP_RULE_UPDATE: + case DPLANE_OP_RULE_DELETE: + zd_dpdk_rule_update(ctx); + break; + + default: + atomic_fetch_add_explicit(&dpdk_stat->ignored_updates, 1, + memory_order_relaxed); + + break; + } +} + + +static int zd_dpdk_process(struct zebra_dplane_provider *prov) +{ + struct zebra_dplane_ctx *ctx; + int counter, limit; + + if (IS_ZEBRA_DEBUG_DPLANE_DPDK_DETAIL) + zlog_debug("processing %s", dplane_provider_get_name(prov)); + + limit = dplane_provider_get_work_limit(prov); + for (counter = 0; counter < limit; counter++) { + ctx = dplane_provider_dequeue_in_ctx(prov); + if (!ctx) + break; + + zd_dpdk_process_update(ctx); + dplane_ctx_set_status(ctx, ZEBRA_DPLANE_REQUEST_SUCCESS); + dplane_provider_enqueue_out_ctx(prov, ctx); + } + + return 0; +} + + +static int zd_dpdk_start(struct zebra_dplane_provider *prov) +{ + if (IS_ZEBRA_DEBUG_DPLANE_DPDK) + zlog_debug("%s start", dplane_provider_get_name(prov)); + + /* XXX - place holder */ + return 0; +} + + +static int zd_dpdk_finish(struct zebra_dplane_provider *prov, bool early) +{ + if (early) { + if (IS_ZEBRA_DEBUG_DPLANE_DPDK) + zlog_debug("%s early finish", + dplane_provider_get_name(prov)); + + return 0; + } + + if (IS_ZEBRA_DEBUG_DPLANE_DPDK) + zlog_debug("%s finish", dplane_provider_get_name(prov)); + + + /* XXX - place holder */ + return 0; +} + + +static int zd_dpdk_plugin_init(struct thread_master *tm) +{ + int ret; + + ret = dplane_provider_register( + plugin_name, DPLANE_PRIO_KERNEL, DPLANE_PROV_FLAGS_DEFAULT, + zd_dpdk_start, zd_dpdk_process, zd_dpdk_finish, dpdk_ctx, NULL); + + if (IS_ZEBRA_DEBUG_DPLANE_DPDK) + zlog_debug("%s register status %d", plugin_name, ret); + + return 0; +} + + +static int zd_dpdk_module_init(void) +{ + hook_register(frr_late_init, zd_dpdk_plugin_init); + return 0; +} + +FRR_MODULE_SETUP(.name = "dplane_dpdk", .version = "0.0.1", + .description = "Data plane plugin using dpdk for hw offload", + .init = zd_dpdk_module_init, ); diff --git a/zebra/dpdk/zebra_dplane_dpdk.h b/zebra/dpdk/zebra_dplane_dpdk.h new file mode 100644 index 0000000000..469fd8c5bc --- /dev/null +++ b/zebra/dpdk/zebra_dplane_dpdk.h @@ -0,0 +1,30 @@ +/* + * Zebra dataplane plugin for DPDK based hw offload + * + * Copyright (C) 2021 Nvidia + * Anuradha Karuppiah + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; see the file COPYING; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef _ZEBRA_DPLANE_DPDK_H +#define _ZEBRA_DPLANE_DPDK_H + +#include + +extern void zd_dpdk_stat_show(struct vty *vty); +extern void zd_dpdk_vty_init(void); + +#endif diff --git a/zebra/dpdk/zebra_dplane_dpdk_private.h b/zebra/dpdk/zebra_dplane_dpdk_private.h new file mode 100644 index 0000000000..24b943dddd --- /dev/null +++ b/zebra/dpdk/zebra_dplane_dpdk_private.h @@ -0,0 +1,43 @@ +/* + * Zebra dataplane plugin for DPDK based hw offload + * + * Copyright (C) 2021 Nvidia + * Anuradha Karuppiah + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; see the file COPYING; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef _ZEBRA_DPLANE_DPDK_PRIVATE_H +#define _ZEBRA_DPLANE_DPDK_PRIVATE_H + +#include + +#include "zebra_dplane_dpdk.h" + + +struct zd_dpdk_stat { + _Atomic uint32_t ignored_updates; + + _Atomic uint32_t rule_adds; + _Atomic uint32_t rule_dels; +}; + +struct zd_dpdk_ctx { + /* Stats */ + struct zd_dpdk_stat stats; + int dpdk_logtype; +}; + +#endif diff --git a/zebra/dpdk/zebra_dplane_dpdk_vty.c b/zebra/dpdk/zebra_dplane_dpdk_vty.c new file mode 100644 index 0000000000..c1fbb20c4c --- /dev/null +++ b/zebra/dpdk/zebra_dplane_dpdk_vty.c @@ -0,0 +1,45 @@ +/* + * Zebra dataplane plugin for DPDK based hw offload + * + * Copyright (C) 2021 Nvidia + * Donald Sharp + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; see the file COPYING; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ +#include + +#include "lib/json.h" +#include "zebra/dpdk/zebra_dplane_dpdk.h" + +#ifndef VTYSH_EXTRACT_PL +#include "zebra/dpdk/zebra_dplane_dpdk_vty_clippy.c" +#endif + +#define ZD_STR "Zebra dataplane information\n" +#define ZD_DPDK_STR "DPDK offload information\n" + +DEFPY(zd_dpdk_show_counters, zd_dpdk_show_counters_cmd, + "show dplane dpdk counters", + SHOW_STR ZD_STR ZD_DPDK_STR "show counters\n") +{ + zd_dpdk_stat_show(vty); + + return CMD_SUCCESS; +} + +void zd_dpdk_vty_init(void) +{ + install_element(VIEW_NODE, &zd_dpdk_show_counters_cmd); +} diff --git a/zebra/subdir.am b/zebra/subdir.am index e895ffa65f..89836f3ba4 100644 --- a/zebra/subdir.am +++ b/zebra/subdir.am @@ -20,6 +20,7 @@ vtysh_scan += \ zebra/zebra_vty.c \ zebra/zserv.c \ zebra/zebra_vrf.c \ + zebra/dpdk/zebra_dplane_dpdk_vty.c \ # end # can be loaded as DSO - always include for vtysh @@ -139,6 +140,7 @@ clippy_scan += \ zebra/zebra_vty.c \ zebra/zebra_srv6_vty.c \ zebra/zebra_vrf.c \ + zebra/dpdk/zebra_dplane_dpdk_vty.c \ # end noinst_HEADERS += \ @@ -200,6 +202,8 @@ noinst_HEADERS += \ zebra/zebra_evpn_mh.h \ zebra/zebra_neigh.h \ zebra/zserv.h \ + zebra/dpdk/zebra_dplane_dpdk.h \ + zebra/dpdk/zebra_dplane_dpdk_private.h \ # end zebra_zebra_irdp_la_SOURCES = \ @@ -257,3 +261,10 @@ zebra_zebra_SOURCES += \ zebra/debug_nl.c \ # end endif + +if DP_DPDK +module_LTLIBRARIES += zebra/zebra_dplane_dpdk.la +endif + +zebra_zebra_dplane_dpdk_la_SOURCES = zebra/dpdk/zebra_dplane_dpdk.c zebra/dpdk/zebra_dplane_dpdk_vty.c +zebra_zebra_dplane_dpdk_la_LDFLAGS = -avoid-version -module -shared -export-dynamic -L/usr/local/lib -v