Merge pull request #6950 from opensourcerouting/bfd-distributed-v3

bfdd: distributed BFD

bfdd/bfd.c

@@ -216,6 +216,9 @@ void bfd_session_apply(struct bfd_session *bs)
 	    && (bs->timers.desired_min_tx != min_tx
 		|| bs->timers.required_min_rx != min_rx))
 		bfd_set_polling(bs);
+
+	/* Send updated information to data plane. */
+	bfd_dplane_update_session(bs);
 }
 
 void bfd_profile_remove(struct bfd_session *bs)
@@ -293,6 +296,10 @@ int bfd_session_enable(struct bfd_session *bs)
 	struct vrf *vrf = NULL;
 	int psock;
 
+	/* We are using data plane, we don't need software. */
+	if (bs->bdc)
+		return 0;
+
 	/*
 	 * If the interface or VRF doesn't exist, then we must register
 	 * the session but delay its start.
@@ -332,9 +339,14 @@ int bfd_session_enable(struct bfd_session *bs)
 		bs->vrf = vrf_lookup_by_id(VRF_DEFAULT);
 	assert(bs->vrf);
 
-	if (bs->key.ifname[0]
-	    && CHECK_FLAG(bs->flags, BFD_SESS_FLAG_MH) == 0)
-		bs->ifp = ifp;
+	/* Assign interface pointer (if any). */
+	bs->ifp = ifp;
+
+	/* Attempt to use data plane. */
+	if (bglobal.bg_use_dplane && bfd_dplane_add_session(bs) == 0) {
+		control_notify_config(BCM_NOTIFY_CONFIG_ADD, bs);
+		return 0;
+	}
 
 	/* Sanity check: don't leak open sockets. */
 	if (bs->sock != -1) {
@@ -383,6 +395,10 @@ int bfd_session_enable(struct bfd_session *bs)
  */
 void bfd_session_disable(struct bfd_session *bs)
 {
+	/* We are using data plane, we don't need software. */
+	if (bs->bdc)
+		return;
+
 	/* Free up socket resources. */
 	if (bs->sock != -1) {
 		close(bs->sock);
@@ -393,8 +409,6 @@ void bfd_session_disable(struct bfd_session *bs)
 	bfd_recvtimer_delete(bs);
 	bfd_xmttimer_delete(bs);
 	ptm_bfd_echo_stop(bs);
-	bs->vrf = NULL;
-	bs->ifp = NULL;
 
 	/* Set session down so it doesn't report UP and disabled. */
 	ptm_bfd_sess_dn(bs, BD_PATH_DOWN);
@@ -804,6 +818,9 @@ void bfd_session_free(struct bfd_session *bs)
 
 	bfd_session_disable(bs);
 
+	/* Remove session from data plane if any. */
+	bfd_dplane_delete_session(bs);
+
 	bfd_key_delete(bs->key);
 	bfd_id_delete(bs->discrs.my_discr);
 
@@ -1267,14 +1284,18 @@ void bfd_set_echo(struct bfd_session *bs, bool echo)
 		SET_FLAG(bs->flags, BFD_SESS_FLAG_ECHO);
 
 		/* Activate/update echo receive timeout timer. */
-		bs_echo_timer_handler(bs);
+		if (bs->bdc == NULL)
+			bs_echo_timer_handler(bs);
 	} else {
 		/* Check if echo mode is already disabled. */
 		if (!CHECK_FLAG(bs->flags, BFD_SESS_FLAG_ECHO))
 			return;
 
 		UNSET_FLAG(bs->flags, BFD_SESS_FLAG_ECHO);
-		ptm_bfd_echo_stop(bs);
+
+		/* Deactivate timeout timer. */
+		if (bs->bdc == NULL)
+			ptm_bfd_echo_stop(bs);
 	}
 }
 
@@ -1299,6 +1320,14 @@ void bfd_set_shutdown(struct bfd_session *bs, bool shutdown)
 
 		SET_FLAG(bs->flags, BFD_SESS_FLAG_SHUTDOWN);
 
+		/* Handle data plane shutdown case. */
+		if (bs->bdc) {
+			bs->ses_state = PTM_BFD_ADM_DOWN;
+			bfd_dplane_update_session(bs);
+			control_notify(bs, bs->ses_state);
+			return;
+		}
+
 		/* Disable all events. */
 		bfd_recvtimer_delete(bs);
 		bfd_echo_recvtimer_delete(bs);
@@ -1319,6 +1348,14 @@ void bfd_set_shutdown(struct bfd_session *bs, bool shutdown)
 
 		UNSET_FLAG(bs->flags, BFD_SESS_FLAG_SHUTDOWN);
 
+		/* Handle data plane shutdown case. */
+		if (bs->bdc) {
+			bs->ses_state = PTM_BFD_DOWN;
+			bfd_dplane_update_session(bs);
+			control_notify(bs, bs->ses_state);
+			return;
+		}
+
 		/* Change and notify state change. */
 		bs->ses_state = PTM_BFD_DOWN;
 		control_notify(bs, bs->ses_state);
@@ -2028,6 +2065,16 @@ static int bfd_vrf_enable(struct vrf *vrf)
 		bvrf = XCALLOC(MTYPE_BFDD_VRF, sizeof(struct bfd_vrf_global));
 		bvrf->vrf = vrf;
 		vrf->info = (void *)bvrf;
+
+		/* Disable sockets if using data plane. */
+		if (bglobal.bg_use_dplane) {
+			bvrf->bg_shop = -1;
+			bvrf->bg_mhop = -1;
+			bvrf->bg_shop6 = -1;
+			bvrf->bg_mhop6 = -1;
+			bvrf->bg_echo = -1;
+			bvrf->bg_echov6 = -1;
+		}
 	} else
 		bvrf = vrf->info;
 
@@ -2049,25 +2096,24 @@ static int bfd_vrf_enable(struct vrf *vrf)
 		if (!bvrf->bg_echov6)
 			bvrf->bg_echov6 = bp_echov6_socket(vrf);
 
-		/* Add descriptors to the event loop. */
-		if (!bvrf->bg_ev[0])
-			thread_add_read(master, bfd_recv_cb, bvrf, bvrf->bg_shop,
-					&bvrf->bg_ev[0]);
-		if (!bvrf->bg_ev[1])
-			thread_add_read(master, bfd_recv_cb, bvrf, bvrf->bg_mhop,
-					&bvrf->bg_ev[1]);
+		if (!bvrf->bg_ev[0] && bvrf->bg_shop != -1)
+			thread_add_read(master, bfd_recv_cb, bvrf,
+					bvrf->bg_shop, &bvrf->bg_ev[0]);
+		if (!bvrf->bg_ev[1] && bvrf->bg_mhop != -1)
+			thread_add_read(master, bfd_recv_cb, bvrf,
+					bvrf->bg_mhop, &bvrf->bg_ev[1]);
 		if (!bvrf->bg_ev[2] && bvrf->bg_shop6 != -1)
-			thread_add_read(master, bfd_recv_cb, bvrf, bvrf->bg_shop6,
-					&bvrf->bg_ev[2]);
+			thread_add_read(master, bfd_recv_cb, bvrf,
+					bvrf->bg_shop6, &bvrf->bg_ev[2]);
 		if (!bvrf->bg_ev[3] && bvrf->bg_mhop6 != -1)
-			thread_add_read(master, bfd_recv_cb, bvrf, bvrf->bg_mhop6,
-					&bvrf->bg_ev[3]);
-		if (!bvrf->bg_ev[4])
-			thread_add_read(master, bfd_recv_cb, bvrf, bvrf->bg_echo,
-					&bvrf->bg_ev[4]);
+			thread_add_read(master, bfd_recv_cb, bvrf,
+					bvrf->bg_mhop6, &bvrf->bg_ev[3]);
+		if (!bvrf->bg_ev[4] && bvrf->bg_echo != -1)
+			thread_add_read(master, bfd_recv_cb, bvrf,
+					bvrf->bg_echo, &bvrf->bg_ev[4]);
 		if (!bvrf->bg_ev[5] && bvrf->bg_echov6 != -1)
-			thread_add_read(master, bfd_recv_cb, bvrf, bvrf->bg_echov6,
-					&bvrf->bg_ev[5]);
+			thread_add_read(master, bfd_recv_cb, bvrf,
+					bvrf->bg_echov6, &bvrf->bg_ev[5]);
 	}
 	if (vrf->vrf_id != VRF_DEFAULT) {
 		bfdd_zclient_register(vrf->vrf_id);

bfdd/bfd.h

@@ -269,6 +269,7 @@ struct bfd_session {
 	struct bfd_key key;
 	struct peer_label *pl;
 
+	struct bfd_dplane_ctx *bdc;
 	struct sockaddr_any local_address;
 	struct interface *ifp;
 	struct vrf *vrf;
@@ -424,6 +425,10 @@ struct bfd_vrf_global {
 	struct thread *bg_ev[6];
 };
 
+/* Forward declaration of data plane context struct. */
+struct bfd_dplane_ctx;
+TAILQ_HEAD(dplane_queue, bfd_dplane_ctx);
+
 struct bfd_global {
 	int bg_csock;
 	struct thread *bg_csockev;
@@ -441,7 +446,15 @@ struct bfd_global {
 	 */
 	bool bg_shutdown;
 
+	/* Distributed BFD items. */
+	bool bg_use_dplane;
+	int bg_dplane_sock;
+	struct thread *bg_dplane_sockev;
+	struct dplane_queue bg_dplaneq;
+
 	/* Debug options. */
+	/* Show distributed BFD debug messages. */
+	bool debug_dplane;
 	/* Show all peer state changes events. */
 	bool debug_peer_event;
 	/*
@@ -742,4 +755,56 @@ void bfd_session_update_vrf_name(struct bfd_session *bs, struct vrf *vrf);
 
 int ptm_bfd_notify(struct bfd_session *bs, uint8_t notify_state);
 
+/*
+ * dplane.c
+ */
+
+/**
+ * Initialize BFD data plane infrastructure for distributed BFD implementation.
+ *
+ * \param sa socket address.
+ * \param salen socket address structure length.
+ * \param client `true` means connecting socket, `false` listening socket.
+ */
+void bfd_dplane_init(const struct sockaddr *sa, socklen_t salen, bool client);
+
+/**
+ * Attempts to delegate the BFD session liveness detection to hardware.
+ *
+ * \param bs the BFD session data structure.
+ *
+ * \returns
+ * `0` on success and BFD daemon should do nothing or `-1` on failure
+ * and we should fallback to software implementation.
+ */
+int bfd_dplane_add_session(struct bfd_session *bs);
+
+/**
+ * Send new session settings to data plane.
+ *
+ * \param bs the BFD session to update.
+ */
+int bfd_dplane_update_session(const struct bfd_session *bs);
+
+/**
+ * Deletes session from data plane.
+ *
+ * \param bs the BFD session to delete.
+ *
+ * \returns `0` on success otherwise `-1`.
+ */
+int bfd_dplane_delete_session(struct bfd_session *bs);
+
+/**
+ * Asks the data plane for updated counters and update the session data
+ * structure.
+ *
+ * \param bs the BFD session that needs updating.
+ *
+ * \returns `0` on success otherwise `-1` on failure.
+ */
+int bfd_dplane_update_session_counters(struct bfd_session *bs);
+
+void bfd_dplane_show_counters(struct vty *vty);
+
 #endif /* _BFD_H_ */

bfdd/bfdd.c

@@ -20,12 +20,20 @@
 
 #include <zebra.h>
 
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+
+#include <err.h>
+
 #include "filter.h"
 #include "if.h"
 #include "vrf.h"
 
 #include "bfd.h"
 #include "bfdd_nb.h"
+#include "bfddp_packet.h"
 #include "lib/version.h"
 #include "lib/command.h"
 
@@ -129,8 +137,10 @@ FRR_DAEMON_INFO(bfdd, BFD, .vty_port = 2617,
 		.n_yang_modules = array_size(bfdd_yang_modules))
 
 #define OPTION_CTLSOCK 1001
+#define OPTION_DPLANEADDR 2000
 static const struct option longopts[] = {
 	{"bfdctl", required_argument, NULL, OPTION_CTLSOCK},
+	{"dplaneaddr", required_argument, NULL, OPTION_DPLANEADDR},
 	{0}
 };
 
@@ -160,6 +170,143 @@ const struct bfd_state_str_list state_list[] = {
 	{.str = NULL},
 };
 
+static uint16_t
+parse_port(const char *str)
+{
+	char *nulbyte;
+	long rv;
+
+	errno = 0;
+	rv = strtol(str, &nulbyte, 10);
+	/* No conversion performed. */
+	if (rv == 0 && errno == EINVAL) {
+		fprintf(stderr, "invalid BFD data plane address port: %s\n",
+			str);
+		exit(0);
+	}
+	/* Invalid number range. */
+	if ((rv <= 0 || rv >= 65535) || errno == ERANGE) {
+		fprintf(stderr, "invalid BFD data plane port range: %s\n",
+			str);
+		exit(0);
+	}
+	/* There was garbage at the end of the string. */
+	if (*nulbyte != 0) {
+		fprintf(stderr, "invalid BFD data plane port: %s\n",
+			str);
+		exit(0);
+	}
+
+	return (uint16_t)rv;
+}
+
+static void
+distributed_bfd_init(const char *arg)
+{
+	char *sptr, *saux;
+	bool is_client = false;
+	size_t slen;
+	socklen_t salen;
+	char addr[64];
+	char type[64];
+	union {
+		struct sockaddr_in sin;
+		struct sockaddr_in6 sin6;
+		struct sockaddr_un sun;
+	} sa;
+
+	/* Basic parsing: find ':' to figure out type part and address part. */
+	sptr = strchr(arg, ':');
+	if (sptr == NULL) {
+		fprintf(stderr, "invalid BFD data plane socket: %s\n", arg);
+		exit(1);
+	}
+
+	/* Calculate type string length. */
+	slen = (size_t)(sptr - arg);
+
+	/* Copy the address part. */
+	sptr++;
+	strlcpy(addr, sptr, sizeof(addr));
+
+	/* Copy type part. */
+	strlcpy(type, arg, slen + 1);
+
+	/* Reset address data. */
+	memset(&sa, 0, sizeof(sa));
+
+	/* Fill the address information. */
+	if (strcmp(type, "unix") == 0 || strcmp(type, "unixc") == 0) {
+		if (strcmp(type, "unixc") == 0)
+			is_client = true;
+
+		salen = sizeof(sa.sun);
+		sa.sun.sun_family = AF_UNIX;
+		strlcpy(sa.sun.sun_path, addr, sizeof(sa.sun.sun_path));
+	} else if (strcmp(type, "ipv4") == 0 || strcmp(type, "ipv4c") == 0) {
+		if (strcmp(type, "ipv4c") == 0)
+			is_client = true;
+
+		salen = sizeof(sa.sin);
+		sa.sin.sin_family = AF_INET;
+
+		/* Parse port if any. */
+		sptr = strchr(addr, ':');
+		if (sptr == NULL) {
+			sa.sin.sin_port = htons(BFD_DATA_PLANE_DEFAULT_PORT);
+		} else {
+			*sptr = 0;
+			sa.sin.sin_port = htons(parse_port(sptr + 1));
+		}
+
+		if (inet_pton(AF_INET, addr, &sa.sin.sin_addr) != 1)
+			errx(1, "%s: inet_pton: invalid address %s", __func__,
+			     addr);
+	} else if (strcmp(type, "ipv6") == 0 || strcmp(type, "ipv6c") == 0) {
+		if (strcmp(type, "ipv6c") == 0)
+			is_client = true;
+
+		salen = sizeof(sa.sin6);
+		sa.sin6.sin6_family = AF_INET6;
+
+		/* Check for IPv6 enclosures '[]' */
+		sptr = &addr[0];
+		if (*sptr != '[')
+			errx(1, "%s: invalid IPv6 address format: %s", __func__,
+			     addr);
+
+		saux = strrchr(addr, ']');
+		if (saux == NULL)
+			errx(1, "%s: invalid IPv6 address format: %s", __func__,
+			     addr);
+
+		/* Consume the '[]:' part. */
+		slen = saux - sptr;
+		memmove(addr, addr + 1, slen);
+		addr[slen - 1] = 0;
+
+		/* Parse port if any. */
+		saux++;
+		sptr = strrchr(saux, ':');
+		if (sptr == NULL) {
+			sa.sin6.sin6_port = htons(BFD_DATA_PLANE_DEFAULT_PORT);
+		} else {
+			*sptr = 0;
+			sa.sin6.sin6_port = htons(parse_port(sptr + 1));
+		}
+
+		if (inet_pton(AF_INET6, addr, &sa.sin6.sin6_addr) != 1)
+			errx(1, "%s: inet_pton: invalid address %s", __func__,
+			     addr);
+	} else {
+		fprintf(stderr, "invalid BFD data plane socket type: %s\n",
+			type);
+		exit(1);
+	}
+
+	/* Initialize BFD data plane listening socket. */
+	bfd_dplane_init((struct sockaddr *)&sa, salen, is_client);
+}
 
 static void bg_init(void)
 {
@@ -185,7 +332,7 @@ static void bg_init(void)
 
 int main(int argc, char *argv[])
 {
-	char ctl_path[512];
+	char ctl_path[512], dplane_addr[512];
 	bool ctlsockused = false;
 	int opt;
 
@@ -194,7 +341,8 @@ int main(int argc, char *argv[])
 
 	frr_preinit(&bfdd_di, argc, argv);
 	frr_opt_add("", longopts,
-		    "      --bfdctl       Specify bfdd control socket\n");
+		    "      --bfdctl       Specify bfdd control socket\n"
+		    "      --dplaneaddr   Specify BFD data plane address\n");
 
 	snprintf(ctl_path, sizeof(ctl_path), BFDD_CONTROL_SOCKET,
 		 "", "");
@@ -208,6 +356,10 @@ int main(int argc, char *argv[])
 			strlcpy(ctl_path, optarg, sizeof(ctl_path));
 			ctlsockused = true;
 			break;
+		case OPTION_DPLANEADDR:
+			strlcpy(dplane_addr, optarg, sizeof(dplane_addr));
+			bglobal.bg_use_dplane = true;
+			break;
 
 		default:
 			frr_help_exit(1);
@@ -248,6 +400,10 @@ int main(int argc, char *argv[])
 	/* read configuration file and daemonize  */
 	frr_config_fork();
 
+	/* Initialize BFD data plane listening socket. */
+	if (bglobal.bg_use_dplane)
+		distributed_bfd_init(dplane_addr);
+
 	frr_run(master);
 	/* NOTREACHED */
 

bfdd/bfdd_vty.c

@@ -348,6 +348,11 @@ static void _display_peer_counter(struct vty *vty, struct bfd_session *bs)
 {
 	_display_peer_header(vty, bs);
 
+	/* Ask data plane for updated counters. */
+	if (bfd_dplane_update_session_counters(bs) == -1)
+		zlog_debug("%s: failed to update BFD session counters (%s)",
+			   __func__, bs_to_string(bs));
+
 	vty_out(vty, "\t\tControl packet input: %" PRIu64 " packets\n",
 		bs->stats.rx_ctrl_pkt);
 	vty_out(vty, "\t\tControl packet output: %" PRIu64 " packets\n",
@@ -369,6 +374,11 @@ static struct json_object *__display_peer_counters_json(struct bfd_session *bs)
 {
 	struct json_object *jo = _peer_json_header(bs);
 
+	/* Ask data plane for updated counters. */
+	if (bfd_dplane_update_session_counters(bs) == -1)
+		zlog_debug("%s: failed to update BFD session counters (%s)",
+			   __func__, bs_to_string(bs));
+
 	json_object_int_add(jo, "control-packet-input", bs->stats.rx_ctrl_pkt);
 	json_object_int_add(jo, "control-packet-output", bs->stats.tx_ctrl_pkt);
 	json_object_int_add(jo, "echo-packet-input", bs->stats.rx_echo_pkt);
@@ -748,6 +758,28 @@ DEFPY(bfd_show_peers_brief, bfd_show_peers_brief_cmd,
 	return CMD_SUCCESS;
 }
 
+DEFPY(show_bfd_distributed, show_bfd_distributed_cmd,
+      "show bfd distributed",
+      SHOW_STR
+      "Bidirection Forwarding Detection\n"
+      "Show BFD data plane (distributed BFD) statistics\n")
+{
+	bfd_dplane_show_counters(vty);
+	return CMD_SUCCESS;
+}
+
+DEFPY(
+	bfd_debug_distributed, bfd_debug_distributed_cmd,
+	"[no] debug bfd distributed",
+	NO_STR
+	DEBUG_STR
+	"Bidirection Forwarding Detection\n"
+	"BFD data plane (distributed BFD) debugging\n")
+{
+	bglobal.debug_dplane = !no;
+	return CMD_SUCCESS;
+}
+
 DEFPY(
 	bfd_debug_peer, bfd_debug_peer_cmd,
 	"[no] debug bfd peer",
@@ -888,6 +920,8 @@ DEFUN_NOSH(show_debugging_bfd,
 	   "BFD daemon\n")
 {
 	vty_out(vty, "BFD debugging status:\n");
+	if (bglobal.debug_dplane)
+		vty_out(vty, "  Distributed BFD debugging is on.\n");
 	if (bglobal.debug_peer_event)
 		vty_out(vty, "  Peer events debugging is on.\n");
 	if (bglobal.debug_zebra)
@@ -919,6 +953,11 @@ static int bfdd_write_config(struct vty *vty)
 	struct lyd_node *dnode;
 	int written = 0;
 
+	if (bglobal.debug_dplane) {
+		vty_out(vty, "debug bfd distributed\n");
+		written = 1;
+	}
+
 	if (bglobal.debug_peer_event) {
 		vty_out(vty, "debug bfd peer\n");
 		written = 1;
@@ -951,12 +990,15 @@ void bfdd_vty_init(void)
 	install_element(ENABLE_NODE, &bfd_show_peers_cmd);
 	install_element(ENABLE_NODE, &bfd_show_peer_cmd);
 	install_element(ENABLE_NODE, &bfd_show_peers_brief_cmd);
+	install_element(ENABLE_NODE, &show_bfd_distributed_cmd);
 	install_element(ENABLE_NODE, &show_debugging_bfd_cmd);
 
+	install_element(ENABLE_NODE, &bfd_debug_distributed_cmd);
 	install_element(ENABLE_NODE, &bfd_debug_peer_cmd);
 	install_element(ENABLE_NODE, &bfd_debug_zebra_cmd);
 	install_element(ENABLE_NODE, &bfd_debug_network_cmd);
 
+	install_element(CONFIG_NODE, &bfd_debug_distributed_cmd);
 	install_element(CONFIG_NODE, &bfd_debug_peer_cmd);
 	install_element(CONFIG_NODE, &bfd_debug_zebra_cmd);
 	install_element(CONFIG_NODE, &bfd_debug_network_cmd);

bfdd/bfddp_packet.h

@@ -0,0 +1,381 @@
+/*
+ * BFD Data Plane protocol messages header.
+ *
+ * Copyright (C) 2020 Network Device Education Foundation, Inc. ("NetDEF")
+ *                    Rafael F. Zalamena
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the “Software”), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+/**
+ * \file bfddp_packet.h
+ */
+#ifndef BFD_DP_PACKET_H
+#define BFD_DP_PACKET_H
+
+#include <netinet/in.h>
+
+#include <stdint.h>
+
+/*
+ * Protocol definitions.
+ */
+
+/**
+ * BFD protocol version as defined in RFC5880 Section 4.1 Generic BFD Control
+ * Packet Format.
+ */
+#define BFD_PROTOCOL_VERSION 1
+
+/** Default data plane port. */
+#define BFD_DATA_PLANE_DEFAULT_PORT 50700
+
+/** BFD single hop UDP port, as defined in RFC 5881 Section 4. Encapsulation. */
+#define BFD_SINGLE_HOP_PORT 3784
+
+/** BFD multi hop UDP port, as defined in RFC 5883 Section 5. Encapsulation. */
+#define BFD_MULTI_HOP_PORT 4784
+
+/** Default slow start multiplier. */
+#define SLOWSTART_DMULT 3
+/** Default slow start transmission speed. */
+#define SLOWSTART_TX 1000000u
+/** Default slow start receive speed. */
+#define SLOWSTART_RX 1000000u
+/** Default slow start echo receive speed. */
+#define SLOWSTART_ERX 0u
+
+/*
+ * BFD single hop source UDP ports. As defined in RFC 5881 Section 4.
+ * Encapsulation.
+ */
+#define BFD_SOURCE_PORT_BEGIN 49152
+#define BFD_SOURCE_PORT_END 65535
+
+/** BFD data plane protocol version. */
+#define BFD_DP_VERSION 1
+
+/** BFD data plane message types. */
+enum bfddp_message_type {
+	/** Ask for BFD daemon or data plane for echo packet. */
+	ECHO_REQUEST = 0,
+	/** Answer a ECHO_REQUEST packet. */
+	ECHO_REPLY = 1,
+	/** Add or update BFD peer session. */
+	DP_ADD_SESSION = 2,
+	/** Delete BFD peer session. */
+	DP_DELETE_SESSION = 3,
+	/** Tell BFD daemon state changed: timer expired or session down. */
+	BFD_STATE_CHANGE = 4,
+
+	/** Ask for BFD session counters. */
+	DP_REQUEST_SESSION_COUNTERS = 5,
+	/** Tell BFD daemon about counters values. */
+	BFD_SESSION_COUNTERS = 6,
+};
+
+/**
+ * `ECHO_REQUEST`/`ECHO_REPLY` data payload.
+ *
+ * Data plane might use whatever precision it wants for `dp_time`
+ * field, however if you want to be able to tell the delay between
+ * data plane packet send and BFD daemon packet processing you should
+ * use `gettimeofday()` and have the data plane clock synchronized with
+ * BFD daemon (not a problem if data plane runs in the same system).
+ *
+ * Normally data plane will only check the time stamp it sent to determine
+ * the whole packet trip time.
+ */
+struct bfddp_echo {
+	/** Filled by data plane. */
+	uint64_t dp_time;
+	/** Filled by BFD daemon. */
+	uint64_t bfdd_time;
+};
+
+
+/** BFD session flags. */
+enum bfddp_session_flag {
+	/** Set when using multi hop. */
+	SESSION_MULTIHOP = (1 << 0),
+	/** Set when using demand mode. */
+	SESSION_DEMAND = (1 << 1),
+	/** Set when using cbit (Control Plane Independent). */
+	SESSION_CBIT = (1 << 2),
+	/** Set when using echo mode. */
+	SESSION_ECHO = (1 << 3),
+	/** Set when using IPv6. */
+	SESSION_IPV6 = (1 << 4),
+	/** Set when using passive mode. */
+	SESSION_PASSIVE = (1 << 5),
+	/** Set when session is administrative down. */
+	SESSION_SHUTDOWN = (1 << 6),
+};
+
+/**
+ * `DP_ADD_SESSION`/`DP_DELETE_SESSION` data payload.
+ *
+ * `lid` is unique in BFD daemon so it might be used as key for data
+ * structures lookup.
+ */
+struct bfddp_session {
+	/** Important session flags. \see bfddp_session_flag. */
+	uint32_t flags;
+	/**
+	 * Session source address.
+	 *
+	 * Check `flags` field for `SESSION_IPV6` before using as IPv6.
+	 */
+	struct in6_addr src;
+	/**
+	 * Session destination address.
+	 *
+	 * Check `flags` field for `SESSION_IPV6` before using as IPv6.
+	 */
+	struct in6_addr dst;
+
+	/** Local discriminator. */
+	uint32_t lid;
+	/**
+	 * Minimum desired transmission interval (in microseconds) without
+	 * jitter.
+	 */
+	uint32_t min_tx;
+	/**
+	 * Required minimum receive interval rate (in microseconds) without
+	 * jitter.
+	 */
+	uint32_t min_rx;
+	/**
+	 * Required minimum echo receive interval rate (in microseconds)
+	 * without jitter.
+	 */
+	uint32_t min_echo_rx;
+	/** Amount of milliseconds to wait before starting the session */
+	uint32_t hold_time;
+
+	/** Minimum TTL. */
+	uint8_t ttl;
+	/** Detection multiplier. */
+	uint8_t detect_mult;
+	/** Reserved / zeroed. */
+	uint16_t zero;
+
+	/** Interface index (set to `0` when unavailable). */
+	uint32_t ifindex;
+	/** Interface name (empty when unavailable). */
+	char ifname[64];
+
+	/* TODO: missing authentication. */
+};
+
+/** BFD packet state values as defined in RFC 5880, Section 4.1. */
+enum bfd_state_value {
+	/** Session is administratively down. */
+	STATE_ADMINDOWN = 0,
+	/** Session is down or went down. */
+	STATE_DOWN = 1,
+	/** Session is initializing. */
+	STATE_INIT = 2,
+	/** Session is up. */
+	STATE_UP = 3,
+};
+
+/** BFD diagnostic field values as defined in RFC 5880, Section 4.1. */
+enum bfd_diagnostic_value {
+	/** Nothing was diagnosed. */
+	DIAG_NOTHING = 0,
+	/** Control detection time expired. */
+	DIAG_CONTROL_EXPIRED = 1,
+	/** Echo function failed. */
+	DIAG_ECHO_FAILED = 2,
+	/** Neighbor signaled down. */
+	DIAG_DOWN = 3,
+	/** Forwarding plane reset. */
+	DIAG_FP_RESET = 4,
+	/** Path down. */
+	DIAG_PATH_DOWN = 5,
+	/** Concatenated path down. */
+	DIAG_CONCAT_PATH_DOWN = 6,
+	/** Administratively down. */
+	DIAG_ADMIN_DOWN = 7,
+	/** Reverse concatenated path down. */
+	DIAG_REV_CONCAT_PATH_DOWN = 8,
+};
+
+/** BFD remote state flags. */
+enum bfd_remote_flags {
+	/** Control Plane Independent bit. */
+	RBIT_CPI = (1 << 0),
+	/** Demand mode bit. */
+	RBIT_DEMAND = (1 << 1),
+	/** Multipoint bit. */
+	RBIT_MP = (1 << 2),
+};
+
+/**
+ * `BFD_STATE_CHANGE` data payload.
+ */
+struct bfddp_state_change {
+	/** Local discriminator. */
+	uint32_t lid;
+	/** Remote discriminator. */
+	uint32_t rid;
+	/** Remote configurations/bits set. \see bfd_remote_flags. */
+	uint32_t remote_flags;
+	/** Remote minimum desired transmission interval. */
+	uint32_t desired_tx;
+	/** Remote minimum receive interval. */
+	uint32_t required_rx;
+	/** Remote minimum echo receive interval. */
+	uint32_t required_echo_rx;
+	/** Remote state. \see bfd_state_values.*/
+	uint8_t state;
+	/** Remote diagnostics (if any) */
+	uint8_t diagnostics;
+	/** Remote detection multiplier. */
+	uint8_t detection_multiplier;
+};
+
+/**
+ * BFD control packet state bits definition.
+ */
+enum bfddp_control_state_bits {
+	/** Used to request connection establishment signal. */
+	STATE_POLL_BIT = (1 << 5),
+	/** Finalizes the connection establishment signal. */
+	STATE_FINAL_BIT = (1 << 4),
+	/** Signalizes that forward plane doesn't depend on control plane. */
+	STATE_CPI_BIT = (1 << 3),
+	/** Signalizes the use of authentication. */
+	STATE_AUTH_BIT = (1 << 2),
+	/** Signalizes that peer is using demand mode. */
+	STATE_DEMAND_BIT = (1 << 1),
+	/** Used in RFC 8562 implementation. */
+	STATE_MULTI_BIT = (1 << 0),
+};
+
+/**
+ * BFD control packet.
+ *
+ * As defined in 'RFC 5880 Section 4.1 Generic BFD Control Packet Format'.
+ */
+struct bfddp_control_packet {
+	/** (3 bits version << 5) | (5 bits diag). */
+	uint8_t version_diag;
+	/**
+	 * (2 bits state << 6) | (6 bits flags)
+	 *
+	 * \see bfd_state_value, bfddp_control_state_bits.
+	 */
+	uint8_t state_bits;
+	/** Detection multiplier. */
+	uint8_t detection_multiplier;
+	/** Packet length in bytes. */
+	uint8_t length;
+	/** Our discriminator. */
+	uint32_t local_id;
+	/** Remote system discriminator. */
+	uint32_t remote_id;
+	/** Desired minimum send interval in microseconds. */
+	uint32_t desired_tx;
+	/** Desired minimum receive interval in microseconds. */
+	uint32_t required_rx;
+	/** Desired minimum echo receive interval in microseconds. */
+	uint32_t required_echo_rx;
+};
+
+/**
+ * The protocol wire message header structure.
+ */
+struct bfddp_message_header {
+	/** Protocol version format. \see BFD_DP_VERSION. */
+	uint8_t version;
+	/** Reserved / zero field. */
+	uint8_t zero;
+	/** Message contents type. \see bfddp_message_type. */
+	uint16_t type;
+	/**
+	 * Message identification (to pair request/response).
+	 *
+	 * The ID `0` is reserved for asynchronous messages (e.g. unrequested
+	 * messages).
+	 */
+	uint16_t id;
+	/** Message length. */
+	uint16_t length;
+};
+
+/**
+ * Data plane session counters request.
+ *
+ * Message type: `DP_REQUEST_SESSION_COUNTERS`.
+ */
+struct bfddp_request_counters {
+	/** Session local discriminator. */
+	uint32_t lid;
+};
+
+/**
+ * BFD session counters reply.
+ *
+ * Message type: `BFD_SESSION_COUNTERS`.
+ */
+struct bfddp_session_counters {
+	/** Session local discriminator. */
+	uint32_t lid;
+
+	/** Control packet bytes input. */
+	uint64_t control_input_bytes;
+	/** Control packets input. */
+	uint64_t control_input_packets;
+	/** Control packet bytes output. */
+	uint64_t control_output_bytes;
+	/** Control packets output. */
+	uint64_t control_output_packets;
+
+	/** Echo packet bytes input. */
+	uint64_t echo_input_bytes;
+	/** Echo packets input. */
+	uint64_t echo_input_packets;
+	/** Echo packet bytes output. */
+	uint64_t echo_output_bytes;
+	/** Echo packets output. */
+	uint64_t echo_output_packets;
+};
+
+/**
+ * The protocol wire messages structure.
+ */
+struct bfddp_message {
+	/** Message header. \see bfddp_message_header. */
+	struct bfddp_message_header header;
+
+	/** Message payload. \see bfddp_message_type. */
+	union {
+		struct bfddp_echo echo;
+		struct bfddp_session session;
+		struct bfddp_state_change state;
+		struct bfddp_control_packet control;
+		struct bfddp_request_counters counters_req;
+		struct bfddp_session_counters session_counters;
+	} data;
+};
+
+#endif /* BFD_DP_PACKET_H */

bfdd/ptm_adapter.c

@@ -703,7 +703,7 @@ static void bfdd_sessions_disable_interface(struct interface *ifp)
 			continue;
 
 		bfd_session_disable(bs);
-
+		bs->ifp = NULL;
 	}
 }
 
@@ -752,6 +752,7 @@ void bfdd_sessions_disable_vrf(struct vrf *vrf)
 			continue;
 
 		bfd_session_disable(bs);
+		bs->vrf = NULL;
 	}
 }
 

bfdd/subdir.am

@@ -22,10 +22,18 @@ bfdd_libbfd_a_SOURCES = \
 	bfdd/bfd_packet.c \
 	bfdd/config.c \
 	bfdd/control.c \
+	bfdd/dplane.c \
 	bfdd/event.c \
 	bfdd/ptm_adapter.c \
 	# end
 
+# Install headers so it can be used by external data plane
+# implementations.
+bfdd_headersdir = $(pkgincludedir)/bfdd
+bfdd_headers_HEADERS = \
+	bfdd/bfddp_packet.h \
+	# end
+
 clippy_scan += \
 	bfdd/bfdd_cli.c \
 	bfdd/bfdd_vty.c \

doc/user/bfd.rst

@@ -50,6 +50,40 @@ may also be specified (:ref:`common-invocation-options`).
    This option overrides the location addition that the -N option provides
    to the bfdd.sock
 
+.. option:: --dplaneaddr <type>:<address>[<:port>]
+
+   Configure the distributed BFD data plane listening socket bind address.
+
+   One would expect the data plane to run in the same machine as FRR, so
+   the suggested configuration would be:
+
+      --dplaneaddr unix:/var/run/frr/bfdd_dplane.sock
+
+   Or using IPv4:
+
+      --dplaneaddr ipv4:127.0.0.1
+
+   Or using IPv6:
+
+      --dplaneaddr ipv6:[::1]
+
+   It is also possible to specify a port (for IPv4/IPv6 only):
+
+     --dplaneaddr ipv6:[::1]:50701
+
+   (if ommited the default port is ``50700``).
+
+   It is also possible to operate in client mode (instead of listening for
+   connections). To connect to a data plane server append the letter 'c' to
+   the protocol, example:
+
+     --dplaneaddr ipv4c:127.0.0.1
+
+.. note::
+
+   When using UNIX sockets don't forget to check the file permissions
+   before attempting to use it.
+
 
 .. _bfd-commands:
 
@@ -113,6 +147,12 @@ BFDd Commands
 
     Show all configured BFD peers information and current status in brief.
 
+.. index:: show bfd distributed
+.. clicmd:: show bfd distributed
+
+   Show the BFD data plane (distributed BFD) statistics.
+
+
 .. _bfd-peer-config:
 
 Peer / Profile Configuration
@@ -603,6 +643,68 @@ You can also clear packet counters per session with the following commands, only
                 Session down events: 0
                 Zebra notifications: 4
 
+
+.. _bfd-distributed:
+
+Distributed BFD
+===============
+
+The distributed BFD is the separation of the BFD protocol control plane from
+the data plane. FRR implements its own BFD data plane protocol so vendors can
+study and include it in their own software/hardware without having to modify
+the FRR source code. The protocol definitions can be found at
+``bfdd/bfddp_packet.h`` header (or the installed
+``/usr/include/frr/bfdd/bfddp_packet.h``).
+
+To use this feature the BFD daemon needs to be started using the command line
+option :option:`--dplaneaddr`. When operating using this option the BFD daemon
+will not attempt to establish BFD sessions, but it will offload all its work to
+the data plane that is (or will be) connected. Data plane reconnection is also
+supported.
+
+The BFD data plane will be responsible for:
+
+* Sending/receiving the BFD protocol control/echo packets
+
+* Notifying BFD sessions state changes
+
+* Keeping the number of packets/bytes received/transmitted per session
+
+
+The FRR BFD daemon will be responsible for:
+
+* Adding/updating BFD session settings
+
+* Asking for BFD session counters
+
+* Redistributing the state changes to the integrated protocols (``bgpd``,
+  ``ospfd`` etc...)
+
+
+BFD daemon will also keep record of data plane communication statistics with
+the command :clicmd:`show bfd distributed`.
+
+Sample output:
+
+::
+
+   frr# show bfd distributed
+               Data plane
+               ==========
+          File descriptor: 16
+              Input bytes: 1296
+         Input bytes peak: 72
+           Input messages: 42
+      Input current usage: 0
+             Output bytes: 568
+        Output bytes peak: 136
+          Output messages: 19
+       Output full events: 0
+     Output current usage: 0
+
+
+.. _bfd-debugging:
+
 Debugging
 =========
 
@@ -619,6 +721,16 @@ sure you have `debugging` level enabled:
 You may also fine tune the debug messages by selecting one or more of the
 debug levels:
 
+.. index:: debug bfd distributed
+.. clicmd:: [no] debug bfd distributed
+
+   Toggle BFD data plane (distributed BFD) debugging.
+
+   Activates the following debug messages:
+
+   * Data plane received / send messages
+   * Connection events
+
 .. index:: debug bfd network
 .. clicmd:: [no] debug bfd network
 

lib/stream.c

@@ -1372,3 +1372,19 @@ void stream_fifo_free(struct stream_fifo *fifo)
 	stream_fifo_deinit(fifo);
 	XFREE(MTYPE_STREAM_FIFO, fifo);
 }
+
+void stream_pulldown(struct stream *s)
+{
+	size_t rlen = STREAM_READABLE(s);
+
+	/* No more data, so just move the pointers. */
+	if (rlen == 0) {
+		stream_reset(s);
+		return;
+	}
+
+	/* Move the available data to the beginning. */
+	memmove(s->data, &s->data[s->getp], rlen);
+	s->getp = 0;
+	s->endp = rlen;
+}

lib/stream.h

@@ -262,6 +262,16 @@ extern int stream_empty(struct stream *); /* is the stream empty? */
 /* debugging */
 extern void stream_hexdump(const struct stream *s);
 
+/**
+ * Reorganize the buffer data so it can fit more. This function is normally
+ * called right after stream data is consumed so we can read more data
+ * (the functions that consume data start with `stream_get*()` and macros
+ * `STREAM_GET*()`).
+ *
+ * \param s stream pointer.
+ */
+extern void stream_pulldown(struct stream *s);
+
 /* deprecated */
 extern uint8_t *stream_pnt(struct stream *);
 

redhat/frr.spec.in

@@ -747,6 +747,10 @@ sed -i 's/ -M rpki//' %{_sysconfdir}/frr/daemons
 %{_includedir}/%{name}/*.h
 %dir %{_includedir}/%{name}/ospfd
 %{_includedir}/%{name}/ospfd/*.h
+%if %{with_bfdd}
+    %dir %{_includedir}/%{name}/bfdd
+    %{_includedir}/%{name}/bfdd/bfddp_packet.h
+%endif
 %if %{with_ospfapi}
     %dir %{_includedir}/%{name}/ospfapi
     %{_includedir}/%{name}/ospfapi/*.h

zebra/dplane_fpm_nl.c

@@ -190,31 +190,6 @@ static int fpm_rib_reset(struct thread *t);
 static int fpm_rmac_send(struct thread *t);
 static int fpm_rmac_reset(struct thread *t);
 
-/*
- * Helper functions.
- */
-
-/**
- * Reorganizes the data on the buffer so it can fit more data.
- *
- * @param s stream pointer.
- */
-static void stream_pulldown(struct stream *s)
-{
-	size_t rlen = STREAM_READABLE(s);
-
-	/* No more data, so just move the pointers. */
-	if (rlen == 0) {
-		stream_reset(s);
-		return;
-	}
-
-	/* Move the available data to the beginning. */
-	memmove(s->data, &s->data[s->getp], rlen);
-	s->getp = 0;
-	s->endp = rlen;
-}
-
 /*
  * CLI.
  */