zebra: use dataplane to read interface NETCONF info

Use the dataplane to query and read interface NETCONF data;
add netconf-oriented data to the dplane context object, and
add accessors for it. Add handler for incoming update
processing.

Signed-off-by: Mark Stapp <mstapp@nvidia.com>

zebra/if_netlink.c

@@ -2080,10 +2080,6 @@ void interface_list(struct zebra_ns *zns)
 	netlink_nexthop_read(zns);
 
 	interface_addr_lookup_netlink(zns);
-
-	/* Read some other properties via the 'netconf' apis */
-	netconf_lookup_netlink(zns);
-
 }
 
 #endif /* GNU_LINUX */

zebra/interface.c

@@ -1318,6 +1318,56 @@ done:
 	dplane_ctx_fini(&ctx);
 }
 
+/*
+ * Handle netconf change from a dplane context object; runs in the main
+ * pthread so it can update zebra data structs.
+ */
+int zebra_if_netconf_update_ctx(struct zebra_dplane_ctx *ctx)
+{
+	struct zebra_ns *zns;
+	struct interface *ifp;
+	struct zebra_if *zif;
+	enum dplane_netconf_status_e mpls;
+	int ret = 0;
+
+	zns = zebra_ns_lookup(dplane_ctx_get_netconf_ns_id(ctx));
+	if (zns == NULL) {
+		ret = -1;
+		goto done;
+	}
+
+	ifp = if_lookup_by_index_per_ns(zns,
+					dplane_ctx_get_netconf_ifindex(ctx));
+	if (ifp == NULL) {
+		ret = -1;
+		goto done;
+	}
+
+	zif = ifp->info;
+	if (zif == NULL) {
+		ret = -1;
+		goto done;
+	}
+
+	mpls = dplane_ctx_get_netconf_mpls(ctx);
+
+	if (mpls == DPLANE_NETCONF_STATUS_ENABLED)
+		zif->mpls = true;
+	else if (mpls == DPLANE_NETCONF_STATUS_DISABLED)
+		zif->mpls = false;
+
+	if (IS_ZEBRA_DEBUG_KERNEL)
+		zlog_debug("%s: if %s, ifindex %d, mpls %s",
+			   __func__, ifp->name, ifp->ifindex,
+			   (zif->mpls ? "ON" : "OFF"));
+
+done:
+	/* Free the context */
+	dplane_ctx_fini(&ctx);
+
+	return ret;
+}
+
 /* Dump if address information to vty. */
 static void connected_dump_vty(struct vty *vty, json_object *json,
 			       struct connected *connected)

zebra/interface.h

@@ -523,6 +523,7 @@ extern void zebra_l2_unmap_slave_from_bond(struct zebra_if *zif);
 extern const char *zebra_protodown_rc_str(enum protodown_reasons protodown_rc,
 					  char *pd_buf, uint32_t pd_buf_len);
 void zebra_if_addr_update_ctx(struct zebra_dplane_ctx *ctx);
+int zebra_if_netconf_update_ctx(struct zebra_dplane_ctx *ctx);
 
 #ifdef HAVE_PROC_NET_DEV
 extern void ifstat_update_proc(void);

zebra/kernel_netlink.c

@@ -372,13 +372,11 @@ static int netlink_information_fetch(struct nlmsghdr *h, ns_id_t ns_id,
 	case RTM_DELNEXTHOP:
 		return netlink_nexthop_change(h, ns_id, startup);
 
-	case RTM_NEWNETCONF:
-	case RTM_DELNETCONF:
-		return netlink_netconf_change(h, ns_id, startup);
-
 	/* Messages handled in the dplane thread */
 	case RTM_NEWADDR:
 	case RTM_DELADDR:
+	case RTM_NEWNETCONF:
+	case RTM_DELNETCONF:
 		return 0;
 
 	default:
@@ -1616,13 +1614,15 @@ void kernel_init(struct zebra_ns *zns)
 		RTMGRP_NEIGH                   |
 		((uint32_t) 1 << (RTNLGRP_IPV4_RULE - 1)) |
 		((uint32_t) 1 << (RTNLGRP_IPV6_RULE - 1)) |
-		((uint32_t) 1 << (RTNLGRP_NEXTHOP - 1)) |
+		((uint32_t) 1 << (RTNLGRP_NEXTHOP - 1));
-		((uint32_t) 1 << (RTNLGRP_IPV4_NETCONF - 1)) |
-		((uint32_t) 1 << (RTNLGRP_MPLS_NETCONF - 1));
 
 	dplane_groups = (RTMGRP_LINK            |
 			 RTMGRP_IPV4_IFADDR     |
-			 RTMGRP_IPV6_IFADDR);
+			 RTMGRP_IPV6_IFADDR     |
+			 ((uint32_t) 1 << (RTNLGRP_IPV4_NETCONF - 1)) |
+			 ((uint32_t) 1 << (RTNLGRP_IPV6_NETCONF - 1)) |
+			 ((uint32_t) 1 << (RTNLGRP_MPLS_NETCONF - 1)));
+
 
 	snprintf(zns->netlink.name, sizeof(zns->netlink.name),
 		 "netlink-listen (NS %u)", zns->ns_id);

zebra/netconf_netlink.c

@@ -37,25 +37,53 @@
 
 static struct rtattr *netconf_rta(struct netconfmsg *ncm)
 {
-	return (struct rtattr *)((char *)ncm
+	return (struct rtattr *)((char *)ncm +
-				 + NLMSG_ALIGN(sizeof(struct netconfmsg)));
+				 NLMSG_ALIGN(sizeof(struct netconfmsg)));
 }
 
+/*
+ * Handle netconf update about a single interface: create dplane
+ * context, and enqueue for processing in the main zebra pthread.
+ */
+static int netlink_netconf_dplane_update(ns_id_t ns_id, ifindex_t ifindex,
+					 enum dplane_netconf_status_e mpls_on,
+					 enum dplane_netconf_status_e mcast_on)
+{
+	struct zebra_dplane_ctx *ctx;
+
+	ctx = dplane_ctx_alloc();
+	dplane_ctx_set_op(ctx, DPLANE_OP_INTF_NETCONFIG);
+	dplane_ctx_set_netconf_ns_id(ctx, ns_id);
+	dplane_ctx_set_netconf_ifindex(ctx, ifindex);
+
+	dplane_ctx_set_netconf_mpls(ctx, mpls_on);
+	dplane_ctx_set_netconf_mcast(ctx, mcast_on);
+
+	/* Enqueue ctx for main pthread to process */
+	dplane_provider_enqueue_to_zebra(ctx);
+
+	return 0;
+}
+
+/*
+ * Parse and process an incoming netlink netconf update.
+ */
 int netlink_netconf_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 {
 	struct netconfmsg *ncm;
 	struct rtattr *tb[NETCONFA_MAX + 1] = {};
 	int len;
 	ifindex_t ifindex;
-	bool mpls_on = false;
+	uint32_t ival;
-	bool mc_on = false;
+	enum dplane_netconf_status_e mpls_on = DPLANE_NETCONF_STATUS_UNKNOWN;
+	enum dplane_netconf_status_e mcast_on = DPLANE_NETCONF_STATUS_UNKNOWN;
 
 	if (h->nlmsg_type != RTM_NEWNETCONF && h->nlmsg_type != RTM_DELNETCONF)
 		return 0;
 
 	len = h->nlmsg_len - NLMSG_LENGTH(sizeof(struct netconfmsg));
 	if (len < 0) {
-		zlog_err("%s: Message received from netlink is of a broken size: %d %zu",
+		zlog_err("%s: Message received from netlink is of a broken size: %d, min %zu",
 			 __func__, h->nlmsg_len,
 			 (size_t)NLMSG_LENGTH(sizeof(struct netconfmsg)));
 		return -1;
@@ -66,8 +94,7 @@ int netlink_netconf_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 	netlink_parse_rtattr(tb, NETCONFA_MAX, netconf_rta(ncm), len);
 
 	if (!tb[NETCONFA_IFINDEX]) {
-		zlog_err("%s: Message received from netlink that we expected to receive an interface on",
+		zlog_err("NETCONF message received from netlink without an ifindex");
-			__func__);
 		return 0;
 	}
 
@@ -81,59 +108,68 @@ int netlink_netconf_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 		 * for all and default interfaces.  I am not 100% sure
 		 * what that is yet, or where we would store it.
 		 */
+		if (IS_ZEBRA_DEBUG_KERNEL)
+			zlog_debug("%s: Ignoring global ifindex %d",
+				   __func__, ifindex);
+
 		return 0;
 	default:
 		break;
 	}
+
 	if (tb[NETCONFA_INPUT]) {
-		mpls_on = *(bool *)RTA_DATA(tb[NETCONFA_INPUT]);
+		ival = *(uint32_t *)RTA_DATA(tb[NETCONFA_INPUT]);
-		/* Create a context and pass it up for processing */
+		if (ival != 0)
+			mpls_on = DPLANE_NETCONF_STATUS_ENABLED;
+		else
+			mpls_on = DPLANE_NETCONF_STATUS_DISABLED;
 	}
 
 	if (tb[NETCONFA_MC_FORWARDING]) {
-		mc_on = *(bool *)RTA_DATA(tb[NETCONFA_MC_FORWARDING]);
+		ival = *(uint32_t *)RTA_DATA(tb[NETCONFA_MC_FORWARDING]);
-		/* Create a context and pass it up for processing */
+		if (ival != 0)
+			mcast_on = DPLANE_NETCONF_STATUS_ENABLED;
+		else
+			mcast_on = DPLANE_NETCONF_STATUS_DISABLED;
 	}
 
 	if (IS_ZEBRA_DEBUG_KERNEL)
-		zlog_debug("Interface %u is mpls on: %d multicast on: %d",
+		zlog_debug("%s: interface %u is mpls on: %d multicast on: %d",
-			   ifindex, mpls_on, mc_on);
+			   __func__, ifindex, mpls_on, mcast_on);
+
+	/* Create a dplane context and pass it along for processing */
+	netlink_netconf_dplane_update(ns_id, ifindex, mpls_on, mcast_on);
 
 	return 0;
 }
 
-static int netlink_request_netconf(struct nlsock *netlink_cmd)
+/*
+ * Request info from the host OS. This only sends the request; any replies
+ * are processed asynchronously.
+ */
+int netlink_request_netconf(int sockfd)
 {
+	struct nlsock *nls;
 	struct {
 		struct nlmsghdr n;
 		struct netconfmsg ncm;
 		char buf[1024];
-	} req;
+	} req = {};
+
+	nls = kernel_netlink_nlsock_lookup(sockfd);
+
+	if (IS_ZEBRA_DEBUG_KERNEL)
+		zlog_debug("%s: nlsock %s", __func__, nls ? nls->name : "NULL");
+
+	if (nls == NULL)
+		return -1;
 
-	memset(&req, 0, sizeof(req));
 	req.n.nlmsg_len = NLMSG_LENGTH(sizeof(struct netconfmsg));
 	req.n.nlmsg_flags = NLM_F_DUMP | NLM_F_REQUEST;
 	req.n.nlmsg_type = RTM_GETNETCONF;
 	req.ncm.ncm_family = AF_UNSPEC;
 
-	return netlink_request(netlink_cmd, &req);
+	return netlink_request(nls, &req);
-}
-
-int netconf_lookup_netlink(struct zebra_ns *zns)
-{
-	int ret;
-	struct zebra_dplane_info dp_info;
-	struct nlsock *netlink_cmd = &zns->netlink_cmd;
-
-	zebra_dplane_info_from_zns(&dp_info, zns, true);
-
-	ret = netlink_request_netconf(netlink_cmd);
-	if (ret < 0)
-		return ret;
-
-	ret = netlink_parse_info(netlink_netconf_change, netlink_cmd, &dp_info,
-				 0, 1);
-	return ret;
 }
 
 #endif	/* HAVE_NETLINK */

zebra/netconf_netlink.h

@@ -26,14 +26,17 @@
 
 #ifdef HAVE_NETLINK /* Netlink-only module */
 
+#include "zebra/zebra_ns.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+/* Parse and handle a NETCONF message. */
 extern int netlink_netconf_change(struct nlmsghdr *h, ns_id_t ns_id,
 				  int startup);
-
+/* Request info from the host OS. */
-extern int netconf_lookup_netlink(struct zebra_ns *zns);
+int netlink_request_netconf(int sockfd);
 
 
 #ifdef __cplusplus

zebra/zebra_dplane.c

@@ -28,6 +28,7 @@
 #include "lib/memory.h"
 #include "lib/queue.h"
 #include "lib/zebra.h"
+#include "zebra/netconf_netlink.h"
 #include "zebra/zebra_router.h"
 #include "zebra/zebra_dplane.h"
 #include "zebra/zebra_vxlan_private.h"
@@ -430,6 +431,9 @@ PREDECL_DLIST(zns_info_list);
 struct dplane_zns_info {
 	struct zebra_dplane_info info;
 
+	/* Request data from the OS */
+	struct thread *t_request;
+
 	/* Read event */
 	struct thread *t_read;
 
@@ -554,8 +558,6 @@ DECLARE_DLIST(zns_info_list, struct dplane_zns_info, link);
 
 /* Prototypes */
 static void dplane_thread_loop(struct thread *event);
-static void dplane_info_from_zns(struct zebra_dplane_info *ns_info,
-				 struct zebra_ns *zns);
 static enum zebra_dplane_result lsp_update_internal(struct zebra_lsp *lsp,
 						    enum dplane_op_e op);
 static enum zebra_dplane_result pw_update_internal(struct zebra_pw *pw,
@@ -2384,6 +2386,22 @@ uint32_t dplane_get_in_queue_len(void)
 				    memory_order_seq_cst);
 }
 
+/*
+ * Internal helper that copies information from a zebra ns object; this is
+ * called in the zebra main pthread context as part of dplane ctx init.
+ */
+static void ctx_info_from_zns(struct zebra_dplane_info *ns_info,
+			      struct zebra_ns *zns)
+{
+	ns_info->ns_id = zns->ns_id;
+
+#if defined(HAVE_NETLINK)
+	ns_info->is_cmd = true;
+	ns_info->sock = zns->netlink_dplane_out.sock;
+	ns_info->seq = zns->netlink_dplane_out.seq;
+#endif /* NETLINK */
+}
+
 /*
  * Common dataplane context init with zebra namespace info.
  */
@@ -2391,7 +2409,7 @@ static int dplane_ctx_ns_init(struct zebra_dplane_ctx *ctx,
 			      struct zebra_ns *zns,
 			      bool is_update)
 {
-	dplane_info_from_zns(&(ctx->zd_ns_info), zns);
+	ctx_info_from_zns(&(ctx->zd_ns_info), zns); /*  */
 
 	ctx->zd_is_update = is_update;
 
@@ -4923,21 +4941,6 @@ bool dplane_provider_is_threaded(const struct zebra_dplane_provider *prov)
 	return (prov->dp_flags & DPLANE_PROV_FLAG_THREADED);
 }
 
-/*
- * Internal helper that copies information from a zebra ns object; this is
- * called in the zebra main pthread context as part of dplane ctx init.
- */
-static void dplane_info_from_zns(struct zebra_dplane_info *ns_info,
-				 struct zebra_ns *zns)
-{
-	ns_info->ns_id = zns->ns_id;
-
-#if defined(HAVE_NETLINK)
-	ns_info->is_cmd = true;
-	ns_info->sock = zns->netlink_dplane_out.sock;
-#endif /* NETLINK */
-}
-
 #ifdef HAVE_NETLINK
 /*
  * Callback when an OS (netlink) incoming event read is ready. This runs
@@ -4953,6 +4956,40 @@ static void dplane_incoming_read(struct thread *event)
 	thread_add_read(zdplane_info.dg_master, dplane_incoming_read, zi,
 			zi->info.sock, &zi->t_read);
 }
+
+/*
+ * Callback in the dataplane pthread that requests info from the OS and
+ * initiates netlink reads.
+ */
+static void dplane_incoming_request(struct thread *event)
+{
+	struct dplane_zns_info *zi = THREAD_ARG(event);
+
+	/* Start read task */
+	thread_add_read(zdplane_info.dg_master, dplane_incoming_read, zi,
+			zi->info.sock, &zi->t_read);
+
+	/* Send requests */
+	netlink_request_netconf(zi->info.sock);
+}
+
+/*
+ * Initiate requests for existing info from the OS. This is called by the
+ * main pthread, but we want all activity on the dplane netlink socket to
+ * take place on the dplane pthread, so we schedule an event to accomplish
+ * that.
+ */
+static void dplane_kernel_info_request(struct dplane_zns_info *zi)
+{
+	/* If we happen to encounter an enabled zns before the dplane
+	 * pthread is running, we'll initiate this later on.
+	 */
+	if (zdplane_info.dg_master)
+		thread_add_event(zdplane_info.dg_master,
+				 dplane_incoming_request, zi, 0,
+				 &zi->t_request);
+}
+
 #endif /* HAVE_NETLINK */
 
 /*
@@ -4996,11 +5033,10 @@ void zebra_dplane_ns_enable(struct zebra_ns *zns, bool enabled)
 		zi->info.is_cmd = false;
 		zi->info.sock = zns->netlink_dplane_in.sock;
 
-		/* Start read task for the dplane pthread. */
+		/* Initiate requests for existing info from the OS, and
-		if (zdplane_info.dg_master)
+		 * begin reading from the netlink socket.
-			thread_add_read(zdplane_info.dg_master,
+		 */
-					dplane_incoming_read, zi, zi->info.sock,
+		dplane_kernel_info_request(zi);
-					&zi->t_read);
 #endif
 	} else if (zi) {
 		if (IS_ZEBRA_DEBUG_DPLANE)
@@ -5010,9 +5046,14 @@ void zebra_dplane_ns_enable(struct zebra_ns *zns, bool enabled)
 		/* Stop reading, free memory */
 		zns_info_list_del(&zdplane_info.dg_zns_list, zi);
 
-		if (zdplane_info.dg_master)
+		/* Stop any outstanding tasks */
+		if (zdplane_info.dg_master) {
+			thread_cancel_async(zdplane_info.dg_master,
+					    &zi->t_request, NULL);
+
 			thread_cancel_async(zdplane_info.dg_master, &zi->t_read,
 					    NULL);
+		}
 
 		XFREE(MTYPE_DP_NS, zi);
 	}
@@ -5692,8 +5733,10 @@ static void dplane_check_shutdown_status(struct thread *event)
 	frr_each_safe (zns_info_list, &zdplane_info.dg_zns_list, zi) {
 		zns_info_list_del(&zdplane_info.dg_zns_list, zi);
 
-		if (zdplane_info.dg_master)
+		if (zdplane_info.dg_master) {
 			thread_cancel(&zi->t_read);
+			thread_cancel(&zi->t_request);
+		}
 
 		XFREE(MTYPE_DP_NS, zi);
 	}
@@ -6023,11 +6066,12 @@ void zebra_dplane_start(void)
 	thread_add_event(zdplane_info.dg_master, dplane_thread_loop, NULL, 0,
 			 &zdplane_info.dg_t_update);
 
-	/* Enqueue reads if necessary */
+	/* Enqueue requests and reads if necessary */
 	frr_each (zns_info_list, &zdplane_info.dg_zns_list, zi) {
 #if defined(HAVE_NETLINK)
 		thread_add_read(zdplane_info.dg_master, dplane_incoming_read,
 				zi, zi->info.sock, &zi->t_read);
+		dplane_kernel_info_request(zi);
 #endif
 	}
 

zebra/zebra_rib.c

@@ -4320,6 +4320,10 @@ static void rib_process_dplane_results(struct thread *thread)
 				zebra_if_addr_update_ctx(ctx);
 				break;
 
+			case DPLANE_OP_INTF_NETCONFIG:
+				zebra_if_netconf_update_ctx(ctx);
+				break;
+
 			/* Some op codes not handled here */
 			case DPLANE_OP_ADDR_INSTALL:
 			case DPLANE_OP_ADDR_UNINSTALL:
@@ -4334,7 +4338,6 @@ static void rib_process_dplane_results(struct thread *thread)
 			case DPLANE_OP_BR_PORT_UPDATE:
 			case DPLANE_OP_NEIGH_TABLE_UPDATE:
 			case DPLANE_OP_GRE_SET:
-			case DPLANE_OP_INTF_NETCONFIG:
 			case DPLANE_OP_NONE:
 				/* Don't expect this: just return the struct? */
 				dplane_ctx_fini(&ctx);