zebra: use the dataplane to read netlink intf addr changes

Read incoming interface address change notifications in the
dplane pthread; enqueue the events to the main pthread
for processing. This is netlink-only for now - the bsd
kernel socket path remains unchanged.

Signed-off-by: Mark Stapp <mjs.ietf@gmail.com>

zebra/interface.c

@@ -1205,6 +1205,16 @@ void zebra_if_set_protodown(struct interface *ifp, bool down)
 #endif
 }
 
+/* Handler for incoming intf address change events */
+void zebra_if_addr_update_ctx(struct zebra_dplane_ctx *ctx)
+{
+#ifdef HAVE_NETLINK
+	netlink_interface_addr_ctx(ctx);
+#else
+	dplane_ctx_fini(&ctx);
+#endif /* HAVE_NETLINK */
+}
+
 /* Dump if address information to vty. */
 static void connected_dump_vty(struct vty *vty, json_object *json,
 			       struct connected *connected)

zebra/interface.h

@@ -513,6 +513,7 @@ extern void zebra_l2_map_slave_to_bond(struct zebra_if *zif, vrf_id_t vrf);
 extern void zebra_l2_unmap_slave_from_bond(struct zebra_if *zif);
 extern const char *zebra_protodown_rc_str(enum protodown_reasons protodown_rc,
 					  char *pd_buf, uint32_t pd_buf_len);
+void zebra_if_addr_update_ctx(struct zebra_dplane_ctx *ctx);
 
 #ifdef HAVE_PROC_NET_DEV
 extern void ifstat_update_proc(void);

zebra/kernel_netlink.c

@@ -324,6 +324,10 @@ static int netlink_socket(struct nlsock *nl, unsigned long groups,
 	return ret;
 }
 
+/*
+ * Dispatch an incoming netlink message; used by the zebra main pthread's
+ * netlink event reader.
+ */
 static int netlink_information_fetch(struct nlmsghdr *h, ns_id_t ns_id,
 				     int startup)
 {
@@ -345,10 +349,6 @@ static int netlink_information_fetch(struct nlmsghdr *h, ns_id_t ns_id,
 		return netlink_link_change(h, ns_id, startup);
 	case RTM_DELLINK:
 		return netlink_link_change(h, ns_id, startup);
-	case RTM_NEWADDR:
-		return netlink_interface_addr(h, ns_id, startup);
-	case RTM_DELADDR:
-		return netlink_interface_addr(h, ns_id, startup);
 	case RTM_NEWNEIGH:
 	case RTM_DELNEIGH:
 	case RTM_GETNEIGH:
@@ -361,6 +361,12 @@ static int netlink_information_fetch(struct nlmsghdr *h, ns_id_t ns_id,
 		return netlink_nexthop_change(h, ns_id, startup);
 	case RTM_DELNEXTHOP:
 		return netlink_nexthop_change(h, ns_id, startup);
+
+	/* Messages handled in the dplane thread */
+	case RTM_NEWADDR:
+	case RTM_DELADDR:
+		return 0;
+
 	default:
 		/*
 		 * If we have received this message then
@@ -378,6 +384,32 @@ static int netlink_information_fetch(struct nlmsghdr *h, ns_id_t ns_id,
 	return 0;
 }
 
+/*
+ * Dispatch an incoming netlink message; used by the dataplane pthread's
+ * netlink event reader code.
+ */
+static int dplane_netlink_information_fetch(struct nlmsghdr *h, ns_id_t ns_id,
+					    int startup)
+{
+	/*
+	 * Dispatch the incoming messages that the dplane pthread handles
+	 */
+	switch (h->nlmsg_type) {
+	case RTM_NEWADDR:
+	case RTM_DELADDR:
+		return netlink_interface_addr_dplane(h, ns_id, startup);
+
+	/* TODO */
+	case RTM_NEWLINK:
+	case RTM_DELLINK:
+
+	default:
+		break;
+	}
+
+	return 0;
+}
+
 static int kernel_read(struct thread *thread)
 {
 	struct zebra_ns *zns = (struct zebra_ns *)THREAD_ARG(thread);
@@ -388,13 +420,24 @@ static int kernel_read(struct thread *thread)
 
 	netlink_parse_info(netlink_information_fetch, &zns->netlink, &dp_info,
 			   5, 0);
-	zns->t_netlink = NULL;
+
 	thread_add_read(zrouter.master, kernel_read, zns, zns->netlink.sock,
 			&zns->t_netlink);
 
 	return 0;
 }
 
+/*
+ * Called by the dplane pthread to read incoming OS messages and dispatch them.
+ */
+int kernel_dplane_read(struct zebra_dplane_info *info)
+{
+	netlink_parse_info(dplane_netlink_information_fetch, &info->nls, info,
+			   5, 0);
+
+	return 0;
+}
+
 /*
  * Filter out messages from self that occur on listener socket,
  * caused by our actions on the command socket(s)
@@ -476,9 +519,8 @@ static void netlink_install_filter(int sock, uint32_t pid, uint32_t dplane_pid)
 			     safe_strerror(errno));
 }
 
-void netlink_parse_rtattr_flags(struct rtattr **tb, int max,
-				struct rtattr *rta, int len,
-				unsigned short flags)
+void netlink_parse_rtattr_flags(struct rtattr **tb, int max, struct rtattr *rta,
+				int len, unsigned short flags)
 {
 	unsigned short type;
 
@@ -800,8 +842,7 @@ static int netlink_recv_msg(const struct nlsock *nl, struct msghdr msg,
  * ignored, -1 otherwise.
  */
 static int netlink_parse_error(const struct nlsock *nl, struct nlmsghdr *h,
-			       const struct zebra_dplane_info *zns,
-			       bool startup)
+			       bool is_cmd, bool startup)
 {
 	struct nlmsgerr *err = (struct nlmsgerr *)NLMSG_DATA(h);
 	int errnum = err->error;
@@ -834,7 +875,7 @@ static int netlink_parse_error(const struct nlsock *nl, struct nlmsghdr *h,
 	}
 
 	/* Deal with errors that occur because of races in link handling. */
-	if (zns->is_cmd
+	if (is_cmd
 	    && ((msg_type == RTM_DELROUTE
 		 && (-errnum == ENODEV || -errnum == ESRCH))
 		|| (msg_type == RTM_NEWROUTE
@@ -853,7 +894,7 @@ static int netlink_parse_error(const struct nlsock *nl, struct nlmsghdr *h,
 	 * do not log these as an error.
 	 */
 	if (msg_type == RTM_DELNEIGH
-	    || (zns->is_cmd && msg_type == RTM_NEWROUTE
+	    || (is_cmd && msg_type == RTM_NEWROUTE
 		&& (-errnum == ESRCH || -errnum == ENETUNREACH))) {
 		/*
 		 * This is known to happen in some situations, don't log as
@@ -925,8 +966,9 @@ int netlink_parse_info(int (*filter)(struct nlmsghdr *, ns_id_t, int),
 
 			/* Error handling. */
 			if (h->nlmsg_type == NLMSG_ERROR) {
-				int err = netlink_parse_error(nl, h, zns,
-							      startup);
+				int err = netlink_parse_error(
+					nl, h, zns->is_cmd, startup);
+
 				if (err == 1) {
 					if (!(h->nlmsg_flags & NLM_F_MULTI))
 						return 0;
@@ -938,8 +980,8 @@ int netlink_parse_info(int (*filter)(struct nlmsghdr *, ns_id_t, int),
 			/* OK we got netlink message. */
 			if (IS_ZEBRA_DEBUG_KERNEL)
 				zlog_debug(
-					"netlink_parse_info: %s type %s(%u), len=%d, seq=%u, pid=%u",
-					nl->name,
+					"%s: %s type %s(%u), len=%d, seq=%u, pid=%u",
+					__func__, nl->name,
 					nl_msg_type_to_str(h->nlmsg_type),
 					h->nlmsg_type, h->nlmsg_len,
 					h->nlmsg_seq, h->nlmsg_pid);
@@ -1141,7 +1183,8 @@ static int nl_batch_read_resp(struct nl_batch *bth)
 		}
 
 		if (h->nlmsg_type == NLMSG_ERROR) {
-			int err = netlink_parse_error(nl, h, bth->zns, 0);
+			int err = netlink_parse_error(nl, h, bth->zns->is_cmd,
+						      false);
 
 			if (err == -1)
 				dplane_ctx_set_status(

zebra/kernel_socket.c

@@ -1457,6 +1457,14 @@ void kernel_terminate(struct zebra_ns *zns, bool complete)
 	return;
 }
 
+/*
+ * Called by the dplane pthread to read incoming OS messages and dispatch them.
+ */
+int kernel_dplane_read(struct zebra_dplane_info *info)
+{
+	return 0;
+}
+
 void kernel_update_multi(struct dplane_ctx_q *ctx_list)
 {
 	struct zebra_dplane_ctx *ctx;

zebra/rt.h

@@ -110,6 +110,11 @@ extern int kernel_del_mac_nhg(uint32_t nhg_id);
  */
 extern void kernel_update_multi(struct dplane_ctx_q *ctx_list);
 
+/*
+ * Called by the dplane pthread to read incoming OS messages and dispatch them.
+ */
+int kernel_dplane_read(struct zebra_dplane_info *info);
+
 #ifdef __cplusplus
 }
 #endif

zebra/zebra_dplane.c

@@ -37,11 +37,12 @@
 #include "zebra/zebra_pbr.h"
 #include "printfrr.h"
 
-/* Memory type for context blocks */
+/* Memory types */
 DEFINE_MTYPE_STATIC(ZEBRA, DP_CTX, "Zebra DPlane Ctx");
 DEFINE_MTYPE_STATIC(ZEBRA, DP_INTF, "Zebra DPlane Intf");
 DEFINE_MTYPE_STATIC(ZEBRA, DP_PROV, "Zebra DPlane Provider");
 DEFINE_MTYPE_STATIC(ZEBRA, DP_NETFILTER, "Zebra Netfilter Internal Object");
+DEFINE_MTYPE_STATIC(ZEBRA, DP_NS, "DPlane NSes");
 
 #ifndef AOK
 #  define AOK 0
@@ -402,6 +403,19 @@ struct zebra_dplane_provider {
 	TAILQ_ENTRY(zebra_dplane_provider) dp_prov_link;
 };
 
+/* Declare types for list of zns info objects */
+PREDECL_DLIST(zns_info_list);
+
+struct dplane_zns_info {
+	struct zebra_dplane_info info;
+
+	/* Read event */
+	struct thread *t_read;
+
+	/* List linkage */
+	struct zns_info_list_item link;
+};
+
 /*
  * Globals
  */
@@ -424,6 +438,9 @@ static struct zebra_dplane_globals {
 	/* Ordered list of providers */
 	TAILQ_HEAD(zdg_prov_q, zebra_dplane_provider) dg_providers_q;
 
+	/* List of info about each zns */
+	struct zns_info_list_head dg_zns_list;
+
 	/* Counter used to assign internal ids to providers */
 	uint32_t dg_provider_id;
 
@@ -498,6 +515,9 @@ static struct zebra_dplane_globals {
 
 } zdplane_info;
 
+/* Instantiate zns list type */
+DECLARE_DLIST(zns_info_list, struct dplane_zns_info, link);
+
 /*
  * Lock and unlock for interactions with the zebra 'core' pthread
  */
@@ -4813,6 +4833,88 @@ static void dplane_info_from_zns(struct zebra_dplane_info *ns_info,
 #endif /* NETLINK */
 }
 
+#ifdef HAVE_NETLINK
+/*
+ * Callback when an OS (netlink) incoming event read is ready. This runs
+ * in the dplane pthread.
+ */
+static int dplane_incoming_read(struct thread *event)
+{
+	struct dplane_zns_info *zi = THREAD_ARG(event);
+
+	kernel_dplane_read(&zi->info);
+
+	/* Re-start read task */
+	thread_add_read(zdplane_info.dg_master, dplane_incoming_read, zi,
+			zi->info.nls.sock, &zi->t_read);
+
+	return 0;
+}
+#endif /* HAVE_NETLINK */
+
+/*
+ * Notify dplane when namespaces are enabled and disabled. The dplane
+ * needs to start and stop reading incoming events from the zns. In the
+ * common case where vrfs are _not_ namespaces, there will only be one
+ * of these.
+ *
+ * This is called in the main pthread.
+ */
+void zebra_dplane_ns_enable(struct zebra_ns *zns, bool enabled)
+{
+	struct dplane_zns_info *zi;
+
+	if (IS_ZEBRA_DEBUG_DPLANE)
+		zlog_debug("%s: %s for nsid %u", __func__,
+			   (enabled ? "ENABLED" : "DISABLED"), zns->ns_id);
+
+	/* Search for an existing zns info entry */
+	frr_each (zns_info_list, &zdplane_info.dg_zns_list, zi) {
+		if (zi->info.ns_id == zns->ns_id)
+			break;
+	}
+
+	if (enabled) {
+		/* Create a new entry if necessary; start reading. */
+		if (zi == NULL) {
+			zi = XCALLOC(MTYPE_DP_NS, sizeof(*zi));
+
+			zi->info.ns_id = zns->ns_id;
+
+			zns_info_list_add_tail(&zdplane_info.dg_zns_list, zi);
+
+			if (IS_ZEBRA_DEBUG_DPLANE)
+				zlog_debug("%s: nsid %u, new zi %p", __func__,
+					   zns->ns_id, zi);
+		}
+
+		/* Make sure we're up-to-date with the zns object */
+#if defined(HAVE_NETLINK)
+		zi->info.is_cmd = false;
+		zi->info.nls = zns->netlink_dplane_in;
+
+		/* Start read task for the dplane pthread. */
+		if (zdplane_info.dg_master)
+			thread_add_read(zdplane_info.dg_master,
+					dplane_incoming_read, zi,
+					zi->info.nls.sock, &zi->t_read);
+#endif
+	} else if (zi) {
+		if (IS_ZEBRA_DEBUG_DPLANE)
+			zlog_debug("%s: nsid %u, deleting zi %p", __func__,
+				   zns->ns_id, zi);
+
+		/* Stop reading, free memory */
+		zns_info_list_del(&zdplane_info.dg_zns_list, zi);
+
+		if (zdplane_info.dg_master)
+			thread_cancel_async(zdplane_info.dg_master, &zi->t_read,
+					    NULL);
+
+		XFREE(MTYPE_DP_NS, zi);
+	}
+}
+
 /*
  * Provider api to signal that work/events are available
  * for the dataplane pthread.
@@ -5468,9 +5570,21 @@ done:
  */
 static int dplane_check_shutdown_status(struct thread *event)
 {
+	struct dplane_zns_info *zi;
+
 	if (IS_ZEBRA_DEBUG_DPLANE)
 		zlog_debug("Zebra dataplane shutdown status check called");
 
+	/* Remove any zns info entries as we stop the dplane pthread. */
+	frr_each_safe (zns_info_list, &zdplane_info.dg_zns_list, zi) {
+		zns_info_list_del(&zdplane_info.dg_zns_list, zi);
+
+		if (zdplane_info.dg_master)
+			thread_cancel(&zi->t_read);
+
+		XFREE(MTYPE_DP_NS, zi);
+	}
+
 	if (dplane_work_pending()) {
 		/* Reschedule dplane check on a short timer */
 		thread_add_timer_msec(zdplane_info.dg_master,
@@ -5765,6 +5879,7 @@ static void zebra_dplane_init_internal(void)
 
 	TAILQ_INIT(&zdplane_info.dg_update_ctx_q);
 	TAILQ_INIT(&zdplane_info.dg_providers_q);
+	zns_info_list_init(&zdplane_info.dg_zns_list);
 
 	zdplane_info.dg_updates_per_cycle = DPLANE_DEFAULT_NEW_WORK;
 
@@ -5780,6 +5895,7 @@ static void zebra_dplane_init_internal(void)
  */
 void zebra_dplane_start(void)
 {
+	struct dplane_zns_info *zi;
 	struct zebra_dplane_provider *prov;
 	struct frr_pthread_attr pattr = {
 		.start = frr_pthread_attr_default.start,
@@ -5799,6 +5915,14 @@ void zebra_dplane_start(void)
 	thread_add_event(zdplane_info.dg_master, dplane_thread_loop, NULL, 0,
 			 &zdplane_info.dg_t_update);
 
+	/* Enqueue reads if necessary */
+	frr_each (zns_info_list, &zdplane_info.dg_zns_list, zi) {
+#if defined(HAVE_NETLINK)
+		thread_add_read(zdplane_info.dg_master, dplane_incoming_read,
+				zi, zi->info.nls.sock, &zi->t_read);
+#endif
+	}
+
 	/* Call start callbacks for registered providers */
 
 	DPLANE_LOCK();

zebra/zebra_ns.c

@@ -123,6 +123,7 @@ int zebra_ns_enable(ns_id_t ns_id, void **info)
 	zns->ns_id = ns_id;
 
 	kernel_init(zns);
+	zebra_dplane_ns_enable(zns, true);
 	interface_list(zns);
 	route_read(zns);
 	kernel_read_pbr_rules(zns);
@@ -140,6 +141,8 @@ static int zebra_ns_disable_internal(struct zebra_ns *zns, bool complete)
 {
 	route_table_finish(zns->if_table);
 
+	zebra_dplane_ns_enable(zns, false /*Disable*/);
+
 	kernel_terminate(zns, complete);
 
 	table_manager_disable(zns->ns_id);

zebra/zebra_rib.c

@@ -4209,6 +4209,11 @@ static int rib_process_dplane_results(struct thread *thread)
 				zebra_pbr_dplane_result(ctx);
 				break;
 
+			case DPLANE_OP_INTF_ADDR_ADD:
+			case DPLANE_OP_INTF_ADDR_DEL:
+				zebra_if_addr_update_ctx(ctx);
+				break;
+
 			/* Some op codes not handled here */
 			case DPLANE_OP_ADDR_INSTALL:
 			case DPLANE_OP_ADDR_UNINSTALL:
@@ -4223,8 +4228,6 @@ static int rib_process_dplane_results(struct thread *thread)
 			case DPLANE_OP_BR_PORT_UPDATE:
 			case DPLANE_OP_NEIGH_TABLE_UPDATE:
 			case DPLANE_OP_GRE_SET:
-			case DPLANE_OP_INTF_ADDR_ADD:
-			case DPLANE_OP_INTF_ADDR_DEL:
 			case DPLANE_OP_NONE:
 				/* Don't expect this: just return the struct? */
 				dplane_ctx_fini(&ctx);