zebra: dataplane notifications for system route changes

Add notifications from zebra to the dataplane subsystem when
kernel or connected routes change.

Signed-off-by: Mark Stapp <mjs@voltanet.io>

zebra/zebra_dplane.c

@@ -246,6 +246,9 @@ static struct zebra_dplane_globals {
 	/* Limit number of pending, unprocessed updates */
 	_Atomic uint32_t dg_max_queued_updates;
 
+	/* Control whether system route notifications should be produced. */
+	bool dg_sys_route_notifs;
+
 	/* Limit number of new updates dequeued at once, to pace an
 	 * incoming burst.
 	 */
@@ -326,6 +329,12 @@ static struct zebra_dplane_ctx *dplane_ctx_alloc(void)
 	return p;
 }
 
+/* Enable system route notifications */
+void dplane_enable_sys_route_notifs(void)
+{
+	zdplane_info.dg_sys_route_notifs = true;
+}
+
 /*
  * Free a dataplane results context.
  */
@@ -347,6 +356,8 @@ static void dplane_ctx_free(struct zebra_dplane_ctx **pctx)
 	case DPLANE_OP_ROUTE_INSTALL:
 	case DPLANE_OP_ROUTE_UPDATE:
 	case DPLANE_OP_ROUTE_DELETE:
+	case DPLANE_OP_SYS_ROUTE_ADD:
+	case DPLANE_OP_SYS_ROUTE_DELETE:
 
 		/* Free allocated nexthops */
 		if ((*pctx)->u.rinfo.zd_ng.nexthop) {
@@ -532,6 +543,12 @@ const char *dplane_op2str(enum dplane_op_e op)
 		ret = "PW_UNINSTALL";
 		break;
 
+	case DPLANE_OP_SYS_ROUTE_ADD:
+		ret = "SYS_ROUTE_ADD";
+		break;
+	case DPLANE_OP_SYS_ROUTE_DELETE:
+		ret = "SYS_ROUTE_DEL";
+		break;
 	}
 
 	return ret;
@@ -962,21 +979,25 @@ static int dplane_ctx_route_init(struct zebra_dplane_ctx *ctx,
 	ctx->u.rinfo.zd_afi = info->afi;
 	ctx->u.rinfo.zd_safi = info->safi;
 
-	/* Extract ns info - can't use pointers to 'core' structs */
-	zvrf = vrf_info_lookup(re->vrf_id);
-	zns = zvrf->zns;
-
-	dplane_ctx_ns_init(ctx, zns, (op == DPLANE_OP_ROUTE_UPDATE));
-
 	/* Copy nexthops; recursive info is included too */
 	copy_nexthops(&(ctx->u.rinfo.zd_ng.nexthop), re->ng.nexthop, NULL);
 
-	/* TODO -- maybe use array of nexthops to avoid allocs? */
-
 	/* Ensure that the dplane's nexthops flags are clear. */
 	for (ALL_NEXTHOPS(ctx->u.rinfo.zd_ng, nexthop))
 		UNSET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
 
+	/* Don't need some info when capturing a system notification */
+	if (op == DPLANE_OP_SYS_ROUTE_ADD ||
+	    op == DPLANE_OP_SYS_ROUTE_DELETE) {
+		ret = AOK;
+		goto done;
+	}
+
+	/* Extract ns info - can't use pointers to 'core' structs */
+	zvrf = vrf_info_lookup(re->vrf_id);
+	zns = zvrf->zns;
+	dplane_ctx_ns_init(ctx, zns, (op == DPLANE_OP_ROUTE_UPDATE));
+
 	/* Trying out the sequence number idea, so we can try to detect
 	 * when a result is stale.
 	 */
@@ -1296,6 +1317,54 @@ done:
 	return ret;
 }
 
+/*
+ * Notify the dplane when system/connected routes change.
+ */
+enum zebra_dplane_result dplane_sys_route_add(struct route_node *rn,
+					      struct route_entry *re)
+{
+	enum zebra_dplane_result ret = ZEBRA_DPLANE_REQUEST_FAILURE;
+
+	/* Ignore this event unless a provider plugin has requested it. */
+	if (!zdplane_info.dg_sys_route_notifs) {
+		ret = ZEBRA_DPLANE_REQUEST_SUCCESS;
+		goto done;
+	}
+
+	if (rn == NULL || re == NULL)
+		goto done;
+
+	ret = dplane_route_update_internal(rn, re, NULL,
+					   DPLANE_OP_SYS_ROUTE_ADD);
+
+done:
+	return ret;
+}
+
+/*
+ * Notify the dplane when system/connected routes are deleted.
+ */
+enum zebra_dplane_result dplane_sys_route_del(struct route_node *rn,
+					      struct route_entry *re)
+{
+	enum zebra_dplane_result ret = ZEBRA_DPLANE_REQUEST_FAILURE;
+
+	/* Ignore this event unless a provider plugin has requested it. */
+	if (!zdplane_info.dg_sys_route_notifs) {
+		ret = ZEBRA_DPLANE_REQUEST_SUCCESS;
+		goto done;
+	}
+
+	if (rn == NULL || re == NULL)
+		goto done;
+
+	ret = dplane_route_update_internal(rn, re, NULL,
+					   DPLANE_OP_SYS_ROUTE_DELETE);
+
+done:
+	return ret;
+}
+
 /*
  * Enqueue LSP add for the dataplane.
  */
@@ -1856,6 +1925,12 @@ static int kernel_dplane_process_func(struct zebra_dplane_provider *prov)
 			res = kernel_dplane_pw_update(ctx);
 			break;
 
+		/* Ignore system 'notifications' - the kernel already knows */
+		case DPLANE_OP_SYS_ROUTE_ADD:
+		case DPLANE_OP_SYS_ROUTE_DELETE:
+			res = ZEBRA_DPLANE_REQUEST_SUCCESS;
+			break;
+
 		default:
 			atomic_fetch_add_explicit(
 				&zdplane_info.dg_other_errors, 1,
@@ -1916,6 +1991,11 @@ static int test_dplane_process_func(struct zebra_dplane_provider *prov)
 		if (ctx == NULL)
 			break;
 
+		if (IS_ZEBRA_DEBUG_DPLANE_DETAIL)
+			zlog_debug("dplane provider '%s': op %s",
+				   dplane_provider_get_name(prov),
+				   dplane_op2str(dplane_ctx_get_op(ctx)));
+
 		dplane_ctx_set_status(ctx, ZEBRA_DPLANE_REQUEST_SUCCESS);
 
 		dplane_provider_enqueue_out_ctx(prov, ctx);

zebra/zebra_dplane.h

@@ -114,8 +114,15 @@ enum dplane_op_e {
 	/* Pseudowire update */
 	DPLANE_OP_PW_INSTALL,
 	DPLANE_OP_PW_UNINSTALL,
+
+	/* System route notification */
+	DPLANE_OP_SYS_ROUTE_ADD,
+	DPLANE_OP_SYS_ROUTE_DELETE,
 };
 
+/* Enable system route notifications */
+void dplane_enable_sys_route_notifs(void);
+
 /*
  * The dataplane context struct is used to exchange info between the main zebra
  * context and the dataplane module(s). If these are two independent pthreads,
@@ -249,6 +256,12 @@ enum zebra_dplane_result dplane_route_update(struct route_node *rn,
 enum zebra_dplane_result dplane_route_delete(struct route_node *rn,
 					     struct route_entry *re);
 
+/* Notify the dplane when system/connected routes change */
+enum zebra_dplane_result dplane_sys_route_add(struct route_node *rn,
+					      struct route_entry *re);
+enum zebra_dplane_result dplane_sys_route_del(struct route_node *rn,
+					      struct route_entry *re);
+
 /*
  * Enqueue LSP change operations for the dataplane.
  */

zebra/zebra_rib.c

@@ -2799,6 +2799,12 @@ int rib_add_multipath(afi_t afi, safi_t safi, struct prefix *p,
 			break;
 	}
 
+	/* If this route is kernel/connected route, notify the dataplane. */
+	if (RIB_SYSTEM_ROUTE(re)) {
+		/* Notify dataplane */
+		dplane_sys_route_add(rn, re);
+	}
+
 	/* Link new re to node.*/
 	if (IS_ZEBRA_DEBUG_RIB) {
 		rnode_debug(rn, re->vrf_id,
@@ -3010,6 +3016,11 @@ void rib_delete(afi_t afi, safi_t safi, vrf_id_t vrf_id, int type,
 							       &vtep_ip, p);
 			}
 		}
+
+		/* Notify dplane if system route changes */
+		if (RIB_SYSTEM_ROUTE(re))
+			dplane_sys_route_del(rn, same);
+
 		rib_delnode(rn, same);
 	}
 
@@ -3355,6 +3366,12 @@ static int rib_process_dplane_results(struct thread *thread)
 				handle_pw_result(ctx);
 				break;
 
+			case DPLANE_OP_SYS_ROUTE_ADD:
+			case DPLANE_OP_SYS_ROUTE_DELETE:
+				/* No further processing in zebra for these. */
+				dplane_ctx_fini(&ctx);
+				break;
+
 			default:
 				/* Don't expect this: just return the struct? */
 				dplane_ctx_fini(&ctx);