Merge pull request #6465 from xThaid/iprules

zebra: move ip rule installation to use dplane thread

tests/topotests/lib/topogen.py

@@ -658,7 +658,7 @@ class TopoRouter(TopoGear):
         Possible daemon values are: TopoRouter.RD_ZEBRA, TopoRouter.RD_RIP,
         TopoRouter.RD_RIPNG, TopoRouter.RD_OSPF, TopoRouter.RD_OSPF6,
         TopoRouter.RD_ISIS, TopoRouter.RD_BGP, TopoRouter.RD_LDP,
-        TopoRouter.RD_PIM.
+        TopoRouter.RD_PIM, TopoRouter.RD_PBR.
         """
         daemonstr = self.RD.get(daemon)
         self.logger.info('loading "{}" configuration: {}'.format(daemonstr, source))
@@ -1064,6 +1064,7 @@ def diagnose_env_linux():
             "isisd",
             "pimd",
             "ldpd",
+            "pbrd"
         ]:
             path = os.path.join(frrdir, fname)
             if not os.path.isfile(path):
@@ -1121,6 +1122,7 @@ def diagnose_env_linux():
             "ripngd",
             "isisd",
             "pimd",
+            "pbrd"
         ]:
             path = os.path.join(quaggadir, fname)
             if not os.path.isfile(path):

tests/topotests/lib/topotest.py

@@ -727,6 +727,57 @@ def ip6_route(node):
     return result
 
 
+def ip_rules(node):
+    """
+    Gets a structured return of the command 'ip rule'. It can be used in
+    conjuction with json_cmp() to provide accurate assert explanations.
+
+    Return example:
+    [
+        {
+            "pref": "0"
+            "from": "all"
+        },
+        {
+            "pref": "32766"
+            "from": "all"
+        },
+        {
+            "to": "3.4.5.0/24",
+            "iif": "r1-eth2",
+            "pref": "304",
+            "from": "1.2.0.0/16",
+            "proto": "zebra"
+        }
+    ]
+    """
+    output = normalize_text(node.run("ip rule")).splitlines()
+    result = []
+    for line in output:
+        columns = line.split(" ")
+
+        route = {}
+        # remove last character, since it is ':'
+        pref = columns[0][:-1]
+        route["pref"] = pref
+        prev = None
+        for column in columns:
+            if prev == "from":
+                route["from"] = column
+            if prev == "to":
+                route["to"] = column
+            if prev == "proto":
+                route["proto"] = column
+            if prev == "iif":
+                route["iif"] = column
+            if prev == "fwmark":
+                route["fwmark"] = column
+            prev = column
+
+        result.append(route)
+    return result
+
+
 def sleep(amount, reason=None):
     """
     Sleep wrapper that registers in the log the amount of sleep

tests/topotests/pbr-topo1/r1/linux-rules.json

@@ -0,0 +1,19 @@
+[
+  {
+    "iif": "r1-eth1",
+    "pref": "304",
+    "from": "4.5.6.7"
+  },
+  {
+    "to": "3.4.5.0/24",
+    "iif": "r1-eth2",
+    "pref": "304",
+    "from": "1.2.0.0/16"
+  },
+  {
+    "to": "9.9.9.9",
+    "iif": "r1-eth1",
+    "pref": "309",
+    "from": "all"
+  }
+]

tests/topotests/pbr-topo1/test_pbr_topo1.py

@@ -201,6 +201,28 @@ def test_pbr_flap():
         # Actual output from router
         actual = router.vtysh_cmd("show pbr interface json", isjson=True)
         assertmsg = '"show pbr interface" mismatches on {}'.format(router.name)
+
+        assert topotest.json_cmp(actual, expected) is None, assertmsg
+
+
+def test_rule_linux_installation():
+    "Ensure that rule is installed in the kernel"
+
+    tgen = get_topogen()
+    # Don't run this test if we have any failure.
+    if tgen.routers_have_failure():
+        pytest.skip(tgen.errors)
+
+    logger.info("Checking for installed PBR rules in OS")
+
+    router_list = tgen.routers().values()
+    for router in router_list:
+        rules_file = "{}/{}/linux-rules.json".format(CWD, router.name)
+
+        actual = topotest.ip_rules(router)
+        expected = json.loads(open(rules_file).read())
+
+        assertmsg = "Router {} OS rules mismatch".format(router.name)
         assert topotest.json_cmp(actual, expected) is None, assertmsg
 
 

zebra/rule_netlink.c

@@ -41,6 +41,7 @@
 #include "zebra/rule_netlink.h"
 #include "zebra/zebra_pbr.h"
 #include "zebra/zebra_errors.h"
+#include "zebra/zebra_dplane.h"
 
 /* definitions */
 
@@ -48,11 +49,19 @@
 
 /* Private functions */
 
-/* Install or uninstall specified rule for a specific interface.
+
- * Form netlink message and ship it. Currently, notify status after
+/*
- * waiting for netlink status.
+ * netlink_rule_msg_encode
+ *
+ * Encodes netlink RTM_ADDRULE/RTM_DELRULE message to buffer buf of size buflen.
+ *
+ * Returns -1 on failure or the number of bytes
+ * written to buf.
  */
-static int netlink_rule_update(int cmd, struct zebra_pbr_rule *rule)
+static ssize_t netlink_rule_msg_encode(
+	int cmd, const struct zebra_dplane_ctx *ctx, uint32_t filter_bm,
+	uint32_t priority, uint32_t table, const struct prefix *src_ip,
+	const struct prefix *dst_ip, uint32_t fwmark, void *buf, size_t buflen)
 {
 	uint8_t protocol = RTPROT_ZEBRA;
 	int family;
@@ -60,142 +69,133 @@ static int netlink_rule_update(int cmd, struct zebra_pbr_rule *rule)
 	struct {
 		struct nlmsghdr n;
 		struct fib_rule_hdr frh;
-		char buf[NL_PKT_BUF_SIZE];
+		char buf[];
-	} req;
+	} *req = buf;
-	struct zebra_ns *zns = zebra_ns_lookup(NS_DEFAULT);
+
-	struct sockaddr_nl snl;
+	const char *ifname = dplane_ctx_get_ifname(ctx);
 	char buf1[PREFIX_STRLEN];
 	char buf2[PREFIX_STRLEN];
 
-	memset(&req, 0, sizeof(req) - NL_PKT_BUF_SIZE);
+	memset(req, 0, sizeof(*req));
-	family = PREFIX_FAMILY(&rule->rule.filter.src_ip);
+	family = PREFIX_FAMILY(src_ip);
 	bytelen = (family == AF_INET ? 4 : 16);
 
-	req.n.nlmsg_type = cmd;
+	req->n.nlmsg_type = cmd;
-	req.n.nlmsg_len = NLMSG_LENGTH(sizeof(struct rtmsg));
+	req->n.nlmsg_len = NLMSG_LENGTH(sizeof(struct rtmsg));
-	req.n.nlmsg_flags = NLM_F_REQUEST;
+	req->n.nlmsg_flags = NLM_F_REQUEST;
-	req.n.nlmsg_pid = zns->netlink_cmd.snl.nl_pid;
 
-	req.frh.family = family;
+	req->frh.family = family;
-	req.frh.action = FR_ACT_TO_TBL;
+	req->frh.action = FR_ACT_TO_TBL;
 
-	addattr_l(&req.n, sizeof(req),
+	addattr_l(&req->n, buflen, FRA_PROTOCOL, &protocol, sizeof(protocol));
-		  FRA_PROTOCOL, &protocol, sizeof(protocol));
 
 	/* rule's pref # */
-	addattr32(&req.n, sizeof(req), FRA_PRIORITY, rule->rule.priority);
+	addattr32(&req->n, buflen, FRA_PRIORITY, priority);
 
 	/* interface on which applied */
-	addattr_l(&req.n, sizeof(req), FRA_IFNAME, rule->ifname,
+	addattr_l(&req->n, buflen, FRA_IFNAME, ifname, strlen(ifname) + 1);
-		  strlen(rule->ifname) + 1);
 
 	/* source IP, if specified */
-	if (IS_RULE_FILTERING_ON_SRC_IP(rule)) {
+	if (filter_bm & PBR_FILTER_SRC_IP) {
-		req.frh.src_len = rule->rule.filter.src_ip.prefixlen;
+		req->frh.src_len = src_ip->prefixlen;
-		addattr_l(&req.n, sizeof(req), FRA_SRC,
+		addattr_l(&req->n, buflen, FRA_SRC, &src_ip->u.prefix, bytelen);
-			  &rule->rule.filter.src_ip.u.prefix, bytelen);
 	}
+
 	/* destination IP, if specified */
-	if (IS_RULE_FILTERING_ON_DST_IP(rule)) {
+	if (filter_bm & PBR_FILTER_DST_IP) {
-		req.frh.dst_len = rule->rule.filter.dst_ip.prefixlen;
+		req->frh.dst_len = dst_ip->prefixlen;
-		addattr_l(&req.n, sizeof(req), FRA_DST,
+		addattr_l(&req->n, buflen, FRA_DST, &dst_ip->u.prefix, bytelen);
-			  &rule->rule.filter.dst_ip.u.prefix, bytelen);
 	}
 
 	/* fwmark, if specified */
-	if (IS_RULE_FILTERING_ON_FWMARK(rule)) {
+	if (filter_bm & PBR_FILTER_FWMARK)
-		addattr32(&req.n, sizeof(req), FRA_FWMARK,
+		addattr32(&req->n, buflen, FRA_FWMARK, fwmark);
-			  rule->rule.filter.fwmark);
-	}
 
 	/* Route table to use to forward, if filter criteria matches. */
-	if (rule->rule.action.table < 256)
+	if (table < 256)
-		req.frh.table = rule->rule.action.table;
+		req->frh.table = table;
 	else {
-		req.frh.table = RT_TABLE_UNSPEC;
+		req->frh.table = RT_TABLE_UNSPEC;
-		addattr32(&req.n, sizeof(req), FRA_TABLE,
+		addattr32(&req->n, buflen, FRA_TABLE, table);
-			  rule->rule.action.table);
 	}
 
 	if (IS_ZEBRA_DEBUG_KERNEL)
 		zlog_debug(
 			"Tx %s family %s IF %s(%u) Pref %u Fwmark %u Src %s Dst %s Table %u",
 			nl_msg_type_to_str(cmd), nl_family_to_str(family),
-			rule->ifname, rule->rule.ifindex, rule->rule.priority,
+			ifname, dplane_ctx_get_ifindex(ctx), priority, fwmark,
-			rule->rule.filter.fwmark,
+			prefix2str(src_ip, buf1, sizeof(buf1)),
-			prefix2str(&rule->rule.filter.src_ip, buf1,
+			prefix2str(dst_ip, buf2, sizeof(buf2)), table);
-				   sizeof(buf1)),
-			prefix2str(&rule->rule.filter.dst_ip, buf2,
-				   sizeof(buf2)),
-			rule->rule.action.table);
 
-	/* Ship off the message.
+	return NLMSG_ALIGN(req->n.nlmsg_len);
-	 * Note: Currently, netlink_talk() is a blocking call which returns
-	 * back the status.
-	 */
-	memset(&snl, 0, sizeof(snl));
-	snl.nl_family = AF_NETLINK;
-	return netlink_talk(netlink_talk_filter, &req.n,
-			    &zns->netlink_cmd, zns, 0);
 }
 
+/* Install or uninstall specified rule for a specific interface.
+ * Form netlink message and ship it.
+ */
+static int
+netlink_rule_update_internal(int cmd, const struct zebra_dplane_ctx *ctx,
+			     uint32_t filter_bm, uint32_t priority,
+			     uint32_t table, const struct prefix *src_ip,
+			     const struct prefix *dst_ip, uint32_t fwmark)
+{
+	char buf[NL_PKT_BUF_SIZE];
 
+	netlink_rule_msg_encode(cmd, ctx, filter_bm, priority, table, src_ip,
+				dst_ip, fwmark, buf, sizeof(buf));
+	return netlink_talk_info(netlink_talk_filter, (void *)&buf,
+				 dplane_ctx_get_ns(ctx), 0);
+}
 /* Public functions */
-/*
- * Install specified rule for a specific interface. The preference is what
- * goes in the rule to denote relative ordering; it may or may not be the
- * same as the rule's user-defined sequence number.
- */
-enum zebra_dplane_result kernel_add_pbr_rule(struct zebra_pbr_rule *rule)
-{
-	int ret = 0;
-
-	ret = netlink_rule_update(RTM_NEWRULE, rule);
-	kernel_pbr_rule_add_del_status(rule,
-				       (!ret) ? ZEBRA_DPLANE_INSTALL_SUCCESS
-					      : ZEBRA_DPLANE_INSTALL_FAILURE);
-
-	return ZEBRA_DPLANE_REQUEST_SUCCESS;
-}
 
 /*
- * Uninstall specified rule for a specific interface.
+ * Add, update or delete a rule from the
+ * kernel, using info from a dataplane context.
  */
-enum zebra_dplane_result kernel_del_pbr_rule(struct zebra_pbr_rule *rule)
+enum zebra_dplane_result kernel_pbr_rule_update(struct zebra_dplane_ctx *ctx)
 {
-	int ret = 0;
+	enum dplane_op_e op;
+	int cmd;
+	int ret;
 
-	ret = netlink_rule_update(RTM_DELRULE, rule);
+	op = dplane_ctx_get_op(ctx);
-	kernel_pbr_rule_add_del_status(rule,
+	if (op == DPLANE_OP_RULE_ADD || op == DPLANE_OP_RULE_UPDATE)
-				       (!ret) ? ZEBRA_DPLANE_DELETE_SUCCESS
+		cmd = RTM_NEWRULE;
-					      : ZEBRA_DPLANE_DELETE_FAILURE);
+	else if (op == DPLANE_OP_RULE_DELETE)
+		cmd = RTM_DELRULE;
+	else {
+		flog_err(
+			EC_ZEBRA_PBR_RULE_UPDATE,
+			"Context received for kernel rule update with incorrect OP code (%u)",
+			op);
+		return ZEBRA_DPLANE_REQUEST_FAILURE;
+	}
 
-	return ZEBRA_DPLANE_REQUEST_SUCCESS;
+	ret = netlink_rule_update_internal(
-}
+		cmd, ctx, dplane_ctx_rule_get_filter_bm(ctx),
-
+		dplane_ctx_rule_get_priority(ctx),
-/*
+		dplane_ctx_rule_get_table(ctx), dplane_ctx_rule_get_src_ip(ctx),
- * Update specified rule for a specific interface.
+		dplane_ctx_rule_get_dst_ip(ctx),
- */
+		dplane_ctx_rule_get_fwmark(ctx));
-enum zebra_dplane_result kernel_update_pbr_rule(struct zebra_pbr_rule *old_rule,
-						struct zebra_pbr_rule *new_rule)
-{
-	int ret = 0;
-
-	/* Add the new, updated one */
-	ret = netlink_rule_update(RTM_NEWRULE, new_rule);
 
 	/**
 	 * Delete the old one.
 	 *
 	 * Don't care about this result right?
 	 */
-	netlink_rule_update(RTM_DELRULE, old_rule);
+	if (op == DPLANE_OP_RULE_UPDATE)
+		netlink_rule_update_internal(
+			RTM_DELRULE, ctx,
+			dplane_ctx_rule_get_old_filter_bm(ctx),
+			dplane_ctx_rule_get_old_priority(ctx),
+			dplane_ctx_rule_get_old_table(ctx),
+			dplane_ctx_rule_get_old_src_ip(ctx),
+			dplane_ctx_rule_get_old_dst_ip(ctx),
+			dplane_ctx_rule_get_old_fwmark(ctx));
 
-	kernel_pbr_rule_add_del_status(new_rule,
-				       (!ret) ? ZEBRA_DPLANE_INSTALL_SUCCESS
-					      : ZEBRA_DPLANE_INSTALL_FAILURE);
 
-	return ZEBRA_DPLANE_REQUEST_SUCCESS;
+	return (ret == 0 ? ZEBRA_DPLANE_REQUEST_SUCCESS
+			 : ZEBRA_DPLANE_REQUEST_FAILURE);
 }
 
 /*
@@ -296,14 +296,16 @@ int netlink_rule_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 		 * It should have been flushed on a previous shutdown.
 		 */
 		if (startup && proto == RTPROT_ZEBRA) {
-			int ret;
+			enum zebra_dplane_result ret;
 
-			ret = netlink_rule_update(RTM_DELRULE, &rule);
+			ret = dplane_pbr_rule_delete(&rule);
 
 			zlog_debug(
 				"%s: %s leftover rule: family %s IF %s(%u) Pref %u Src %s Dst %s Table %u",
 				__func__,
-				((ret == 0) ? "Removed" : "Failed to remove"),
+				((ret == ZEBRA_DPLANE_REQUEST_FAILURE)
+					 ? "Failed to remove"
+					 : "Removed"),
 				nl_family_to_str(frh->family), rule.ifname,
 				rule.rule.ifindex, rule.rule.priority,
 				prefix2str(&rule.rule.filter.src_ip, buf1,

zebra/rule_socket.c

@@ -43,26 +43,11 @@
 #include "zebra/zebra_pbr.h"
 #include "zebra/zebra_errors.h"
 
-enum zebra_dplane_result kernel_add_pbr_rule(struct zebra_pbr_rule *rule)
+enum zebra_dplane_result kernel_pbr_rule_update(struct zebra_dplane_ctx *ctx)
 {
 	flog_err(EC_LIB_UNAVAILABLE, "%s not Implemented for this platform",
 		 __func__);
 	return ZEBRA_DPLANE_REQUEST_FAILURE;
 }
 
-enum zebra_dplane_result kernel_del_pbr_rule(struct zebra_pbr_rule *rule)
-{
-	flog_err(EC_LIB_UNAVAILABLE, "%s not Implemented for this platform",
-		 __func__);
-	return ZEBRA_DPLANE_REQUEST_FAILURE;
-}
-
-enum zebra_dplane_result kernel_update_pbr_rule(struct zebra_pbr_rule *old_rule,
-						struct zebra_pbr_rule *new_rule)
-{
-	flog_err(EC_LIB_UNAVAILABLE, "%s not Implemented for this platform",
-		 __PRETTY_FUNCTION__);
-	return ZEBRA_DPLANE_REQUEST_FAILURE;
-}
-
 #endif

zebra/zapi_msg.c

@@ -785,7 +785,7 @@ int zsend_route_notify_owner_ctx(const struct zebra_dplane_ctx *ctx,
 				      note));
 }
 
-void zsend_rule_notify_owner(struct zebra_pbr_rule *rule,
+void zsend_rule_notify_owner(const struct zebra_dplane_ctx *ctx,
 			     enum zapi_rule_notify_owner note)
 {
 	struct listnode *node;
@@ -793,10 +793,11 @@ void zsend_rule_notify_owner(struct zebra_pbr_rule *rule,
 	struct stream *s;
 
 	if (IS_ZEBRA_DEBUG_PACKET)
-		zlog_debug("%s: Notifying %u", __func__, rule->rule.unique);
+		zlog_debug("%s: Notifying %u", __func__,
+			   dplane_ctx_rule_get_unique(ctx));
 
 	for (ALL_LIST_ELEMENTS_RO(zrouter.client_list, node, client)) {
-		if (rule->sock == client->sock)
+		if (dplane_ctx_rule_get_sock(ctx) == client->sock)
 			break;
 	}
 
@@ -807,10 +808,10 @@ void zsend_rule_notify_owner(struct zebra_pbr_rule *rule,
 
 	zclient_create_header(s, ZEBRA_RULE_NOTIFY_OWNER, VRF_DEFAULT);
 	stream_put(s, &note, sizeof(note));
-	stream_putl(s, rule->rule.seq);
+	stream_putl(s, dplane_ctx_rule_get_seq(ctx));
-	stream_putl(s, rule->rule.priority);
+	stream_putl(s, dplane_ctx_rule_get_priority(ctx));
-	stream_putl(s, rule->rule.unique);
+	stream_putl(s, dplane_ctx_rule_get_unique(ctx));
-	stream_putl(s, rule->rule.ifindex);
+	stream_putl(s, dplane_ctx_get_ifindex(ctx));
 
 	stream_putw_at(s, 0, stream_get_endp(s));
 

zebra/zapi_msg.h

@@ -81,7 +81,7 @@ extern int zsend_route_notify_owner(struct route_entry *re,
 extern int zsend_route_notify_owner_ctx(const struct zebra_dplane_ctx *ctx,
 					enum zapi_route_notify_owner note);
 
-extern void zsend_rule_notify_owner(struct zebra_pbr_rule *rule,
+extern void zsend_rule_notify_owner(const struct zebra_dplane_ctx *ctx,
 				    enum zapi_rule_notify_owner note);
 extern void zsend_ipset_notify_owner(struct zebra_pbr_ipset *ipset,
 				     enum zapi_ipset_notify_owner note);

zebra/zebra_dplane.c

@@ -35,6 +35,7 @@
 #include "zebra/zebra_mpls.h"
 #include "zebra/rt.h"
 #include "zebra/debug.h"
+#include "zebra/zebra_pbr.h"
 
 /* Memory type for context blocks */
 DEFINE_MTYPE_STATIC(ZEBRA, DP_CTX, "Zebra DPlane Ctx")
@@ -191,6 +192,36 @@ struct dplane_neigh_info {
 	uint16_t state;
 };
 
+/*
+ * Policy based routing rule info for the dataplane
+ */
+struct dplane_ctx_rule {
+	uint32_t priority;
+
+	/* The route table pointed by this rule */
+	uint32_t table;
+
+	/* Filter criteria */
+	uint32_t filter_bm;
+	uint32_t fwmark;
+	struct prefix src_ip;
+	struct prefix dst_ip;
+};
+
+struct dplane_rule_info {
+	/*
+	 * Originating zclient sock fd, so we can know who to send
+	 * back to.
+	 */
+	int sock;
+
+	int unique;
+	int seq;
+
+	struct dplane_ctx_rule new;
+	struct dplane_ctx_rule old;
+};
+
 /*
  * The context block used to exchange info about route updates across
  * the boundary between the zebra main context (and pthread) and the
@@ -238,6 +269,7 @@ struct zebra_dplane_ctx {
 		struct dplane_intf_info intf;
 		struct dplane_mac_info macinfo;
 		struct dplane_neigh_info neigh;
+		struct dplane_rule_info rule;
 	} u;
 
 	/* Namespace info, used especially for netlink kernel communication */
@@ -361,6 +393,9 @@ static struct zebra_dplane_globals {
 	_Atomic uint32_t dg_neighs_in;
 	_Atomic uint32_t dg_neigh_errors;
 
+	_Atomic uint32_t dg_rules_in;
+	_Atomic uint32_t dg_rule_errors;
+
 	_Atomic uint32_t dg_update_yields;
 
 	/* Dataplane pthread */
@@ -564,6 +599,9 @@ static void dplane_ctx_free_internal(struct zebra_dplane_ctx *ctx)
 	case DPLANE_OP_NEIGH_DELETE:
 	case DPLANE_OP_VTEP_ADD:
 	case DPLANE_OP_VTEP_DELETE:
+	case DPLANE_OP_RULE_ADD:
+	case DPLANE_OP_RULE_DELETE:
+	case DPLANE_OP_RULE_UPDATE:
 	case DPLANE_OP_NONE:
 		break;
 	}
@@ -786,6 +824,16 @@ const char *dplane_op2str(enum dplane_op_e op)
 	case DPLANE_OP_VTEP_DELETE:
 		ret = "VTEP_DELETE";
 		break;
+
+	case DPLANE_OP_RULE_ADD:
+		ret = "RULE_ADD";
+		break;
+	case DPLANE_OP_RULE_DELETE:
+		ret = "RULE_DELETE";
+		break;
+	case DPLANE_OP_RULE_UPDATE:
+		ret = "RULE_UPDATE";
+		break;
 	}
 
 	return ret;
@@ -1517,6 +1565,116 @@ uint16_t dplane_ctx_neigh_get_state(const struct zebra_dplane_ctx *ctx)
 	return ctx->u.neigh.state;
 }
 
+/* Accessors for PBR rule information */
+int dplane_ctx_rule_get_sock(const struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+
+	return ctx->u.rule.sock;
+}
+
+int dplane_ctx_rule_get_unique(const struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+
+	return ctx->u.rule.unique;
+}
+
+int dplane_ctx_rule_get_seq(const struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+
+	return ctx->u.rule.seq;
+}
+
+uint32_t dplane_ctx_rule_get_priority(const struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+
+	return ctx->u.rule.new.priority;
+}
+
+uint32_t dplane_ctx_rule_get_old_priority(const struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+
+	return ctx->u.rule.old.priority;
+}
+
+uint32_t dplane_ctx_rule_get_table(const struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+
+	return ctx->u.rule.new.table;
+}
+
+uint32_t dplane_ctx_rule_get_old_table(const struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+
+	return ctx->u.rule.old.table;
+}
+
+uint32_t dplane_ctx_rule_get_filter_bm(const struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+
+	return ctx->u.rule.new.filter_bm;
+}
+
+uint32_t dplane_ctx_rule_get_old_filter_bm(const struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+
+	return ctx->u.rule.old.filter_bm;
+}
+
+uint32_t dplane_ctx_rule_get_fwmark(const struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+
+	return ctx->u.rule.new.fwmark;
+}
+
+uint32_t dplane_ctx_rule_get_old_fwmark(const struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+
+	return ctx->u.rule.old.fwmark;
+}
+
+const struct prefix *
+dplane_ctx_rule_get_src_ip(const struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+
+	return &(ctx->u.rule.new.src_ip);
+}
+
+const struct prefix *
+dplane_ctx_rule_get_old_src_ip(const struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+
+	return &(ctx->u.rule.old.src_ip);
+}
+
+const struct prefix *
+dplane_ctx_rule_get_dst_ip(const struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+
+	return &(ctx->u.rule.new.dst_ip);
+}
+
+const struct prefix *
+dplane_ctx_rule_get_old_dst_ip(const struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+
+	return &(ctx->u.rule.old.dst_ip);
+}
+
 /*
  * End of dplane context accessors
  */
@@ -1913,6 +2071,76 @@ static int dplane_ctx_pw_init(struct zebra_dplane_ctx *ctx,
 	return AOK;
 }
 
+/**
+ * dplane_ctx_rule_init_single() - Initialize a dataplane representation of a
+ * PBR rule.
+ *
+ * @dplane_rule:	Dataplane internal representation of a rule
+ * @rule:			PBR rule
+ */
+static void dplane_ctx_rule_init_single(struct dplane_ctx_rule *dplane_rule,
+					struct zebra_pbr_rule *rule)
+{
+	dplane_rule->priority = rule->rule.priority;
+	dplane_rule->table = rule->rule.action.table;
+
+	dplane_rule->filter_bm = rule->rule.filter.filter_bm;
+	dplane_rule->fwmark = rule->rule.filter.fwmark;
+	prefix_copy(&(dplane_rule->dst_ip), &rule->rule.filter.dst_ip);
+	prefix_copy(&(dplane_rule->src_ip), &rule->rule.filter.src_ip);
+}
+
+/**
+ * dplane_ctx_rule_init() - Initialize a context block for a PBR rule update.
+ *
+ * @ctx:		Dataplane context to init
+ * @op:			Operation being performed
+ * @new_rule:	PBR rule
+ *
+ * Return:	Result status
+ */
+static int dplane_ctx_rule_init(struct zebra_dplane_ctx *ctx,
+				enum dplane_op_e op,
+				struct zebra_pbr_rule *new_rule,
+				struct zebra_pbr_rule *old_rule)
+{
+	if (IS_ZEBRA_DEBUG_DPLANE_DETAIL) {
+		char buf1[PREFIX_STRLEN];
+		char buf2[PREFIX_STRLEN];
+
+		zlog_debug(
+			"init dplane ctx %s: IF %s(%u) Prio %u Fwmark %u Src %s Dst %s Table %u",
+			dplane_op2str(op), new_rule->ifname,
+			new_rule->rule.ifindex, new_rule->rule.priority,
+			new_rule->rule.filter.fwmark,
+			prefix2str(&new_rule->rule.filter.src_ip, buf1,
+				   sizeof(buf1)),
+			prefix2str(&new_rule->rule.filter.dst_ip, buf2,
+				   sizeof(buf2)),
+			new_rule->rule.action.table);
+	}
+
+	ctx->zd_op = op;
+	ctx->zd_status = ZEBRA_DPLANE_REQUEST_SUCCESS;
+
+	dplane_ctx_ns_init(ctx, zebra_ns_lookup(NS_DEFAULT),
+			   op == DPLANE_OP_RULE_UPDATE);
+
+	ctx->zd_vrf_id = new_rule->vrf_id;
+	memcpy(ctx->zd_ifname, new_rule->ifname, sizeof(new_rule->ifname));
+	ctx->zd_ifindex = new_rule->rule.ifindex;
+
+	ctx->u.rule.sock = new_rule->sock;
+	ctx->u.rule.unique = new_rule->rule.unique;
+	ctx->u.rule.seq = new_rule->rule.seq;
+
+	dplane_ctx_rule_init_single(&ctx->u.rule.new, new_rule);
+	if (op == DPLANE_OP_RULE_UPDATE)
+		dplane_ctx_rule_init_single(&ctx->u.rule.old, old_rule);
+
+	return AOK;
+}
+
 /*
  * Enqueue a new update,
  * and ensure an event is active for the dataplane pthread.
@@ -2839,6 +3067,56 @@ neigh_update_internal(enum dplane_op_e op,
 	return result;
 }
 
+/*
+ * Common helper api for PBR rule updates
+ */
+static enum zebra_dplane_result
+rule_update_internal(enum dplane_op_e op, struct zebra_pbr_rule *new_rule,
+		     struct zebra_pbr_rule *old_rule)
+{
+	enum zebra_dplane_result result = ZEBRA_DPLANE_REQUEST_FAILURE;
+	struct zebra_dplane_ctx *ctx;
+	int ret;
+
+	ctx = dplane_ctx_alloc();
+
+	ret = dplane_ctx_rule_init(ctx, op, new_rule, old_rule);
+	if (ret != AOK)
+		goto done;
+
+	ret = dplane_update_enqueue(ctx);
+
+done:
+	atomic_fetch_add_explicit(&zdplane_info.dg_rules_in, 1,
+				  memory_order_relaxed);
+
+	if (ret == AOK)
+		result = ZEBRA_DPLANE_REQUEST_QUEUED;
+	else {
+		atomic_fetch_add_explicit(&zdplane_info.dg_rule_errors, 1,
+					  memory_order_relaxed);
+		dplane_ctx_free(&ctx);
+	}
+
+	return result;
+}
+
+enum zebra_dplane_result dplane_pbr_rule_add(struct zebra_pbr_rule *rule)
+{
+	return rule_update_internal(DPLANE_OP_RULE_ADD, rule, NULL);
+}
+
+enum zebra_dplane_result dplane_pbr_rule_delete(struct zebra_pbr_rule *rule)
+{
+	return rule_update_internal(DPLANE_OP_RULE_DELETE, rule, NULL);
+}
+
+enum zebra_dplane_result dplane_pbr_rule_update(struct zebra_pbr_rule *old_rule,
+						struct zebra_pbr_rule *new_rule)
+{
+	return rule_update_internal(DPLANE_OP_RULE_UPDATE, new_rule, old_rule);
+}
+
 /*
  * Handler for 'show dplane'
  */
@@ -2909,6 +3187,13 @@ int dplane_show_helper(struct vty *vty, bool detailed)
 	vty_out(vty, "EVPN neigh updates:       %"PRIu64"\n", incoming);
 	vty_out(vty, "EVPN neigh errors:        %"PRIu64"\n", errs);
 
+	incoming = atomic_load_explicit(&zdplane_info.dg_rules_in,
+					memory_order_relaxed);
+	errs = atomic_load_explicit(&zdplane_info.dg_rule_errors,
+				    memory_order_relaxed);
+	vty_out(vty, "Rule updates:             %" PRIu64 "\n", incoming);
+	vty_out(vty, "Rule errors:              %" PRIu64 "\n", errs);
+
 	return CMD_SUCCESS;
 }
 
@@ -3397,6 +3682,29 @@ kernel_dplane_neigh_update(struct zebra_dplane_ctx *ctx)
 	return res;
 }
 
+/*
+ *  Handler for kernel PBR rule updates
+ */
+static enum zebra_dplane_result
+kernel_dplane_rule_update(struct zebra_dplane_ctx *ctx)
+{
+	enum zebra_dplane_result res;
+
+	if (IS_ZEBRA_DEBUG_DPLANE_DETAIL)
+		zlog_debug("Dplane rule update op %s, if %s(%u), ctx %p",
+			   dplane_op2str(dplane_ctx_get_op(ctx)),
+			   dplane_ctx_get_ifname(ctx),
+			   dplane_ctx_get_ifindex(ctx), ctx);
+
+	res = kernel_pbr_rule_update(ctx);
+
+	if (res != ZEBRA_DPLANE_REQUEST_SUCCESS)
+		atomic_fetch_add_explicit(&zdplane_info.dg_rule_errors, 1,
+					  memory_order_relaxed);
+
+	return res;
+}
+
 /*
  * Kernel provider callback
  */
@@ -3470,6 +3778,12 @@ static int kernel_dplane_process_func(struct zebra_dplane_provider *prov)
 			res = kernel_dplane_neigh_update(ctx);
 			break;
 
+		case DPLANE_OP_RULE_ADD:
+		case DPLANE_OP_RULE_DELETE:
+		case DPLANE_OP_RULE_UPDATE:
+			res = kernel_dplane_rule_update(ctx);
+			break;
+
 		/* Ignore 'notifications' - no-op */
 		case DPLANE_OP_SYS_ROUTE_ADD:
 		case DPLANE_OP_SYS_ROUTE_DELETE:

zebra/zebra_dplane.h

@@ -144,6 +144,11 @@ enum dplane_op_e {
 	/* EVPN VTEP updates */
 	DPLANE_OP_VTEP_ADD,
 	DPLANE_OP_VTEP_DELETE,
+
+	/* Policy based routing rule update */
+	DPLANE_OP_RULE_ADD,
+	DPLANE_OP_RULE_DELETE,
+	DPLANE_OP_RULE_UPDATE,
 };
 
 /*
@@ -384,6 +389,27 @@ const struct ethaddr *dplane_ctx_neigh_get_mac(
 uint32_t dplane_ctx_neigh_get_flags(const struct zebra_dplane_ctx *ctx);
 uint16_t dplane_ctx_neigh_get_state(const struct zebra_dplane_ctx *ctx);
 
+/* Accessors for policy based routing rule information */
+int dplane_ctx_rule_get_sock(const struct zebra_dplane_ctx *ctx);
+int dplane_ctx_rule_get_unique(const struct zebra_dplane_ctx *ctx);
+int dplane_ctx_rule_get_seq(const struct zebra_dplane_ctx *ctx);
+uint32_t dplane_ctx_rule_get_priority(const struct zebra_dplane_ctx *ctx);
+uint32_t dplane_ctx_rule_get_old_priority(const struct zebra_dplane_ctx *ctx);
+uint32_t dplane_ctx_rule_get_table(const struct zebra_dplane_ctx *ctx);
+uint32_t dplane_ctx_rule_get_old_table(const struct zebra_dplane_ctx *ctx);
+uint32_t dplane_ctx_rule_get_filter_bm(const struct zebra_dplane_ctx *ctx);
+uint32_t dplane_ctx_rule_get_old_filter_bm(const struct zebra_dplane_ctx *ctx);
+uint32_t dplane_ctx_rule_get_fwmark(const struct zebra_dplane_ctx *ctx);
+uint32_t dplane_ctx_rule_get_old_fwmark(const struct zebra_dplane_ctx *ctx);
+const struct prefix *
+dplane_ctx_rule_get_src_ip(const struct zebra_dplane_ctx *ctx);
+const struct prefix *
+dplane_ctx_rule_get_old_src_ip(const struct zebra_dplane_ctx *ctx);
+const struct prefix *
+dplane_ctx_rule_get_dst_ip(const struct zebra_dplane_ctx *ctx);
+const struct prefix *
+dplane_ctx_rule_get_old_dst_ip(const struct zebra_dplane_ctx *ctx);
+
 /* Namespace info - esp. for netlink communication */
 const struct zebra_dplane_info *dplane_ctx_get_ns(
 	const struct zebra_dplane_ctx *ctx);
@@ -509,6 +535,21 @@ enum zebra_dplane_result dplane_vtep_delete(const struct interface *ifp,
 					    const struct in_addr *ip,
 					    vni_t vni);
 
+/* Forward ref of zebra_pbr_rule */
+struct zebra_pbr_rule;
+
+/*
+ * Enqueue policy based routing rule for the dataplane.
+ * It is possible that the user-defined sequence number and the one in the
+ * forwarding plane may not coincide, hence the API requires a separate
+ * rule priority - maps to preference/FRA_PRIORITY on Linux.
+ */
+enum zebra_dplane_result dplane_pbr_rule_add(struct zebra_pbr_rule *rule);
+enum zebra_dplane_result dplane_pbr_rule_delete(struct zebra_pbr_rule *rule);
+enum zebra_dplane_result
+dplane_pbr_rule_update(struct zebra_pbr_rule *old_rule,
+		       struct zebra_pbr_rule *new_rule);
+
 /* Encode route information into data plane context. */
 int dplane_ctx_route_init(struct zebra_dplane_ctx *ctx, enum dplane_op_e op,
 			  struct route_node *rn, struct route_entry *re);

zebra/zebra_errors.h

@@ -77,6 +77,7 @@ enum zebra_log_refs {
 	EC_ZEBRA_NHG_FIB_UPDATE,
 	EC_ZEBRA_IF_LOOKUP_FAILED,
 	EC_ZEBRA_NS_NO_DEFAULT,
+	EC_ZEBRA_PBR_RULE_UPDATE,
 	/* warnings */
 	EC_ZEBRA_NS_NOTIFY_READ,
 	EC_ZEBRAING_LM_PROTO_MISMATCH,

zebra/zebra_nhg.c

@@ -2563,6 +2563,9 @@ void zebra_nhg_dplane_result(struct zebra_dplane_ctx *ctx)
 	case DPLANE_OP_NEIGH_DELETE:
 	case DPLANE_OP_VTEP_ADD:
 	case DPLANE_OP_VTEP_DELETE:
+	case DPLANE_OP_RULE_ADD:
+	case DPLANE_OP_RULE_DELETE:
+	case DPLANE_OP_RULE_UPDATE:
 	case DPLANE_OP_NONE:
 		break;
 	}

zebra/zebra_pbr.c

@@ -131,7 +131,7 @@ void zebra_pbr_rules_free(void *arg)
 
 	rule = (struct zebra_pbr_rule *)arg;
 
-	(void)kernel_del_pbr_rule(rule);
+	(void)dplane_pbr_rule_delete(rule);
 	XFREE(MTYPE_TMP, rule);
 }
 
@@ -460,7 +460,7 @@ void zebra_pbr_add_rule(struct zebra_pbr_rule *rule)
 
 	/* If found, this is an update */
 	if (found) {
-		(void)kernel_update_pbr_rule(found, rule);
+		(void)dplane_pbr_rule_update(found, rule);
 
 		if (pbr_rule_release(found))
 			zlog_debug(
@@ -468,12 +468,12 @@ void zebra_pbr_add_rule(struct zebra_pbr_rule *rule)
 				__PRETTY_FUNCTION__);
 
 	} else
-		(void)kernel_add_pbr_rule(rule);
+		(void)dplane_pbr_rule_add(rule);
 }
 
 void zebra_pbr_del_rule(struct zebra_pbr_rule *rule)
 {
-	(void)kernel_del_pbr_rule(rule);
+	(void)dplane_pbr_rule_delete(rule);
 
 	if (pbr_rule_release(rule))
 		zlog_debug("%s: Rule being deleted we know nothing about",
@@ -486,7 +486,7 @@ static void zebra_pbr_cleanup_rules(struct hash_bucket *b, void *data)
 	int *sock = data;
 
 	if (rule->sock == *sock) {
-		(void)kernel_del_pbr_rule(rule);
+		(void)dplane_pbr_rule_delete(rule);
 		if (hash_release(zrouter.rules_hash, rule))
 			XFREE(MTYPE_TMP, rule);
 		else
@@ -735,25 +735,29 @@ void zebra_pbr_del_iptable(struct zebra_pbr_iptable *iptable)
 /*
  * Handle success or failure of rule (un)install in the kernel.
  */
-void kernel_pbr_rule_add_del_status(struct zebra_pbr_rule *rule,
+void zebra_pbr_dplane_result(struct zebra_dplane_ctx *ctx)
-				    enum zebra_dplane_status res)
 {
-	switch (res) {
+	enum zebra_dplane_result res;
-	case ZEBRA_DPLANE_INSTALL_SUCCESS:
+	enum dplane_op_e op;
-		zsend_rule_notify_owner(rule, ZAPI_RULE_INSTALLED);
+
-		break;
+	res = dplane_ctx_get_status(ctx);
-	case ZEBRA_DPLANE_INSTALL_FAILURE:
+	op = dplane_ctx_get_op(ctx);
-		zsend_rule_notify_owner(rule, ZAPI_RULE_FAIL_INSTALL);
+	if (op == DPLANE_OP_RULE_ADD || op == DPLANE_OP_RULE_UPDATE)
-		break;
+		zsend_rule_notify_owner(ctx, res == ZEBRA_DPLANE_REQUEST_SUCCESS
-	case ZEBRA_DPLANE_DELETE_SUCCESS:
+						     ? ZAPI_RULE_INSTALLED
-		zsend_rule_notify_owner(rule, ZAPI_RULE_REMOVED);
+						     : ZAPI_RULE_FAIL_INSTALL);
-		break;
+	else if (op == DPLANE_OP_RULE_DELETE)
-	case ZEBRA_DPLANE_DELETE_FAILURE:
+		zsend_rule_notify_owner(ctx, res == ZEBRA_DPLANE_REQUEST_SUCCESS
-		zsend_rule_notify_owner(rule, ZAPI_RULE_FAIL_REMOVE);
+						     ? ZAPI_RULE_REMOVED
-		break;
+						     : ZAPI_RULE_FAIL_REMOVE);
-	case ZEBRA_DPLANE_STATUS_NONE:
+	else
-		break;
+		flog_err(
-	}
+			EC_ZEBRA_PBR_RULE_UPDATE,
+			"Context received in pbr rule dplane result handler with incorrect OP code (%u)",
+			op);
+
+
+	dplane_ctx_fini(&ctx);
 }
 
 /*

zebra/zebra_pbr.h

@@ -170,24 +170,11 @@ void zebra_pbr_add_iptable(struct zebra_pbr_iptable *iptable);
 void zebra_pbr_del_iptable(struct zebra_pbr_iptable *iptable);
 
 /*
- * Install specified rule for a specific interface.
+ * Add, update or delete a rule from the
- * It is possible that the user-defined sequence number and the one in the
+ * kernel, using info from a dataplane context.
- * forwarding plane may not coincide, hence the API requires a separate
- * rule priority - maps to preference/FRA_PRIORITY on Linux.
- */
-extern enum zebra_dplane_result kernel_add_pbr_rule(struct zebra_pbr_rule *rule);
-
-/*
- * Uninstall specified rule for a specific interface.
- */
-extern enum zebra_dplane_result kernel_del_pbr_rule(struct zebra_pbr_rule *rule);
-
-/*
- * Update specified rule for a specific interface.
  */
 extern enum zebra_dplane_result
-kernel_update_pbr_rule(struct zebra_pbr_rule *old_rule,
+kernel_pbr_rule_update(struct zebra_dplane_ctx *ctx);
-		       struct zebra_pbr_rule *new_rule);
 
 /*
  * Get to know existing PBR rules in the kernel - typically called at startup.
@@ -197,8 +184,7 @@ extern void kernel_read_pbr_rules(struct zebra_ns *zns);
 /*
  * Handle success or failure of rule (un)install in the kernel.
  */
-extern void kernel_pbr_rule_add_del_status(struct zebra_pbr_rule *rule,
+extern void zebra_pbr_dplane_result(struct zebra_dplane_ctx *ctx);
-					   enum zebra_dplane_status res);
 
 /*
  * Handle success or failure of ipset kinds (un)install in the kernel.

zebra/zebra_rib.c

@@ -3583,6 +3583,12 @@ static int rib_process_dplane_results(struct thread *thread)
 				zebra_vxlan_handle_result(ctx);
 				break;
 
+			case DPLANE_OP_RULE_ADD:
+			case DPLANE_OP_RULE_DELETE:
+			case DPLANE_OP_RULE_UPDATE:
+				zebra_pbr_dplane_result(ctx);
+				break;
+
 			/* Some op codes not handled here */
 			case DPLANE_OP_ADDR_INSTALL:
 			case DPLANE_OP_ADDR_UNINSTALL: