diff --git a/tests/topotests/lib/topogen.py b/tests/topotests/lib/topogen.py index 414dc17874..efd5b90685 100644 --- a/tests/topotests/lib/topogen.py +++ b/tests/topotests/lib/topogen.py @@ -658,7 +658,7 @@ class TopoRouter(TopoGear): Possible daemon values are: TopoRouter.RD_ZEBRA, TopoRouter.RD_RIP, TopoRouter.RD_RIPNG, TopoRouter.RD_OSPF, TopoRouter.RD_OSPF6, TopoRouter.RD_ISIS, TopoRouter.RD_BGP, TopoRouter.RD_LDP, - TopoRouter.RD_PIM. + TopoRouter.RD_PIM, TopoRouter.RD_PBR. """ daemonstr = self.RD.get(daemon) self.logger.info('loading "{}" configuration: {}'.format(daemonstr, source)) @@ -1064,6 +1064,7 @@ def diagnose_env_linux(): "isisd", "pimd", "ldpd", + "pbrd" ]: path = os.path.join(frrdir, fname) if not os.path.isfile(path): @@ -1121,6 +1122,7 @@ def diagnose_env_linux(): "ripngd", "isisd", "pimd", + "pbrd" ]: path = os.path.join(quaggadir, fname) if not os.path.isfile(path): diff --git a/tests/topotests/lib/topotest.py b/tests/topotests/lib/topotest.py index 6262082193..9d945d5262 100644 --- a/tests/topotests/lib/topotest.py +++ b/tests/topotests/lib/topotest.py @@ -727,6 +727,57 @@ def ip6_route(node): return result +def ip_rules(node): + """ + Gets a structured return of the command 'ip rule'. It can be used in + conjuction with json_cmp() to provide accurate assert explanations. + + Return example: + [ + { + "pref": "0" + "from": "all" + }, + { + "pref": "32766" + "from": "all" + }, + { + "to": "3.4.5.0/24", + "iif": "r1-eth2", + "pref": "304", + "from": "1.2.0.0/16", + "proto": "zebra" + } + ] + """ + output = normalize_text(node.run("ip rule")).splitlines() + result = [] + for line in output: + columns = line.split(" ") + + route = {} + # remove last character, since it is ':' + pref = columns[0][:-1] + route["pref"] = pref + prev = None + for column in columns: + if prev == "from": + route["from"] = column + if prev == "to": + route["to"] = column + if prev == "proto": + route["proto"] = column + if prev == "iif": + route["iif"] = column + if prev == "fwmark": + route["fwmark"] = column + prev = column + + result.append(route) + return result + + def sleep(amount, reason=None): """ Sleep wrapper that registers in the log the amount of sleep diff --git a/tests/topotests/pbr-topo1/__init__.py b/tests/topotests/pbr-topo1/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/topotests/pbr-topo1/r1/linux-rules.json b/tests/topotests/pbr-topo1/r1/linux-rules.json new file mode 100644 index 0000000000..5af4363418 --- /dev/null +++ b/tests/topotests/pbr-topo1/r1/linux-rules.json @@ -0,0 +1,19 @@ +[ + { + "iif": "r1-eth1", + "pref": "304", + "from": "4.5.6.7" + }, + { + "to": "3.4.5.0/24", + "iif": "r1-eth2", + "pref": "304", + "from": "1.2.0.0/16" + }, + { + "to": "9.9.9.9", + "iif": "r1-eth1", + "pref": "309", + "from": "all" + } +] diff --git a/tests/topotests/pbr-topo1/test_pbr_topo1.py b/tests/topotests/pbr-topo1/test_pbr_topo1.py index aedc9ca3fc..7de1cfa519 100755 --- a/tests/topotests/pbr-topo1/test_pbr_topo1.py +++ b/tests/topotests/pbr-topo1/test_pbr_topo1.py @@ -201,6 +201,28 @@ def test_pbr_flap(): # Actual output from router actual = router.vtysh_cmd("show pbr interface json", isjson=True) assertmsg = '"show pbr interface" mismatches on {}'.format(router.name) + + assert topotest.json_cmp(actual, expected) is None, assertmsg + + +def test_rule_linux_installation(): + "Ensure that rule is installed in the kernel" + + tgen = get_topogen() + # Don't run this test if we have any failure. + if tgen.routers_have_failure(): + pytest.skip(tgen.errors) + + logger.info("Checking for installed PBR rules in OS") + + router_list = tgen.routers().values() + for router in router_list: + rules_file = "{}/{}/linux-rules.json".format(CWD, router.name) + + actual = topotest.ip_rules(router) + expected = json.loads(open(rules_file).read()) + + assertmsg = "Router {} OS rules mismatch".format(router.name) assert topotest.json_cmp(actual, expected) is None, assertmsg diff --git a/zebra/rule_netlink.c b/zebra/rule_netlink.c index a5a605f27e..9da008bb61 100644 --- a/zebra/rule_netlink.c +++ b/zebra/rule_netlink.c @@ -41,6 +41,7 @@ #include "zebra/rule_netlink.h" #include "zebra/zebra_pbr.h" #include "zebra/zebra_errors.h" +#include "zebra/zebra_dplane.h" /* definitions */ @@ -48,11 +49,19 @@ /* Private functions */ -/* Install or uninstall specified rule for a specific interface. - * Form netlink message and ship it. Currently, notify status after - * waiting for netlink status. + +/* + * netlink_rule_msg_encode + * + * Encodes netlink RTM_ADDRULE/RTM_DELRULE message to buffer buf of size buflen. + * + * Returns -1 on failure or the number of bytes + * written to buf. */ -static int netlink_rule_update(int cmd, struct zebra_pbr_rule *rule) +static ssize_t netlink_rule_msg_encode( + int cmd, const struct zebra_dplane_ctx *ctx, uint32_t filter_bm, + uint32_t priority, uint32_t table, const struct prefix *src_ip, + const struct prefix *dst_ip, uint32_t fwmark, void *buf, size_t buflen) { uint8_t protocol = RTPROT_ZEBRA; int family; @@ -60,142 +69,133 @@ static int netlink_rule_update(int cmd, struct zebra_pbr_rule *rule) struct { struct nlmsghdr n; struct fib_rule_hdr frh; - char buf[NL_PKT_BUF_SIZE]; - } req; - struct zebra_ns *zns = zebra_ns_lookup(NS_DEFAULT); - struct sockaddr_nl snl; + char buf[]; + } *req = buf; + + const char *ifname = dplane_ctx_get_ifname(ctx); char buf1[PREFIX_STRLEN]; char buf2[PREFIX_STRLEN]; - memset(&req, 0, sizeof(req) - NL_PKT_BUF_SIZE); - family = PREFIX_FAMILY(&rule->rule.filter.src_ip); + memset(req, 0, sizeof(*req)); + family = PREFIX_FAMILY(src_ip); bytelen = (family == AF_INET ? 4 : 16); - req.n.nlmsg_type = cmd; - req.n.nlmsg_len = NLMSG_LENGTH(sizeof(struct rtmsg)); - req.n.nlmsg_flags = NLM_F_REQUEST; - req.n.nlmsg_pid = zns->netlink_cmd.snl.nl_pid; + req->n.nlmsg_type = cmd; + req->n.nlmsg_len = NLMSG_LENGTH(sizeof(struct rtmsg)); + req->n.nlmsg_flags = NLM_F_REQUEST; - req.frh.family = family; - req.frh.action = FR_ACT_TO_TBL; + req->frh.family = family; + req->frh.action = FR_ACT_TO_TBL; - addattr_l(&req.n, sizeof(req), - FRA_PROTOCOL, &protocol, sizeof(protocol)); + addattr_l(&req->n, buflen, FRA_PROTOCOL, &protocol, sizeof(protocol)); /* rule's pref # */ - addattr32(&req.n, sizeof(req), FRA_PRIORITY, rule->rule.priority); + addattr32(&req->n, buflen, FRA_PRIORITY, priority); /* interface on which applied */ - addattr_l(&req.n, sizeof(req), FRA_IFNAME, rule->ifname, - strlen(rule->ifname) + 1); + addattr_l(&req->n, buflen, FRA_IFNAME, ifname, strlen(ifname) + 1); /* source IP, if specified */ - if (IS_RULE_FILTERING_ON_SRC_IP(rule)) { - req.frh.src_len = rule->rule.filter.src_ip.prefixlen; - addattr_l(&req.n, sizeof(req), FRA_SRC, - &rule->rule.filter.src_ip.u.prefix, bytelen); + if (filter_bm & PBR_FILTER_SRC_IP) { + req->frh.src_len = src_ip->prefixlen; + addattr_l(&req->n, buflen, FRA_SRC, &src_ip->u.prefix, bytelen); } + /* destination IP, if specified */ - if (IS_RULE_FILTERING_ON_DST_IP(rule)) { - req.frh.dst_len = rule->rule.filter.dst_ip.prefixlen; - addattr_l(&req.n, sizeof(req), FRA_DST, - &rule->rule.filter.dst_ip.u.prefix, bytelen); + if (filter_bm & PBR_FILTER_DST_IP) { + req->frh.dst_len = dst_ip->prefixlen; + addattr_l(&req->n, buflen, FRA_DST, &dst_ip->u.prefix, bytelen); } /* fwmark, if specified */ - if (IS_RULE_FILTERING_ON_FWMARK(rule)) { - addattr32(&req.n, sizeof(req), FRA_FWMARK, - rule->rule.filter.fwmark); - } + if (filter_bm & PBR_FILTER_FWMARK) + addattr32(&req->n, buflen, FRA_FWMARK, fwmark); /* Route table to use to forward, if filter criteria matches. */ - if (rule->rule.action.table < 256) - req.frh.table = rule->rule.action.table; + if (table < 256) + req->frh.table = table; else { - req.frh.table = RT_TABLE_UNSPEC; - addattr32(&req.n, sizeof(req), FRA_TABLE, - rule->rule.action.table); + req->frh.table = RT_TABLE_UNSPEC; + addattr32(&req->n, buflen, FRA_TABLE, table); } if (IS_ZEBRA_DEBUG_KERNEL) zlog_debug( "Tx %s family %s IF %s(%u) Pref %u Fwmark %u Src %s Dst %s Table %u", nl_msg_type_to_str(cmd), nl_family_to_str(family), - rule->ifname, rule->rule.ifindex, rule->rule.priority, - rule->rule.filter.fwmark, - prefix2str(&rule->rule.filter.src_ip, buf1, - sizeof(buf1)), - prefix2str(&rule->rule.filter.dst_ip, buf2, - sizeof(buf2)), - rule->rule.action.table); + ifname, dplane_ctx_get_ifindex(ctx), priority, fwmark, + prefix2str(src_ip, buf1, sizeof(buf1)), + prefix2str(dst_ip, buf2, sizeof(buf2)), table); - /* Ship off the message. - * Note: Currently, netlink_talk() is a blocking call which returns - * back the status. - */ - memset(&snl, 0, sizeof(snl)); - snl.nl_family = AF_NETLINK; - return netlink_talk(netlink_talk_filter, &req.n, - &zns->netlink_cmd, zns, 0); + return NLMSG_ALIGN(req->n.nlmsg_len); } +/* Install or uninstall specified rule for a specific interface. + * Form netlink message and ship it. + */ +static int +netlink_rule_update_internal(int cmd, const struct zebra_dplane_ctx *ctx, + uint32_t filter_bm, uint32_t priority, + uint32_t table, const struct prefix *src_ip, + const struct prefix *dst_ip, uint32_t fwmark) +{ + char buf[NL_PKT_BUF_SIZE]; + netlink_rule_msg_encode(cmd, ctx, filter_bm, priority, table, src_ip, + dst_ip, fwmark, buf, sizeof(buf)); + return netlink_talk_info(netlink_talk_filter, (void *)&buf, + dplane_ctx_get_ns(ctx), 0); +} /* Public functions */ -/* - * Install specified rule for a specific interface. The preference is what - * goes in the rule to denote relative ordering; it may or may not be the - * same as the rule's user-defined sequence number. - */ -enum zebra_dplane_result kernel_add_pbr_rule(struct zebra_pbr_rule *rule) -{ - int ret = 0; - - ret = netlink_rule_update(RTM_NEWRULE, rule); - kernel_pbr_rule_add_del_status(rule, - (!ret) ? ZEBRA_DPLANE_INSTALL_SUCCESS - : ZEBRA_DPLANE_INSTALL_FAILURE); - - return ZEBRA_DPLANE_REQUEST_SUCCESS; -} /* - * Uninstall specified rule for a specific interface. + * Add, update or delete a rule from the + * kernel, using info from a dataplane context. */ -enum zebra_dplane_result kernel_del_pbr_rule(struct zebra_pbr_rule *rule) +enum zebra_dplane_result kernel_pbr_rule_update(struct zebra_dplane_ctx *ctx) { - int ret = 0; + enum dplane_op_e op; + int cmd; + int ret; - ret = netlink_rule_update(RTM_DELRULE, rule); - kernel_pbr_rule_add_del_status(rule, - (!ret) ? ZEBRA_DPLANE_DELETE_SUCCESS - : ZEBRA_DPLANE_DELETE_FAILURE); + op = dplane_ctx_get_op(ctx); + if (op == DPLANE_OP_RULE_ADD || op == DPLANE_OP_RULE_UPDATE) + cmd = RTM_NEWRULE; + else if (op == DPLANE_OP_RULE_DELETE) + cmd = RTM_DELRULE; + else { + flog_err( + EC_ZEBRA_PBR_RULE_UPDATE, + "Context received for kernel rule update with incorrect OP code (%u)", + op); + return ZEBRA_DPLANE_REQUEST_FAILURE; + } - return ZEBRA_DPLANE_REQUEST_SUCCESS; -} - -/* - * Update specified rule for a specific interface. - */ -enum zebra_dplane_result kernel_update_pbr_rule(struct zebra_pbr_rule *old_rule, - struct zebra_pbr_rule *new_rule) -{ - int ret = 0; - - /* Add the new, updated one */ - ret = netlink_rule_update(RTM_NEWRULE, new_rule); + ret = netlink_rule_update_internal( + cmd, ctx, dplane_ctx_rule_get_filter_bm(ctx), + dplane_ctx_rule_get_priority(ctx), + dplane_ctx_rule_get_table(ctx), dplane_ctx_rule_get_src_ip(ctx), + dplane_ctx_rule_get_dst_ip(ctx), + dplane_ctx_rule_get_fwmark(ctx)); /** * Delete the old one. * * Don't care about this result right? */ - netlink_rule_update(RTM_DELRULE, old_rule); + if (op == DPLANE_OP_RULE_UPDATE) + netlink_rule_update_internal( + RTM_DELRULE, ctx, + dplane_ctx_rule_get_old_filter_bm(ctx), + dplane_ctx_rule_get_old_priority(ctx), + dplane_ctx_rule_get_old_table(ctx), + dplane_ctx_rule_get_old_src_ip(ctx), + dplane_ctx_rule_get_old_dst_ip(ctx), + dplane_ctx_rule_get_old_fwmark(ctx)); - kernel_pbr_rule_add_del_status(new_rule, - (!ret) ? ZEBRA_DPLANE_INSTALL_SUCCESS - : ZEBRA_DPLANE_INSTALL_FAILURE); - return ZEBRA_DPLANE_REQUEST_SUCCESS; + return (ret == 0 ? ZEBRA_DPLANE_REQUEST_SUCCESS + : ZEBRA_DPLANE_REQUEST_FAILURE); } /* @@ -296,14 +296,16 @@ int netlink_rule_change(struct nlmsghdr *h, ns_id_t ns_id, int startup) * It should have been flushed on a previous shutdown. */ if (startup && proto == RTPROT_ZEBRA) { - int ret; + enum zebra_dplane_result ret; - ret = netlink_rule_update(RTM_DELRULE, &rule); + ret = dplane_pbr_rule_delete(&rule); zlog_debug( "%s: %s leftover rule: family %s IF %s(%u) Pref %u Src %s Dst %s Table %u", __func__, - ((ret == 0) ? "Removed" : "Failed to remove"), + ((ret == ZEBRA_DPLANE_REQUEST_FAILURE) + ? "Failed to remove" + : "Removed"), nl_family_to_str(frh->family), rule.ifname, rule.rule.ifindex, rule.rule.priority, prefix2str(&rule.rule.filter.src_ip, buf1, diff --git a/zebra/rule_socket.c b/zebra/rule_socket.c index 219fa7de6f..e629017bdf 100644 --- a/zebra/rule_socket.c +++ b/zebra/rule_socket.c @@ -43,26 +43,11 @@ #include "zebra/zebra_pbr.h" #include "zebra/zebra_errors.h" -enum zebra_dplane_result kernel_add_pbr_rule(struct zebra_pbr_rule *rule) +enum zebra_dplane_result kernel_pbr_rule_update(struct zebra_dplane_ctx *ctx) { flog_err(EC_LIB_UNAVAILABLE, "%s not Implemented for this platform", __func__); return ZEBRA_DPLANE_REQUEST_FAILURE; } -enum zebra_dplane_result kernel_del_pbr_rule(struct zebra_pbr_rule *rule) -{ - flog_err(EC_LIB_UNAVAILABLE, "%s not Implemented for this platform", - __func__); - return ZEBRA_DPLANE_REQUEST_FAILURE; -} - -enum zebra_dplane_result kernel_update_pbr_rule(struct zebra_pbr_rule *old_rule, - struct zebra_pbr_rule *new_rule) -{ - flog_err(EC_LIB_UNAVAILABLE, "%s not Implemented for this platform", - __PRETTY_FUNCTION__); - return ZEBRA_DPLANE_REQUEST_FAILURE; -} - #endif diff --git a/zebra/zapi_msg.c b/zebra/zapi_msg.c index cea8edf752..a40aa8b643 100644 --- a/zebra/zapi_msg.c +++ b/zebra/zapi_msg.c @@ -785,7 +785,7 @@ int zsend_route_notify_owner_ctx(const struct zebra_dplane_ctx *ctx, note)); } -void zsend_rule_notify_owner(struct zebra_pbr_rule *rule, +void zsend_rule_notify_owner(const struct zebra_dplane_ctx *ctx, enum zapi_rule_notify_owner note) { struct listnode *node; @@ -793,10 +793,11 @@ void zsend_rule_notify_owner(struct zebra_pbr_rule *rule, struct stream *s; if (IS_ZEBRA_DEBUG_PACKET) - zlog_debug("%s: Notifying %u", __func__, rule->rule.unique); + zlog_debug("%s: Notifying %u", __func__, + dplane_ctx_rule_get_unique(ctx)); for (ALL_LIST_ELEMENTS_RO(zrouter.client_list, node, client)) { - if (rule->sock == client->sock) + if (dplane_ctx_rule_get_sock(ctx) == client->sock) break; } @@ -807,10 +808,10 @@ void zsend_rule_notify_owner(struct zebra_pbr_rule *rule, zclient_create_header(s, ZEBRA_RULE_NOTIFY_OWNER, VRF_DEFAULT); stream_put(s, ¬e, sizeof(note)); - stream_putl(s, rule->rule.seq); - stream_putl(s, rule->rule.priority); - stream_putl(s, rule->rule.unique); - stream_putl(s, rule->rule.ifindex); + stream_putl(s, dplane_ctx_rule_get_seq(ctx)); + stream_putl(s, dplane_ctx_rule_get_priority(ctx)); + stream_putl(s, dplane_ctx_rule_get_unique(ctx)); + stream_putl(s, dplane_ctx_get_ifindex(ctx)); stream_putw_at(s, 0, stream_get_endp(s)); diff --git a/zebra/zapi_msg.h b/zebra/zapi_msg.h index 6d655e11aa..eb50e3c410 100644 --- a/zebra/zapi_msg.h +++ b/zebra/zapi_msg.h @@ -81,7 +81,7 @@ extern int zsend_route_notify_owner(struct route_entry *re, extern int zsend_route_notify_owner_ctx(const struct zebra_dplane_ctx *ctx, enum zapi_route_notify_owner note); -extern void zsend_rule_notify_owner(struct zebra_pbr_rule *rule, +extern void zsend_rule_notify_owner(const struct zebra_dplane_ctx *ctx, enum zapi_rule_notify_owner note); extern void zsend_ipset_notify_owner(struct zebra_pbr_ipset *ipset, enum zapi_ipset_notify_owner note); diff --git a/zebra/zebra_dplane.c b/zebra/zebra_dplane.c index 568b398924..eb3d48d784 100644 --- a/zebra/zebra_dplane.c +++ b/zebra/zebra_dplane.c @@ -35,6 +35,7 @@ #include "zebra/zebra_mpls.h" #include "zebra/rt.h" #include "zebra/debug.h" +#include "zebra/zebra_pbr.h" /* Memory type for context blocks */ DEFINE_MTYPE_STATIC(ZEBRA, DP_CTX, "Zebra DPlane Ctx") @@ -191,6 +192,36 @@ struct dplane_neigh_info { uint16_t state; }; +/* + * Policy based routing rule info for the dataplane + */ +struct dplane_ctx_rule { + uint32_t priority; + + /* The route table pointed by this rule */ + uint32_t table; + + /* Filter criteria */ + uint32_t filter_bm; + uint32_t fwmark; + struct prefix src_ip; + struct prefix dst_ip; +}; + +struct dplane_rule_info { + /* + * Originating zclient sock fd, so we can know who to send + * back to. + */ + int sock; + + int unique; + int seq; + + struct dplane_ctx_rule new; + struct dplane_ctx_rule old; +}; + /* * The context block used to exchange info about route updates across * the boundary between the zebra main context (and pthread) and the @@ -238,6 +269,7 @@ struct zebra_dplane_ctx { struct dplane_intf_info intf; struct dplane_mac_info macinfo; struct dplane_neigh_info neigh; + struct dplane_rule_info rule; } u; /* Namespace info, used especially for netlink kernel communication */ @@ -361,6 +393,9 @@ static struct zebra_dplane_globals { _Atomic uint32_t dg_neighs_in; _Atomic uint32_t dg_neigh_errors; + _Atomic uint32_t dg_rules_in; + _Atomic uint32_t dg_rule_errors; + _Atomic uint32_t dg_update_yields; /* Dataplane pthread */ @@ -564,6 +599,9 @@ static void dplane_ctx_free_internal(struct zebra_dplane_ctx *ctx) case DPLANE_OP_NEIGH_DELETE: case DPLANE_OP_VTEP_ADD: case DPLANE_OP_VTEP_DELETE: + case DPLANE_OP_RULE_ADD: + case DPLANE_OP_RULE_DELETE: + case DPLANE_OP_RULE_UPDATE: case DPLANE_OP_NONE: break; } @@ -786,6 +824,16 @@ const char *dplane_op2str(enum dplane_op_e op) case DPLANE_OP_VTEP_DELETE: ret = "VTEP_DELETE"; break; + + case DPLANE_OP_RULE_ADD: + ret = "RULE_ADD"; + break; + case DPLANE_OP_RULE_DELETE: + ret = "RULE_DELETE"; + break; + case DPLANE_OP_RULE_UPDATE: + ret = "RULE_UPDATE"; + break; } return ret; @@ -1517,6 +1565,116 @@ uint16_t dplane_ctx_neigh_get_state(const struct zebra_dplane_ctx *ctx) return ctx->u.neigh.state; } +/* Accessors for PBR rule information */ +int dplane_ctx_rule_get_sock(const struct zebra_dplane_ctx *ctx) +{ + DPLANE_CTX_VALID(ctx); + + return ctx->u.rule.sock; +} + +int dplane_ctx_rule_get_unique(const struct zebra_dplane_ctx *ctx) +{ + DPLANE_CTX_VALID(ctx); + + return ctx->u.rule.unique; +} + +int dplane_ctx_rule_get_seq(const struct zebra_dplane_ctx *ctx) +{ + DPLANE_CTX_VALID(ctx); + + return ctx->u.rule.seq; +} + +uint32_t dplane_ctx_rule_get_priority(const struct zebra_dplane_ctx *ctx) +{ + DPLANE_CTX_VALID(ctx); + + return ctx->u.rule.new.priority; +} + +uint32_t dplane_ctx_rule_get_old_priority(const struct zebra_dplane_ctx *ctx) +{ + DPLANE_CTX_VALID(ctx); + + return ctx->u.rule.old.priority; +} + +uint32_t dplane_ctx_rule_get_table(const struct zebra_dplane_ctx *ctx) +{ + DPLANE_CTX_VALID(ctx); + + return ctx->u.rule.new.table; +} + +uint32_t dplane_ctx_rule_get_old_table(const struct zebra_dplane_ctx *ctx) +{ + DPLANE_CTX_VALID(ctx); + + return ctx->u.rule.old.table; +} + +uint32_t dplane_ctx_rule_get_filter_bm(const struct zebra_dplane_ctx *ctx) +{ + DPLANE_CTX_VALID(ctx); + + return ctx->u.rule.new.filter_bm; +} + +uint32_t dplane_ctx_rule_get_old_filter_bm(const struct zebra_dplane_ctx *ctx) +{ + DPLANE_CTX_VALID(ctx); + + return ctx->u.rule.old.filter_bm; +} + +uint32_t dplane_ctx_rule_get_fwmark(const struct zebra_dplane_ctx *ctx) +{ + DPLANE_CTX_VALID(ctx); + + return ctx->u.rule.new.fwmark; +} + +uint32_t dplane_ctx_rule_get_old_fwmark(const struct zebra_dplane_ctx *ctx) +{ + DPLANE_CTX_VALID(ctx); + + return ctx->u.rule.old.fwmark; +} + +const struct prefix * +dplane_ctx_rule_get_src_ip(const struct zebra_dplane_ctx *ctx) +{ + DPLANE_CTX_VALID(ctx); + + return &(ctx->u.rule.new.src_ip); +} + +const struct prefix * +dplane_ctx_rule_get_old_src_ip(const struct zebra_dplane_ctx *ctx) +{ + DPLANE_CTX_VALID(ctx); + + return &(ctx->u.rule.old.src_ip); +} + +const struct prefix * +dplane_ctx_rule_get_dst_ip(const struct zebra_dplane_ctx *ctx) +{ + DPLANE_CTX_VALID(ctx); + + return &(ctx->u.rule.new.dst_ip); +} + +const struct prefix * +dplane_ctx_rule_get_old_dst_ip(const struct zebra_dplane_ctx *ctx) +{ + DPLANE_CTX_VALID(ctx); + + return &(ctx->u.rule.old.dst_ip); +} + /* * End of dplane context accessors */ @@ -1913,6 +2071,76 @@ static int dplane_ctx_pw_init(struct zebra_dplane_ctx *ctx, return AOK; } +/** + * dplane_ctx_rule_init_single() - Initialize a dataplane representation of a + * PBR rule. + * + * @dplane_rule: Dataplane internal representation of a rule + * @rule: PBR rule + */ +static void dplane_ctx_rule_init_single(struct dplane_ctx_rule *dplane_rule, + struct zebra_pbr_rule *rule) +{ + dplane_rule->priority = rule->rule.priority; + dplane_rule->table = rule->rule.action.table; + + dplane_rule->filter_bm = rule->rule.filter.filter_bm; + dplane_rule->fwmark = rule->rule.filter.fwmark; + prefix_copy(&(dplane_rule->dst_ip), &rule->rule.filter.dst_ip); + prefix_copy(&(dplane_rule->src_ip), &rule->rule.filter.src_ip); +} + +/** + * dplane_ctx_rule_init() - Initialize a context block for a PBR rule update. + * + * @ctx: Dataplane context to init + * @op: Operation being performed + * @new_rule: PBR rule + * + * Return: Result status + */ +static int dplane_ctx_rule_init(struct zebra_dplane_ctx *ctx, + enum dplane_op_e op, + struct zebra_pbr_rule *new_rule, + struct zebra_pbr_rule *old_rule) +{ + if (IS_ZEBRA_DEBUG_DPLANE_DETAIL) { + char buf1[PREFIX_STRLEN]; + char buf2[PREFIX_STRLEN]; + + zlog_debug( + "init dplane ctx %s: IF %s(%u) Prio %u Fwmark %u Src %s Dst %s Table %u", + dplane_op2str(op), new_rule->ifname, + new_rule->rule.ifindex, new_rule->rule.priority, + new_rule->rule.filter.fwmark, + prefix2str(&new_rule->rule.filter.src_ip, buf1, + sizeof(buf1)), + prefix2str(&new_rule->rule.filter.dst_ip, buf2, + sizeof(buf2)), + new_rule->rule.action.table); + } + + ctx->zd_op = op; + ctx->zd_status = ZEBRA_DPLANE_REQUEST_SUCCESS; + + dplane_ctx_ns_init(ctx, zebra_ns_lookup(NS_DEFAULT), + op == DPLANE_OP_RULE_UPDATE); + + ctx->zd_vrf_id = new_rule->vrf_id; + memcpy(ctx->zd_ifname, new_rule->ifname, sizeof(new_rule->ifname)); + ctx->zd_ifindex = new_rule->rule.ifindex; + + ctx->u.rule.sock = new_rule->sock; + ctx->u.rule.unique = new_rule->rule.unique; + ctx->u.rule.seq = new_rule->rule.seq; + + dplane_ctx_rule_init_single(&ctx->u.rule.new, new_rule); + if (op == DPLANE_OP_RULE_UPDATE) + dplane_ctx_rule_init_single(&ctx->u.rule.old, old_rule); + + return AOK; +} + /* * Enqueue a new update, * and ensure an event is active for the dataplane pthread. @@ -2839,6 +3067,56 @@ neigh_update_internal(enum dplane_op_e op, return result; } +/* + * Common helper api for PBR rule updates + */ +static enum zebra_dplane_result +rule_update_internal(enum dplane_op_e op, struct zebra_pbr_rule *new_rule, + struct zebra_pbr_rule *old_rule) +{ + enum zebra_dplane_result result = ZEBRA_DPLANE_REQUEST_FAILURE; + struct zebra_dplane_ctx *ctx; + int ret; + + ctx = dplane_ctx_alloc(); + + ret = dplane_ctx_rule_init(ctx, op, new_rule, old_rule); + if (ret != AOK) + goto done; + + ret = dplane_update_enqueue(ctx); + +done: + atomic_fetch_add_explicit(&zdplane_info.dg_rules_in, 1, + memory_order_relaxed); + + if (ret == AOK) + result = ZEBRA_DPLANE_REQUEST_QUEUED; + else { + atomic_fetch_add_explicit(&zdplane_info.dg_rule_errors, 1, + memory_order_relaxed); + dplane_ctx_free(&ctx); + } + + return result; +} + +enum zebra_dplane_result dplane_pbr_rule_add(struct zebra_pbr_rule *rule) +{ + return rule_update_internal(DPLANE_OP_RULE_ADD, rule, NULL); +} + +enum zebra_dplane_result dplane_pbr_rule_delete(struct zebra_pbr_rule *rule) +{ + return rule_update_internal(DPLANE_OP_RULE_DELETE, rule, NULL); +} + +enum zebra_dplane_result dplane_pbr_rule_update(struct zebra_pbr_rule *old_rule, + struct zebra_pbr_rule *new_rule) +{ + return rule_update_internal(DPLANE_OP_RULE_UPDATE, new_rule, old_rule); +} + /* * Handler for 'show dplane' */ @@ -2909,6 +3187,13 @@ int dplane_show_helper(struct vty *vty, bool detailed) vty_out(vty, "EVPN neigh updates: %"PRIu64"\n", incoming); vty_out(vty, "EVPN neigh errors: %"PRIu64"\n", errs); + incoming = atomic_load_explicit(&zdplane_info.dg_rules_in, + memory_order_relaxed); + errs = atomic_load_explicit(&zdplane_info.dg_rule_errors, + memory_order_relaxed); + vty_out(vty, "Rule updates: %" PRIu64 "\n", incoming); + vty_out(vty, "Rule errors: %" PRIu64 "\n", errs); + return CMD_SUCCESS; } @@ -3397,6 +3682,29 @@ kernel_dplane_neigh_update(struct zebra_dplane_ctx *ctx) return res; } +/* + * Handler for kernel PBR rule updates + */ +static enum zebra_dplane_result +kernel_dplane_rule_update(struct zebra_dplane_ctx *ctx) +{ + enum zebra_dplane_result res; + + if (IS_ZEBRA_DEBUG_DPLANE_DETAIL) + zlog_debug("Dplane rule update op %s, if %s(%u), ctx %p", + dplane_op2str(dplane_ctx_get_op(ctx)), + dplane_ctx_get_ifname(ctx), + dplane_ctx_get_ifindex(ctx), ctx); + + res = kernel_pbr_rule_update(ctx); + + if (res != ZEBRA_DPLANE_REQUEST_SUCCESS) + atomic_fetch_add_explicit(&zdplane_info.dg_rule_errors, 1, + memory_order_relaxed); + + return res; +} + /* * Kernel provider callback */ @@ -3470,6 +3778,12 @@ static int kernel_dplane_process_func(struct zebra_dplane_provider *prov) res = kernel_dplane_neigh_update(ctx); break; + case DPLANE_OP_RULE_ADD: + case DPLANE_OP_RULE_DELETE: + case DPLANE_OP_RULE_UPDATE: + res = kernel_dplane_rule_update(ctx); + break; + /* Ignore 'notifications' - no-op */ case DPLANE_OP_SYS_ROUTE_ADD: case DPLANE_OP_SYS_ROUTE_DELETE: diff --git a/zebra/zebra_dplane.h b/zebra/zebra_dplane.h index 7f8049b767..c93b95a6ad 100644 --- a/zebra/zebra_dplane.h +++ b/zebra/zebra_dplane.h @@ -144,6 +144,11 @@ enum dplane_op_e { /* EVPN VTEP updates */ DPLANE_OP_VTEP_ADD, DPLANE_OP_VTEP_DELETE, + + /* Policy based routing rule update */ + DPLANE_OP_RULE_ADD, + DPLANE_OP_RULE_DELETE, + DPLANE_OP_RULE_UPDATE, }; /* @@ -384,6 +389,27 @@ const struct ethaddr *dplane_ctx_neigh_get_mac( uint32_t dplane_ctx_neigh_get_flags(const struct zebra_dplane_ctx *ctx); uint16_t dplane_ctx_neigh_get_state(const struct zebra_dplane_ctx *ctx); +/* Accessors for policy based routing rule information */ +int dplane_ctx_rule_get_sock(const struct zebra_dplane_ctx *ctx); +int dplane_ctx_rule_get_unique(const struct zebra_dplane_ctx *ctx); +int dplane_ctx_rule_get_seq(const struct zebra_dplane_ctx *ctx); +uint32_t dplane_ctx_rule_get_priority(const struct zebra_dplane_ctx *ctx); +uint32_t dplane_ctx_rule_get_old_priority(const struct zebra_dplane_ctx *ctx); +uint32_t dplane_ctx_rule_get_table(const struct zebra_dplane_ctx *ctx); +uint32_t dplane_ctx_rule_get_old_table(const struct zebra_dplane_ctx *ctx); +uint32_t dplane_ctx_rule_get_filter_bm(const struct zebra_dplane_ctx *ctx); +uint32_t dplane_ctx_rule_get_old_filter_bm(const struct zebra_dplane_ctx *ctx); +uint32_t dplane_ctx_rule_get_fwmark(const struct zebra_dplane_ctx *ctx); +uint32_t dplane_ctx_rule_get_old_fwmark(const struct zebra_dplane_ctx *ctx); +const struct prefix * +dplane_ctx_rule_get_src_ip(const struct zebra_dplane_ctx *ctx); +const struct prefix * +dplane_ctx_rule_get_old_src_ip(const struct zebra_dplane_ctx *ctx); +const struct prefix * +dplane_ctx_rule_get_dst_ip(const struct zebra_dplane_ctx *ctx); +const struct prefix * +dplane_ctx_rule_get_old_dst_ip(const struct zebra_dplane_ctx *ctx); + /* Namespace info - esp. for netlink communication */ const struct zebra_dplane_info *dplane_ctx_get_ns( const struct zebra_dplane_ctx *ctx); @@ -509,6 +535,21 @@ enum zebra_dplane_result dplane_vtep_delete(const struct interface *ifp, const struct in_addr *ip, vni_t vni); +/* Forward ref of zebra_pbr_rule */ +struct zebra_pbr_rule; + +/* + * Enqueue policy based routing rule for the dataplane. + * It is possible that the user-defined sequence number and the one in the + * forwarding plane may not coincide, hence the API requires a separate + * rule priority - maps to preference/FRA_PRIORITY on Linux. + */ +enum zebra_dplane_result dplane_pbr_rule_add(struct zebra_pbr_rule *rule); +enum zebra_dplane_result dplane_pbr_rule_delete(struct zebra_pbr_rule *rule); +enum zebra_dplane_result +dplane_pbr_rule_update(struct zebra_pbr_rule *old_rule, + struct zebra_pbr_rule *new_rule); + /* Encode route information into data plane context. */ int dplane_ctx_route_init(struct zebra_dplane_ctx *ctx, enum dplane_op_e op, struct route_node *rn, struct route_entry *re); diff --git a/zebra/zebra_errors.h b/zebra/zebra_errors.h index 395004d0bb..5f2a7a12c6 100644 --- a/zebra/zebra_errors.h +++ b/zebra/zebra_errors.h @@ -77,6 +77,7 @@ enum zebra_log_refs { EC_ZEBRA_NHG_FIB_UPDATE, EC_ZEBRA_IF_LOOKUP_FAILED, EC_ZEBRA_NS_NO_DEFAULT, + EC_ZEBRA_PBR_RULE_UPDATE, /* warnings */ EC_ZEBRA_NS_NOTIFY_READ, EC_ZEBRAING_LM_PROTO_MISMATCH, diff --git a/zebra/zebra_nhg.c b/zebra/zebra_nhg.c index d07ceb652c..8723bd8d30 100644 --- a/zebra/zebra_nhg.c +++ b/zebra/zebra_nhg.c @@ -2563,6 +2563,9 @@ void zebra_nhg_dplane_result(struct zebra_dplane_ctx *ctx) case DPLANE_OP_NEIGH_DELETE: case DPLANE_OP_VTEP_ADD: case DPLANE_OP_VTEP_DELETE: + case DPLANE_OP_RULE_ADD: + case DPLANE_OP_RULE_DELETE: + case DPLANE_OP_RULE_UPDATE: case DPLANE_OP_NONE: break; } diff --git a/zebra/zebra_pbr.c b/zebra/zebra_pbr.c index 62cbcbda4b..c5a7795273 100644 --- a/zebra/zebra_pbr.c +++ b/zebra/zebra_pbr.c @@ -131,7 +131,7 @@ void zebra_pbr_rules_free(void *arg) rule = (struct zebra_pbr_rule *)arg; - (void)kernel_del_pbr_rule(rule); + (void)dplane_pbr_rule_delete(rule); XFREE(MTYPE_TMP, rule); } @@ -460,7 +460,7 @@ void zebra_pbr_add_rule(struct zebra_pbr_rule *rule) /* If found, this is an update */ if (found) { - (void)kernel_update_pbr_rule(found, rule); + (void)dplane_pbr_rule_update(found, rule); if (pbr_rule_release(found)) zlog_debug( @@ -468,12 +468,12 @@ void zebra_pbr_add_rule(struct zebra_pbr_rule *rule) __PRETTY_FUNCTION__); } else - (void)kernel_add_pbr_rule(rule); + (void)dplane_pbr_rule_add(rule); } void zebra_pbr_del_rule(struct zebra_pbr_rule *rule) { - (void)kernel_del_pbr_rule(rule); + (void)dplane_pbr_rule_delete(rule); if (pbr_rule_release(rule)) zlog_debug("%s: Rule being deleted we know nothing about", @@ -486,7 +486,7 @@ static void zebra_pbr_cleanup_rules(struct hash_bucket *b, void *data) int *sock = data; if (rule->sock == *sock) { - (void)kernel_del_pbr_rule(rule); + (void)dplane_pbr_rule_delete(rule); if (hash_release(zrouter.rules_hash, rule)) XFREE(MTYPE_TMP, rule); else @@ -735,25 +735,29 @@ void zebra_pbr_del_iptable(struct zebra_pbr_iptable *iptable) /* * Handle success or failure of rule (un)install in the kernel. */ -void kernel_pbr_rule_add_del_status(struct zebra_pbr_rule *rule, - enum zebra_dplane_status res) +void zebra_pbr_dplane_result(struct zebra_dplane_ctx *ctx) { - switch (res) { - case ZEBRA_DPLANE_INSTALL_SUCCESS: - zsend_rule_notify_owner(rule, ZAPI_RULE_INSTALLED); - break; - case ZEBRA_DPLANE_INSTALL_FAILURE: - zsend_rule_notify_owner(rule, ZAPI_RULE_FAIL_INSTALL); - break; - case ZEBRA_DPLANE_DELETE_SUCCESS: - zsend_rule_notify_owner(rule, ZAPI_RULE_REMOVED); - break; - case ZEBRA_DPLANE_DELETE_FAILURE: - zsend_rule_notify_owner(rule, ZAPI_RULE_FAIL_REMOVE); - break; - case ZEBRA_DPLANE_STATUS_NONE: - break; - } + enum zebra_dplane_result res; + enum dplane_op_e op; + + res = dplane_ctx_get_status(ctx); + op = dplane_ctx_get_op(ctx); + if (op == DPLANE_OP_RULE_ADD || op == DPLANE_OP_RULE_UPDATE) + zsend_rule_notify_owner(ctx, res == ZEBRA_DPLANE_REQUEST_SUCCESS + ? ZAPI_RULE_INSTALLED + : ZAPI_RULE_FAIL_INSTALL); + else if (op == DPLANE_OP_RULE_DELETE) + zsend_rule_notify_owner(ctx, res == ZEBRA_DPLANE_REQUEST_SUCCESS + ? ZAPI_RULE_REMOVED + : ZAPI_RULE_FAIL_REMOVE); + else + flog_err( + EC_ZEBRA_PBR_RULE_UPDATE, + "Context received in pbr rule dplane result handler with incorrect OP code (%u)", + op); + + + dplane_ctx_fini(&ctx); } /* diff --git a/zebra/zebra_pbr.h b/zebra/zebra_pbr.h index 83797b9521..4bc0f40037 100644 --- a/zebra/zebra_pbr.h +++ b/zebra/zebra_pbr.h @@ -170,24 +170,11 @@ void zebra_pbr_add_iptable(struct zebra_pbr_iptable *iptable); void zebra_pbr_del_iptable(struct zebra_pbr_iptable *iptable); /* - * Install specified rule for a specific interface. - * It is possible that the user-defined sequence number and the one in the - * forwarding plane may not coincide, hence the API requires a separate - * rule priority - maps to preference/FRA_PRIORITY on Linux. - */ -extern enum zebra_dplane_result kernel_add_pbr_rule(struct zebra_pbr_rule *rule); - -/* - * Uninstall specified rule for a specific interface. - */ -extern enum zebra_dplane_result kernel_del_pbr_rule(struct zebra_pbr_rule *rule); - -/* - * Update specified rule for a specific interface. + * Add, update or delete a rule from the + * kernel, using info from a dataplane context. */ extern enum zebra_dplane_result -kernel_update_pbr_rule(struct zebra_pbr_rule *old_rule, - struct zebra_pbr_rule *new_rule); +kernel_pbr_rule_update(struct zebra_dplane_ctx *ctx); /* * Get to know existing PBR rules in the kernel - typically called at startup. @@ -197,8 +184,7 @@ extern void kernel_read_pbr_rules(struct zebra_ns *zns); /* * Handle success or failure of rule (un)install in the kernel. */ -extern void kernel_pbr_rule_add_del_status(struct zebra_pbr_rule *rule, - enum zebra_dplane_status res); +extern void zebra_pbr_dplane_result(struct zebra_dplane_ctx *ctx); /* * Handle success or failure of ipset kinds (un)install in the kernel. diff --git a/zebra/zebra_rib.c b/zebra/zebra_rib.c index 0fc716db17..75619520dc 100644 --- a/zebra/zebra_rib.c +++ b/zebra/zebra_rib.c @@ -3583,6 +3583,12 @@ static int rib_process_dplane_results(struct thread *thread) zebra_vxlan_handle_result(ctx); break; + case DPLANE_OP_RULE_ADD: + case DPLANE_OP_RULE_DELETE: + case DPLANE_OP_RULE_UPDATE: + zebra_pbr_dplane_result(ctx); + break; + /* Some op codes not handled here */ case DPLANE_OP_ADDR_INSTALL: case DPLANE_OP_ADDR_UNINSTALL: