zebra: read in and sweep rules on startup

On startup of zebra, read in all ipv4/ipv6 rules from
the kernel and remove any with the zebra proto.

If there are any, this means we failed to remove them
on shutdown due to a crash or something. Without this,
users have to manually remove them with iproute2 or some
such and its really annoying.

Signed-off-by: Stephen Worley <sworley@cumulusnetworks.com>

zebra/rule_netlink.c

@@ -204,6 +204,10 @@ enum zebra_dplane_result kernel_update_pbr_rule(struct zebra_pbr_rule *old_rule,
  * DELs are notified up, if other attributes indicate it may be a
  * notification of interest. The expectation is that if this corresponds
  * to a PBR rule added by FRR, it will be readded.
+ *
+ * If startup and we see a rule we created, delete it as its leftover
+ * from a previous instance and should have been removed on shutdown.
+ *
  */
 int netlink_rule_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 {
@@ -215,15 +219,12 @@ int netlink_rule_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 	struct zebra_pbr_rule rule = {};
 	char buf1[PREFIX_STRLEN];
 	char buf2[PREFIX_STRLEN];
+	uint8_t proto = 0;
 
 	/* Basic validation followed by extracting attributes. */
 	if (h->nlmsg_type != RTM_NEWRULE && h->nlmsg_type != RTM_DELRULE)
 		return 0;
 
-	/* TBD */
-	if (h->nlmsg_type == RTM_NEWRULE)
-		return 0;
-
 	len = h->nlmsg_len - NLMSG_LENGTH(sizeof(struct fib_rule_hdr));
 	if (len < 0) {
 		zlog_err(
@@ -247,19 +248,6 @@ int netlink_rule_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 	memset(tb, 0, sizeof(tb));
 	netlink_parse_rtattr(tb, FRA_MAX, RTM_RTA(frh), len);
 
-	/* TBD: We don't care about rules not specifying an IIF. */
-	if (tb[FRA_IFNAME] == NULL)
-		return 0;
-
-	ifname = (char *)RTA_DATA(tb[FRA_IFNAME]);
-	zns = zebra_ns_lookup(ns_id);
-
-	/* If we don't know the interface, we don't care. */
-	if (!if_lookup_by_name_per_ns(zns, ifname))
-		return 0;
-
-	strlcpy(rule.ifname, ifname, sizeof(rule.ifname));
-
 	if (tb[FRA_PRIORITY])
 		rule.rule.priority = *(uint32_t *)RTA_DATA(tb[FRA_PRIORITY]);
 
@@ -292,6 +280,49 @@ int netlink_rule_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 	else
 		rule.rule.action.table = frh->table;
 
+	/* TBD: We don't care about rules not specifying an IIF. */
+	if (tb[FRA_IFNAME] == NULL)
+		return 0;
+
+	if (tb[FRA_PROTOCOL])
+		proto = *(uint8_t *)RTA_DATA(tb[FRA_PROTOCOL]);
+
+	ifname = (char *)RTA_DATA(tb[FRA_IFNAME]);
+	strlcpy(rule.ifname, ifname, sizeof(rule.ifname));
+
+	if (h->nlmsg_type == RTM_NEWRULE) {
+		/*
+		 * If we see a rule at startup we created, delete it now.
+		 * It should have been flushed on a previous shutdown.
+		 */
+		if (startup && proto == RTPROT_ZEBRA) {
+			int ret;
+
+			ret = netlink_rule_update(RTM_DELRULE, &rule);
+
+			zlog_debug(
+				"%s: %s leftover rule: family %s IF %s(%u) Pref %u Src %s Dst %s Table %u",
+				__func__,
+				((ret == 0) ? "Removed" : "Failed to remove"),
+				nl_family_to_str(frh->family), rule.ifname,
+				rule.rule.ifindex, rule.rule.priority,
+				prefix2str(&rule.rule.filter.src_ip, buf1,
+					   sizeof(buf1)),
+				prefix2str(&rule.rule.filter.dst_ip, buf2,
+					   sizeof(buf2)),
+				rule.rule.action.table);
+		}
+
+		/* TBD */
+		return 0;
+	}
+
+	zns = zebra_ns_lookup(ns_id);
+
+	/* If we don't know the interface, we don't care. */
+	if (!if_lookup_by_name_per_ns(zns, ifname))
+		return 0;
+
 	if (IS_ZEBRA_DEBUG_KERNEL)
 		zlog_debug(
 			"Rx %s family %s IF %s(%u) Pref %u Src %s Dst %s Table %u",
@@ -307,13 +338,52 @@ int netlink_rule_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 	return kernel_pbr_rule_del(&rule);
 }
 
+/*
+ * Request rules from the kernel
+ */
+static int netlink_request_rules(struct zebra_ns *zns, int family, int type)
+{
+	struct {
+		struct nlmsghdr n;
+		struct fib_rule_hdr frh;
+		char buf[NL_PKT_BUF_SIZE];
+	} req;
+
+	memset(&req, 0, sizeof(req));
+	req.n.nlmsg_type = type;
+	req.n.nlmsg_flags = NLM_F_ROOT | NLM_F_MATCH | NLM_F_REQUEST;
+	req.n.nlmsg_len = NLMSG_LENGTH(sizeof(struct fib_rule_hdr));
+	req.frh.family = family;
+
+	return netlink_request(&zns->netlink_cmd, &req.n);
+}
+
 /*
  * Get to know existing PBR rules in the kernel - typically called at startup.
- * TBD.
  */
 int netlink_rules_read(struct zebra_ns *zns)
 {
-	return 0;
+	int ret;
+	struct zebra_dplane_info dp_info;
+
+	zebra_dplane_info_from_zns(&dp_info, zns, true);
+
+	ret = netlink_request_rules(zns, AF_INET, RTM_GETRULE);
+	if (ret < 0)
+		return ret;
+
+	ret = netlink_parse_info(netlink_rule_change, &zns->netlink_cmd,
+				 &dp_info, 0, 1);
+	if (ret < 0)
+		return ret;
+
+	ret = netlink_request_rules(zns, AF_INET6, RTM_GETRULE);
+	if (ret < 0)
+		return ret;
+
+	ret = netlink_parse_info(netlink_rule_change, &zns->netlink_cmd,
+				 &dp_info, 0, 1);
+	return ret;
 }
 
 #endif /* HAVE_NETLINK */

zebra/zebra_ns.c

@@ -126,6 +126,7 @@ int zebra_ns_enable(ns_id_t ns_id, void **info)
 	kernel_init(zns);
 	interface_list(zns);
 	route_read(zns);
+	kernel_read_pbr_rules(zns);
 
 	/* Initiate Table Manager per ZNS */
 	table_manager_enable(ns_id);