zebra: install neigh entries on SVD

Install neigh entries always on SVD if it exists in
zebra. If zebra is using a Single Vxlan Device, we must
duplicate the install of our neigh entries to it so that
vxlan communication can also work across it in the downstream VNI
case.

Signed-off-by: Stephen Worley <sworley@nvidia.com>

zebra/zebra_evpn_neigh.h

@@ -63,6 +63,9 @@ struct zebra_neigh {
 
 	struct zebra_evpn *zevpn;
 
+	/* Refcnt - Only used by SVD neighs currently */
+	uint32_t refcnt;
+
 	uint32_t flags;
 #define ZEBRA_NEIGH_LOCAL 0x01
 #define ZEBRA_NEIGH_REMOTE 0x02

zebra/zebra_vxlan.c

@@ -73,6 +73,9 @@ DEFINE_HOOK(zebra_rmac_update,
 /* config knobs */
 static bool accept_bgp_seq = true;
 
+/* Single VXlan Device Global Neigh Table */
+struct hash *svd_nh_table;
+
 /* static function declarations */
 static void zevpn_print_neigh_hash_all_evpn(struct hash_bucket *bucket,
 					    void **args);
@@ -93,6 +96,11 @@ static int zl3vni_nh_del(struct zebra_l3vni *zl3vni, struct zebra_neigh *n);
 static int zl3vni_nh_install(struct zebra_l3vni *zl3vni, struct zebra_neigh *n);
 static int zl3vni_nh_uninstall(struct zebra_l3vni *zl3vni,
 			       struct zebra_neigh *n);
+static struct zebra_neigh *svd_nh_add(const struct ipaddr *vtep_ip,
+				      const struct ethaddr *rmac);
+static int svd_nh_del(struct zebra_neigh *n);
+static int svd_nh_install(struct zebra_l3vni *zl3vni, struct zebra_neigh *n);
+static int svd_nh_uninstall(struct zebra_l3vni *zl3vni, struct zebra_neigh *n);
 
 /* l3-vni rmac related APIs */
 static void zl3vni_print_rmac_hash(struct hash_bucket *, void *);
@@ -1446,9 +1454,9 @@ static void zl3vni_remote_rmac_del(struct zebra_l3vni *zl3vni,
 }
 
 /*
- * Look up nh hash entry on a l3-vni.
+ * Common code for look up of nh hash entry.
  */
-static struct zebra_neigh *zl3vni_nh_lookup(struct zebra_l3vni *zl3vni,
+static struct zebra_neigh *_nh_lookup(struct zebra_l3vni *zl3vni,
 				      const struct ipaddr *ip)
 {
 	struct zebra_neigh tmp;
@@ -1456,11 +1464,31 @@ static struct zebra_neigh *zl3vni_nh_lookup(struct zebra_l3vni *zl3vni,
 
 	memset(&tmp, 0, sizeof(tmp));
 	memcpy(&tmp.ip, ip, sizeof(struct ipaddr));
+
+	if (zl3vni)
 		n = hash_lookup(zl3vni->nh_table, &tmp);
+	else
+		n = hash_lookup(svd_nh_table, &tmp);
 
 	return n;
 }
 
+/*
+ * Look up nh hash entry on a l3-vni.
+ */
+static struct zebra_neigh *zl3vni_nh_lookup(struct zebra_l3vni *zl3vni,
+					    const struct ipaddr *ip)
+{
+	return _nh_lookup(zl3vni, ip);
+}
+
+/*
+ * Look up nh hash entry on a SVD.
+ */
+static struct zebra_neigh *svd_nh_lookup(const struct ipaddr *ip)
+{
+	return _nh_lookup(NULL, ip);
+}
 
 /*
  * Callback to allocate NH hash entry on L3-VNI.
@@ -1477,9 +1505,9 @@ static void *zl3vni_nh_alloc(void *p)
 }
 
 /*
- * Add neighbor entry.
+ * Common code for neigh add.
  */
-static struct zebra_neigh *zl3vni_nh_add(struct zebra_l3vni *zl3vni,
+static struct zebra_neigh *_nh_add(struct zebra_l3vni *zl3vni,
 				   const struct ipaddr *ip,
 				   const struct ethaddr *mac)
 {
@@ -1488,7 +1516,13 @@ static struct zebra_neigh *zl3vni_nh_add(struct zebra_l3vni *zl3vni,
 
 	memset(&tmp_n, 0, sizeof(tmp_n));
 	memcpy(&tmp_n.ip, ip, sizeof(struct ipaddr));
+
+	if (zl3vni)
 		n = hash_get(zl3vni->nh_table, &tmp_n, zl3vni_nh_alloc);
+	else
+		n = hash_get(svd_nh_table, &tmp_n, zl3vni_nh_alloc);
+
+	assert(n);
 
 	RB_INIT(host_rb_tree_entry, &n->host_rb);
 
@@ -1499,6 +1533,16 @@ static struct zebra_neigh *zl3vni_nh_add(struct zebra_l3vni *zl3vni,
 	return n;
 }
 
+/*
+ * Add neighbor entry.
+ */
+static struct zebra_neigh *zl3vni_nh_add(struct zebra_l3vni *zl3vni,
+					 const struct ipaddr *ip,
+					 const struct ethaddr *mac)
+{
+	return _nh_add(zl3vni, ip, mac);
+}
+
 /*
  * Delete neighbor entry.
  */
@@ -1521,14 +1565,38 @@ static int zl3vni_nh_del(struct zebra_l3vni *zl3vni, struct zebra_neigh *n)
 }
 
 /*
- * Install remote nh as neigh into the kernel.
+ * Add Single VXlan Device neighbor entry.
  */
-static int zl3vni_nh_install(struct zebra_l3vni *zl3vni, struct zebra_neigh *n)
+static struct zebra_neigh *svd_nh_add(const struct ipaddr *ip,
+				      const struct ethaddr *mac)
+{
+	return _nh_add(NULL, ip, mac);
+}
+
+/*
+ * Del Single VXlan Device neighbor entry.
+ */
+static int svd_nh_del(struct zebra_neigh *n)
+{
+	if (n->refcnt > 0)
+		return -1;
+
+	hash_release(svd_nh_table, n);
+	XFREE(MTYPE_L3NEIGH, n);
+
+	return 0;
+}
+
+/*
+ * Common code to install remote nh as neigh into the kernel.
+ */
+static int _nh_install(struct zebra_l3vni *zl3vni, struct interface *ifp,
+		       struct zebra_neigh *n)
 {
 	uint8_t flags;
 	int ret = 0;
 
-	if (!is_l3vni_oper_up(zl3vni))
+	if (zl3vni && !is_l3vni_oper_up(zl3vni))
 		return -1;
 
 	if (!(n->flags & ZEBRA_NEIGH_REMOTE)
@@ -1539,31 +1607,63 @@ static int zl3vni_nh_install(struct zebra_l3vni *zl3vni, struct zebra_neigh *n)
 	if (n->flags & ZEBRA_NEIGH_ROUTER_FLAG)
 		flags |= DPLANE_NTF_ROUTER;
 
-	dplane_rem_neigh_add(zl3vni->svi_if, &n->ip, &n->emac, flags,
+	dplane_rem_neigh_add(ifp, &n->ip, &n->emac, flags,
 			     false /*was_static*/);
 
 	return ret;
 }
 
+/*
+ * Common code to uninstall remote nh from the kernel.
+ */
+static int _nh_uninstall(struct interface *ifp, struct zebra_neigh *n)
+{
+	if (!(n->flags & ZEBRA_NEIGH_REMOTE)
+	    || !(n->flags & ZEBRA_NEIGH_REMOTE_NH))
+		return 0;
+
+	if (!ifp || !if_is_operative(ifp))
+		return 0;
+
+	dplane_rem_neigh_delete(ifp, &n->ip);
+
+	return 0;
+}
+
+/*
+ * Install remote nh as neigh into the kernel.
+ */
+static int zl3vni_nh_install(struct zebra_l3vni *zl3vni, struct zebra_neigh *n)
+{
+	return _nh_install(zl3vni, zl3vni->svi_if, n);
+}
+
 /*
  * Uninstall remote nh from the kernel.
  */
 static int zl3vni_nh_uninstall(struct zebra_l3vni *zl3vni,
 			       struct zebra_neigh *n)
 {
-	if (!(n->flags & ZEBRA_NEIGH_REMOTE)
+	return _nh_uninstall(zl3vni->svi_if, n);
-	    || !(n->flags & ZEBRA_NEIGH_REMOTE_NH))
-		return 0;
-
-	if (!zl3vni->svi_if || !if_is_operative(zl3vni->svi_if))
-		return 0;
-
-	dplane_rem_neigh_delete(zl3vni->svi_if, &n->ip);
-
-	return 0;
 }
 
-/* add remote vtep as a neigh entry */
+/*
+ * Install SVD remote nh as neigh into the kernel.
+ */
+static int svd_nh_install(struct zebra_l3vni *zl3vni, struct zebra_neigh *n)
+{
+	return _nh_install(zl3vni, zl3vni->vxlan_if, n);
+}
+
+/*
+ * Uninstall SVD remote nh from the kernel.
+ */
+static int svd_nh_uninstall(struct zebra_l3vni *zl3vni, struct zebra_neigh *n)
+{
+	return _nh_uninstall(zl3vni->vxlan_if, n);
+}
+
+/* Add remote vtep as a neigh entry */
 static int zl3vni_remote_nh_add(struct zebra_l3vni *zl3vni,
 				const struct ipaddr *vtep_ip,
 				const struct ethaddr *rmac,
@@ -1601,7 +1701,7 @@ static int zl3vni_remote_nh_add(struct zebra_l3vni *zl3vni,
 	return 0;
 }
 
-/* handle nh neigh delete */
+/* Del remote vtep as a neigh entry */
 static void zl3vni_remote_nh_del(struct zebra_l3vni *zl3vni,
 				 struct zebra_neigh *nh,
 				 struct prefix *host_prefix)
@@ -1617,6 +1717,91 @@ static void zl3vni_remote_nh_del(struct zebra_l3vni *zl3vni,
 	}
 }
 
+/* Add remote vtep as a SVD neigh entry */
+static int svd_remote_nh_add(struct zebra_l3vni *zl3vni,
+			     const struct ipaddr *vtep_ip,
+			     const struct ethaddr *rmac,
+			     const struct prefix *host_prefix)
+{
+	struct zebra_neigh *nh = NULL;
+
+	/* SVD backed VNI check */
+	if (!IS_ZL3VNI_SVD_BACKED(zl3vni))
+		return 0;
+
+	/* Create the SVD next hop entry, or update its mac, if necessary. */
+	nh = svd_nh_lookup(vtep_ip);
+	if (!nh) {
+		nh = svd_nh_add(vtep_ip, rmac);
+		if (!nh) {
+			zlog_debug(
+				"Failed to add NH %pIA as SVD Neigh (RMAC %pEA prefix %pFX)",
+				vtep_ip, rmac, host_prefix);
+			return -1;
+		}
+
+	} else if (memcmp(&nh->emac, rmac, ETH_ALEN) != 0) {
+		if (IS_ZEBRA_DEBUG_VXLAN)
+			zlog_debug(
+				"SVD RMAC change(%pEA --> %pEA) for nexthop %pIA, prefix %pFX",
+				&nh->emac, rmac, vtep_ip, host_prefix);
+
+		memcpy(&nh->emac, rmac, ETH_ALEN);
+		/* install (update) the nh neigh in kernel */
+		svd_nh_install(zl3vni, nh);
+
+		/* Don't increment refcnt change */
+		return 0;
+	}
+
+	nh->refcnt++;
+
+	if (IS_ZEBRA_DEBUG_VXLAN)
+		zlog_debug("SVD NH ADD refcnt (%u) for nexthop %pIA",
+			   nh->refcnt, vtep_ip);
+
+	/*
+	 * Install the nh neigh in kernel if this is the first time we
+	 * have seen it.
+	 */
+	if (nh->refcnt == 1)
+		svd_nh_install(zl3vni, nh);
+
+	return 0;
+}
+
+/* Del remote vtep as a SVD neigh entry */
+static int svd_remote_nh_del(struct zebra_l3vni *zl3vni,
+			     const struct ipaddr *vtep_ip)
+{
+	struct zebra_neigh *nh;
+
+	/* SVD backed VNI check */
+	if (!IS_ZL3VNI_SVD_BACKED(zl3vni))
+		return 0;
+
+	nh = svd_nh_lookup(vtep_ip);
+	if (!nh) {
+		zlog_debug("Failed to del NH %pIA as SVD Neigh", vtep_ip);
+
+		return -1;
+	}
+
+	nh->refcnt--;
+
+	if (IS_ZEBRA_DEBUG_VXLAN)
+		zlog_debug("SVD NH Del refcnt (%u) for nexthop %pIA",
+			   nh->refcnt, vtep_ip);
+
+	/* Last refcnt on NH, remove it completely. */
+	if (nh->refcnt == 0) {
+		svd_nh_uninstall(zl3vni, nh);
+		svd_nh_del(nh);
+	}
+
+	return 0;
+}
+
 /* handle neigh update from kernel - the only thing of interest is to
  * readd stale entries.
  */
@@ -2323,6 +2508,9 @@ void zebra_vxlan_evpn_vrf_route_add(vrf_id_t vrf_id, const struct ethaddr *rmac,
 	 */
 	zl3vni_remote_nh_add(zl3vni, vtep_ip, rmac, host_prefix);
 
+	/* Add SVD next hop neighbor */
+	svd_remote_nh_add(zl3vni, vtep_ip, rmac, host_prefix);
+
 	/*
 	 * if the remote vtep is a ipv4 mapped ipv6 address convert it to ipv4
 	 * address. Rmac is programmed against the ipv4 vtep because we only
@@ -2366,6 +2554,9 @@ void zebra_vxlan_evpn_vrf_route_del(vrf_id_t vrf_id,
 	/* delete the next hop entry */
 	zl3vni_remote_nh_del(zl3vni, nh, host_prefix);
 
+	/* Delete SVD next hop entry */
+	svd_remote_nh_del(zl3vni, vtep_ip);
+
 	/* delete the rmac entry */
 	if (zrmac)
 		zl3vni_remote_rmac_del(zl3vni, zrmac, vtep_ip);
@@ -5476,6 +5667,9 @@ void zebra_vxlan_init(void)
 {
 	zrouter.l3vni_table = hash_create(l3vni_hash_keymake, l3vni_hash_cmp,
 					  "Zebra VRF L3 VNI table");
+
+	svd_nh_table = zebra_neigh_db_create("Zebra SVD next-hop table");
+
 	zrouter.evpn_vrf = NULL;
 	zebra_evpn_mh_init();
 }
@@ -5499,6 +5693,42 @@ ifindex_t get_l3vni_svi_ifindex(vrf_id_t vrf_id)
 	return zl3vni->svi_if->ifindex;
 }
 
+/* get the l3vni vxlan ifindex */
+ifindex_t get_l3vni_vxlan_ifindex(vrf_id_t vrf_id)
+{
+	struct zebra_l3vni *zl3vni = NULL;
+
+	zl3vni = zl3vni_from_vrf(vrf_id);
+	if (!zl3vni || !is_l3vni_oper_up(zl3vni))
+		return 0;
+
+	return zl3vni->vxlan_if->ifindex;
+}
+
+/* get the l3vni vni */
+vni_t get_l3vni_vni(vrf_id_t vrf_id)
+{
+	struct zebra_l3vni *zl3vni = NULL;
+
+	zl3vni = zl3vni_from_vrf(vrf_id);
+	if (!zl3vni || !is_l3vni_oper_up(zl3vni))
+		return 0;
+
+	return zl3vni->vni;
+}
+
+/* is the vrf l3vni SVD backed? */
+bool is_vrf_l3vni_svd_backed(vrf_id_t vrf_id)
+{
+	struct zebra_l3vni *zl3vni = NULL;
+
+	zl3vni = zl3vni_from_vrf(vrf_id);
+	if (!zl3vni || !is_l3vni_oper_up(zl3vni))
+		return false;
+
+	return IS_ZL3VNI_SVD_BACKED(zl3vni);
+}
+
 /************************** vxlan SG cache management ************************/
 /* Inform PIM about the mcast group */
 static int zebra_vxlan_sg_send(struct zebra_vrf *zvrf,

zebra/zebra_vxlan.h

@@ -89,6 +89,9 @@ extern void zebra_vxlan_sg_replay(ZAPI_HANDLER_ARGS);
 
 extern int is_l3vni_for_prefix_routes_only(vni_t vni);
 extern ifindex_t get_l3vni_svi_ifindex(vrf_id_t vrf_id);
+extern ifindex_t get_l3vni_vxlan_ifindex(vrf_id_t vrf_id);
+extern vni_t get_l3vni_vni(vrf_id_t vrf_id);
+extern bool is_vrf_l3vni_svd_backed(vrf_id_t vrf_id);
 extern int zebra_vxlan_vrf_delete(struct zebra_vrf *zvrf);
 extern int zebra_vxlan_vrf_enable(struct zebra_vrf *zvrf);
 extern int zebra_vxlan_vrf_disable(struct zebra_vrf *zvrf);

zebra/zebra_vxlan_private.h

@@ -76,6 +76,10 @@ struct zebra_l3vni {
 	struct hash *nh_table;
 };
 
+#define IS_ZL3VNI_SVD_BACKED(zl3vni)                                           \
+	(zl3vni->vxlan_if && zl3vni->vxlan_if->info &&                         \
+	 IS_ZEBRA_VXLAN_IF_SVD((struct zebra_if *)zl3vni->vxlan_if->info))
+
 /* get the vx-intf name for l3vni */
 static inline const char *zl3vni_vxlan_if_name(struct zebra_l3vni *zl3vni)
 {