Merge pull request #8154 from AnuradhaKaruppiah/evpn-mh-irb-2

bgpd, lib, zebra: Complete support for sym-IRB with EVPN-MH

bgpd/bgp_attr.h

@@ -243,6 +243,15 @@ struct attr {
 	 */
 #define ATTR_ES_PEER_ROUTER (1 << 4)
 
+	/* These two flags are only set on L3 routes installed in a
+	 * VRF as a result of EVPN MAC-IP route
+	 * XXX - while splitting up per-family attrs these need to be
+	 * classified as non-EVPN
+	 */
+#define ATTR_ES_L3_NHG_USE (1 << 5)
+#define ATTR_ES_L3_NHG_ACTIVE (1 << 6)
+#define ATTR_ES_L3_NHG (ATTR_ES_L3_NHG_USE | ATTR_ES_L3_NHG_ACTIVE)
+
 	/* route tag */
 	route_tag_t tag;
 

bgpd/bgp_evpn.c

@@ -65,11 +65,6 @@ DEFINE_QOBJ_TYPE(bgp_evpn_es);
  * Static function declarations
  */
 static int delete_all_vni_routes(struct bgp *bgp, struct bgpevpn *vpn);
-static void bgp_evpn_update_type2_route_entry(struct bgp *bgp,
-					      struct bgpevpn *vpn,
-					      struct bgp_dest *dest,
-					      struct bgp_path_info *local_pi,
-					      const char *caller);
 static struct in_addr zero_vtep_ip;
 
 /*
@@ -1602,8 +1597,8 @@ static int update_evpn_route_entry(struct bgp *bgp, struct bgpevpn *vpn,
 		}
 	}
 
-	/* MAC-IP routes in the VNI route table are linked to the
+	/* local MAC-IP routes in the VNI table are linked to
-	 * destination ES
+	 * the destination ES
 	 */
 	if (route_change && vpn_rt
 	    && (evp->prefix.route_type == BGP_EVPN_MAC_IP_ROUTE))
@@ -1669,6 +1664,18 @@ static void evpn_cleanup_local_non_best_route(struct bgp *bgp,
 	evpn_zebra_reinstall_best_route(bgp, vpn, dest);
 }
 
+static inline bool bgp_evpn_route_add_l3_ecomm_ok(struct bgpevpn *vpn,
+						  const struct prefix_evpn *p,
+						  esi_t *esi)
+{
+	return p->prefix.route_type == BGP_EVPN_MAC_IP_ROUTE
+	       && (is_evpn_prefix_ipaddr_v4(p)
+		   || !IN6_IS_ADDR_LINKLOCAL(
+			   &p->prefix.macip_addr.ip.ipaddr_v6))
+	       && CHECK_FLAG(vpn->flags, VNI_FLAG_USE_TWO_LABELS)
+	       && bgpevpn_get_l3vni(vpn) && bgp_evpn_es_add_l3_ecomm_ok(esi);
+}
+
 /*
  * Create or update EVPN route (of type based on prefix) for specified VNI
  * and schedule for processing.
@@ -1738,12 +1745,8 @@ static int update_evpn_route(struct bgp *bgp, struct bgpevpn *vpn,
 	 * IPv4 or IPv6 global addresses and we're advertising L3VNI with
 	 * these routes.
 	 */
-	if (p->prefix.route_type == BGP_EVPN_MAC_IP_ROUTE &&
+	add_l3_ecomm = bgp_evpn_route_add_l3_ecomm_ok(
-	    (is_evpn_prefix_ipaddr_v4(p) ||
+		vpn, p, (attr.es_flags & ATTR_ES_IS_LOCAL) ? &attr.esi : NULL);
-	     !IN6_IS_ADDR_LINKLOCAL(&p->prefix.macip_addr.ip.ipaddr_v6)) &&
-	    CHECK_FLAG(vpn->flags, VNI_FLAG_USE_TWO_LABELS) &&
-	    bgpevpn_get_l3vni(vpn))
-		add_l3_ecomm = 1;
 
 	/* Set up extended community. */
 	build_evpn_route_extcomm(vpn, &attr, add_l3_ecomm);
@@ -1930,8 +1933,7 @@ static int delete_evpn_route(struct bgp *bgp, struct bgpevpn *vpn,
 	return 0;
 }
 
-static void bgp_evpn_update_type2_route_entry(struct bgp *bgp,
+void bgp_evpn_update_type2_route_entry(struct bgp *bgp, struct bgpevpn *vpn,
-					      struct bgpevpn *vpn,
 				       struct bgp_dest *dest,
 				       struct bgp_path_info *local_pi,
 				       const char *caller)
@@ -1977,12 +1979,9 @@ static void bgp_evpn_update_type2_route_entry(struct bgp *bgp,
 	/* Add L3 VNI RTs and RMAC for non IPv6 link-local if
 	 * using L3 VNI for type-2 routes also.
 	 */
-	if ((is_evpn_prefix_ipaddr_v4(evp) ||
+	add_l3_ecomm = bgp_evpn_route_add_l3_ecomm_ok(
-				!IN6_IS_ADDR_LINKLOCAL(
+		vpn, evp,
-				&evp->prefix.macip_addr.ip.ipaddr_v6)) &&
+		(attr.es_flags & ATTR_ES_IS_LOCAL) ? &attr.esi : NULL);
-			CHECK_FLAG(vpn->flags, VNI_FLAG_USE_TWO_LABELS) &&
-			bgpevpn_get_l3vni(vpn))
-		add_l3_ecomm = 1;
 
 	/* Set up extended community. */
 	build_evpn_route_extcomm(vpn, &attr, add_l3_ecomm);
@@ -2379,6 +2378,8 @@ static int install_evpn_route_entry_in_vrf(struct bgp *bgp_vrf,
 	afi_t afi = 0;
 	safi_t safi = 0;
 	bool new_pi = false;
+	bool use_l3nhg = false;
+	bool is_l3nhg_active = false;
 
 	memset(pp, 0, sizeof(struct prefix));
 	ip_prefix_from_evpn_prefix(evp, pp);
@@ -2414,6 +2415,13 @@ static int install_evpn_route_entry_in_vrf(struct bgp *bgp_vrf,
 	else
 		attr.flag |= ATTR_FLAG_BIT(BGP_ATTR_NEXT_HOP);
 
+	bgp_evpn_es_vrf_use_nhg(bgp_vrf, &parent_pi->attr->esi, &use_l3nhg,
+				&is_l3nhg_active, NULL);
+	if (use_l3nhg)
+		attr.es_flags |= ATTR_ES_L3_NHG_USE;
+	if (is_l3nhg_active)
+		attr.es_flags |= ATTR_ES_L3_NHG_ACTIVE;
+
 	/* Check if route entry is already present. */
 	for (pi = bgp_dest_get_bgp_path_info(dest); pi; pi = pi->next)
 		if (pi->extra
@@ -2454,6 +2462,9 @@ static int install_evpn_route_entry_in_vrf(struct bgp *bgp_vrf,
 	/* as it is an importation, change nexthop */
 	bgp_path_info_set_flag(dest, pi, BGP_PATH_ANNC_NH_SELF);
 
+	/* Link path to evpn nexthop */
+	bgp_evpn_path_nh_add(bgp_vrf, pi);
+
 	bgp_aggregate_increment(bgp_vrf, bgp_dest_get_prefix(dest), pi, afi,
 				safi);
 
@@ -2487,6 +2498,8 @@ static int install_evpn_route_entry(struct bgp *bgp, struct bgpevpn *vpn,
 	struct attr *attr_new;
 	int ret;
 	struct prefix_evpn ad_evp;
+	bool old_local_es = false;
+	bool new_local_es;
 
 	/* EAD prefix in the global table doesn't include the VTEP-IP so
 	 * we need to create a different copy for the VNI
@@ -2509,6 +2522,7 @@ static int install_evpn_route_entry(struct bgp *bgp, struct bgpevpn *vpn,
 		/* Create an info */
 		pi = bgp_create_evpn_bgp_path_info(parent_pi, dest,
 						    parent_pi->attr);
+		new_local_es = bgp_evpn_attr_is_local_es(pi->attr);
 	} else {
 		if (attrhash_cmp(pi->attr, parent_pi->attr)
 		    && !CHECK_FLAG(pi->flags, BGP_PATH_REMOVED)) {
@@ -2527,17 +2541,29 @@ static int install_evpn_route_entry(struct bgp *bgp, struct bgpevpn *vpn,
 		if (!IPV4_ADDR_SAME(&pi->attr->nexthop, &attr_new->nexthop))
 			SET_FLAG(pi->flags, BGP_PATH_IGP_CHANGED);
 
+		old_local_es = bgp_evpn_attr_is_local_es(pi->attr);
+		new_local_es = bgp_evpn_attr_is_local_es(attr_new);
+		/* If ESI is different or if its type has changed we
+		 * need to reinstall the path in zebra
+		 */
+		if ((old_local_es != new_local_es)
+		    || memcmp(&pi->attr->esi, &attr_new->esi,
+			      sizeof(attr_new->esi))) {
+
+			if (BGP_DEBUG(evpn_mh, EVPN_MH_RT))
+				zlog_debug("VNI %d path %pFX chg to %s es",
+					   vpn->vni, &pi->net->p,
+					   new_local_es ? "local"
+							: "non-local");
+			bgp_path_info_set_flag(dest, pi, BGP_PATH_ATTR_CHANGED);
+		}
+
 		/* Unintern existing, set to new. */
 		bgp_attr_unintern(&pi->attr);
 		pi->attr = attr_new;
 		pi->uptime = bgp_clock();
 	}
 
-	/* MAC-IP routes in the VNI table are linked to the destination ES */
-	if (p->prefix.route_type == BGP_EVPN_MAC_IP_ROUTE)
-		bgp_evpn_path_es_link(pi, vpn->vni,
-				      bgp_evpn_attr_get_esi(pi->attr));
-
 	/* Perform route selection and update zebra, if required. */
 	ret = evpn_route_select_install(bgp, vpn, dest);
 
@@ -2547,10 +2573,9 @@ static int install_evpn_route_entry(struct bgp *bgp, struct bgpevpn *vpn,
 	 * from sync-path to remote-path)
 	 */
 	local_pi = bgp_evpn_route_get_local_path(bgp, dest);
-	if (local_pi && bgp_evpn_attr_is_local_es(local_pi->attr))
+	if (local_pi && (old_local_es || new_local_es))
 		bgp_evpn_update_type2_route_entry(bgp, vpn, dest, local_pi,
 						  __func__);
-
 	bgp_dest_unlock_node(dest);
 
 	return ret;
@@ -2619,6 +2644,9 @@ static int uninstall_evpn_route_entry_in_vrf(struct bgp *bgp_vrf,
 	/* Mark entry for deletion */
 	bgp_path_info_delete(dest, pi);
 
+	/* Unlink path to evpn nexthop */
+	bgp_evpn_path_nh_del(bgp_vrf, pi);
+
 	/* Perform route selection and update zebra, if required. */
 	bgp_process(bgp_vrf, dest, afi, safi);
 
@@ -2853,11 +2881,11 @@ static int bgp_evpn_route_rmac_self_check(struct bgp *bgp_vrf,
 
 /* don't import hosts that are locally attached */
 static inline bool
-bgp_evpn_skip_vrf_import_of_local_es(const struct prefix_evpn *evp,
+bgp_evpn_skip_vrf_import_of_local_es(struct bgp *bgp_vrf,
+				     const struct prefix_evpn *evp,
 				     struct bgp_path_info *pi, int install)
 {
 	esi_t *esi;
-	struct in_addr nh;
 
 	if (evp->prefix.route_type == BGP_EVPN_MAC_IP_ROUTE) {
 		esi = bgp_evpn_attr_get_esi(pi->attr);
@@ -2875,31 +2903,53 @@ bgp_evpn_skip_vrf_import_of_local_es(const struct prefix_evpn *evp,
 			}
 			return true;
 		}
-
-		/* Don't import routes with ES as destination if the nexthop
-		 * has not been advertised via the EAD-ES
-		 */
-		if (pi->attr)
-			nh = pi->attr->nexthop;
-		else
-			nh.s_addr = INADDR_ANY;
-		if (install && !bgp_evpn_es_is_vtep_active(esi, nh)) {
-			if (BGP_DEBUG(evpn_mh, EVPN_MH_RT)) {
-				char esi_buf[ESI_STR_LEN];
-
-				zlog_debug(
-					"vrf %s of evpn prefix %pFX skipped, nh %pI4 inactive in es %s",
-					install ? "import" : "unimport", evp,
-					&nh,
-					esi_to_str(esi, esi_buf,
-						   sizeof(esi_buf)));
-			}
-			return true;
-		}
 	}
 	return false;
 }
 
+/*
+ * Install or uninstall a mac-ip route in the provided vrf if
+ * there is a rt match
+ */
+int bgp_evpn_route_entry_install_if_vrf_match(struct bgp *bgp_vrf,
+					      struct bgp_path_info *pi,
+					      int install)
+{
+	int ret = 0;
+	const struct prefix_evpn *evp =
+		(const struct prefix_evpn *)bgp_dest_get_prefix(pi->net);
+
+	/* Consider "valid" remote routes applicable for
+	 * this VRF.
+	 */
+	if (!(CHECK_FLAG(pi->flags, BGP_PATH_VALID)
+	      && pi->type == ZEBRA_ROUTE_BGP
+	      && pi->sub_type == BGP_ROUTE_NORMAL))
+		return 0;
+
+	if (is_route_matching_for_vrf(bgp_vrf, pi)) {
+		if (bgp_evpn_route_rmac_self_check(bgp_vrf, evp, pi))
+			return 0;
+
+		/* don't import hosts that are locally attached */
+		if (install
+		    && !bgp_evpn_skip_vrf_import_of_local_es(bgp_vrf, evp, pi,
+							     install))
+			ret = install_evpn_route_entry_in_vrf(bgp_vrf, evp, pi);
+		else
+			ret = uninstall_evpn_route_entry_in_vrf(bgp_vrf, evp,
+								pi);
+
+		if (ret)
+			flog_err(EC_BGP_EVPN_FAIL,
+				 "Failed to %s EVPN %pFX route in VRF %s",
+				 install ? "install" : "uninstall", evp,
+				 vrf_id_to_name(bgp_vrf->vrf_id));
+	}
+
+	return ret;
+}
+
 /*
  * Install or uninstall mac-ip routes are appropriate for this
  * particular VRF.
@@ -2949,49 +2999,13 @@ static int install_uninstall_routes_for_vrf(struct bgp *bgp_vrf, int install)
 
 			for (pi = bgp_dest_get_bgp_path_info(dest); pi;
 			     pi = pi->next) {
-				/* Consider "valid" remote routes applicable for
+				ret = bgp_evpn_route_entry_install_if_vrf_match(
-				 * this VRF.
+					bgp_vrf, pi, install);
-				 */
+				if (ret)
-				if (!(CHECK_FLAG(pi->flags, BGP_PATH_VALID)
-				      && pi->type == ZEBRA_ROUTE_BGP
-				      && pi->sub_type == BGP_ROUTE_NORMAL))
-					continue;
-
-				/* don't import hosts that are locally attached
-				 */
-				if (bgp_evpn_skip_vrf_import_of_local_es(
-					    evp, pi, install))
-					continue;
-
-				if (is_route_matching_for_vrf(bgp_vrf, pi)) {
-					if (bgp_evpn_route_rmac_self_check(
-								bgp_vrf, evp, pi))
-						continue;
-
-					if (install)
-						ret = install_evpn_route_entry_in_vrf(
-							bgp_vrf, evp, pi);
-					else
-						ret = uninstall_evpn_route_entry_in_vrf(
-							bgp_vrf, evp, pi);
-
-					if (ret) {
-						flog_err(
-							EC_BGP_EVPN_FAIL,
-							"Failed to %s EVPN %pFX route in VRF %s",
-							install ? "install"
-								: "uninstall",
-							evp,
-							vrf_id_to_name(
-								bgp_vrf->vrf_id));
-						bgp_dest_unlock_node(rd_dest);
-						bgp_dest_unlock_node(dest);
 					return ret;
 			}
 		}
 	}
-		}
-	}
 
 	return 0;
 }
@@ -3169,14 +3183,13 @@ static int install_uninstall_route_in_vrfs(struct bgp *bgp_def, afi_t afi,
 		 || is_evpn_prefix_ipaddr_v6(evp)))
 		return 0;
 
-	/* don't import hosts that are locally attached */
-	if (bgp_evpn_skip_vrf_import_of_local_es(evp, pi, install))
-		return 0;
-
 	for (ALL_LIST_ELEMENTS(vrfs, node, nnode, bgp_vrf)) {
 		int ret;
 
-		if (install)
+		/* don't import hosts that are locally attached */
+		if (install
+		    && !bgp_evpn_skip_vrf_import_of_local_es(bgp_vrf, evp, pi,
+							     install))
 			ret = install_evpn_route_entry_in_vrf(bgp_vrf, evp, pi);
 		else
 			ret = uninstall_evpn_route_entry_in_vrf(bgp_vrf, evp,
@@ -3291,6 +3304,13 @@ static int bgp_evpn_install_uninstall_table(struct bgp *bgp, afi_t afi,
 		if (sub_type != ECOMMUNITY_ROUTE_TARGET)
 			continue;
 
+		/* non-local MAC-IP routes in the global route table are linked
+		 * to the destination ES
+		 */
+		if (evp->prefix.route_type == BGP_EVPN_MAC_IP_ROUTE)
+			bgp_evpn_path_es_link(pi, 0,
+					      bgp_evpn_attr_get_esi(pi->attr));
+
 		/*
 		 * macip routes (type-2) are imported into VNI and VRF tables.
 		 * IMET route is imported into VNI table.
@@ -3370,6 +3390,18 @@ static int install_uninstall_evpn_route(struct bgp *bgp, afi_t afi, safi_t safi,
 						true, true);
 }
 
+void bgp_evpn_import_type2_route(struct bgp_path_info *pi, int import)
+{
+	struct bgp *bgp_evpn;
+
+	bgp_evpn = bgp_get_evpn();
+	if (!bgp_evpn)
+		return;
+
+	install_uninstall_evpn_route(bgp_evpn, AFI_L2VPN, SAFI_EVPN,
+				     &pi->net->p, pi, import);
+}
+
 /* Import the pi into vrf routing tables */
 void bgp_evpn_import_route_in_vrfs(struct bgp_path_info *pi, int import)
 {
@@ -3723,7 +3755,7 @@ static int process_type2_route(struct peer *peer, afi_t afi, safi_t safi,
 	if (attr) {
 		STREAM_GET(&attr->esi, pkt, sizeof(esi_t));
 
-		if (bgp_evpn_is_esi_local(&attr->esi))
+		if (bgp_evpn_is_esi_local_and_non_bypass(&attr->esi))
 			attr->es_flags |= ATTR_ES_IS_LOCAL;
 		else
 			attr->es_flags &= ~ATTR_ES_IS_LOCAL;
@@ -5776,11 +5808,14 @@ void bgp_evpn_init(struct bgp *bgp)
 
 	/* Default BUM handling is to do head-end replication. */
 	bgp->vxlan_flood_ctrl = VXLAN_FLOOD_HEAD_END_REPL;
+
+	bgp_evpn_nh_init(bgp);
 }
 
 void bgp_evpn_vrf_delete(struct bgp *bgp_vrf)
 {
 	bgp_evpn_unmap_vrf_from_its_rts(bgp_vrf);
+	bgp_evpn_nh_finish(bgp_vrf);
 }
 
 /*

bgpd/bgp_evpn.h

@@ -206,5 +206,4 @@ extern void bgp_evpn_init(struct bgp *bgp);
 extern int bgp_evpn_get_type5_prefixlen(const struct prefix *pfx);
 extern bool bgp_evpn_is_prefix_nht_supported(const struct prefix *pfx);
 extern void update_advertise_vrf_routes(struct bgp *bgp_vrf);
-
 #endif /* _QUAGGA_BGP_EVPN_H */

bgpd/bgp_evpn_mh.h

@@ -105,8 +105,17 @@ struct bgp_evpn_es {
 
 	/* List of MAC-IP VNI paths using this ES as destination -
 	 * element is bgp_path_info_extra->es_info
+	 * Note: Only local/zebra-added MACIP paths in the VNI
+	 * routing table are linked to this list
 	 */
-	struct list *macip_path_list;
+	struct list *macip_evi_path_list;
+
+	/* List of MAC-IP paths in the global routing table using this
+	 * ES as destination - data is bgp_path_info_extra->es_info
+	 * Note: Only non-local/imported MACIP paths in the global
+	 * routing table are linked to this list
+	 */
+	struct list *macip_global_path_list;
 
 	/* Number of remote VNIs referencing this ES */
 	uint32_t remote_es_evi_cnt;
@@ -241,6 +250,26 @@ struct bgp_evpn_es_evi_vtep {
 	struct bgp_evpn_es_vtep *es_vtep;
 };
 
+/* A nexthop is created when a path (imported from an EVPN type-2 route)
+ * is added to the VRF route table using that nexthop.
+ * It is added on first pi reference and removed on last pi deref.
+ */
+struct bgp_evpn_nh {
+	/* backpointer to the VRF */
+	struct bgp *bgp_vrf;
+	/* nexthop/VTEP IP */
+	struct ipaddr ip;
+	/* description for easy logging */
+	char nh_str[INET6_ADDRSTRLEN];
+	struct ethaddr rmac;
+	/* pi from which we are pulling the nh RMAC */
+	struct bgp_path_info *ref_pi;
+	/* List of VRF paths using this nexthop */
+	struct list *pi_list;
+	uint8_t flags;
+#define BGP_EVPN_NH_READY_FOR_ZEBRA (1 << 0)
+};
+
 /* multihoming information stored in bgp_master */
 #define bgp_mh_info (bm->mh_info)
 struct bgp_evpn_mh_info {
@@ -273,6 +302,12 @@ struct bgp_evpn_mh_info {
 	/* Skip EAD-EVI advertisements by turning off this knob */
 	bool ead_evi_tx;
 #define BGP_EVPN_MH_EAD_EVI_TX_DEF true
+	/* If the Local ES is inactive we advertise the MAC-IP without the
+	 * L3 ecomm
+	 */
+	bool suppress_l3_ecomm_on_inactive_es;
+	/* Setup EVPN PE nexthops and their RMAC in bgpd */
+	bool bgp_evpn_nh_setup;
 };
 
 /****************************************************************************/
@@ -330,6 +365,12 @@ static inline uint32_t bgp_evpn_attr_get_df_pref(struct attr *attr)
 	return (attr) ? attr->df_pref : 0;
 }
 
+static inline bool bgp_evpn_local_es_is_active(struct bgp_evpn_es *es)
+{
+	return (es->flags & BGP_EVPNES_OPER_UP)
+	       && !(es->flags & BGP_EVPNES_BYPASS);
+}
+
 /****************************************************************************/
 extern int bgp_evpn_es_route_install_uninstall(struct bgp *bgp,
 		struct bgp_evpn_es *es, afi_t afi, safi_t safi,
@@ -362,21 +403,28 @@ void bgp_evpn_es_evi_show_vni(struct vty *vty, vni_t vni,
 		bool uj, bool detail);
 void bgp_evpn_es_evi_show(struct vty *vty, bool uj, bool detail);
 struct bgp_evpn_es *bgp_evpn_es_find(const esi_t *esi);
-extern bool bgp_evpn_is_esi_local(esi_t *esi);
 extern void bgp_evpn_vrf_es_init(struct bgp *bgp_vrf);
+extern bool bgp_evpn_is_esi_local_and_non_bypass(esi_t *esi);
 extern void bgp_evpn_es_vrf_deref(struct bgp_evpn_es_evi *es_evi);
 extern void bgp_evpn_es_vrf_ref(struct bgp_evpn_es_evi *es_evi,
 				struct bgp *bgp_vrf);
-extern void bgp_evpn_path_es_info_free(struct bgp_path_es_info *es_info);
+extern void bgp_evpn_path_mh_info_free(struct bgp_path_mh_info *mh_info);
-extern void bgp_evpn_path_es_unlink(struct bgp_path_es_info *es_info);
 extern void bgp_evpn_path_es_link(struct bgp_path_info *pi, vni_t vni,
 				  esi_t *esi);
-extern bool bgp_evpn_es_is_vtep_active(esi_t *esi, struct in_addr nh);
 extern bool bgp_evpn_path_es_use_nhg(struct bgp *bgp_vrf,
 				     struct bgp_path_info *pi, uint32_t *nhg_p);
 extern void bgp_evpn_es_vrf_show(struct vty *vty, bool uj,
 				 struct bgp_evpn_es *es);
 extern void bgp_evpn_es_vrf_show_esi(struct vty *vty, esi_t *esi, bool uj);
 extern void bgp_evpn_switch_ead_evi_rx(void);
+extern bool bgp_evpn_es_add_l3_ecomm_ok(esi_t *esi);
+extern void bgp_evpn_es_vrf_use_nhg(struct bgp *bgp_vrf, esi_t *esi,
+				    bool *use_l3nhg, bool *is_l3nhg_active,
+				    struct bgp_evpn_es_vrf **es_vrf_p);
+extern void bgp_evpn_nh_init(struct bgp *bgp_vrf);
+extern void bgp_evpn_nh_finish(struct bgp *bgp_vrf);
+extern void bgp_evpn_nh_show(struct vty *vty, bool uj);
+extern void bgp_evpn_path_nh_add(struct bgp *bgp_vrf, struct bgp_path_info *pi);
+extern void bgp_evpn_path_nh_del(struct bgp *bgp_vrf, struct bgp_path_info *pi);
 
 #endif /* _FRR_BGP_EVPN_MH_H */

bgpd/bgp_evpn_private.h

@@ -631,4 +631,13 @@ bgp_global_evpn_node_lookup(struct bgp_table *table, afi_t afi, safi_t safi,
 			    const struct prefix_evpn *evp,
 			    struct prefix_rd *prd);
 extern void bgp_evpn_import_route_in_vrfs(struct bgp_path_info *pi, int import);
+extern void bgp_evpn_update_type2_route_entry(struct bgp *bgp,
+					      struct bgpevpn *vpn,
+					      struct bgp_node *rn,
+					      struct bgp_path_info *local_pi,
+					      const char *caller);
+extern int bgp_evpn_route_entry_install_if_vrf_match(struct bgp *bgp_vrf,
+						     struct bgp_path_info *pi,
+						     int install);
+extern void bgp_evpn_import_type2_route(struct bgp_path_info *pi, int import);
 #endif /* _BGP_EVPN_PRIVATE_H */

bgpd/bgp_evpn_vty.c

@@ -688,7 +688,8 @@ static void show_esi_routes(struct bgp *bgp,
 
 /* Display all MAC-IP VNI routes linked to an ES */
 static void bgp_evpn_show_routes_mac_ip_es(struct vty *vty, esi_t *esi,
-					   json_object *json, int detail)
+					   json_object *json, int detail,
+					   bool global_table)
 {
 	struct bgp_node *rn;
 	struct bgp_path_info *pi;
@@ -709,11 +710,17 @@ static void bgp_evpn_show_routes_mac_ip_es(struct vty *vty, esi_t *esi,
 		json_paths = json_object_new_array();
 
 	RB_FOREACH (es, bgp_es_rb_head, &bgp_mh_info->es_rb_tree) {
+		struct list *es_list;
 
 		if (esi && memcmp(esi, &es->esi, sizeof(*esi)))
 			continue;
 
-		for (ALL_LIST_ELEMENTS_RO(es->macip_path_list, node, es_info)) {
+		if (global_table)
+			es_list = es->macip_global_path_list;
+		else
+			es_list = es->macip_evi_path_list;
+
+		for (ALL_LIST_ELEMENTS_RO(es_list, node, es_info)) {
 			json_object *json_path = NULL;
 
 			pi = es_info->pi;
@@ -758,6 +765,18 @@ static void bgp_evpn_show_routes_mac_ip_es(struct vty *vty, esi_t *esi,
 	}
 }
 
+static void bgp_evpn_show_routes_mac_ip_evi_es(struct vty *vty, esi_t *esi,
+					       json_object *json, int detail)
+{
+	return bgp_evpn_show_routes_mac_ip_es(vty, esi, json, detail, false);
+}
+
+static void bgp_evpn_show_routes_mac_ip_global_es(struct vty *vty, esi_t *esi,
+						  json_object *json, int detail)
+{
+	return bgp_evpn_show_routes_mac_ip_es(vty, esi, json, detail, true);
+}
+
 static void show_vni_routes(struct bgp *bgp, struct bgpevpn *vpn, int type,
 			    struct vty *vty, struct in_addr vtep_ip,
 			    json_object *json, int detail)
@@ -4100,6 +4119,21 @@ DEFPY(show_bgp_l2vpn_evpn_es_vrf, show_bgp_l2vpn_evpn_es_vrf_cmd,
 	return CMD_SUCCESS;
 }
 
+DEFPY(show_bgp_l2vpn_evpn_nh,
+      show_bgp_l2vpn_evpn_nh_cmd,
+      "show bgp l2vpn evpn next-hops [json$uj]",
+      SHOW_STR
+      BGP_STR
+      L2VPN_HELP_STR
+      EVPN_HELP_STR
+      "Nexthops\n"
+      JSON_STR)
+{
+	bgp_evpn_nh_show(vty, uj);
+
+	return CMD_SUCCESS;
+}
+
 /*
  * Display EVPN neighbor summary.
  */
@@ -4658,12 +4692,12 @@ DEFUN(show_bgp_l2vpn_evpn_route_vni_all,
 }
 
 DEFPY_HIDDEN(
-	show_bgp_l2vpn_evpn_route_mac_ip_es,
+	show_bgp_l2vpn_evpn_route_mac_ip_evi_es,
-	show_bgp_l2vpn_evpn_route_mac_ip_es_cmd,
+	show_bgp_l2vpn_evpn_route_mac_ip_evi_es_cmd,
-	"show bgp l2vpn evpn route mac-ip-es [NAME$esi_str|detail$detail] [json$uj]",
+	"show bgp l2vpn evpn route mac-ip-evi-es [NAME$esi_str|detail$detail] [json$uj]",
 	SHOW_STR BGP_STR L2VPN_HELP_STR EVPN_HELP_STR
 	"EVPN route information\n"
-	"MAC IP routes linked to the ES\n"
+	"MAC IP routes in the EVI tables linked to the ES\n"
 	"ES ID\n"
 	"Detailed information\n" JSON_STR)
 {
@@ -4683,7 +4717,44 @@ DEFPY_HIDDEN(
 
 	if (uj)
 		json = json_object_new_object();
-	bgp_evpn_show_routes_mac_ip_es(vty, esi_p, json, !!detail);
+	bgp_evpn_show_routes_mac_ip_evi_es(vty, esi_p, json, !!detail);
+	if (uj) {
+		vty_out(vty, "%s\n",
+			json_object_to_json_string_ext(
+				json, JSON_C_TO_STRING_PRETTY));
+		json_object_free(json);
+	}
+
+	return CMD_SUCCESS;
+}
+
+DEFPY_HIDDEN(
+	show_bgp_l2vpn_evpn_route_mac_ip_global_es,
+	show_bgp_l2vpn_evpn_route_mac_ip_global_es_cmd,
+	"show bgp l2vpn evpn route mac-ip-global-es [NAME$esi_str|detail$detail] [json$uj]",
+	SHOW_STR BGP_STR L2VPN_HELP_STR EVPN_HELP_STR
+	"EVPN route information\n"
+	"MAC IP routes in the global table linked to the ES\n"
+	"ES ID\n"
+	"Detailed information\n" JSON_STR)
+{
+	esi_t esi;
+	esi_t *esi_p;
+	json_object *json = NULL;
+
+	if (esi_str) {
+		if (!str_to_esi(esi_str, &esi)) {
+			vty_out(vty, "%%Malformed ESI\n");
+			return CMD_WARNING;
+		}
+		esi_p = &esi;
+	} else {
+		esi_p = NULL;
+	}
+
+	if (uj)
+		json = json_object_new_object();
+	bgp_evpn_show_routes_mac_ip_global_es(vty, esi_p, json, !!detail);
 	if (uj) {
 		vty_out(vty, "%s\n",
 			json_object_to_json_string_ext(
@@ -5957,6 +6028,7 @@ void bgp_ethernetvpn_init(void)
 	install_element(VIEW_NODE, &show_bgp_l2vpn_evpn_es_cmd);
 	install_element(VIEW_NODE, &show_bgp_l2vpn_evpn_es_evi_cmd);
 	install_element(VIEW_NODE, &show_bgp_l2vpn_evpn_es_vrf_cmd);
+	install_element(VIEW_NODE, &show_bgp_l2vpn_evpn_nh_cmd);
 	install_element(VIEW_NODE, &show_bgp_l2vpn_evpn_vni_cmd);
 	install_element(VIEW_NODE, &show_bgp_l2vpn_evpn_summary_cmd);
 	install_element(VIEW_NODE, &show_bgp_l2vpn_evpn_route_cmd);
@@ -5968,7 +6040,10 @@ void bgp_ethernetvpn_init(void)
 			&show_bgp_l2vpn_evpn_route_vni_multicast_cmd);
 	install_element(VIEW_NODE, &show_bgp_l2vpn_evpn_route_vni_macip_cmd);
 	install_element(VIEW_NODE, &show_bgp_l2vpn_evpn_route_vni_all_cmd);
-	install_element(VIEW_NODE, &show_bgp_l2vpn_evpn_route_mac_ip_es_cmd);
+	install_element(VIEW_NODE,
+			&show_bgp_l2vpn_evpn_route_mac_ip_evi_es_cmd);
+	install_element(VIEW_NODE,
+			&show_bgp_l2vpn_evpn_route_mac_ip_global_es_cmd);
 	install_element(VIEW_NODE, &show_bgp_l2vpn_evpn_import_rt_cmd);
 	install_element(VIEW_NODE, &show_bgp_l2vpn_evpn_vrf_import_rt_cmd);
 

bgpd/bgp_memory.c

@@ -119,7 +119,10 @@ DEFINE_MTYPE(BGPD, LCOMMUNITY_VAL, "Large Community value");
 DEFINE_MTYPE(BGPD, BGP_EVPN, "BGP EVPN Information");
 DEFINE_MTYPE(BGPD, BGP_EVPN_MH_INFO, "BGP EVPN MH Information");
 DEFINE_MTYPE(BGPD, BGP_EVPN_ES_VTEP, "BGP EVPN ES VTEP");
+DEFINE_MTYPE(BGPD, BGP_EVPN_PATH_MH_INFO, "BGP EVPN PATH MH Information");
 DEFINE_MTYPE(BGPD, BGP_EVPN_PATH_ES_INFO, "BGP EVPN PATH ES Information");
+DEFINE_MTYPE(BGPD, BGP_EVPN_PATH_NH_INFO, "BGP EVPN PATH NH Information");
+DEFINE_MTYPE(BGPD, BGP_EVPN_NH, "BGP EVPN Nexthop");
 DEFINE_MTYPE(BGPD, BGP_EVPN_ES_EVI_VTEP, "BGP EVPN ES-EVI VTEP");
 DEFINE_MTYPE(BGPD, BGP_EVPN_ES, "BGP EVPN ESI Information");
 DEFINE_MTYPE(BGPD, BGP_EVPN_ES_EVI, "BGP EVPN ES-per-EVI Information");

bgpd/bgp_memory.h

@@ -118,6 +118,9 @@ DECLARE_MTYPE(BGP_EVPN_ES_EVI);
 DECLARE_MTYPE(BGP_EVPN_ES_VRF);
 DECLARE_MTYPE(BGP_EVPN_ES_VTEP);
 DECLARE_MTYPE(BGP_EVPN_PATH_ES_INFO);
+DECLARE_MTYPE(BGP_EVPN_PATH_MH_INFO);
+DECLARE_MTYPE(BGP_EVPN_PATH_NH_INFO);
+DECLARE_MTYPE(BGP_EVPN_NH);
 DECLARE_MTYPE(BGP_EVPN_ES_EVI_VTEP);
 
 DECLARE_MTYPE(BGP_EVPN);

bgpd/bgp_route.c

@@ -250,8 +250,8 @@ void bgp_path_info_extra_free(struct bgp_path_info_extra **extra)
 	if (e->aggr_suppressors)
 		list_delete(&e->aggr_suppressors);
 
-	if (e->es_info)
+	if (e->mh_info)
-		bgp_evpn_path_es_info_free(e->es_info);
+		bgp_evpn_path_mh_info_free(e->mh_info);
 
 	if ((*extra)->bgp_fs_iprule)
 		list_delete(&((*extra)->bgp_fs_iprule));
@@ -8856,15 +8856,17 @@ void route_vty_out(struct vty *vty, const struct prefix *p,
 		if (safi == SAFI_EVPN) {
 			struct bgp_path_es_info *path_es_info = NULL;
 
-			if (path->extra)
-				path_es_info = path->extra->es_info;
-
 			if (bgp_evpn_is_esi_valid(&attr->esi)) {
 				/* XXX - add these params to the json out */
 				vty_out(vty, "%*s", 20, " ");
 				vty_out(vty, "ESI:%s",
 					esi_to_str(&attr->esi, esi_buf,
 						   sizeof(esi_buf)));
+
+				if (path->extra && path->extra->mh_info)
+					path_es_info =
+						path->extra->mh_info->es_info;
+
 				if (path_es_info && path_es_info->es)
 					vty_out(vty, " VNI: %u",
 						path_es_info->vni);
@@ -9626,12 +9628,20 @@ void route_vty_out_detail(struct vty *vty, struct bgp *bgp,
 					buf1, sizeof(buf1));
 				if (is_pi_family_evpn(parent_ri)) {
 					vty_out(vty,
-						"  Imported from %s:%pFX, VNI %s\n",
+						"  Imported from %s:%pFX, VNI %s",
 						buf1,
 						(struct prefix_evpn *)
 							bgp_dest_get_prefix(
 								dest),
 						tag_buf);
+					if (attr->es_flags & ATTR_ES_L3_NHG)
+						vty_out(vty, ", L3NHG %s",
+							(attr->es_flags
+							 & ATTR_ES_L3_NHG_ACTIVE)
+								? "active"
+								: "inactive");
+					vty_out(vty, "\n");
+
 				} else
 					vty_out(vty,
 						"  Imported from %s:%pFX\n",

bgpd/bgp_route.h

@@ -102,7 +102,9 @@ enum bgp_show_adj_route_type {
 #define BGP_NLRI_PARSE_ERROR_EVPN_TYPE1_SIZE -15
 #define BGP_NLRI_PARSE_ERROR -32
 
-/* MAC-IP/type-2 path_info in the VNI routing table is linked to the
+/* 1. local MAC-IP/type-2 paths in the VNI routing table are linked to the
+ * destination ES
+ * 2. remote MAC-IP paths in the global routing table are linked to the
  * destination ES
  */
 struct bgp_path_es_info {
@@ -113,6 +115,27 @@ struct bgp_path_es_info {
 	struct bgp_evpn_es *es;
 	/* memory used for linking the path to the destination ES */
 	struct listnode es_listnode;
+	uint8_t flags;
+/* Path is linked to the VNI list */
+#define BGP_EVPN_PATH_ES_INFO_VNI_LIST (1 << 0)
+/* Path is linked to the global list */
+#define BGP_EVPN_PATH_ES_INFO_GLOBAL_LIST (1 << 1)
+};
+
+/* IP paths imported into the VRF from an EVPN route source
+ * are linked to the nexthop/VTEP IP
+ */
+struct bgp_path_evpn_nh_info {
+	/* back pointer to the route */
+	struct bgp_path_info *pi;
+	struct bgp_evpn_nh *nh;
+	/* memory used for linking the path to the nexthop */
+	struct listnode nh_listnode;
+};
+
+struct bgp_path_mh_info {
+	struct bgp_path_es_info *es_info;
+	struct bgp_path_evpn_nh_info *nh_info;
 };
 
 /* Ancillary information to struct bgp_path_info,
@@ -202,7 +225,7 @@ struct bgp_path_info_extra {
 	/* presence of FS pbr iprule based entry */
 	struct list *bgp_fs_iprule;
 	/* Destination Ethernet Segment links for EVPN MH */
-	struct bgp_path_es_info *es_info;
+	struct bgp_path_mh_info *mh_info;
 };
 
 struct bgp_path_info {

bgpd/bgpd.h

@@ -664,6 +664,9 @@ struct bgp {
 	/* RB tree of ES-VRFs */
 	struct bgp_es_vrf_rb_head es_vrf_rb_tree;
 
+	/* Hash table of EVPN nexthops maintained per-tenant-VRF */
+	struct hash *evpn_nh_table;
+
 	/* vrf flags */
 	uint32_t vrf_flags;
 #define BGP_VRF_AUTO                        (1 << 0)

lib/log.c

@@ -462,7 +462,9 @@ static const struct zebra_desc_table command_types[] = {
 	DESC_ENTRY(ZEBRA_NHG_DEL),
 	DESC_ENTRY(ZEBRA_NHG_NOTIFY_OWNER),
 	DESC_ENTRY(ZEBRA_ROUTE_NOTIFY_REQUEST),
-	DESC_ENTRY(ZEBRA_CLIENT_CLOSE_NOTIFY)};
+	DESC_ENTRY(ZEBRA_CLIENT_CLOSE_NOTIFY),
+	DESC_ENTRY(ZEBRA_EVPN_REMOTE_NH_ADD),
+	DESC_ENTRY(ZEBRA_EVPN_REMOTE_NH_DEL)};
 #undef DESC_ENTRY
 
 static const struct zebra_desc_table unknown = {0, "unknown", '?'};

lib/zclient.h

@@ -213,6 +213,8 @@ typedef enum {
 	ZEBRA_NHG_ADD,
 	ZEBRA_NHG_DEL,
 	ZEBRA_NHG_NOTIFY_OWNER,
+	ZEBRA_EVPN_REMOTE_NH_ADD,
+	ZEBRA_EVPN_REMOTE_NH_DEL,
 	ZEBRA_ERROR,
 	ZEBRA_CLIENT_CAPABILITIES,
 	ZEBRA_OPAQUE_MESSAGE,

zebra/zapi_msg.c

@@ -3350,6 +3350,8 @@ void (*const zserv_handlers[])(ZAPI_HANDLER_ARGS) = {
 	[ZEBRA_NHG_ADD] = zread_nhg_add,
 	[ZEBRA_NHG_DEL] = zread_nhg_del,
 	[ZEBRA_ROUTE_NOTIFY_REQUEST] = zread_route_notify_request,
+	[ZEBRA_EVPN_REMOTE_NH_ADD] = zebra_evpn_proc_remote_nh,
+	[ZEBRA_EVPN_REMOTE_NH_DEL] = zebra_evpn_proc_remote_nh,
 };
 
 /*

zebra/zebra_evpn_mh.c

@@ -3867,6 +3867,47 @@ static void zebra_evpn_mh_startup_delay_timer_start(const char *rc)
 	}
 }
 
+/*****************************************************************************
+ * Nexthop management: nexthops associated with Type-2 routes that have
+ * an ES as destination are consolidated by BGP into a per-VRF nh->rmac
+ * mapping which is the installed as a remote neigh/fdb entry with a
+ * dummy (type-1) prefix referencing it.
+ * This handling is needed because Type-2 routes with ES as dest use NHG
+ * that are setup using EAD routes (i.e. such NHGs do not include the
+ * RMAC info).
+ ****************************************************************************/
+void zebra_evpn_proc_remote_nh(ZAPI_HANDLER_ARGS)
+{
+	struct stream *s;
+	vrf_id_t vrf_id;
+	struct ipaddr nh;
+	struct ethaddr rmac;
+	struct prefix_evpn dummy_prefix;
+
+	s = msg;
+	vrf_id = stream_getl(s);
+	stream_get(&nh, s, sizeof(nh));
+
+	memset(&dummy_prefix, 0, sizeof(dummy_prefix));
+	dummy_prefix.family = AF_EVPN;
+	dummy_prefix.prefixlen = (sizeof(struct evpn_addr) * 8);
+	dummy_prefix.prefix.route_type = 1; /* XXX - fixup to type-1 def */
+
+	if (hdr->command == ZEBRA_EVPN_REMOTE_NH_ADD) {
+		stream_get(&rmac, s, sizeof(rmac));
+		if (IS_ZEBRA_DEBUG_EVPN_MH_ES)
+			zlog_debug("evpn remote nh %d %pIA rmac %pEA add",
+				   vrf_id, &nh, &rmac);
+		zebra_vxlan_evpn_vrf_route_add(vrf_id, &rmac, &nh,
+					       (struct prefix *)&dummy_prefix);
+	} else {
+		if (IS_ZEBRA_DEBUG_EVPN_MH_ES)
+			zlog_debug("evpn remote nh %d %pIA del", vrf_id, &nh);
+		zebra_vxlan_evpn_vrf_route_del(vrf_id, &nh,
+					       (struct prefix *)&dummy_prefix);
+	}
+}
+
 /*****************************************************************************/
 void zebra_evpn_mh_config_write(struct vty *vty)
 {

zebra/zebra_evpn_mh.h

@@ -382,5 +382,6 @@ extern void zebra_evpn_acc_bd_svi_set(struct zebra_if *vlan_zif,
 extern void zebra_evpn_acc_bd_svi_mac_add(struct interface *vlan_if);
 extern void zebra_evpn_es_bypass_update(struct zebra_evpn_es *es,
 					struct interface *ifp, bool bypass);
+extern void zebra_evpn_proc_remote_nh(ZAPI_HANDLER_ARGS);
 
 #endif /* _ZEBRA_EVPN_MH_H */

zebra/zebra_vxlan.c

@@ -149,6 +149,11 @@ static int host_rb_entry_compare(const struct host_rb_entry *hle1,
 	} else if (hle1->p.family == AF_INET6) {
 		return memcmp(&hle1->p.u.prefix6, &hle2->p.u.prefix6,
 			      IPV6_MAX_BYTELEN);
+	} else if (hle1->p.family == AF_EVPN) {
+		/* a single dummy prefix of route_type BGP_EVPN_AD_ROUTE is
+		 * used for all nexthops associated with a non-zero ESI
+		 */
+		return 0;
 	} else {
 		zlog_debug("%s: Unexpected family type: %d", __func__,
 			   hle1->p.family);