zebra: support for lacp bypass with EVPN MH

Feature overview:
=================
A 802.3ad bond can be setup to allow lacp-bypass. This is done to enable
servers to pxe boot without a LACP license i.e. allows the bond to go oper
up (with a single link) without LACP converging.

If an ES-bond is oper-up in an "LACP-bypass" state MH treats it as a non-ES
bond. This involves the following special handling -
1. If the bond is in a bypass-state the associated ES is placed in a
bypass state.
2. If an ES is in a bypass state -
a. DF election is disabled (i.e. assumed DF)
b. SPH filter is not installed.
3. MACs learnt via the host bond are advertised with a zero ESI.
When the ES moves out of "bypass" the MACs are moved from a zero-ESI to
the correct non-zero id. This is treated as a local station move.

Implementation:
===============
When (a) an ES is detached from a hostbond or (b) an ES-bond goes into
LACP bypass zebra deletes all the local macs (with that ES as destination)
in the kernel and its local db. BGP re-sends any imported MAC-IP routes
that may exist with this ES destination as remote routes i.e. zebra can
end up programming a MAC that was perviously local as remote pointing
to a VTEP-ECMP group.

When an ES is attached to a hostbond or an ES-bond goes
LACP-up (out of bypss) zebra again deletes all the local macs in the
kernel and its local db. At this point BGP resends any imported MAC-IP
routes that may exist with this ES destination as sync routes i.e.
zebra can end up programming a MAC that was perviously remote
as local pointing to an access port.

Signed-off-by: Anuradha Karuppiah <anuradhak@cumulusnetworks.com>

zebra/if_netlink.c

@@ -719,6 +719,20 @@ static void netlink_proc_dplane_if_protodown(struct zebra_if *zif,
 	}
 }
 
+static uint8_t netlink_parse_lacp_bypass(struct rtattr **linkinfo)
+{
+	uint8_t bypass = 0;
+	struct rtattr *mbrinfo[IFLA_BOND_SLAVE_MAX + 1];
+
+	parse_rtattr_nested(mbrinfo, IFLA_BOND_SLAVE_MAX,
+			    linkinfo[IFLA_INFO_SLAVE_DATA]);
+	if (mbrinfo[IFLA_BOND_SLAVE_AD_RX_BYPASS])
+		bypass = *(uint8_t *)RTA_DATA(
+			mbrinfo[IFLA_BOND_SLAVE_AD_RX_BYPASS]);
+
+	return bypass;
+}
+
 /*
  * Called from interface_lookup_netlink().  This function is only used
  * during bootstrap.
@@ -743,6 +757,7 @@ static int netlink_interface(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 	ifindex_t bond_ifindex = IFINDEX_INTERNAL;
 	struct zebra_if *zif;
 	ns_id_t link_nsid = ns_id;
+	uint8_t bypass = 0;
 
 	zns = zebra_ns_lookup(ns_id);
 	ifi = NLMSG_DATA(h);
@@ -816,6 +831,7 @@ static int netlink_interface(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 		} else if (slave_kind && (strcmp(slave_kind, "bond") == 0)) {
 			zif_slave_type = ZEBRA_IF_SLAVE_BOND;
 			bond_ifindex = *(ifindex_t *)RTA_DATA(tb[IFLA_MASTER]);
+			bypass = netlink_parse_lacp_bypass(linkinfo);
 		} else
 			zif_slave_type = ZEBRA_IF_SLAVE_OTHER;
 	}
@@ -882,7 +898,7 @@ static int netlink_interface(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 	if (IS_ZEBRA_IF_BRIDGE_SLAVE(ifp))
 		zebra_l2if_update_bridge_slave(ifp, bridge_ifindex, ns_id);
 	else if (IS_ZEBRA_IF_BOND_SLAVE(ifp))
-		zebra_l2if_update_bond_slave(ifp, bond_ifindex);
+		zebra_l2if_update_bond_slave(ifp, bond_ifindex, !!bypass);
 
 	if (tb[IFLA_PROTO_DOWN]) {
 		uint8_t protodown;
@@ -1322,6 +1338,7 @@ int netlink_link_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 	struct zebra_if *zif;
 	ns_id_t link_nsid = ns_id;
 	ifindex_t master_infindex = IFINDEX_INTERNAL;
+	uint8_t bypass = 0;
 
 	zns = zebra_ns_lookup(ns_id);
 	ifi = NLMSG_DATA(h);
@@ -1423,6 +1440,7 @@ int netlink_link_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 				zif_slave_type = ZEBRA_IF_SLAVE_BOND;
 				master_infindex = bond_ifindex =
 					*(ifindex_t *)RTA_DATA(tb[IFLA_MASTER]);
+				bypass = netlink_parse_lacp_bypass(linkinfo);
 			} else
 				zif_slave_type = ZEBRA_IF_SLAVE_OTHER;
 		}
@@ -1484,7 +1502,8 @@ int netlink_link_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 							       bridge_ifindex,
 							       ns_id);
 			else if (IS_ZEBRA_IF_BOND_SLAVE(ifp))
-				zebra_l2if_update_bond_slave(ifp, bond_ifindex);
+				zebra_l2if_update_bond_slave(ifp, bond_ifindex,
+							     !!bypass);
 
 			if (tb[IFLA_PROTO_DOWN]) {
 				uint8_t protodown;
@@ -1597,7 +1616,8 @@ int netlink_link_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 							       bridge_ifindex,
 							       ns_id);
 			else if (IS_ZEBRA_IF_BOND_SLAVE(ifp) || was_bond_slave)
-				zebra_l2if_update_bond_slave(ifp, bond_ifindex);
+				zebra_l2if_update_bond_slave(ifp, bond_ifindex,
+							     !!bypass);
 
 			if (tb[IFLA_PROTO_DOWN]) {
 				uint8_t protodown;
@@ -1633,6 +1653,8 @@ int netlink_link_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 
 		if (IS_ZEBRA_IF_BOND(ifp))
 			zebra_l2if_update_bond(ifp, false);
+		if (IS_ZEBRA_IF_BOND_SLAVE(ifp))
+			zebra_l2if_update_bond_slave(ifp, bond_ifindex, false);
 		/* Special handling for bridge or VxLAN interfaces. */
 		if (IS_ZEBRA_IF_BRIDGE(ifp))
 			zebra_l2_bridge_del(ifp);

zebra/interface.c

@@ -1604,6 +1604,9 @@ static void if_dump_vty(struct vty *vty, struct interface *ifp)
 		}
 	}
 
+	if (zebra_if->flags & ZIF_FLAG_LACP_BYPASS)
+		vty_out(vty, "  LACP bypass: on\n");
+
 	zebra_evpn_if_es_print(vty, zebra_if);
 	vty_out(vty, "  protodown: %s %s\n",
 		(zebra_if->flags & ZIF_FLAG_PROTODOWN) ? "on" : "off",

zebra/interface.h

@@ -286,6 +286,9 @@ struct zebra_es_if_info {
 	esi_t esi;
 
 	uint16_t df_pref;
+	uint8_t flags;
+#define ZIF_CFG_ES_FLAG_BYPASS (1 << 0)
+
 	struct zebra_evpn_es *es; /* local ES */
 };
 
@@ -297,7 +300,13 @@ enum zebra_if_flags {
 	ZIF_FLAG_EVPN_MH_UPLINK_OPER_UP = (1 << 1),
 
 	/* Dataplane protodown-on */
-	ZIF_FLAG_PROTODOWN = (1 << 2)
+	ZIF_FLAG_PROTODOWN = (1 << 2),
+
+	/* LACP bypass state is set by the dataplane on a bond member
+	 * and inherited by the bond (if one or more bond members are in
+	 * a bypass state the bond is placed in a bypass state)
+	 */
+	ZIF_FLAG_LACP_BYPASS = (1 << 3)
 };
 
 /* `zebra' daemon local interface structure. */

zebra/zebra_dplane.c

@@ -3250,6 +3250,24 @@ enum zebra_dplane_result dplane_local_mac_add(const struct interface *ifp,
 	return result;
 }
 
+/*
+ * Enqueue local mac del
+ */
+enum zebra_dplane_result
+dplane_local_mac_del(const struct interface *ifp,
+		     const struct interface *bridge_ifp, vlanid_t vid,
+		     const struct ethaddr *mac)
+{
+	enum zebra_dplane_result result;
+	struct in_addr vtep_ip;
+
+	vtep_ip.s_addr = 0;
+
+	/* Use common helper api */
+	result = mac_update_common(DPLANE_OP_MAC_DELETE, ifp, bridge_ifp, vid,
+				   mac, vtep_ip, false, 0, 0);
+	return result;
+}
 /*
  * Public api to init an empty context - either newly-allocated or
  * reset/cleared - for a MAC update.

zebra/zebra_dplane.h

@@ -582,6 +582,11 @@ enum zebra_dplane_result dplane_local_mac_add(const struct interface *ifp,
 					uint32_t set_static,
 					uint32_t set_inactive);
 
+enum zebra_dplane_result
+dplane_local_mac_del(const struct interface *ifp,
+		     const struct interface *bridge_ifp, vlanid_t vid,
+		     const struct ethaddr *mac);
+
 enum zebra_dplane_result dplane_rem_mac_del(const struct interface *ifp,
 					const struct interface *bridge_ifp,
 					vlanid_t vid,

zebra/zebra_evpn_mac.c

@@ -753,14 +753,10 @@ static char *zebra_evpn_print_mac_flags(zebra_mac_t *mac, char *flags_buf,
 					size_t flags_buf_sz)
 {
 	snprintf(flags_buf, flags_buf_sz, "%s%s%s%s",
-			mac->sync_neigh_cnt ?
+		 mac->sync_neigh_cnt ? "N" : "",
-			"N" : "",
+		 (mac->flags & ZEBRA_MAC_ES_PEER_ACTIVE) ? "P" : "",
-			(mac->flags & ZEBRA_MAC_ES_PEER_ACTIVE) ?
+		 (mac->flags & ZEBRA_MAC_ES_PEER_PROXY) ? "X" : "",
-			"P" : "",
+		 (mac->flags & ZEBRA_MAC_LOCAL_INACTIVE) ? "I" : "");
-			(mac->flags & ZEBRA_MAC_ES_PEER_PROXY) ?
-			"X" : "",
-			(mac->flags & ZEBRA_MAC_LOCAL_INACTIVE) ?
-			"I" : "");
 
 	return flags_buf;
 }
@@ -1796,6 +1792,7 @@ static bool zebra_evpn_local_mac_update_fwd_info(zebra_mac_t *mac,
 	bool es_change;
 	ns_id_t local_ns_id = NS_DEFAULT;
 	struct zebra_vrf *zvrf;
+	struct zebra_evpn_es *es;
 
 	zvrf = zebra_vrf_lookup_by_id(ifp->vrf_id);
 	if (zvrf && zvrf->zns)
@@ -1803,7 +1800,10 @@ static bool zebra_evpn_local_mac_update_fwd_info(zebra_mac_t *mac,
 
 	memset(&mac->fwd_info, 0, sizeof(mac->fwd_info));
 
-	es_change = zebra_evpn_es_mac_ref_entry(mac, zif->es_info.es);
+	es = zif->es_info.es;
+	if (es && (es->flags & ZEBRA_EVPNES_BYPASS))
+		es = NULL;
+	es_change = zebra_evpn_es_mac_ref_entry(mac, es);
 
 	if (!mac->es) {
 		/* if es is set fwd_info is not-relevant/taped-out */
@@ -2064,9 +2064,8 @@ int zebra_evpn_add_update_local_mac(struct zebra_vrf *zvrf, zebra_evpn_t *zevpn,
 				    struct interface *ifp,
 				    struct ethaddr *macaddr, vlanid_t vid,
 				    bool sticky, bool local_inactive,
-				    bool dp_static)
+				    bool dp_static, zebra_mac_t *mac)
 {
-	zebra_mac_t *mac;
 	char buf[ETHER_ADDR_STRLEN];
 	bool mac_sticky = false;
 	bool inform_client = false;
@@ -2083,6 +2082,7 @@ int zebra_evpn_add_update_local_mac(struct zebra_vrf *zvrf, zebra_evpn_t *zevpn,
 
 	assert(ifp);
 	/* Check if we need to create or update or it is a NO-OP. */
+	if (!mac)
 		mac = zebra_evpn_mac_lookup(zevpn, macaddr);
 	if (!mac) {
 		if (IS_ZEBRA_DEBUG_VXLAN || IS_ZEBRA_DEBUG_EVPN_MH_MAC)
@@ -2134,6 +2134,8 @@ int zebra_evpn_add_update_local_mac(struct zebra_vrf *zvrf, zebra_evpn_t *zevpn,
 			old_static = zebra_evpn_mac_is_static(mac);
 			if (CHECK_FLAG(mac->flags, ZEBRA_MAC_STICKY))
 				mac_sticky = true;
+			es_change = zebra_evpn_local_mac_update_fwd_info(
+				mac, ifp, vid);
 
 			/*
 			 * Update any changes and if changes are relevant to
@@ -2142,7 +2144,7 @@ int zebra_evpn_add_update_local_mac(struct zebra_vrf *zvrf, zebra_evpn_t *zevpn,
 			if (mac_sticky == sticky && old_ifp == ifp
 			    && old_vid == vid
 			    && old_local_inactive == local_inactive
-			    && dp_static == old_static) {
+			    && dp_static == old_static && !es_change) {
 				if (IS_ZEBRA_DEBUG_VXLAN)
 					zlog_debug(
 						"        Add/Update %sMAC %s intf %s(%u) VID %u -> VNI %u%s, "
@@ -2166,14 +2168,16 @@ int zebra_evpn_add_update_local_mac(struct zebra_vrf *zvrf, zebra_evpn_t *zevpn,
 				inform_client = true;
 			}
 
-			es_change = zebra_evpn_local_mac_update_fwd_info(
-				mac, ifp, vid);
 			/* If an es_change is detected we need to advertise
 			 * the route with a sequence that is one
 			 * greater. This is need to indicate a mac-move
 			 * to the ES peers
 			 */
 			if (es_change) {
+				/* update the sequence number only if the entry
+				 * is locally active
+				 */
+				if (!local_inactive)
 					mac->loc_seq = mac->loc_seq + 1;
 				/* force drop the peer/sync info as it is
 				 * simply no longer relevant
@@ -2308,7 +2312,8 @@ int zebra_evpn_add_update_local_mac(struct zebra_vrf *zvrf, zebra_evpn_t *zevpn,
 	return 0;
 }
 
-int zebra_evpn_del_local_mac(zebra_evpn_t *zevpn, zebra_mac_t *mac)
+int zebra_evpn_del_local_mac(zebra_evpn_t *zevpn, zebra_mac_t *mac,
+			     bool clear_static)
 {
 	char buf[ETHER_ADDR_STRLEN];
 	bool old_bgp_ready;
@@ -2321,7 +2326,7 @@ int zebra_evpn_del_local_mac(zebra_evpn_t *zevpn, zebra_mac_t *mac)
 			   listcount(mac->neigh_list));
 
 	old_bgp_ready = zebra_evpn_mac_is_ready_for_bgp(mac->flags);
-	if (zebra_evpn_mac_is_static(mac)) {
+	if (!clear_static && zebra_evpn_mac_is_static(mac)) {
 		/* this is a synced entry and can only be removed when the
 		 * es-peers stop advertising it.
 		 */
@@ -2356,6 +2361,9 @@ int zebra_evpn_del_local_mac(zebra_evpn_t *zevpn, zebra_mac_t *mac)
 		return 0;
 	}
 
+	/* flush the peer info */
+	zebra_evpn_mac_clear_sync_info(mac);
+
 	/* Update all the neigh entries associated with this mac */
 	zebra_evpn_process_neigh_on_local_mac_del(zevpn, mac);
 

zebra/zebra_evpn_mac.h

@@ -260,8 +260,9 @@ int zebra_evpn_add_update_local_mac(struct zebra_vrf *zvrf, zebra_evpn_t *zevpn,
 				    struct interface *ifp,
 				    struct ethaddr *macaddr, vlanid_t vid,
 				    bool sticky, bool local_inactive,
-				    bool dp_static);
+				    bool dp_static, zebra_mac_t *mac);
-int zebra_evpn_del_local_mac(zebra_evpn_t *zevpn, zebra_mac_t *mac);
+int zebra_evpn_del_local_mac(zebra_evpn_t *zevpn, zebra_mac_t *mac,
+			     bool clear_static);
 int zebra_evpn_mac_gw_macip_add(struct interface *ifp, zebra_evpn_t *zevpn,
 				struct ipaddr *ip, zebra_mac_t **macp,
 				struct ethaddr *macaddr, vlanid_t vlan_id,

zebra/zebra_evpn_mh.c

@@ -1548,22 +1548,32 @@ static bool zebra_evpn_es_br_port_dplane_update(struct zebra_evpn_es *es,
 		return false;
 
 	if (IS_ZEBRA_DEBUG_EVPN_MH_ES)
-		zlog_debug("es %s br-port dplane update by %s", es->esi_str, caller);
+		zlog_debug("es %s br-port dplane update by %s", es->esi_str,
+			   caller);
 	backup_nhg_id = (es->flags & ZEBRA_EVPNES_NHG_ACTIVE) ? es->nhg_id : 0;
 
 	memset(&sph_filters, 0, sizeof(sph_filters));
+	if (es->flags & ZEBRA_EVPNES_BYPASS) {
+		if (IS_ZEBRA_DEBUG_EVPN_MH_ES)
+			zlog_debug(
+				"es %s SPH filter disabled as it is in bypass",
+				es->esi_str);
+	} else {
 		if (listcount(es->es_vtep_list) > ES_VTEP_MAX_CNT) {
 			zlog_warn("es %s vtep count %d exceeds filter cnt %d",
 				  es->esi_str, listcount(es->es_vtep_list),
 				  ES_VTEP_MAX_CNT);
 		} else {
-		for (ALL_LIST_ELEMENTS_RO(es->es_vtep_list, node, es_vtep)) {
+			for (ALL_LIST_ELEMENTS_RO(es->es_vtep_list, node,
-			if (es_vtep->flags & ZEBRA_EVPNES_VTEP_DEL_IN_PROG)
+						  es_vtep)) {
+				if (es_vtep->flags
+				    & ZEBRA_EVPNES_VTEP_DEL_IN_PROG)
 					continue;
 				sph_filters[sph_filter_cnt] = es_vtep->vtep_ip;
 				++sph_filter_cnt;
 			}
 		}
+	}
 
 	dplane_br_port_update(es->zif->ifp, !!(es->flags & ZEBRA_EVPNES_NON_DF),
 			      sph_filter_cnt, sph_filters, backup_nhg_id);
@@ -1609,6 +1619,7 @@ static bool zebra_evpn_es_run_df_election(struct zebra_evpn_es *es,
 	 * is no need to setup the BUM block filter
 	 */
 	if (!(es->flags & ZEBRA_EVPNES_LOCAL)
+	    || (es->flags & ZEBRA_EVPNES_BYPASS)
 	    || !zmh_info->es_originator_ip.s_addr)
 		return zebra_evpn_es_df_change(es, new_non_df, caller,
 					       "not-ready");
@@ -1834,6 +1845,7 @@ static int zebra_evpn_es_send_add_to_client(struct zebra_evpn_es *es)
 	struct zserv *client;
 	struct stream *s;
 	uint8_t oper_up;
+	bool bypass;
 
 	client = zserv_find_client(ZEBRA_ROUTE_BGP, 0);
 	/* BGP may not be running. */
@@ -1848,14 +1860,17 @@ static int zebra_evpn_es_send_add_to_client(struct zebra_evpn_es *es)
 	oper_up = !!(es->flags & ZEBRA_EVPNES_OPER_UP);
 	stream_putc(s, oper_up);
 	stream_putw(s, es->df_pref);
+	bypass = !!(es->flags & ZEBRA_EVPNES_BYPASS);
+	stream_putc(s, bypass);
 
 	/* Write packet size. */
 	stream_putw_at(s, 0, stream_get_endp(s));
 
 	if (IS_ZEBRA_DEBUG_EVPN_MH_ES)
-		zlog_debug("send add local es %s %pI4 active %u df_pref %u to %s",
+		zlog_debug(
-			   es->esi_str, &zmh_info->es_originator_ip,
+			"send add local es %s %pI4 active %u df_pref %u%s to %s",
-			   oper_up, es->df_pref,
+			es->esi_str, &zmh_info->es_originator_ip, oper_up,
+			es->df_pref, bypass ? " bypass" : "",
 			zebra_route_string(client->proto));
 
 	client->local_es_add_cnt++;
@@ -1980,18 +1995,50 @@ static void zebra_evpn_es_setup_evis(struct zebra_evpn_es *es)
 	}
 }
 
-static void zebra_evpn_es_local_mac_update(struct zebra_evpn_es *es,
+static void zebra_evpn_flush_local_mac(zebra_mac_t *mac, struct interface *ifp)
-		bool force_clear_static)
+{
+	struct zebra_if *zif;
+	struct interface *br_ifp;
+	vlanid_t vid;
+
+	zif = ifp->info;
+	br_ifp = zif->brslave_info.br_if;
+	if (!br_ifp)
+		return;
+
+	if (mac->zevpn->vxlan_if) {
+		zif = mac->zevpn->vxlan_if->info;
+		vid = zif->l2info.vxl.access_vlan;
+	} else {
+		vid = 0;
+	}
+
+	/* delete the local mac from the dataplane */
+	dplane_local_mac_del(ifp, br_ifp, vid, &mac->macaddr);
+	/* delete the local mac in zebra */
+	zebra_evpn_del_local_mac(mac->zevpn, mac, true);
+}
+
+static void zebra_evpn_es_flush_local_macs(struct zebra_evpn_es *es,
+					   struct interface *ifp, bool add)
 {
 	zebra_mac_t *mac;
 	struct listnode	*node;
+	struct listnode *nnode;
 
-	for (ALL_LIST_ELEMENTS_RO(es->mac_list, node, mac)) {
+	for (ALL_LIST_ELEMENTS(es->mac_list, node, nnode, mac)) {
-		if (CHECK_FLAG(mac->flags, ZEBRA_MAC_ES_PEER_ACTIVE)) {
+		if (!CHECK_FLAG(mac->flags, ZEBRA_MAC_LOCAL))
-			zebra_evpn_sync_mac_dp_install(
+			continue;
-				mac, false /* set_inactive */,
+
-				force_clear_static, __func__);
+		/* If ES is being attached/detached from the access port we
-		}
+		 * need to clear local activity and peer activity and start
+		 * over */
+		if (IS_ZEBRA_DEBUG_EVPN_MH_MAC)
+			zlog_debug("VNI %u mac %pEA update; local ES %s %s",
+				   mac->zevpn->vni,
+				   &mac->macaddr,
+				   es->esi_str, add ? "add" : "del");
+		zebra_evpn_flush_local_mac(mac, ifp);
 	}
 }
 
@@ -2134,6 +2181,10 @@ static void zebra_evpn_es_local_info_set(struct zebra_evpn_es *es,
 	if (zif->brslave_info.bridge_ifindex != IFINDEX_INTERNAL)
 		es->flags |= ZEBRA_EVPNES_BR_PORT;
 
+	/* inherit the bypass flag from the interface */
+	if (zif->flags & ZIF_FLAG_LACP_BYPASS)
+		es->flags |= ZEBRA_EVPNES_BYPASS;
+
 	/* setup base-vni if one doesn't already exist; the ES will get sent
 	 * to BGP as a part of that process
 	 */
@@ -2165,11 +2216,9 @@ static void zebra_evpn_es_local_info_set(struct zebra_evpn_es *es,
 	 */
 	zebra_evpn_es_setup_evis(es);
 	/* if there any local macs referring to the ES as dest we
-	 * need to set the static reference on them if the MAC is
+	 * need to clear the contents and start over
-	 * synced from an ES peer
 	 */
-	zebra_evpn_es_local_mac_update(es,
+	zebra_evpn_es_flush_local_macs(es, zif->ifp, true);
-			false /* force_clear_static */);
 
 	/* inherit EVPN protodown flags on the access port */
 	zebra_evpn_mh_update_protodown_es(es, true /*resync_dplane*/);
@@ -2184,33 +2233,34 @@ static void zebra_evpn_es_local_info_clear(struct zebra_evpn_es **esp)
 	if (!(es->flags & ZEBRA_EVPNES_LOCAL))
 		return;
 
+	zif = es->zif;
+
+	/* if there any local macs referring to the ES as dest we
+	 * need to clear the contents and start over
+	 */
+	zebra_evpn_es_flush_local_macs(es, zif->ifp, false);
+
 	es->flags &= ~(ZEBRA_EVPNES_LOCAL | ZEBRA_EVPNES_READY_FOR_BGP);
 
 	THREAD_OFF(es->df_delay_timer);
 
-	/* remove the DF filter */
-	dplane_updated = zebra_evpn_es_run_df_election(es, __func__);
-
 	/* clear EVPN protodown flags on the access port */
 	zebra_evpn_mh_clear_protodown_es(es);
 
-	/* if there any local macs referring to the ES as dest we
+	/* remove the DF filter */
-	 * need to clear the static reference on them
+	dplane_updated = zebra_evpn_es_run_df_election(es, __func__);
-	 */
-	zebra_evpn_es_local_mac_update(es,
-			true /* force_clear_static */);
 
 	/* flush the BUM filters and backup NHG */
 	if (!dplane_updated)
 		zebra_evpn_es_br_port_dplane_clear(es);
 
 	/* clear the es from the parent interface */
-	zif = es->zif;
 	zif->es_info.es = NULL;
 	es->zif = NULL;
 
 	/* clear all local flags associated with the ES */
-	es->flags &= ~(ZEBRA_EVPNES_OPER_UP | ZEBRA_EVPNES_BR_PORT);
+	es->flags &= ~(ZEBRA_EVPNES_OPER_UP | ZEBRA_EVPNES_BR_PORT
+		       | ZEBRA_EVPNES_BYPASS);
 
 	/* remove from the ES list */
 	list_delete_node(zmh_info->local_es_list, &es->local_es_listnode);
@@ -2623,6 +2673,82 @@ static void zebra_evpn_es_df_pref_update(struct zebra_if *zif, uint16_t df_pref)
 		zebra_evpn_es_send_add_to_client(es);
 }
 
+/* If bypass mode on an es changed we set all local macs to
+ * inactive and drop the sync info
+ */
+static void zebra_evpn_es_bypass_update_macs(struct zebra_evpn_es *es,
+					     struct interface *ifp, bool bypass)
+{
+	zebra_mac_t *mac;
+	struct listnode *node;
+	struct listnode *nnode;
+
+	/* Flush all MACs linked to the ES */
+	for (ALL_LIST_ELEMENTS(es->mac_list, node, nnode, mac)) {
+		if (!CHECK_FLAG(mac->flags, ZEBRA_MAC_LOCAL))
+			continue;
+
+		if (IS_ZEBRA_DEBUG_EVPN_MH_MAC)
+			zlog_debug("VNI %u mac %pEA %s update es %s",
+				   mac->zevpn->vni,
+				   &mac->macaddr,
+				   bypass ? "bypass" : "non-bypass",
+				   es->esi_str);
+		zebra_evpn_flush_local_mac(mac, ifp);
+	}
+}
+
+void zebra_evpn_es_bypass_update(struct zebra_evpn_es *es,
+				 struct interface *ifp, bool bypass)
+{
+	bool old_bypass;
+	bool dplane_updated;
+
+	old_bypass = !!(es->flags & ZEBRA_EVPNES_BYPASS);
+	if (old_bypass == bypass)
+		return;
+
+	if (bypass)
+		es->flags |= ZEBRA_EVPNES_BYPASS;
+	else
+		es->flags &= ~ZEBRA_EVPNES_BYPASS;
+
+	if (IS_ZEBRA_DEBUG_EVPN_MH_ES)
+		zlog_debug("bond %s es %s lacp bypass changed to %s", ifp->name,
+			   es->esi_str, bypass ? "on" : "off");
+
+	/* send bypass update to BGP */
+	if (es->flags & ZEBRA_EVPNES_READY_FOR_BGP)
+		zebra_evpn_es_send_add_to_client(es);
+
+	zebra_evpn_es_bypass_update_macs(es, ifp, bypass);
+
+	/* re-run DF election */
+	dplane_updated = zebra_evpn_es_run_df_election(es, __func__);
+
+	/* disable SPH filter */
+	if (!dplane_updated && (es->flags & ZEBRA_EVPNES_LOCAL)
+	    && (listcount(es->es_vtep_list) > ES_VTEP_MAX_CNT))
+		zebra_evpn_es_br_port_dplane_update(es, __func__);
+}
+
+static void zebra_evpn_es_bypass_cfg_update(struct zebra_if *zif, bool bypass)
+{
+	bool old_bypass = !!(zif->es_info.flags & ZIF_CFG_ES_FLAG_BYPASS);
+
+	if (old_bypass == bypass)
+		return;
+
+	if (bypass)
+		zif->es_info.flags |= ZIF_CFG_ES_FLAG_BYPASS;
+	else
+		zif->es_info.flags &= ~ZIF_CFG_ES_FLAG_BYPASS;
+
+
+	if (zif->es_info.es)
+		zebra_evpn_es_bypass_update(zif->es_info.es, zif->ifp, bypass);
+}
+
 
 /* Only certain types of access ports can be setup as an Ethernet Segment */
 bool zebra_evpn_is_if_es_capable(struct zebra_if *zif)
@@ -2818,7 +2944,7 @@ static void zebra_evpn_es_json_vtep_fill(struct zebra_evpn_es *es,
 static void zebra_evpn_es_show_entry(struct vty *vty, struct zebra_evpn_es *es,
 				     json_object *json_array)
 {
-	char type_str[4];
+	char type_str[5];
 	char vtep_str[ES_VTEP_LIST_STR_SZ];
 
 	if (json_array) {
@@ -2839,6 +2965,8 @@ static void zebra_evpn_es_show_entry(struct vty *vty, struct zebra_evpn_es *es,
 				json_array_string_add(json_flags, "remote");
 			if (es->flags & ZEBRA_EVPNES_NON_DF)
 				json_array_string_add(json_flags, "nonDF");
+			if (es->flags & ZEBRA_EVPNES_BYPASS)
+				json_array_string_add(json_flags, "bypass");
 			json_object_object_add(json, "flags", json_flags);
 		}
 
@@ -2860,6 +2988,8 @@ static void zebra_evpn_es_show_entry(struct vty *vty, struct zebra_evpn_es *es,
 			strlcat(type_str, "R", sizeof(type_str));
 		if (es->flags & ZEBRA_EVPNES_NON_DF)
 			strlcat(type_str, "N", sizeof(type_str));
+		if (es->flags & ZEBRA_EVPNES_BYPASS)
+			strlcat(type_str, "B", sizeof(type_str));
 
 		zebra_evpn_es_vtep_str(vtep_str, es, sizeof(vtep_str));
 
@@ -2897,6 +3027,8 @@ static void zebra_evpn_es_show_entry_detail(struct vty *vty,
 				json_array_string_add(json_flags, "remote");
 			if (es->flags & ZEBRA_EVPNES_NON_DF)
 				json_array_string_add(json_flags, "nonDF");
+			if (es->flags & ZEBRA_EVPNES_BYPASS)
+				json_array_string_add(json_flags, "bypass");
 			if (es->flags & ZEBRA_EVPNES_READY_FOR_BGP)
 				json_array_string_add(json_flags,
 						      "readyForBgp");
@@ -2952,6 +3084,8 @@ static void zebra_evpn_es_show_entry_detail(struct vty *vty,
 		vty_out(vty, " Ready for BGP: %s\n",
 				(es->flags & ZEBRA_EVPNES_READY_FOR_BGP) ?
 				"yes" : "no");
+		if (es->flags & ZEBRA_EVPNES_BYPASS)
+			vty_out(vty, " LACP bypass: on\n");
 		vty_out(vty, " VNI Count: %d\n", listcount(es->es_evi_list));
 		vty_out(vty, " MAC Count: %d\n", listcount(es->mac_list));
 		if (es->flags & ZEBRA_EVPNES_LOCAL)
@@ -2991,7 +3125,7 @@ void zebra_evpn_es_show(struct vty *vty, bool uj)
 	if (uj) {
 		json_array = json_object_new_array();
 	} else {
-		vty_out(vty, "Type: L local, R remote, N non-DF\n");
+		vty_out(vty, "Type: B bypass, L local, R remote, N non-DF\n");
 		vty_out(vty, "%-30s %-4s %-21s %s\n",
 				"ESI", "Type", "ES-IF", "VTEPs");
 	}
@@ -3097,12 +3231,35 @@ int zebra_evpn_mh_if_write(struct vty *vty, struct interface *ifp)
 #ifndef VTYSH_EXTRACT_PL
 #include "zebra/zebra_evpn_mh_clippy.c"
 #endif
+/* CLI for setting an ES in bypass mode */
+DEFPY_HIDDEN(zebra_evpn_es_bypass, zebra_evpn_es_bypass_cmd,
+	     "[no] evpn mh bypass",
+	     NO_STR "EVPN\n" EVPN_MH_VTY_STR "set bypass mode\n")
+{
+	VTY_DECLVAR_CONTEXT(interface, ifp);
+	struct zebra_if *zif;
+
+	zif = ifp->info;
+
+	if (no) {
+		zebra_evpn_es_bypass_cfg_update(zif, false);
+	} else {
+		if (!zebra_evpn_is_if_es_capable(zif)) {
+			vty_out(vty,
+				"%%DF bypass cannot be associated with this interface type\n");
+			return CMD_WARNING;
+		}
+		zebra_evpn_es_bypass_cfg_update(zif, true);
+	}
+	return CMD_SUCCESS;
+}
+
 /* CLI for configuring DF preference part for an ES */
 DEFPY(zebra_evpn_es_pref, zebra_evpn_es_pref_cmd,
       "[no$no] evpn mh es-df-pref [(1-65535)$df_pref]",
       NO_STR "EVPN\n" EVPN_MH_VTY_STR
 	     "preference value used for DF election\n"
-	     "ID\n")
+	     "pref\n")
 {
 	VTY_DECLVAR_CONTEXT(interface, ifp);
 	struct zebra_if *zif;
@@ -3767,6 +3924,7 @@ void zebra_evpn_interface_init(void)
 	install_element(INTERFACE_NODE, &zebra_evpn_es_id_cmd);
 	install_element(INTERFACE_NODE, &zebra_evpn_es_sys_mac_cmd);
 	install_element(INTERFACE_NODE, &zebra_evpn_es_pref_cmd);
+	install_element(INTERFACE_NODE, &zebra_evpn_es_bypass_cmd);
 	install_element(INTERFACE_NODE, &zebra_evpn_mh_uplink_cmd);
 }
 

zebra/zebra_evpn_mh.h

@@ -60,6 +60,10 @@ struct zebra_evpn_es {
  * filter, SPH filter and backup NHG for fast-failover
  */
 #define ZEBRA_EVPNES_BR_PORT (1 << 6)
+/* ES is in bypass mode i.e. must not be advertised. ES-bypass is set
+ * when the associated host bond goes into LACP bypass
+ */
+#define ZEBRA_EVPNES_BYPASS (1 << 7)
 
 	/* memory used for adding the es to zmh_info->es_rb_tree */
 	RB_ENTRY(zebra_evpn_es) rb_node;
@@ -376,5 +380,7 @@ extern void zebra_evpn_l2_nh_show(struct vty *vty, bool uj);
 extern void zebra_evpn_acc_bd_svi_set(struct zebra_if *vlan_zif,
 				      struct zebra_if *br_zif, bool is_up);
 extern void zebra_evpn_acc_bd_svi_mac_add(struct interface *vlan_if);
+extern void zebra_evpn_es_bypass_update(struct zebra_evpn_es *es,
+					struct interface *ifp, bool bypass);
 
 #endif /* _ZEBRA_EVPN_MH_H */

zebra/zebra_evpn_neigh.c

@@ -327,9 +327,9 @@ int zebra_evpn_neigh_send_add_to_client(vni_t vni, struct ipaddr *ip,
 	if (CHECK_FLAG(neigh_flags, ZEBRA_NEIGH_SVI_IP))
 		SET_FLAG(flags, ZEBRA_MACIP_TYPE_SVI_IP);
 
-	return zebra_evpn_macip_send_msg_to_client(
+	return zebra_evpn_macip_send_msg_to_client(vni, macaddr, ip, flags, seq,
-		vni, macaddr, ip, flags, seq, ZEBRA_NEIGH_ACTIVE,
+						   ZEBRA_NEIGH_ACTIVE, zmac->es,
-		zmac ? zmac->es : NULL, ZEBRA_MACIP_ADD);
+						   ZEBRA_MACIP_ADD);
 }
 
 /*

zebra/zebra_l2.c

@@ -110,6 +110,44 @@ void zebra_l2_unmap_slave_from_bridge(struct zebra_l2info_brslave *br_slave)
 	br_slave->br_if = NULL;
 }
 
+/* If any of the bond members are in bypass state the bond is placed
+ * in bypass state
+ */
+static void zebra_l2_bond_lacp_bypass_eval(struct zebra_if *bond_zif)
+{
+	struct listnode *node;
+	struct zebra_if *bond_mbr;
+	bool old_bypass = !!(bond_zif->flags & ZIF_FLAG_LACP_BYPASS);
+	bool new_bypass = false;
+
+	if (bond_zif->bond_info.mbr_zifs) {
+		for (ALL_LIST_ELEMENTS_RO(bond_zif->bond_info.mbr_zifs, node,
+					  bond_mbr)) {
+			if (bond_mbr->flags & ZIF_FLAG_LACP_BYPASS) {
+				new_bypass = true;
+				break;
+			}
+		}
+	}
+
+	if (old_bypass == new_bypass)
+		return;
+
+	if (IS_ZEBRA_DEBUG_EVPN_MH_ES || IS_ZEBRA_DEBUG_EVENT)
+		zlog_debug("bond %s lacp bypass changed to %s",
+			   bond_zif->ifp->name, new_bypass ? "on" : "off");
+
+	if (new_bypass)
+		bond_zif->flags |= ZIF_FLAG_LACP_BYPASS;
+	else
+		bond_zif->flags &= ~ZIF_FLAG_LACP_BYPASS;
+
+	if (bond_zif->es_info.es)
+		zebra_evpn_es_bypass_update(bond_zif->es_info.es, bond_zif->ifp,
+					    new_bypass);
+}
+
+/* Returns true if member was newly linked to bond */
 void zebra_l2_map_slave_to_bond(struct zebra_if *zif, vrf_id_t vrf_id)
 {
 	struct interface *bond_if;
@@ -138,6 +176,7 @@ void zebra_l2_map_slave_to_bond(struct zebra_if *zif, vrf_id_t vrf_id)
 			if (zebra_evpn_is_es_bond(bond_if))
 				zebra_evpn_mh_update_protodown_bond_mbr(
 					zif, false /*clear*/, __func__);
+			zebra_l2_bond_lacp_bypass_eval(bond_zif);
 		}
 	} else {
 		if (IS_ZEBRA_DEBUG_EVPN_MH_ES || IS_ZEBRA_DEBUG_EVENT)
@@ -170,6 +209,7 @@ void zebra_l2_unmap_slave_from_bond(struct zebra_if *zif)
 							__func__);
 	listnode_delete(bond_zif->bond_info.mbr_zifs, zif);
 	bond_slave->bond_if = NULL;
+	zebra_l2_bond_lacp_bypass_eval(bond_zif);
 }
 
 void zebra_l2if_update_bond(struct interface *ifp, bool add)
@@ -378,14 +418,36 @@ void zebra_l2if_update_bridge_slave(struct interface *ifp,
 	}
 }
 
-void zebra_l2if_update_bond_slave(struct interface *ifp, ifindex_t bond_ifindex)
+void zebra_l2if_update_bond_slave(struct interface *ifp, ifindex_t bond_ifindex,
+				  bool new_bypass)
 {
 	struct zebra_if *zif;
 	ifindex_t old_bond_ifindex;
+	bool old_bypass;
+	struct zebra_l2info_bondslave *bond_mbr;
 
 	zif = ifp->info;
 	assert(zif);
 
+	old_bypass = !!(zif->flags & ZIF_FLAG_LACP_BYPASS);
+	if (old_bypass != new_bypass) {
+		if (IS_ZEBRA_DEBUG_EVPN_MH_ES || IS_ZEBRA_DEBUG_EVENT)
+			zlog_debug("bond-mbr %s lacp bypass changed to %s",
+				   zif->ifp->name, new_bypass ? "on" : "off");
+
+		if (new_bypass)
+			zif->flags |= ZIF_FLAG_LACP_BYPASS;
+		else
+			zif->flags &= ~ZIF_FLAG_LACP_BYPASS;
+
+		bond_mbr = &zif->bondslave_info;
+		if (bond_mbr->bond_if) {
+			struct zebra_if *bond_zif = bond_mbr->bond_if->info;
+
+			zebra_l2_bond_lacp_bypass_eval(bond_zif);
+		}
+	}
+
 	old_bond_ifindex = zif->bondslave_info.bond_ifindex;
 	if (old_bond_ifindex == bond_ifindex)
 		return;

zebra/zebra_l2.h

@@ -107,7 +107,7 @@ extern void zebra_l2if_update_bridge_slave(struct interface *ifp,
 					   ns_id_t ns_id);
 
 extern void zebra_l2if_update_bond_slave(struct interface *ifp,
-					 ifindex_t bond_ifindex);
+					 ifindex_t bond_ifindex, bool bypass);
 extern void zebra_vlan_bitmap_compute(struct interface *ifp,
 		uint32_t vid_start, uint16_t vid_end);
 extern void zebra_vlan_mbr_re_eval(struct interface *ifp,

zebra/zebra_vxlan.c

@@ -2854,9 +2854,8 @@ void zebra_vxlan_print_macs_vni(struct vty *vty, struct zebra_vrf *zvrf,
 			num_macs);
 		vty_out(vty,
 			"Flags: N=sync-neighs, I=local-inactive, P=peer-active, X=peer-proxy\n");
-			vty_out(vty, "%-17s %-6s %-5s %-30s %-5s %s\n", "MAC",
+		vty_out(vty, "%-17s %-6s %-5s %-30s %-5s %s\n", "MAC", "Type",
-				"Type", "Flags", "Intf/Remote ES/VTEP",
+			"Flags", "Intf/Remote ES/VTEP", "VLAN", "Seq #'s");
-				"VLAN", "Seq #'s");
 	} else
 		json_object_int_add(json, "numMacs", num_macs);
 
@@ -4123,7 +4122,8 @@ int zebra_vxlan_dp_network_mac_del(struct interface *ifp,
 		if (IS_ZEBRA_DEBUG_VXLAN || IS_ZEBRA_DEBUG_EVPN_MH_MAC)
 			zlog_debug("dpDel local-nw-MAC %pEA VNI %u", macaddr,
 				   vni);
-		return zebra_evpn_del_local_mac(zevpn, mac);
+
+		zebra_evpn_del_local_mac(zevpn, mac, false);
 	}
 
 	return 0;
@@ -4160,7 +4160,7 @@ int zebra_vxlan_local_mac_del(struct interface *ifp, struct interface *br_if,
 	if (!CHECK_FLAG(mac->flags, ZEBRA_MAC_LOCAL))
 		return 0;
 
-	return zebra_evpn_del_local_mac(zevpn, mac);
+	return zebra_evpn_del_local_mac(zevpn, mac, false);
 }
 
 /*
@@ -4209,7 +4209,7 @@ int zebra_vxlan_local_mac_add_update(struct interface *ifp,
 
 	return zebra_evpn_add_update_local_mac(zvrf, zevpn, ifp, macaddr, vid,
 					       sticky, local_inactive,
-					       dp_static);
+					       dp_static, NULL);
 }
 
 /*