Merge pull request #6266 from mjstapp/backup_nhgs

zebra: backup nexthops/nhlfes for LSPs

lib/zclient.c

@@ -1542,6 +1542,39 @@ int zapi_backup_nexthop_from_nexthop(struct zapi_nexthop *znh,
 	return ret;
 }
 
+/*
+ * Format some info about a zapi nexthop, for debug or logging.
+ */
+const char *zapi_nexthop2str(const struct zapi_nexthop *znh, char *buf,
+			     int bufsize)
+{
+	char tmp[INET6_ADDRSTRLEN];
+
+	switch (znh->type) {
+	case NEXTHOP_TYPE_IFINDEX:
+		snprintf(buf, bufsize, "if %u", znh->ifindex);
+		break;
+	case NEXTHOP_TYPE_IPV4:
+	case NEXTHOP_TYPE_IPV4_IFINDEX:
+		inet_ntop(AF_INET, &znh->gate.ipv4, tmp, sizeof(tmp));
+		snprintf(buf, bufsize, "%s if %u", tmp, znh->ifindex);
+		break;
+	case NEXTHOP_TYPE_IPV6:
+	case NEXTHOP_TYPE_IPV6_IFINDEX:
+		inet_ntop(AF_INET6, &znh->gate.ipv6, tmp, sizeof(tmp));
+		snprintf(buf, bufsize, "%s if %u", tmp, znh->ifindex);
+		break;
+	case NEXTHOP_TYPE_BLACKHOLE:
+		snprintf(buf, bufsize, "blackhole");
+		break;
+	default:
+		snprintf(buf, bufsize, "unknown");
+		break;
+	}
+
+	return buf;
+}
+
 /*
  * Decode the nexthop-tracking update message
  */
@@ -2811,6 +2844,27 @@ int zapi_labels_encode(struct stream *s, int cmd, struct zapi_labels *zl)
 			return -1;
 	}
 
+	if (CHECK_FLAG(zl->message, ZAPI_LABELS_HAS_BACKUPS)) {
+
+		if (zl->backup_nexthop_num > MULTIPATH_NUM) {
+			flog_err(
+				EC_LIB_ZAPI_ENCODE,
+				"%s: label %u: can't encode %u nexthops (maximum is %u)",
+				__func__, zl->local_label, zl->nexthop_num,
+				MULTIPATH_NUM);
+			return -1;
+		}
+		stream_putw(s, zl->backup_nexthop_num);
+
+		for (int i = 0; i < zl->backup_nexthop_num; i++) {
+			znh = &zl->backup_nexthops[i];
+
+			if (zapi_nexthop_encode(s, znh, 0) < 0)
+				return -1;
+		}
+
+	}
+
 	/* Put length at the first point of the stream. */
 	stream_putw_at(s, 0, stream_get_endp(s));
 
@@ -2885,6 +2939,28 @@ int zapi_labels_decode(struct stream *s, struct zapi_labels *zl)
 			return -1;
 	}
 
+	if (CHECK_FLAG(zl->message, ZAPI_LABELS_HAS_BACKUPS)) {
+		STREAM_GETW(s, zl->backup_nexthop_num);
+
+		if (zl->backup_nexthop_num > MULTIPATH_NUM) {
+			flog_warn(
+				EC_LIB_ZAPI_ENCODE,
+				"%s: Prefix %pFX has %d backup nexthops, but we can only use the first %d",
+				__func__, &zl->route.prefix,
+				zl->backup_nexthop_num,	MULTIPATH_NUM);
+		}
+
+		zl->backup_nexthop_num = MIN(MULTIPATH_NUM,
+					     zl->backup_nexthop_num);
+
+		for (int i = 0; i < zl->backup_nexthop_num; i++) {
+			znh = &zl->backup_nexthops[i];
+
+			if (zapi_nexthop_decode(s, znh, 0) < 0)
+				return -1;
+		}
+	}
+
 	return 0;
 stream_failure:
 	return -1;

lib/zclient.h

@@ -484,7 +484,8 @@ struct zapi_route {
 
 struct zapi_labels {
 	uint8_t message;
-#define ZAPI_LABELS_FTN      0x01
+#define ZAPI_LABELS_FTN           0x01
+#define ZAPI_LABELS_HAS_BACKUPS   0x02
 	enum lsp_types_t type;
 	mpls_label_t local_label;
 	struct {
@@ -495,6 +496,10 @@ struct zapi_labels {
 
 	uint16_t nexthop_num;
 	struct zapi_nexthop nexthops[MULTIPATH_NUM];
+
+	/* Backup nexthops, if present */
+	uint16_t backup_nexthop_num;
+	struct zapi_nexthop backup_nexthops[MULTIPATH_NUM];
 };
 
 struct zapi_pw {
@@ -799,6 +804,8 @@ int zapi_backup_nexthop_from_nexthop(struct zapi_nexthop *znh,
 				     const struct nexthop *nh);
 extern bool zapi_nexthop_update_decode(struct stream *s,
 				       struct zapi_route *nhr);
+const char *zapi_nexthop2str(const struct zapi_nexthop *znh, char *buf,
+			     int bufsize);
 
 /* Decode the zebra error message */
 extern bool zapi_error_decode(struct stream *s, enum zebra_error_types *error);

sharpd/sharp_vty.c

@@ -410,6 +410,8 @@ DEFPY(sharp_lsp_prefix_v4, sharp_lsp_prefix_v4_cmd,
       "Instance\n")
 {
 	struct nexthop_group_cmd *nhgc = NULL;
+	struct nexthop_group_cmd *backup_nhgc = NULL;
+	struct nexthop_group *backup_nhg = NULL;
 	struct prefix p = {};
 	int type = 0;
 
@@ -441,9 +443,23 @@ DEFPY(sharp_lsp_prefix_v4, sharp_lsp_prefix_v4_cmd,
 		return CMD_WARNING;
 	}
 
+	/* Use group's backup nexthop info if present */
+	if (nhgc->backup_list_name[0]) {
+		backup_nhgc = nhgc_find(nhgc->backup_list_name);
+
+		if (!backup_nhgc) {
+			vty_out(vty,
+				"%% Backup group %s not found for group %s\n",
+				nhgc->backup_list_name,
+				nhgname);
+			return CMD_WARNING;
+		}
+		backup_nhg = &(backup_nhgc->nhg);
+	}
+
 	if (sharp_install_lsps_helper(true, pfx->family > 0 ? &p : NULL,
 				      type, instance, inlabel,
-				      &(nhgc->nhg)) == 0)
+				      &(nhgc->nhg), backup_nhg) == 0)
 		return CMD_SUCCESS;
 	else {
 		vty_out(vty, "%% LSP install failed!\n");
@@ -454,7 +470,7 @@ DEFPY(sharp_lsp_prefix_v4, sharp_lsp_prefix_v4_cmd,
 DEFPY(sharp_remove_lsp_prefix_v4, sharp_remove_lsp_prefix_v4_cmd,
       "sharp remove lsp \
         (0-100000)$inlabel\
-        nexthop-group NHGNAME$nhgname\
+        [nexthop-group NHGNAME$nhgname] \
         [prefix A.B.C.D/M$pfx\
        " FRR_IP_REDIST_STR_SHARPD "$type_str [instance (0-255)$instance]]",
       "Sharp Routing Protocol\n"
@@ -472,6 +488,7 @@ DEFPY(sharp_remove_lsp_prefix_v4, sharp_remove_lsp_prefix_v4_cmd,
 	struct nexthop_group_cmd *nhgc = NULL;
 	struct prefix p = {};
 	int type = 0;
+	struct nexthop_group *nhg = NULL;
 
 	/* We're offered a v4 prefix */
 	if (pfx->family > 0 && type_str) {
@@ -489,21 +506,24 @@ DEFPY(sharp_remove_lsp_prefix_v4, sharp_remove_lsp_prefix_v4_cmd,
 		return CMD_WARNING;
 	}
 
-	nhgc = nhgc_find(nhgname);
+	if (nhgname) {
-	if (!nhgc) {
+		nhgc = nhgc_find(nhgname);
-		vty_out(vty, "%%  Nexthop-group '%s' does not exist\n",
+		if (!nhgc) {
-			nhgname);
+			vty_out(vty, "%%  Nexthop-group '%s' does not exist\n",
-		return CMD_WARNING;
+				nhgname);
-	}
+			return CMD_WARNING;
+		}
 
-	if (nhgc->nhg.nexthop == NULL) {
+		if (nhgc->nhg.nexthop == NULL) {
-		vty_out(vty, "%%  Nexthop-group '%s' is empty\n", nhgname);
+			vty_out(vty, "%%  Nexthop-group '%s' is empty\n",
-		return CMD_WARNING;
+				nhgname);
+			return CMD_WARNING;
+		}
+		nhg = &(nhgc->nhg);
 	}
 
 	if (sharp_install_lsps_helper(false, pfx->family > 0 ? &p : NULL,
-				      type, instance, inlabel,
+				      type, instance, inlabel, nhg, NULL) == 0)
-				      &(nhgc->nhg)) == 0)
 		return CMD_SUCCESS;
 	else {
 		vty_out(vty, "%% LSP remove failed!\n");

sharpd/sharp_zebra.c

@@ -89,7 +89,8 @@ static int sharp_ifp_down(struct interface *ifp)
 
 int sharp_install_lsps_helper(bool install_p, const struct prefix *p,
 			      uint8_t type, int instance, uint32_t in_label,
-			      const struct nexthop_group *nhg)
+			      const struct nexthop_group *nhg,
+			      const struct nexthop_group *backup_nhg)
 {
 	struct zapi_labels zl = {};
 	struct zapi_nexthop *znh;
@@ -106,32 +107,68 @@ int sharp_install_lsps_helper(bool install_p, const struct prefix *p,
 		zl.route.instance = instance;
 	}
 
+	/* List of nexthops is optional for delete */
 	i = 0;
-	for (ALL_NEXTHOPS_PTR(nhg, nh)) {
+	if (nhg) {
-		znh = &zl.nexthops[i];
+		for (ALL_NEXTHOPS_PTR(nhg, nh)) {
+			znh = &zl.nexthops[i];
 
-		/* Must have labels to be useful */
+			/* Must have labels to be useful */
-		if (nh->nh_label == NULL || nh->nh_label->num_labels == 0)
+			if (nh->nh_label == NULL ||
-			continue;
+			    nh->nh_label->num_labels == 0)
+				continue;
 
-		if (nh->type == NEXTHOP_TYPE_IFINDEX ||
+			if (nh->type == NEXTHOP_TYPE_IFINDEX ||
-		    nh->type == NEXTHOP_TYPE_BLACKHOLE)
+			    nh->type == NEXTHOP_TYPE_BLACKHOLE)
-			/* Hmm - can't really deal with these types */
+				/* Hmm - can't really deal with these types */
-			continue;
+				continue;
 
-		ret = zapi_nexthop_from_nexthop(znh, nh);
+			ret = zapi_nexthop_from_nexthop(znh, nh);
-		if (ret < 0)
+			if (ret < 0)
-			return -1;
+				return -1;
 
-		i++;
+			i++;
+		}
 	}
 
-	/* Whoops - no nexthops isn't very useful */
+	/* Whoops - no nexthops isn't very useful for install */
-	if (i == 0)
+	if (i == 0 && install_p)
 		return -1;
 
 	zl.nexthop_num = i;
 
+	/* Add optional backup nexthop info. Since these are used by index,
+	 * we can't just skip over an invalid backup nexthop: we will
+	 * invalidate the entire operation.
+	 */
+	if (backup_nhg != NULL) {
+		i = 0;
+		for (ALL_NEXTHOPS_PTR(backup_nhg, nh)) {
+			znh = &zl.backup_nexthops[i];
+
+			/* Must have labels to be useful */
+			if (nh->nh_label == NULL ||
+			    nh->nh_label->num_labels == 0)
+				return -1;
+
+			if (nh->type == NEXTHOP_TYPE_IFINDEX ||
+			    nh->type == NEXTHOP_TYPE_BLACKHOLE)
+				/* Hmm - can't really deal with these types */
+				return -1;
+
+			ret = zapi_nexthop_from_nexthop(znh, nh);
+			if (ret < 0)
+				return -1;
+
+			i++;
+		}
+
+		if (i > 0)
+			SET_FLAG(zl.message, ZAPI_LABELS_HAS_BACKUPS);
+
+		zl.backup_nexthop_num = i;
+	}
+
 	if (install_p)
 		ret = zebra_send_mpls_labels(zclient, ZEBRA_MPLS_LABELS_ADD,
 					     &zl);

sharpd/sharp_zebra.h

@@ -42,5 +42,7 @@ extern void sharp_remove_routes_helper(struct prefix *p, vrf_id_t vrf_id,
 
 int sharp_install_lsps_helper(bool install_p, const struct prefix *p,
 			      uint8_t type, int instance, uint32_t in_label,
-			      const struct nexthop_group *nhg);
+			      const struct nexthop_group *nhg,
+			      const struct nexthop_group *backup_nhg);
+
 #endif

zebra/rt_netlink.c

@@ -3504,6 +3504,7 @@ enum zebra_dplane_result kernel_neigh_update_ctx(struct zebra_dplane_ctx *ctx)
 int netlink_mpls_multipath(int cmd, struct zebra_dplane_ctx *ctx)
 {
 	mpls_lse_t lse;
+	const struct nhlfe_list_head *head;
 	const zebra_nhlfe_t *nhlfe;
 	struct nexthop *nexthop = NULL;
 	unsigned int nexthop_num;
@@ -3524,7 +3525,8 @@ int netlink_mpls_multipath(int cmd, struct zebra_dplane_ctx *ctx)
 	 * or multipath case.
 	 */
 	nexthop_num = 0;
-	for (nhlfe = dplane_ctx_get_nhlfe(ctx); nhlfe; nhlfe = nhlfe->next) {
+	head = dplane_ctx_get_nhlfe_list(ctx);
+	frr_each(nhlfe_list_const, head, nhlfe) {
 		nexthop = nhlfe->nexthop;
 		if (!nexthop)
 			continue;
@@ -3579,8 +3581,7 @@ int netlink_mpls_multipath(int cmd, struct zebra_dplane_ctx *ctx)
 				    routedesc);
 
 		nexthop_num = 0;
-		for (nhlfe = dplane_ctx_get_nhlfe(ctx);
+		frr_each(nhlfe_list_const, head, nhlfe) {
-		     nhlfe; nhlfe = nhlfe->next) {
 			nexthop = nhlfe->nexthop;
 			if (!nexthop)
 				continue;
@@ -3618,8 +3619,7 @@ int netlink_mpls_multipath(int cmd, struct zebra_dplane_ctx *ctx)
 				    routedesc);
 
 		nexthop_num = 0;
-		for (nhlfe = dplane_ctx_get_nhlfe(ctx);
+		frr_each(nhlfe_list_const, head, nhlfe) {
-		     nhlfe; nhlfe = nhlfe->next) {
 			nexthop = nhlfe->nexthop;
 			if (!nexthop)
 				continue;

zebra/zapi_msg.c

@@ -1998,6 +1998,7 @@ static void zread_mpls_labels_add(ZAPI_HANDLER_ARGS)
 {
 	struct stream *s;
 	struct zapi_labels zl;
+	int ret;
 
 	/* Get input stream.  */
 	s = msg;
@@ -2011,20 +2012,11 @@ static void zread_mpls_labels_add(ZAPI_HANDLER_ARGS)
 	if (!mpls_enabled)
 		return;
 
-	for (int i = 0; i < zl.nexthop_num; i++) {
+	ret = mpls_zapi_labels_process(true, zvrf, &zl);
-		struct zapi_nexthop *znh;
+	if (ret < 0) {
-
+		if (IS_ZEBRA_DEBUG_RECV)
-		znh = &zl.nexthops[i];
+			zlog_debug("%s: Error processing zapi request",
-
+				   __func__);
-		mpls_lsp_install(zvrf, zl.type, zl.local_label,
-				 znh->label_num, znh->labels,
-				 znh->type, &znh->gate, znh->ifindex);
-
-		if (CHECK_FLAG(zl.message, ZAPI_LABELS_FTN))
-			mpls_ftn_update(1, zvrf, zl.type, &zl.route.prefix,
-					znh->type, &znh->gate, znh->ifindex,
-					zl.route.type, zl.route.instance,
-					znh->labels[0]);
 	}
 }
 
@@ -2042,6 +2034,7 @@ static void zread_mpls_labels_delete(ZAPI_HANDLER_ARGS)
 {
 	struct stream *s;
 	struct zapi_labels zl;
+	int ret;
 
 	/* Get input stream.  */
 	s = msg;
@@ -2056,21 +2049,11 @@ static void zread_mpls_labels_delete(ZAPI_HANDLER_ARGS)
 		return;
 
 	if (zl.nexthop_num > 0) {
-		for (int i = 0; i < zl.nexthop_num; i++) {
+		ret = mpls_zapi_labels_process(false /*delete*/, zvrf, &zl);
-			struct zapi_nexthop *znh;
+		if (ret < 0) {
-
+			if (IS_ZEBRA_DEBUG_RECV)
-			znh = &zl.nexthops[i];
+				zlog_debug("%s: Error processing zapi request",
-			mpls_lsp_uninstall(zvrf, zl.type, zl.local_label,
+					   __func__);
-					   znh->type, &znh->gate,
-					   znh->ifindex);
-
-			if (CHECK_FLAG(zl.message, ZAPI_LABELS_FTN))
-				mpls_ftn_update(0, zvrf, zl.type,
-						&zl.route.prefix, znh->type,
-						&znh->gate, znh->ifindex,
-						zl.route.type,
-						zl.route.instance,
-						znh->labels[0]);
 		}
 	} else {
 		mpls_lsp_uninstall_all_vrf(zvrf, zl.type, zl.local_label);
@@ -2110,26 +2093,16 @@ static void zread_mpls_labels_replace(ZAPI_HANDLER_ARGS)
 	if (!mpls_enabled)
 		return;
 
+	/* This removes everything, then re-adds from the client's
+	 * zapi message. Since the LSP will be processed later, on this
+	 * this same pthread, all of the changes will 'appear' at once.
+	 */
 	mpls_lsp_uninstall_all_vrf(zvrf, zl.type, zl.local_label);
 	if (CHECK_FLAG(zl.message, ZAPI_LABELS_FTN))
 		mpls_ftn_uninstall(zvrf, zl.type, &zl.route.prefix,
 				   zl.route.type, zl.route.instance);
 
-	for (int i = 0; i < zl.nexthop_num; i++) {
+	mpls_zapi_labels_process(true, zvrf, &zl);
-		struct zapi_nexthop *znh;
-
-		znh = &zl.nexthops[i];
-		mpls_lsp_install(zvrf, zl.type, zl.local_label,
-				 znh->label_num, znh->labels, znh->type,
-				 &znh->gate, znh->ifindex);
-
-		if (CHECK_FLAG(zl.message, ZAPI_LABELS_FTN)) {
-			mpls_ftn_update(1, zvrf, zl.type, &zl.route.prefix,
-					znh->type, &znh->gate, znh->ifindex,
-					zl.route.type, zl.route.instance,
-					znh->labels[0]);
-		}
-	}
 }
 
 /* Send response to a table manager connect request to client */

zebra/zebra_dplane.c

@@ -33,6 +33,7 @@
 #include "zebra/zebra_router.h"
 #include "zebra/zebra_dplane.h"
 #include "zebra/zebra_vxlan_private.h"
+#include "zebra/zebra_mpls.h"
 #include "zebra/rt.h"
 #include "zebra/debug.h"
 
@@ -510,20 +511,28 @@ static void dplane_ctx_free_internal(struct zebra_dplane_ctx *ctx)
 	case DPLANE_OP_LSP_DELETE:
 	case DPLANE_OP_LSP_NOTIFY:
 	{
-		zebra_nhlfe_t *nhlfe, *next;
+		zebra_nhlfe_t *nhlfe;
 
-		/* Free allocated NHLFEs */
+		/* Unlink and free allocated NHLFEs */
-		for (nhlfe = ctx->u.lsp.nhlfe_list; nhlfe; nhlfe = next) {
+		frr_each_safe(nhlfe_list, &ctx->u.lsp.nhlfe_list, nhlfe) {
-			next = nhlfe->next;
+			nhlfe_list_del(&ctx->u.lsp.nhlfe_list, nhlfe);
-
+			zebra_mpls_nhlfe_free(nhlfe);
-			zebra_mpls_nhlfe_del(nhlfe);
 		}
 
-		/* Clear pointers in lsp struct, in case we're cacheing
+		/* Unlink and free allocated backup NHLFEs, if present */
+		frr_each_safe(nhlfe_list,
+			      &(ctx->u.lsp.backup_nhlfe_list), nhlfe) {
+			nhlfe_list_del(&ctx->u.lsp.backup_nhlfe_list,
+				       nhlfe);
+			zebra_mpls_nhlfe_free(nhlfe);
+		}
+
+		/* Clear pointers in lsp struct, in case we're caching
 		 * free context structs.
 		 */
-		ctx->u.lsp.nhlfe_list = NULL;
+		nhlfe_list_init(&ctx->u.lsp.nhlfe_list);
 		ctx->u.lsp.best_nhlfe = NULL;
+		nhlfe_list_init(&ctx->u.lsp.backup_nhlfe_list);
 
 		break;
 	}
@@ -1217,11 +1226,18 @@ void dplane_ctx_set_lsp_flags(struct zebra_dplane_ctx *ctx,
 	ctx->u.lsp.flags = flags;
 }
 
-const zebra_nhlfe_t *dplane_ctx_get_nhlfe(const struct zebra_dplane_ctx *ctx)
+const struct nhlfe_list_head *dplane_ctx_get_nhlfe_list(
+	const struct zebra_dplane_ctx *ctx)
 {
 	DPLANE_CTX_VALID(ctx);
+	return &(ctx->u.lsp.nhlfe_list);
+}
 
-	return ctx->u.lsp.nhlfe_list;
+const struct nhlfe_list_head *dplane_ctx_get_backup_nhlfe_list(
+	const struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+	return &(ctx->u.lsp.backup_nhlfe_list);
 }
 
 zebra_nhlfe_t *dplane_ctx_add_nhlfe(struct zebra_dplane_ctx *ctx,
@@ -1230,7 +1246,7 @@ zebra_nhlfe_t *dplane_ctx_add_nhlfe(struct zebra_dplane_ctx *ctx,
 				    union g_addr *gate,
 				    ifindex_t ifindex,
 				    uint8_t num_labels,
-				    mpls_label_t out_labels[])
+				    mpls_label_t *out_labels)
 {
 	zebra_nhlfe_t *nhlfe;
 
@@ -1243,6 +1259,26 @@ zebra_nhlfe_t *dplane_ctx_add_nhlfe(struct zebra_dplane_ctx *ctx,
 	return nhlfe;
 }
 
+zebra_nhlfe_t *dplane_ctx_add_backup_nhlfe(struct zebra_dplane_ctx *ctx,
+					   enum lsp_types_t lsp_type,
+					   enum nexthop_types_t nh_type,
+					   union g_addr *gate,
+					   ifindex_t ifindex,
+					   uint8_t num_labels,
+					   mpls_label_t *out_labels)
+{
+	zebra_nhlfe_t *nhlfe;
+
+	DPLANE_CTX_VALID(ctx);
+
+	nhlfe = zebra_mpls_lsp_add_backup_nhlfe(&(ctx->u.lsp),
+						lsp_type, nh_type, gate,
+						ifindex, num_labels,
+						out_labels);
+
+	return nhlfe;
+}
+
 const zebra_nhlfe_t *
 dplane_ctx_get_best_nhlfe(const struct zebra_dplane_ctx *ctx)
 {
@@ -1729,27 +1765,21 @@ static int dplane_ctx_lsp_init(struct zebra_dplane_ctx *ctx,
 
 	memset(&ctx->u.lsp, 0, sizeof(ctx->u.lsp));
 
+	nhlfe_list_init(&(ctx->u.lsp.nhlfe_list));
+	nhlfe_list_init(&(ctx->u.lsp.backup_nhlfe_list));
 	ctx->u.lsp.ile = lsp->ile;
 	ctx->u.lsp.addr_family = lsp->addr_family;
 	ctx->u.lsp.num_ecmp = lsp->num_ecmp;
 	ctx->u.lsp.flags = lsp->flags;
 
 	/* Copy source LSP's nhlfes, and capture 'best' nhlfe */
-	for (nhlfe = lsp->nhlfe_list; nhlfe; nhlfe = nhlfe->next) {
+	frr_each(nhlfe_list, &lsp->nhlfe_list, nhlfe) {
 		/* Not sure if this is meaningful... */
 		if (nhlfe->nexthop == NULL)
 			continue;
 
-		new_nhlfe =
+		new_nhlfe = zebra_mpls_lsp_add_nh(&(ctx->u.lsp), nhlfe->type,
-			zebra_mpls_lsp_add_nhlfe(
+						  nhlfe->nexthop);
-				&(ctx->u.lsp),
-				nhlfe->type,
-				nhlfe->nexthop->type,
-				&(nhlfe->nexthop->gate),
-				nhlfe->nexthop->ifindex,
-				nhlfe->nexthop->nh_label->num_labels,
-				nhlfe->nexthop->nh_label->label);
-
 		if (new_nhlfe == NULL || new_nhlfe->nexthop == NULL) {
 			ret = ENOMEM;
 			break;
@@ -1763,9 +1793,32 @@ static int dplane_ctx_lsp_init(struct zebra_dplane_ctx *ctx,
 			ctx->u.lsp.best_nhlfe = new_nhlfe;
 	}
 
+	if (ret != AOK)
+		goto done;
+
+	/* Capture backup nhlfes/nexthops */
+	frr_each(nhlfe_list, &lsp->backup_nhlfe_list, nhlfe) {
+		/* Not sure if this is meaningful... */
+		if (nhlfe->nexthop == NULL)
+			continue;
+
+		new_nhlfe = zebra_mpls_lsp_add_backup_nh(&(ctx->u.lsp),
+							 nhlfe->type,
+							 nhlfe->nexthop);
+		if (new_nhlfe == NULL || new_nhlfe->nexthop == NULL) {
+			ret = ENOMEM;
+			break;
+		}
+
+		/* Need to copy flags too */
+		new_nhlfe->flags = nhlfe->flags;
+		new_nhlfe->nexthop->flags = nhlfe->nexthop->flags;
+	}
+
 	/* On error the ctx will be cleaned-up, so we don't need to
 	 * deal with any allocated nhlfe or nexthop structs here.
 	 */
+done:
 
 	return ret;
 }

zebra/zebra_dplane.h

@@ -310,14 +310,26 @@ void dplane_ctx_set_addr_family(struct zebra_dplane_ctx *ctx,
 uint32_t dplane_ctx_get_lsp_flags(const struct zebra_dplane_ctx *ctx);
 void dplane_ctx_set_lsp_flags(struct zebra_dplane_ctx *ctx,
 			      uint32_t flags);
-const zebra_nhlfe_t *dplane_ctx_get_nhlfe(const struct zebra_dplane_ctx *ctx);
+const struct nhlfe_list_head *dplane_ctx_get_nhlfe_list(
+	const struct zebra_dplane_ctx *ctx);
+const struct nhlfe_list_head *dplane_ctx_get_backup_nhlfe_list(
+	const struct zebra_dplane_ctx *ctx);
+
 zebra_nhlfe_t *dplane_ctx_add_nhlfe(struct zebra_dplane_ctx *ctx,
 				    enum lsp_types_t lsp_type,
 				    enum nexthop_types_t nh_type,
 				    union g_addr *gate,
 				    ifindex_t ifindex,
 				    uint8_t num_labels,
-				    mpls_label_t out_labels[]);
+				    mpls_label_t *out_labels);
+
+zebra_nhlfe_t *dplane_ctx_add_backup_nhlfe(struct zebra_dplane_ctx *ctx,
+					   enum lsp_types_t lsp_type,
+					   enum nexthop_types_t nh_type,
+					   union g_addr *gate,
+					   ifindex_t ifindex,
+					   uint8_t num_labels,
+					   mpls_label_t *out_labels);
 
 const zebra_nhlfe_t *dplane_ctx_get_best_nhlfe(
 	const struct zebra_dplane_ctx *ctx);

zebra/zebra_mpls.h

@@ -55,6 +55,10 @@ typedef struct zebra_nhlfe_t_ zebra_nhlfe_t;
 typedef struct zebra_lsp_t_ zebra_lsp_t;
 typedef struct zebra_fec_t_ zebra_fec_t;
 
+/* Declare LSP nexthop list types */
+PREDECL_DLIST(snhlfe_list);
+PREDECL_DLIST(nhlfe_list);
+
 /*
  * (Outgoing) nexthop label forwarding entry configuration
  */
@@ -71,9 +75,8 @@ struct zebra_snhlfe_t_ {
 	/* Backpointer to base entry. */
 	zebra_slsp_t *slsp;
 
-	/* Pointers to more outgoing information for same in-label */
+	/* Linkage for LSPs' lists */
-	zebra_snhlfe_t *next;
+	struct snhlfe_list_item list;
-	zebra_snhlfe_t *prev;
 };
 
 /*
@@ -96,10 +99,12 @@ struct zebra_nhlfe_t_ {
 #define NHLFE_FLAG_MULTIPATH   (1 << 2)
 #define NHLFE_FLAG_DELETED     (1 << 3)
 #define NHLFE_FLAG_INSTALLED   (1 << 4)
+#define NHLFE_FLAG_IS_BACKUP   (1 << 5)
 
-	zebra_nhlfe_t *next;
-	zebra_nhlfe_t *prev;
 	uint8_t distance;
+
+	/* Linkage for LSPs' lists */
+	struct nhlfe_list_item list;
 };
 
 /*
@@ -117,7 +122,7 @@ struct zebra_slsp_t_ {
 	zebra_ile_t ile;
 
 	/* List of outgoing nexthop static configuration */
-	zebra_snhlfe_t *snhlfe_list;
+	struct snhlfe_list_head snhlfe_list;
 };
 
 /*
@@ -127,11 +132,18 @@ struct zebra_lsp_t_ {
 	/* Incoming label */
 	zebra_ile_t ile;
 
-	/* List of NHLFE, pointer to best and num equal-cost. */
+	/* List of NHLFEs, pointer to best, and num equal-cost. */
-	zebra_nhlfe_t *nhlfe_list;
+	struct nhlfe_list_head nhlfe_list;
+
 	zebra_nhlfe_t *best_nhlfe;
 	uint32_t num_ecmp;
 
+	/* Backup nhlfes, if present. The nexthop in a primary/active nhlfe
+	 * refers to its backup (if any) by index, so the order of this list
+	 * is significant.
+	 */
+	struct nhlfe_list_head backup_nhlfe_list;
+
 	/* Flags */
 	uint32_t flags;
 #define LSP_FLAG_SCHEDULED        (1 << 0)
@@ -164,6 +176,9 @@ struct zebra_fec_t_ {
 	struct list *client_list;
 };
 
+/* Declare typesafe list apis/macros */
+DECLARE_DLIST(nhlfe_list, struct zebra_nhlfe_t_, list);
+
 /* Function declarations. */
 
 /*
@@ -201,10 +216,31 @@ zebra_nhlfe_t *zebra_mpls_lsp_add_nhlfe(zebra_lsp_t *lsp,
 					union g_addr *gate,
 					ifindex_t ifindex,
 					uint8_t num_labels,
-					mpls_label_t out_labels[]);
+					const mpls_label_t *out_labels);
+
+/* Add or update a backup NHLFE for an LSP; return the object */
+zebra_nhlfe_t *zebra_mpls_lsp_add_backup_nhlfe(zebra_lsp_t *lsp,
+					       enum lsp_types_t lsp_type,
+					       enum nexthop_types_t gtype,
+					       union g_addr *gate,
+					       ifindex_t ifindex,
+					       uint8_t num_labels,
+					       const mpls_label_t *out_labels);
+
+/*
+ * Add NHLFE or backup NHLFE to an LSP based on a nexthop. These just maintain
+ * the LSP and NHLFE objects; nothing is scheduled for processing.
+ * Return: the newly-added object
+ */
+zebra_nhlfe_t *zebra_mpls_lsp_add_nh(zebra_lsp_t *lsp,
+				     enum lsp_types_t lsp_type,
+				     const struct nexthop *nh);
+zebra_nhlfe_t *zebra_mpls_lsp_add_backup_nh(zebra_lsp_t *lsp,
+					    enum lsp_types_t lsp_type,
+					    const struct nexthop *nh);
 
 /* Free an allocated NHLFE */
-void zebra_mpls_nhlfe_del(zebra_nhlfe_t *nhlfe);
+void zebra_mpls_nhlfe_free(zebra_nhlfe_t *nhlfe);
 
 int zebra_mpls_fec_register(struct zebra_vrf *zvrf, struct prefix *p,
 			    uint32_t label, uint32_t label_index,
@@ -266,12 +302,11 @@ void zebra_mpls_print_fec(struct vty *vty, struct zebra_vrf *zvrf,
 			  struct prefix *p);
 
 /*
- * Install/uninstall a FEC-To-NHLFE (FTN) binding.
+ * Handle zapi request to install/uninstall LSP and
+ * (optionally) FEC-To-NHLFE (FTN) bindings.
  */
-int mpls_ftn_update(int add, struct zebra_vrf *zvrf, enum lsp_types_t type,
+int mpls_zapi_labels_process(bool add_p, struct zebra_vrf *zvrf,
-		    struct prefix *prefix, enum nexthop_types_t gtype,
+			     const struct zapi_labels *zl);
-		    union g_addr *gate, ifindex_t ifindex, uint8_t route_type,
-		    unsigned short route_instance, mpls_label_t out_label);
 
 /*
  * Uninstall all NHLFEs bound to a single FEC.
@@ -287,7 +322,7 @@ int mpls_ftn_uninstall(struct zebra_vrf *zvrf, enum lsp_types_t type,
  */
 int mpls_lsp_install(struct zebra_vrf *zvrf, enum lsp_types_t type,
 		     mpls_label_t in_label, uint8_t num_out_labels,
-		     mpls_label_t out_labels[], enum nexthop_types_t gtype,
+		     const mpls_label_t *out_labels, enum nexthop_types_t gtype,
 		     const union g_addr *gate, ifindex_t ifindex);
 
 /*

zebra/zebra_mpls_openbsd.c

@@ -239,8 +239,9 @@ static int kernel_send_rtmsg_v6(int action, mpls_label_t in_label,
 
 static int kernel_lsp_cmd(struct zebra_dplane_ctx *ctx)
 {
+	const struct nhlfe_list_head *head;
 	const zebra_nhlfe_t *nhlfe;
-	struct nexthop *nexthop = NULL;
+	const struct nexthop *nexthop = NULL;
 	unsigned int nexthop_num = 0;
 	int action;
 
@@ -258,7 +259,8 @@ static int kernel_lsp_cmd(struct zebra_dplane_ctx *ctx)
 		return -1;
 	}
 
-	for (nhlfe = dplane_ctx_get_nhlfe(ctx); nhlfe; nhlfe = nhlfe->next) {
+	head = dplane_ctx_get_nhlfe_list(ctx);
+	frr_each(nhlfe_list_const, head, nhlfe) {
 		nexthop = nhlfe->nexthop;
 		if (!nexthop)
 			continue;

zebra/zebra_nhg.c

@@ -371,8 +371,12 @@ struct nhg_hash_entry *zebra_nhg_alloc(void)
 	return nhe;
 }
 
-static struct nhg_hash_entry *zebra_nhg_copy(const struct nhg_hash_entry *orig,
+/*
-					     uint32_t id)
+ * Allocate new nhe and make shallow copy of 'orig'; no
+ * recursive info is copied.
+ */
+struct nhg_hash_entry *zebra_nhe_copy(const struct nhg_hash_entry *orig,
+				      uint32_t id)
 {
 	struct nhg_hash_entry *nhe;
 
@@ -401,7 +405,7 @@ static void *zebra_nhg_hash_alloc(void *arg)
 	struct nhg_hash_entry *nhe = NULL;
 	struct nhg_hash_entry *copy = arg;
 
-	nhe = zebra_nhg_copy(copy, copy->id);
+	nhe = zebra_nhe_copy(copy, copy->id);
 
 	/* Mark duplicate nexthops in a group at creation time. */
 	nexthop_group_mark_duplicates(&(nhe->nhg));
@@ -1140,7 +1144,7 @@ static int nhg_ctx_process_new(struct nhg_ctx *ctx)
 			 * their attributes are unhashable.
 			 */
 
-			kernel_nhe = zebra_nhg_copy(nhe, id);
+			kernel_nhe = zebra_nhe_copy(nhe, id);
 
 			if (IS_ZEBRA_DEBUG_NHG_DETAIL)
 				zlog_debug("%s: copying kernel nhe (%u), dup of %u",
@@ -2245,7 +2249,7 @@ int nexthop_active_update(struct route_node *rn, struct route_entry *re)
 	/* Make a local copy of the existing nhe, so we don't work on/modify
 	 * the shared nhe.
 	 */
-	curr_nhe = zebra_nhg_copy(re->nhe, re->nhe->id);
+	curr_nhe = zebra_nhe_copy(re->nhe, re->nhe->id);
 
 	if (IS_ZEBRA_DEBUG_NHG_DETAIL)
 		zlog_debug("%s: re %p nhe %p (%u), curr_nhe %p",

zebra/zebra_nhg.h

@@ -202,6 +202,12 @@ void zebra_nhg_hash_free(void *p);
 void zebra_nhe_init(struct nhg_hash_entry *nhe, afi_t afi,
 		    const struct nexthop *nh);
 
+/*
+ * Shallow copy of 'orig', into new/allocated nhe.
+ */
+struct nhg_hash_entry *zebra_nhe_copy(const struct nhg_hash_entry *orig,
+				      uint32_t id);
+
 /* Allocate, free backup nexthop info objects */
 struct nhg_backup_info *zebra_nhg_backup_alloc(void);
 void zebra_nhg_backup_free(struct nhg_backup_info **p);

zebra/zebra_rib.c

@@ -2276,6 +2276,7 @@ rib_dest_t *zebra_rib_create_dest(struct route_node *rn)
 
 	dest = XCALLOC(MTYPE_RIB_DEST, sizeof(rib_dest_t));
 	rnh_list_init(&dest->nht);
+	re_list_init(&dest->routes);
 	route_lock_node(rn); /* rn route table reference */
 	rn->info = dest;
 	dest->rnode = rn;