zebra: Use ROUTE_ENTRY_INSTALLED as decision for route is installed

zebra is using NEXTHOP_FLAG_FIB as the basis of whether or not
a route_entry is installed.  This is problematic in that we plan
to separate out nexthop handling from route installation.  So modify
the code to keep track of whether or not a route_entry is installed/failed.

This basically means that every place we set/unset NEXTHOP_FLAG_FIB, we
actually also set/unset ROUTE_ENTRY_INSTALLED on the route_entry.
Additionally where we check for route installed via NEXTHOP_FLAG_FIB
switch over to checking if the route think's it is installed.

Signed-off-by: Donald Sharp <sharpd@cumulusnetworks.com>

zebra/zebra_fpm_netlink.c

@@ -276,7 +276,7 @@ static int netlink_route_info_fill(netlink_route_info_t *ri, int cmd,
 		if ((cmd == RTM_NEWROUTE
 		     && CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_ACTIVE))
 		    || (cmd == RTM_DELROUTE
-			&& CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB))) {
+			&& CHECK_FLAG(re->status, ROUTE_ENTRY_INSTALLED))) {
 			netlink_route_info_add_nh(ri, nexthop);
 		}
 	}

zebra/zebra_rib.c

@@ -566,8 +566,8 @@ static int nexthop_active(afi_t afi, struct route_entry *re,
 		} else if (CHECK_FLAG(re->flags, ZEBRA_FLAG_ALLOW_RECURSION)) {
 			resolved = 0;
 			for (ALL_NEXTHOPS(match->ng, newhop)) {
-				if (!CHECK_FLAG(newhop->flags,
+				if (!CHECK_FLAG(match->status,
-						NEXTHOP_FLAG_FIB))
+						ROUTE_ENTRY_INSTALLED))
 					continue;
 				if (CHECK_FLAG(newhop->flags,
 					       NEXTHOP_FLAG_RECURSIVE))
@@ -592,8 +592,8 @@ static int nexthop_active(afi_t afi, struct route_entry *re,
 		} else if (re->type == ZEBRA_ROUTE_STATIC) {
 			resolved = 0;
 			for (ALL_NEXTHOPS(match->ng, newhop)) {
-				if (!CHECK_FLAG(newhop->flags,
+				if (!CHECK_FLAG(match->status,
-						NEXTHOP_FLAG_FIB))
+						ROUTE_ENTRY_INSTALLED))
 					continue;
 
 				if (set) {
@@ -637,7 +637,6 @@ struct route_entry *rib_match(afi_t afi, safi_t safi, vrf_id_t vrf_id,
 	struct route_table *table;
 	struct route_node *rn;
 	struct route_entry *match = NULL;
-	struct nexthop *newhop;
 
 	/* Lookup table.  */
 	table = zebra_vrf_table(afi, safi, vrf_id);
@@ -677,14 +676,8 @@ struct route_entry *rib_match(afi_t afi, safi_t safi, vrf_id_t vrf_id,
 				route_lock_node(rn);
 		} else {
 			if (match->type != ZEBRA_ROUTE_CONNECT) {
-				int found = 0;
+				if (!CHECK_FLAG(match->status,
-				for (ALL_NEXTHOPS(match->ng, newhop))
+						ROUTE_ENTRY_INSTALLED))
-					if (CHECK_FLAG(newhop->flags,
-						       NEXTHOP_FLAG_FIB)) {
-						found = 1;
-						break;
-					}
-				if (!found)
 					return NULL;
 			}
 
@@ -774,7 +767,6 @@ struct route_entry *rib_lookup_ipv4(struct prefix_ipv4 *p, vrf_id_t vrf_id)
 	struct route_table *table;
 	struct route_node *rn;
 	struct route_entry *match = NULL;
-	struct nexthop *nexthop;
 	rib_dest_t *dest;
 
 	/* Lookup table.  */
@@ -802,9 +794,8 @@ struct route_entry *rib_lookup_ipv4(struct prefix_ipv4 *p, vrf_id_t vrf_id)
 	if (match->type == ZEBRA_ROUTE_CONNECT)
 		return match;
 
-	for (ALL_NEXTHOPS(match->ng, nexthop))
+	if (CHECK_FLAG(match->status, ROUTE_ENTRY_INSTALLED))
-		if (CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB))
+		return match;
-			return match;
 
 	return NULL;
 }
@@ -1119,6 +1110,7 @@ void rib_uninstall_kernel(struct route_node *rn, struct route_entry *re)
 	struct zebra_vrf *zvrf = vrf_info_lookup(re->vrf_id);
 
 	if (info->safi != SAFI_UNICAST) {
+		UNSET_FLAG(re->status, ROUTE_ENTRY_INSTALLED);
 		for (ALL_NEXTHOPS(re->ng, nexthop))
 			UNSET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
 		return;
@@ -1171,6 +1163,8 @@ static void rib_uninstall(struct route_node *rn, struct route_entry *re)
 
 		if (!RIB_SYSTEM_ROUTE(re))
 			rib_uninstall_kernel(rn, re);
+		else
+			UNSET_FLAG(re->status, ROUTE_ENTRY_INSTALLED);
 
 		dest->selected_fib = NULL;
 
@@ -1301,6 +1295,7 @@ static void rib_process_del_fib(struct zebra_vrf *zvrf, struct route_node *rn,
 	if (!RIB_SYSTEM_ROUTE(old))
 		rib_uninstall_kernel(rn, old);
 	else {
+		UNSET_FLAG(old->status, ROUTE_ENTRY_INSTALLED);
 		/*
 		 * We are setting this to NULL here
 		 * because that is what we traditionally
@@ -1381,6 +1376,7 @@ static void rib_process_update_fib(struct zebra_vrf *zvrf,
 
 				rib_install_kernel(rn, new, old);
 			} else {
+				UNSET_FLAG(new->status, ROUTE_ENTRY_INSTALLED);
 				/*
 				 * We do not need to install the
 				 * selected route because it
@@ -1400,7 +1396,13 @@ static void rib_process_update_fib(struct zebra_vrf *zvrf,
 				if (RIB_SYSTEM_ROUTE(new)) {
 					if (!RIB_SYSTEM_ROUTE(old))
 						rib_uninstall_kernel(rn, old);
+					else
+						UNSET_FLAG(
+							old->status,
+							ROUTE_ENTRY_INSTALLED);
 				} else {
+					UNSET_FLAG(old->status,
+						   ROUTE_ENTRY_INSTALLED);
 					for (nexthop = old->ng.nexthop; nexthop;
 					     nexthop = nexthop->next)
 						UNSET_FLAG(nexthop->flags,
@@ -1437,8 +1439,10 @@ static void rib_process_update_fib(struct zebra_vrf *zvrf,
 
 			if (!RIB_SYSTEM_ROUTE(old))
 				rib_uninstall_kernel(rn, old);
-			else
+			else {
+				UNSET_FLAG(old->status, ROUTE_ENTRY_INSTALLED);
 				dest->selected_fib = NULL;
+			}
 		}
 	} else {
 		/*
@@ -1450,18 +1454,9 @@ static void rib_process_update_fib(struct zebra_vrf *zvrf,
 		 * is ready
 		 * to add routes.
 		 */
-		if (!RIB_SYSTEM_ROUTE(new)) {
+		if (!RIB_SYSTEM_ROUTE(new)
-			bool in_fib = false;
+		    && !CHECK_FLAG(new->status, ROUTE_ENTRY_INSTALLED))
-
+			rib_install_kernel(rn, new, NULL);
-			for (ALL_NEXTHOPS(new->ng, nexthop))
-				if (CHECK_FLAG(nexthop->flags,
-					       NEXTHOP_FLAG_FIB)) {
-					in_fib = true;
-					break;
-				}
-			if (!in_fib)
-				rib_install_kernel(rn, new, NULL);
-		}
 	}
 
 	/* Update prior route. */
@@ -1823,6 +1818,8 @@ static void rib_process_result(struct zebra_dplane_ctx *ctx)
 	enum dplane_op_e op;
 	enum zebra_dplane_result status;
 	const struct prefix *dest_pfx, *src_pfx;
+	bool re_stale = false;
+	bool old_re_stale = false;
 
 	/* Locate rn and re(s) from ctx */
 
@@ -1907,7 +1904,7 @@ static void rib_process_result(struct zebra_dplane_ctx *ctx)
 			zlog_debug("%u:%s Stale dplane result for re %p",
 				   dplane_ctx_get_vrf(ctx), dest_str, re);
 		}
-		re = NULL;
+		re_stale = true;
 	}
 
 	if (old_re &&
@@ -1916,7 +1913,7 @@ static void rib_process_result(struct zebra_dplane_ctx *ctx)
 			zlog_debug("%u:%s Stale dplane result for old_re %p",
 				   dplane_ctx_get_vrf(ctx), dest_str, old_re);
 		}
-		old_re = NULL;
+		old_re_stale = true;
 	}
 
 	/*
@@ -1934,12 +1931,24 @@ static void rib_process_result(struct zebra_dplane_ctx *ctx)
 	case DPLANE_OP_ROUTE_INSTALL:
 	case DPLANE_OP_ROUTE_UPDATE:
 		if (status == ZEBRA_DPLANE_REQUEST_SUCCESS) {
+			if (re) {
+				UNSET_FLAG(re->status, ROUTE_ENTRY_FAILED);
+				SET_FLAG(re->status, ROUTE_ENTRY_INSTALLED);
+			}
+			if (old_re) {
+				UNSET_FLAG(old_re->status, ROUTE_ENTRY_FAILED);
+				UNSET_FLAG(old_re->status,
+					   ROUTE_ENTRY_INSTALLED);
+			}
 			/* Update zebra nexthop FIB flag for each
 			 * nexthop that was installed.
 			 */
 			for (ALL_NEXTHOPS_PTR(dplane_ctx_get_ng(ctx),
 					      ctx_nexthop)) {
 
+				if (!re)
+					continue;
+
 				for (ALL_NEXTHOPS(re->ng, nexthop)) {
 					if (nexthop_same(ctx_nexthop, nexthop))
 						break;
@@ -1976,15 +1985,21 @@ static void rib_process_result(struct zebra_dplane_ctx *ctx)
 			 * 'old' context info was stale, 'old_re' will be
 			 * NULL here and that delete will not be sent.
 			 */
-			redistribute_update(dest_pfx, src_pfx, re, old_re);
+			if (re)
+				redistribute_update(dest_pfx, src_pfx,
+						    re, old_re);
 
 			/* Notify route owner */
-			zsend_route_notify_owner(re, dest_pfx,
+			zsend_route_notify_owner_ctx(ctx, ZAPI_ROUTE_INSTALLED);
-						 ZAPI_ROUTE_INSTALLED);
 
 		} else {
-			zsend_route_notify_owner(re, dest_pfx,
+			if (re)
-						 ZAPI_ROUTE_FAIL_INSTALL);
+				SET_FLAG(re->status, ROUTE_ENTRY_FAILED);
+			if (old_re)
+				SET_FLAG(old_re->status, ROUTE_ENTRY_FAILED);
+			if (re)
+				zsend_route_notify_owner(re, dest_pfx,
+							 ZAPI_ROUTE_FAIL_INSTALL);
 
 			zlog_warn("%u:%s: Route install failed",
 				  dplane_ctx_get_vrf(ctx),
@@ -1993,17 +2008,25 @@ static void rib_process_result(struct zebra_dplane_ctx *ctx)
 		}
 		break;
 	case DPLANE_OP_ROUTE_DELETE:
+		if (re)
+			SET_FLAG(re->status, ROUTE_ENTRY_FAILED);
 		/*
 		 * In the delete case, the zebra core datastructs were
 		 * updated (or removed) at the time the delete was issued,
 		 * so we're just notifying the route owner.
 		 */
 		if (status == ZEBRA_DPLANE_REQUEST_SUCCESS) {
+			if (re) {
+				UNSET_FLAG(re->status, ROUTE_ENTRY_INSTALLED);
+				UNSET_FLAG(re->status, ROUTE_ENTRY_FAILED);
+			}
 			zsend_route_notify_owner_ctx(ctx, ZAPI_ROUTE_REMOVED);
 
 			if (zvrf)
 				zvrf->removals++;
 		} else {
+			if (re)
+				SET_FLAG(re->status, ROUTE_ENTRY_FAILED);
 			zsend_route_notify_owner_ctx(ctx,
 						     ZAPI_ROUTE_REMOVE_FAIL);
 
@@ -2527,7 +2550,7 @@ void _route_entry_dump(const char *func, union prefixconstptr pp,
 			   (CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_ACTIVE)
 				    ? "ACTIVE "
 				    : ""),
-			   (CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB)
+			   (CHECK_FLAG(re->status, ROUTE_ENTRY_INSTALLED)
 				    ? "FIB "
 				    : ""),
 			   (CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_RECURSIVE)
@@ -2708,9 +2731,11 @@ int rib_add_multipath(afi_t afi, safi_t safi, struct prefix *p,
 	}
 
 	/* If this route is kernel route, set FIB flag to the route. */
-	if (RIB_SYSTEM_ROUTE(re))
+	if (RIB_SYSTEM_ROUTE(re)) {
+		SET_FLAG(re->status, ROUTE_ENTRY_INSTALLED);
 		for (nexthop = re->ng.nexthop; nexthop; nexthop = nexthop->next)
 			SET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
+	}
 
 	/* Link new re to node.*/
 	if (IS_ZEBRA_DEBUG_RIB) {
@@ -2847,6 +2872,7 @@ void rib_delete(afi_t afi, safi_t safi, vrf_id_t vrf_id, int type,
 					rn, fib, fib->type);
 			}
 			if (allow_delete) {
+				UNSET_FLAG(fib->status, ROUTE_ENTRY_INSTALLED);
 				/* Unset flags. */
 				for (rtnh = fib->ng.nexthop; rtnh;
 				     rtnh = rtnh->next)
@@ -3093,6 +3119,7 @@ void rib_sweep_table(struct route_table *table)
 			 * to a different spot (ie startup )
 			 * this decision needs to be revisited
 			 */
+			SET_FLAG(re->status, ROUTE_ENTRY_INSTALLED);
 			for (ALL_NEXTHOPS(re->ng, nexthop))
 				SET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
 

zebra/zebra_rnh.c

@@ -370,14 +370,10 @@ static void zebra_rnh_eval_import_check_entry(vrf_id_t vrfid, afi_t afi,
 	struct zserv *client;
 	char bufn[INET6_ADDRSTRLEN];
 	struct listnode *node;
-	struct nexthop *nexthop;
 
 	if (re && (rnh->state == NULL)) {
-		for (ALL_NEXTHOPS(re->ng, nexthop))
+		if (CHECK_FLAG(re->status, ROUTE_ENTRY_INSTALLED))
-			if (CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB)) {
+			state_changed = 1;
-				state_changed = 1;
-				break;
-			}
 	} else if (!re && (rnh->state != NULL))
 		state_changed = 1;
 
@@ -511,9 +507,10 @@ static void zebra_rnh_process_pbr_tables(afi_t afi, struct route_node *nrn,
  * check in a couple of places, so this is a single home for the logic we
  * use.
  */
-static bool rnh_nexthop_valid(const struct nexthop *nh)
+static bool rnh_nexthop_valid(const struct route_entry *re,
+			      const struct nexthop *nh)
 {
-	return (CHECK_FLAG(nh->flags, NEXTHOP_FLAG_FIB)
+	return (CHECK_FLAG(re->status, ROUTE_ENTRY_INSTALLED)
 		&& CHECK_FLAG(nh->flags, NEXTHOP_FLAG_ACTIVE));
 }
 
@@ -566,7 +563,7 @@ zebra_rnh_resolve_nexthop_entry(struct zebra_vrf *zvrf, afi_t afi,
 			 * have an installed nexthop to be useful.
 			 */
 			for (ALL_NEXTHOPS(re->ng, nexthop)) {
-				if (rnh_nexthop_valid(nexthop))
+				if (rnh_nexthop_valid(re, nexthop))
 					break;
 			}
 
@@ -820,6 +817,7 @@ static void copy_state(struct rnh *rnh, struct route_entry *re,
 	state->distance = re->distance;
 	state->metric = re->metric;
 	state->vrf_id = re->vrf_id;
+	state->status = re->status;
 
 	route_entry_copy_nexthops(state, re->ng.nexthop);
 	rnh->state = state;
@@ -895,7 +893,7 @@ static int send_client(struct rnh *rnh, struct zserv *client, rnh_type_t type,
 		nump = stream_get_endp(s);
 		stream_putc(s, 0);
 		for (ALL_NEXTHOPS(re->ng, nh))
-			if (rnh_nexthop_valid(nh)) {
+			if (rnh_nexthop_valid(re, nh)) {
 				stream_putl(s, nh->vrf_id);
 				stream_putc(s, nh->type);
 				switch (nh->type) {

zebra/zebra_vty.c

@@ -163,13 +163,23 @@ DEFUN (show_ip_rpf_addr,
 
 static char re_status_output_char(struct route_entry *re, struct nexthop *nhop)
 {
-	if (CHECK_FLAG(nhop->flags, NEXTHOP_FLAG_FIB)) {
+	if (CHECK_FLAG(re->status, ROUTE_ENTRY_INSTALLED)) {
-		if (CHECK_FLAG(nhop->flags, NEXTHOP_FLAG_DUPLICATE))
+		if (!CHECK_FLAG(nhop->flags, NEXTHOP_FLAG_DUPLICATE))
-			return ' ';
-		else
 			return '*';
+		else
+			return ' ';
 	}
 
+	if (CHECK_FLAG(re->status, ROUTE_ENTRY_FAILED)) {
+		if (CHECK_FLAG(re->status, ROUTE_ENTRY_QUEUED))
+			return 'q';
+
+		return 'f';
+	}
+
+	if (CHECK_FLAG(re->status, ROUTE_ENTRY_QUEUED))
+		return 'q';
+
 	return ' ';
 }
 
@@ -400,6 +410,21 @@ static void vty_show_ip_route(struct vty *vty, struct route_node *rn,
 				    re->distance);
 		json_object_int_add(json_route, "metric", re->metric);
 
+		if (CHECK_FLAG(re->status, ROUTE_ENTRY_INSTALLED))
+			json_object_boolean_true_add(json_route, "installed");
+
+		if (CHECK_FLAG(re->status, ROUTE_ENTRY_FAILED))
+			json_object_boolean_true_add(json_route, "failed");
+
+		if (CHECK_FLAG(re->status, ROUTE_ENTRY_QUEUED))
+			json_object_boolean_true_add(json_route, "queued");
+
+		if (re->type != ZEBRA_ROUTE_CONNECT) {
+			json_object_int_add(json_route, "distance",
+					    re->distance);
+			json_object_int_add(json_route, "metric", re->metric);
+		}
+
 		if (re->tag)
 			json_object_int_add(json_route, "tag", re->tag);
 
@@ -1409,21 +1434,20 @@ static void vty_show_ip_route_summary_prefix(struct vty *vty,
 			 * In case of ECMP, count only once.
 			 */
 			cnt = 0;
+			if (CHECK_FLAG(re->status, ROUTE_ENTRY_INSTALLED)) {
+				fib_cnt[ZEBRA_ROUTE_TOTAL]++;
+				fib_cnt[re->type]++;
+			}
 			for (nexthop = re->ng.nexthop; (!cnt && nexthop);
 			     nexthop = nexthop->next) {
 				cnt++;
 				rib_cnt[ZEBRA_ROUTE_TOTAL]++;
 				rib_cnt[re->type]++;
-				if (CHECK_FLAG(nexthop->flags,
-					       NEXTHOP_FLAG_FIB)) {
-					fib_cnt[ZEBRA_ROUTE_TOTAL]++;
-					fib_cnt[re->type]++;
-				}
 				if (re->type == ZEBRA_ROUTE_BGP
 				    && CHECK_FLAG(re->flags, ZEBRA_FLAG_IBGP)) {
 					rib_cnt[ZEBRA_ROUTE_IBGP]++;
-					if (CHECK_FLAG(nexthop->flags,
+					if (CHECK_FLAG(re->status,
-						       NEXTHOP_FLAG_FIB))
+						       ROUTE_ENTRY_INSTALLED))
 						fib_cnt[ZEBRA_ROUTE_IBGP]++;
 				}
 			}