zebra: Add kernel level graceful restart

<Initial Code from Praveen Chaudhary>

Add the a `--graceful_restart X` flag to zebra start that
now creates a timer that pops in X seconds and will go
through and remove all routes that are older than startup.

If graceful_restart is not specified then we will just pop
a timer that cleans everything up immediately.

Signed-off-by: Praveen Chaudhary <pchaudhary@linkedin.com>
Signed-off-by: Donald Sharp <sharpd@cumulusnetworks.com>

doc/user/zebra.rst

@@ -27,6 +27,13 @@ Besides the common invocation options (:ref:`common-invocation-options`), the
 
    When zebra starts up, don't delete old self inserted routes.
 
+.. option:: -K TIME, --graceful_restart TIME
+
+   If this option is specified, the graceful restart time is TIME seconds.
+   Zebra, when started, will read in routes.  Those routes that Zebra
+   identifies that it was the originator of will be swept in TIME seconds.
+   If no time is specified then we will sweep those routes immediately.
+
 .. option:: -r, --retain
 
    When program terminates, do not flush routes installed by *zebra* from the

zebra/main.c

@@ -77,6 +77,8 @@ int allow_delete = 0;
 /* Don't delete kernel route. */
 int keep_kernel_mode = 0;
 
+int graceful_restart;
+
 bool v6_rr_semantics = false;
 
 #ifdef HAVE_NETLINK
@@ -95,6 +97,7 @@ struct option longopts[] = {
 	{"label_socket", no_argument, NULL, 'l'},
 	{"retain", no_argument, NULL, 'r'},
 	{"vrfdefaultname", required_argument, NULL, 'o'},
+	{"graceful_restart", required_argument, NULL, 'K'},
 #ifdef HAVE_NETLINK
 	{"vrfwnetns", no_argument, NULL, 'n'},
 	{"nl-bufsize", required_argument, NULL, 's'},
@@ -262,13 +265,14 @@ int main(int argc, char **argv)
 	char *netlink_fuzzing = NULL;
 #endif /* HANDLE_NETLINK_FUZZING */
 
+	graceful_restart = 0;
 	vrf_configure_backend(VRF_BACKEND_VRF_LITE);
 	logicalrouter_configure_backend(LOGICALROUTER_BACKEND_NETNS);
 
 	frr_preinit(&zebra_di, argc, argv);
 
 	frr_opt_add(
-		"bakz:e:l:o:r"
+		"bakz:e:l:o:rK:"
 #ifdef HAVE_NETLINK
 		"s:n"
 #endif
@@ -280,24 +284,25 @@ int main(int argc, char **argv)
 #endif /* HANDLE_NETLINK_FUZZING */
 		,
 		longopts,
-		"  -b, --batch           Runs in batch mode\n"
+		"  -b, --batch              Runs in batch mode\n"
-		"  -a, --allow_delete    Allow other processes to delete zebra routes\n"
+		"  -a, --allow_delete       Allow other processes to delete zebra routes\n"
-		"  -z, --socket          Set path of zebra socket\n"
+		"  -z, --socket             Set path of zebra socket\n"
-		"  -e, --ecmp            Specify ECMP to use.\n"
+		"  -e, --ecmp               Specify ECMP to use.\n"
-		"  -l, --label_socket    Socket to external label manager\n"
+		"  -l, --label_socket       Socket to external label manager\n"
-		"  -k, --keep_kernel     Don't delete old routes which were installed by zebra.\n"
+		"  -k, --keep_kernel        Don't delete old routes which were installed by zebra.\n"
-		"  -r, --retain          When program terminates, retain added route by zebra.\n"
+		"  -r, --retain             When program terminates, retain added route by zebra.\n"
-		"  -o, --vrfdefaultname  Set default VRF name.\n"
+		"  -o, --vrfdefaultname     Set default VRF name.\n"
+		"  -K, --graceful_restart   Graceful restart at the kernel level, timer in seconds for expiration\n"
 #ifdef HAVE_NETLINK
-		"  -n, --vrfwnetns       Use NetNS as VRF backend\n"
+		"  -n, --vrfwnetns          Use NetNS as VRF backend\n"
-		"  -s, --nl-bufsize      Set netlink receive buffer size\n"
+		"  -s, --nl-bufsize         Set netlink receive buffer size\n"
-		"      --v6-rr-semantics Use v6 RR semantics\n"
+		"      --v6-rr-semantics    Use v6 RR semantics\n"
 #endif /* HAVE_NETLINK */
 #if defined(HANDLE_ZAPI_FUZZING)
-		"  -c <file>             Bypass normal startup and use this file for testing of zapi\n"
+		"  -c <file>                Bypass normal startup and use this file for testing of zapi\n"
 #endif /* HANDLE_ZAPI_FUZZING */
 #if defined(HANDLE_NETLINK_FUZZING)
-		"  -w <file>             Bypass normal startup and use this file for testing of netlink input\n"
+		"  -w <file>                Bypass normal startup and use this file for testing of netlink input\n"
 #endif /* HANDLE_NETLINK_FUZZING */
 	);
 
@@ -317,6 +322,10 @@ int main(int argc, char **argv)
 			allow_delete = 1;
 			break;
 		case 'k':
+			if (graceful_restart) {
+				zlog_err("Graceful Restart initiated, we cannot keep the existing kernel routes");
+				return 1;
+			}
 			keep_kernel_mode = 1;
 			break;
 		case 'e':
@@ -348,6 +357,13 @@ int main(int argc, char **argv)
 		case 'r':
 			retain_mode = 1;
 			break;
+		case 'K':
+			if (keep_kernel_mode) {
+				zlog_err("Keep Kernel mode specified, graceful restart incompatible");
+				return 1;
+			}
+			graceful_restart = atoi(optarg);
+			break;
 #ifdef HAVE_NETLINK
 		case 's':
 			nl_rcvbufsize = atoi(optarg);
@@ -435,8 +451,10 @@ int main(int argc, char **argv)
 	*  will be equal to the current getpid(). To know about such routes,
 	* we have to have route_read() called before.
 	*/
+	zrouter.startup_time = monotime(NULL);
 	if (!keep_kernel_mode)
-		rib_sweep_route();
+		thread_add_timer(zrouter.master, rib_sweep_route,
+				 NULL, graceful_restart, NULL);
 
 	/* Needed for BSD routing socket. */
 	pid = getpid();

zebra/rib.h

@@ -400,7 +400,7 @@ extern struct route_entry *rib_lookup_ipv4(struct prefix_ipv4 *p,
 extern void rib_update(vrf_id_t vrf_id, rib_update_event_t event);
 extern void rib_update_table(struct route_table *table,
 			     rib_update_event_t event);
-extern void rib_sweep_route(void);
+extern int rib_sweep_route(struct thread *t);
 extern void rib_sweep_table(struct route_table *table);
 extern void rib_close_table(struct route_table *table);
 extern void rib_init(void);

zebra/zebra_rib.c

@@ -3127,6 +3127,7 @@ void rib_sweep_table(struct route_table *table)
 
 	for (rn = route_top(table); rn; rn = srcdest_route_next(rn)) {
 		RNODE_FOREACH_RE_SAFE (rn, re, next) {
+
 			if (IS_ZEBRA_DEBUG_RIB)
 				route_entry_dump(&rn->p, NULL, re);
 
@@ -3136,6 +3137,14 @@ void rib_sweep_table(struct route_table *table)
 			if (!CHECK_FLAG(re->flags, ZEBRA_FLAG_SELFROUTE))
 				continue;
 
+			/*
+			 * If routes are older than startup_time then
+			 * we know we read them in from the kernel.
+			 * As such we can safely remove them.
+			 */
+			if (zrouter.startup_time < re->uptime)
+				continue;
+
 			/*
 			 * So we are starting up and have received
 			 * routes from the kernel that we have installed
@@ -3165,7 +3174,7 @@ void rib_sweep_table(struct route_table *table)
 }
 
 /* Sweep all RIB tables.  */
-void rib_sweep_route(void)
+int rib_sweep_route(struct thread *t)
 {
 	struct vrf *vrf;
 	struct zebra_vrf *zvrf;
@@ -3179,6 +3188,8 @@ void rib_sweep_route(void)
 	}
 
 	zebra_router_sweep_route();
+
+	return 0;
 }
 
 /* Remove specific by protocol routes from 'table'. */

zebra/zebra_router.h

@@ -112,8 +112,15 @@ struct zebra_router {
 	struct zebra_vrf *evpn_vrf;
 
 	uint32_t multipath_num;
+
+	/*
+	 * Time for when we sweep the rib from old routes
+	 */
+	time_t startup_time;
 };
 
+#define GRACEFUL_RESTART_TIME 60
+
 extern struct zebra_router zrouter;
 
 extern void zebra_router_init(void);