From d4644d4196a74ef406a21b6fa6eb4a64b045bde3 Mon Sep 17 00:00:00 2001 From: Donald Sharp Date: Thu, 9 May 2019 20:06:13 -0400 Subject: [PATCH 1/2] zebra: Add kernel level graceful restart Add the a `--graceful_restart X` flag to zebra start that now creates a timer that pops in X seconds and will go through and remove all routes that are older than startup. If graceful_restart is not specified then we will just pop a timer that cleans everything up immediately. Signed-off-by: Praveen Chaudhary Signed-off-by: Donald Sharp --- doc/user/zebra.rst | 7 +++++++ zebra/main.c | 48 ++++++++++++++++++++++++++++++-------------- zebra/rib.h | 2 +- zebra/zebra_rib.c | 13 +++++++++++- zebra/zebra_router.h | 7 +++++++ 5 files changed, 60 insertions(+), 17 deletions(-) diff --git a/doc/user/zebra.rst b/doc/user/zebra.rst index f38db9d241..59fcad4cd9 100644 --- a/doc/user/zebra.rst +++ b/doc/user/zebra.rst @@ -27,6 +27,13 @@ Besides the common invocation options (:ref:`common-invocation-options`), the When zebra starts up, don't delete old self inserted routes. +.. option:: -K TIME, --graceful_restart TIME + + If this option is specified, the graceful restart time is TIME seconds. + Zebra, when started, will read in routes. Those routes that Zebra + identifies that it was the originator of will be swept in TIME seconds. + If no time is specified then we will sweep those routes immediately. + .. option:: -r, --retain When program terminates, do not flush routes installed by *zebra* from the diff --git a/zebra/main.c b/zebra/main.c index cff5e06933..af1f6084b3 100644 --- a/zebra/main.c +++ b/zebra/main.c @@ -77,6 +77,8 @@ int allow_delete = 0; /* Don't delete kernel route. */ int keep_kernel_mode = 0; +int graceful_restart; + bool v6_rr_semantics = false; #ifdef HAVE_NETLINK @@ -95,6 +97,7 @@ struct option longopts[] = { {"label_socket", no_argument, NULL, 'l'}, {"retain", no_argument, NULL, 'r'}, {"vrfdefaultname", required_argument, NULL, 'o'}, + {"graceful_restart", required_argument, NULL, 'K'}, #ifdef HAVE_NETLINK {"vrfwnetns", no_argument, NULL, 'n'}, {"nl-bufsize", required_argument, NULL, 's'}, @@ -262,13 +265,14 @@ int main(int argc, char **argv) char *netlink_fuzzing = NULL; #endif /* HANDLE_NETLINK_FUZZING */ + graceful_restart = 0; vrf_configure_backend(VRF_BACKEND_VRF_LITE); logicalrouter_configure_backend(LOGICALROUTER_BACKEND_NETNS); frr_preinit(&zebra_di, argc, argv); frr_opt_add( - "bakz:e:l:o:r" + "bakz:e:l:o:rK:" #ifdef HAVE_NETLINK "s:n" #endif @@ -280,24 +284,25 @@ int main(int argc, char **argv) #endif /* HANDLE_NETLINK_FUZZING */ , longopts, - " -b, --batch Runs in batch mode\n" - " -a, --allow_delete Allow other processes to delete zebra routes\n" - " -z, --socket Set path of zebra socket\n" - " -e, --ecmp Specify ECMP to use.\n" - " -l, --label_socket Socket to external label manager\n" - " -k, --keep_kernel Don't delete old routes which were installed by zebra.\n" - " -r, --retain When program terminates, retain added route by zebra.\n" - " -o, --vrfdefaultname Set default VRF name.\n" + " -b, --batch Runs in batch mode\n" + " -a, --allow_delete Allow other processes to delete zebra routes\n" + " -z, --socket Set path of zebra socket\n" + " -e, --ecmp Specify ECMP to use.\n" + " -l, --label_socket Socket to external label manager\n" + " -k, --keep_kernel Don't delete old routes which were installed by zebra.\n" + " -r, --retain When program terminates, retain added route by zebra.\n" + " -o, --vrfdefaultname Set default VRF name.\n" + " -K, --graceful_restart Graceful restart at the kernel level, timer in seconds for expiration\n" #ifdef HAVE_NETLINK - " -n, --vrfwnetns Use NetNS as VRF backend\n" - " -s, --nl-bufsize Set netlink receive buffer size\n" - " --v6-rr-semantics Use v6 RR semantics\n" + " -n, --vrfwnetns Use NetNS as VRF backend\n" + " -s, --nl-bufsize Set netlink receive buffer size\n" + " --v6-rr-semantics Use v6 RR semantics\n" #endif /* HAVE_NETLINK */ #if defined(HANDLE_ZAPI_FUZZING) - " -c Bypass normal startup and use this file for testing of zapi\n" + " -c Bypass normal startup and use this file for testing of zapi\n" #endif /* HANDLE_ZAPI_FUZZING */ #if defined(HANDLE_NETLINK_FUZZING) - " -w Bypass normal startup and use this file for testing of netlink input\n" + " -w Bypass normal startup and use this file for testing of netlink input\n" #endif /* HANDLE_NETLINK_FUZZING */ ); @@ -317,6 +322,10 @@ int main(int argc, char **argv) allow_delete = 1; break; case 'k': + if (graceful_restart) { + zlog_err("Graceful Restart initiated, we cannot keep the existing kernel routes"); + return 1; + } keep_kernel_mode = 1; break; case 'e': @@ -348,6 +357,13 @@ int main(int argc, char **argv) case 'r': retain_mode = 1; break; + case 'K': + if (keep_kernel_mode) { + zlog_err("Keep Kernel mode specified, graceful restart incompatible"); + return 1; + } + graceful_restart = atoi(optarg); + break; #ifdef HAVE_NETLINK case 's': nl_rcvbufsize = atoi(optarg); @@ -435,8 +451,10 @@ int main(int argc, char **argv) * will be equal to the current getpid(). To know about such routes, * we have to have route_read() called before. */ + zrouter.startup_time = monotime(NULL); if (!keep_kernel_mode) - rib_sweep_route(); + thread_add_timer(zrouter.master, rib_sweep_route, + NULL, graceful_restart, NULL); /* Needed for BSD routing socket. */ pid = getpid(); diff --git a/zebra/rib.h b/zebra/rib.h index ca0801c209..0353c9bb99 100644 --- a/zebra/rib.h +++ b/zebra/rib.h @@ -400,7 +400,7 @@ extern struct route_entry *rib_lookup_ipv4(struct prefix_ipv4 *p, extern void rib_update(vrf_id_t vrf_id, rib_update_event_t event); extern void rib_update_table(struct route_table *table, rib_update_event_t event); -extern void rib_sweep_route(void); +extern int rib_sweep_route(struct thread *t); extern void rib_sweep_table(struct route_table *table); extern void rib_close_table(struct route_table *table); extern void rib_init(void); diff --git a/zebra/zebra_rib.c b/zebra/zebra_rib.c index 8f27316669..391917ec68 100644 --- a/zebra/zebra_rib.c +++ b/zebra/zebra_rib.c @@ -3127,6 +3127,7 @@ void rib_sweep_table(struct route_table *table) for (rn = route_top(table); rn; rn = srcdest_route_next(rn)) { RNODE_FOREACH_RE_SAFE (rn, re, next) { + if (IS_ZEBRA_DEBUG_RIB) route_entry_dump(&rn->p, NULL, re); @@ -3136,6 +3137,14 @@ void rib_sweep_table(struct route_table *table) if (!CHECK_FLAG(re->flags, ZEBRA_FLAG_SELFROUTE)) continue; + /* + * If routes are older than startup_time then + * we know we read them in from the kernel. + * As such we can safely remove them. + */ + if (zrouter.startup_time < re->uptime) + continue; + /* * So we are starting up and have received * routes from the kernel that we have installed @@ -3165,7 +3174,7 @@ void rib_sweep_table(struct route_table *table) } /* Sweep all RIB tables. */ -void rib_sweep_route(void) +int rib_sweep_route(struct thread *t) { struct vrf *vrf; struct zebra_vrf *zvrf; @@ -3179,6 +3188,8 @@ void rib_sweep_route(void) } zebra_router_sweep_route(); + + return 0; } /* Remove specific by protocol routes from 'table'. */ diff --git a/zebra/zebra_router.h b/zebra/zebra_router.h index b3def297ac..6c9f3a0f28 100644 --- a/zebra/zebra_router.h +++ b/zebra/zebra_router.h @@ -112,8 +112,15 @@ struct zebra_router { struct zebra_vrf *evpn_vrf; uint32_t multipath_num; + + /* + * Time for when we sweep the rib from old routes + */ + time_t startup_time; }; +#define GRACEFUL_RESTART_TIME 60 + extern struct zebra_router zrouter; extern void zebra_router_init(void); From 33656d2db287fd293be0cfc59885e2b8ebc2ecad Mon Sep 17 00:00:00 2001 From: Donald Sharp Date: Sat, 11 May 2019 03:50:11 -0400 Subject: [PATCH 2/2] doc, zebra: Remove `keep_kernel_mode` from zebra This code doees this: a) Imagine ospf installs a route into zebra. Zebra crashes and we restart FRR. If we are using the -k option on zebra than all routes are re-read in, including this OSPF route. b) Now imagine at the same time that zebra is starting backup ospf on a different router looses a link to the this route. c) Since zebra was run with -k this OSPF route is read back in but never replaced and we now have a route pointing out an interface to other routers that cannot handle it. We should never allow users to implement bad options from zebra's perspective that allow them to put themselves into a clear problem state and additionally we have *absolutely* no mechanism to ever fix that broken route without special human interaction. Signed-off-by: Donald Sharp --- doc/user/zebra.rst | 4 ---- zebra/main.c | 22 +++------------------- 2 files changed, 3 insertions(+), 23 deletions(-) diff --git a/doc/user/zebra.rst b/doc/user/zebra.rst index 59fcad4cd9..40d8949297 100644 --- a/doc/user/zebra.rst +++ b/doc/user/zebra.rst @@ -23,10 +23,6 @@ Besides the common invocation options (:ref:`common-invocation-options`), the Runs in batch mode. *zebra* parses configuration file and terminates immediately. -.. option:: -k, --keep_kernel - - When zebra starts up, don't delete old self inserted routes. - .. option:: -K TIME, --graceful_restart TIME If this option is specified, the graceful restart time is TIME seconds. diff --git a/zebra/main.c b/zebra/main.c index af1f6084b3..5797a5846a 100644 --- a/zebra/main.c +++ b/zebra/main.c @@ -74,9 +74,6 @@ int retain_mode = 0; /* Allow non-quagga entities to delete quagga routes */ int allow_delete = 0; -/* Don't delete kernel route. */ -int keep_kernel_mode = 0; - int graceful_restart; bool v6_rr_semantics = false; @@ -272,7 +269,7 @@ int main(int argc, char **argv) frr_preinit(&zebra_di, argc, argv); frr_opt_add( - "bakz:e:l:o:rK:" + "baz:e:l:o:rK:" #ifdef HAVE_NETLINK "s:n" #endif @@ -289,7 +286,6 @@ int main(int argc, char **argv) " -z, --socket Set path of zebra socket\n" " -e, --ecmp Specify ECMP to use.\n" " -l, --label_socket Socket to external label manager\n" - " -k, --keep_kernel Don't delete old routes which were installed by zebra.\n" " -r, --retain When program terminates, retain added route by zebra.\n" " -o, --vrfdefaultname Set default VRF name.\n" " -K, --graceful_restart Graceful restart at the kernel level, timer in seconds for expiration\n" @@ -321,13 +317,6 @@ int main(int argc, char **argv) case 'a': allow_delete = 1; break; - case 'k': - if (graceful_restart) { - zlog_err("Graceful Restart initiated, we cannot keep the existing kernel routes"); - return 1; - } - keep_kernel_mode = 1; - break; case 'e': zrouter.multipath_num = atoi(optarg); if (zrouter.multipath_num > MULTIPATH_NUM @@ -358,10 +347,6 @@ int main(int argc, char **argv) retain_mode = 1; break; case 'K': - if (keep_kernel_mode) { - zlog_err("Keep Kernel mode specified, graceful restart incompatible"); - return 1; - } graceful_restart = atoi(optarg); break; #ifdef HAVE_NETLINK @@ -452,9 +437,8 @@ int main(int argc, char **argv) * we have to have route_read() called before. */ zrouter.startup_time = monotime(NULL); - if (!keep_kernel_mode) - thread_add_timer(zrouter.master, rib_sweep_route, - NULL, graceful_restart, NULL); + thread_add_timer(zrouter.master, rib_sweep_route, + NULL, graceful_restart, NULL); /* Needed for BSD routing socket. */ pid = getpid();