From f4c14c3a80d690f3fd589a04844c8355a207629e Mon Sep 17 00:00:00 2001
From: Martin Winter <mwinter@opensourcerouting.org>
Date: Wed, 12 Apr 2017 16:00:43 -0700
Subject: [PATCH 01/19] snapcraft: Improve README.usage.md based on feedback
 received

- Fix snap connect (it's now called core, not ubuntu-core)
- Add section on MPLS configuration
- Add FAQ topic on ospfd/ospf6d crashing before privs are assigned to snap
- Add pointer to official webpage

Signed-off-by: Martin Winter <mwinter@opensourcerouting.org>
---
 snapcraft/README.usage.md | 59 +++++++++++++++++++++++++++++++++++----
 1 file changed, 54 insertions(+), 5 deletions(-)

diff --git a/snapcraft/README.usage.md b/snapcraft/README.usage.md
index aaff59438a..c678c8805c 100644
--- a/snapcraft/README.usage.md
+++ b/snapcraft/README.usage.md
@@ -3,7 +3,7 @@ Using the FRRouting Snap
 
 After installing the Snap, the priviledged plug need to be connected:
 
-    snap connect frr:network-control ubuntu-core:network-control
+    snap connect frr:network-control core:network-control
 
 Enabling/Disabling FRRouting Daemons
 -------------------------------------------
@@ -53,25 +53,74 @@ depend on them). These are mainly intended to debug the Snap
 - `frr.ldpd-debug`:
     Starts ldpd daemon in foreground
 
+MPLS (LDP)
+----------
+The MPLS forwarding requires a Linux Kernel version 4.5 or newer and
+specific MPLS kernel modules loaded. It will be auto-detected by
+FRR. You can check the detected setup with the `show mpls status`
+command from within `frr.vtysh`
+
+The following kernel modules `mpls-router` and `mpls-iptunnel`
+need to be loaded. On Ubuntu 16.04, this can be done by editing 
+'/etc/modules-load.d/modules.conf' and add the following lines:
+
+	# Load MPLS Kernel Modules
+	mpls-router
+	mpls-iptunnel
+
+For other distributions, please check the documentation on loading
+modules. You need to either reboot or use `modprobe` to manually load
+the modules as well before MPLS will be available.
+
+In addition to this, the MPLS Label-Processing needs to be enabled
+with `sysctl` on the required interfaces. Assuming the interfaces
+are named `eth0`, `eth1` and `eth2`, then the additional lines in
+`/etc/sysctl.conf` will enable it on a Ubuntu 16.04 system:
+
+	# Enable MPLS Label processing on all interfaces
+	net.mpls.conf.eth0.input=1
+	net.mpls.conf.eth1.input=1
+	net.mpls.conf.eth2.input=1
+	net.mpls.platform_labels=100000
+
+These settings require either a reboot or a manual configuration with
+`sysctl` as well.
+
 FAQ
 ---
 - frr.vtysh displays `--MORE--` on long output. How to suppress this?
     - Define `VTYSH_PAGER` to `cat` (default is `more`). (Ie add 
       `export VTYSH_PAGER=cat` to the end of your `.profile`)
 
+- ospfd / ospf6d are not running after installation
+	- Installing a new snap starts the daemons, but at this time they
+	  may not have the required priviledged access. Make sure you 
+	  issue the `snap connect` command as given above (can be verified
+	  with `snap interfaces`) and **THEN** restart the daemons (or
+	  reboot the system). 
+	  This is a limitation of any snap package at this time which
+	  requires priviledged interfaces (ie to manipulate routing tables)
+	
 Sourcecode available
 ====================
 
 The source for this SNAP is available as part of the FRRouting
-Source Code Distribution. 
+Source Code Distribution under `GPLv2 or later`
 
-    https://github.com/frrouting/frr.git
+<https://github.com/frrouting/frr.git>
 
-Instructions for rebuilding the snap are in `README.snap_build.md`
+Instructions for rebuilding the snap are in `snapcraft/README.snap_build.md`
+
+*Please checkout the desired branch before following the instructions
+as they may have changed between versions of FRR*
+
+Official Webpage for FRR
+========================
+
+Official webpage for FRR is at <https://www.frrouting.org/>
 
 Feedback welcome
 ================
 
 Please send Feedback about this snap to Martin Winter at 
 `mwinter@opensourcerouting.org`
-

From 6f2a494a8304a8480896de0cfba5b07c15316330 Mon Sep 17 00:00:00 2001
From: Martin Winter <mwinter@opensourcerouting.org>
Date: Thu, 20 Apr 2017 16:58:24 -0700
Subject: [PATCH 02/19] snapcraft: Update snapcraft requirements for FRR 3.0
 (from 2.0)

Signed-off-by: Martin Winter <mwinter@opensourcerouting.org>
---
 snapcraft/snapcraft.yaml.in | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/snapcraft/snapcraft.yaml.in b/snapcraft/snapcraft.yaml.in
index a20a1d0392..d0a421c62b 100644
--- a/snapcraft/snapcraft.yaml.in
+++ b/snapcraft/snapcraft.yaml.in
@@ -148,7 +148,6 @@ parts:
            - gawk
            - libreadline-dev
            - texinfo
-           - dejagnu
            - libncurses5-dev
            - texlive-latex-base
            - texlive-generic-recommended
@@ -161,6 +160,10 @@ parts:
            - chrpath
            - pkg-config
            - libjson-c-dev
+           - libc-ares-dev
+           - bison
+           - flex
+           - python3-dev
         stage-packages:
            - coreutils
            - iproute2

From 5ae4018e5d536c4ca7a34358897e2b7feef64954 Mon Sep 17 00:00:00 2001
From: Martin Winter <mwinter@opensourcerouting.org>
Date: Thu, 20 Apr 2017 17:24:12 -0700
Subject: [PATCH 03/19] snapcraft: Fix another old leftover of ubuntu-core in
 build doc

Signed-off-by: Martin Winter <mwinter@opensourcerouting.org>
---
 snapcraft/README.snap_build.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/snapcraft/README.snap_build.md b/snapcraft/README.snap_build.md
index c4db51bd6a..e1f2ce82da 100644
--- a/snapcraft/README.snap_build.md
+++ b/snapcraft/README.snap_build.md
@@ -47,8 +47,10 @@ Installing the snap
 
     Connect the priviledged `network-control` plug to the snap:
 
-        snap connect frr:network-control ubuntu-core:network-control
+        snap connect frr:network-control core:network-control
 
+See README.usage.md for more details on setting up and using the snap
+        
 DONE.
 
 The Snap will be auto-started and running. 

From ae4989487412cc455bfed48a6ceab63423ba5f08 Mon Sep 17 00:00:00 2001
From: Martin Winter <mwinter@opensourcerouting.org>
Date: Thu, 20 Apr 2017 18:18:40 -0700
Subject: [PATCH 04/19] snapcraft: Add nhrpd to snap

Signed-off-by: Martin Winter <mwinter@opensourcerouting.org>
---
 snapcraft/defaults/nhrpd.conf.default |  0
 snapcraft/scripts/Makefile            |  2 +-
 snapcraft/scripts/nhrpd-service       | 12 ++++++++++++
 snapcraft/snapcraft.yaml.in           | 18 ++++++++++++++++--
 4 files changed, 29 insertions(+), 3 deletions(-)
 create mode 100644 snapcraft/defaults/nhrpd.conf.default
 create mode 100644 snapcraft/scripts/nhrpd-service

diff --git a/snapcraft/defaults/nhrpd.conf.default b/snapcraft/defaults/nhrpd.conf.default
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/snapcraft/scripts/Makefile b/snapcraft/scripts/Makefile
index 00984c2afa..0f9a24b649 100644
--- a/snapcraft/scripts/Makefile
+++ b/snapcraft/scripts/Makefile
@@ -11,4 +11,4 @@ install:
 	install -D -m 0755 isisd-service $(DESTDIR)/bin/
 	install -D -m 0755 pimd-service $(DESTDIR)/bin/
 	install -D -m 0755 ldpd-service $(DESTDIR)/bin/
-
+	install -D -m 0755 nhrpd-service $(DESTDIR)/bin/
diff --git a/snapcraft/scripts/nhrpd-service b/snapcraft/scripts/nhrpd-service
new file mode 100644
index 0000000000..a3ba5e3fe9
--- /dev/null
+++ b/snapcraft/scripts/nhrpd-service
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+set -e -x
+
+if ! [ -e $SNAP_DATA/nhrpd.conf ]; then
+    cp $SNAP/etc/frr/nhrpd.conf.default $SNAP_DATA/nhrpd.conf
+fi
+exec $SNAP/sbin/nhrpd \
+    -f $SNAP_DATA/nhrpd.conf \
+    --pid_file $SNAP_DATA/nhrpd.pid \
+    --socket $SNAP_DATA/zsock \
+    --vty_socket $SNAP_DATA
diff --git a/snapcraft/snapcraft.yaml.in b/snapcraft/snapcraft.yaml.in
index d0a421c62b..54a1cfd4f0 100644
--- a/snapcraft/snapcraft.yaml.in
+++ b/snapcraft/snapcraft.yaml.in
@@ -83,6 +83,13 @@ apps:
             - network
             - network-bind
             - network-control
+    nhrpd:
+        command: bin/nhrpd-service
+        daemon: simple
+        plugs:
+            - network
+            - network-bind
+            - network-control
     zebra-debug:
         command: sbin/zebra -f $SNAP_DATA/zebra.conf --pid_file $SNAP_DATA/zebra.pid --socket $SNAP_DATA/zsock --vty_socket $SNAP_DATA
         plugs:
@@ -132,12 +139,18 @@ apps:
             - network-bind
             - network-control
     ldpd-debug:
-        command: sbin/ldpd -f $SNAP_DATA/pimd.conf --pid_file $SNAP_DATA/pimd.pid --socket $SNAP_DATA/zsock --ctl_socket $SNAP_DATA --vty_socket $SNAP_DATA
+        command: sbin/ldpd -f $SNAP_DATA/ldpd.conf --pid_file $SNAP_DATA/ldpd.pid --socket $SNAP_DATA/zsock --ctl_socket $SNAP_DATA --vty_socket $SNAP_DATA
         plugs:
             - network
             - network-bind
             - network-control
-
+    nhrpd-debug:
+        command: sbin/nhrpd -f $SNAP_DATA/nhrpd.conf --pid_file $SNAP_DATA/nhrpd.pid --socket $SNAP_DATA/zsock --vty_socket $SNAP_DATA
+        plugs:
+            - network
+            - network-bind
+            - network-control
+            
 parts:
     frr: 
         build-packages: 
@@ -215,6 +228,7 @@ parts:
             ripd.conf.default: etc/frr/ripd.conf.default
             ripngd.conf.default: etc/frr/ripngd.conf.default
             ldpd.conf.default: etc/frr/ldpd.conf.default
+            nhrpd.conf.default: etc/frr/nhrpd.conf.default
             vtysh.conf.default: etc/frr/vtysh.conf.default
     frr-scripts:
         plugin: make

From 80b4df3b082f5f51afc0add1f2e59dc5822deef1 Mon Sep 17 00:00:00 2001
From: Martin Winter <mwinter@opensourcerouting.org>
Date: Thu, 20 Apr 2017 23:03:03 -0700
Subject: [PATCH 05/19] lib: Add CLI option --moduledir to override default
 module location (needed for snap support)

Signed-off-by: Martin Winter <mwinter@opensourcerouting.org>
---
 lib/libfrr.c | 21 ++++++++++++++++++---
 lib/libfrr.h |  2 ++
 lib/module.c |  6 +++---
 lib/module.h |  2 +-
 4 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/lib/libfrr.c b/lib/libfrr.c
index 64f8be2ca6..de83e14f0e 100644
--- a/lib/libfrr.c
+++ b/lib/libfrr.c
@@ -34,6 +34,7 @@ DEFINE_HOOK(frr_late_init, (struct thread_master *tm), (tm))
 
 const char frr_sysconfdir[] = SYSCONFDIR;
 const char frr_vtydir[] = DAEMON_VTY_DIR;
+const char frr_moduledir[] = MODULE_PATH;
 
 char config_default[256];
 static char pidfile_default[256];
@@ -61,7 +62,8 @@ static void opt_extend(const struct optspec *os)
 }
 
 
-#define OPTION_VTYSOCK 1000
+#define OPTION_VTYSOCK   1000
+#define OPTION_MODULEDIR 1002
 
 static const struct option lo_always[] = {
 	{ "help",        no_argument,       NULL, 'h' },
@@ -69,6 +71,7 @@ static const struct option lo_always[] = {
 	{ "daemon",      no_argument,       NULL, 'd' },
 	{ "module",      no_argument,       NULL, 'M' },
 	{ "vty_socket",  required_argument, NULL, OPTION_VTYSOCK },
+	{ "moduledir",   required_argument, NULL, OPTION_MODULEDIR },
 	{ NULL }
 };
 static const struct optspec os_always = {
@@ -77,7 +80,8 @@ static const struct optspec os_always = {
 	"  -v, --version      Print program version\n"
 	"  -d, --daemon       Runs in daemon mode\n"
 	"  -M, --module       Load specified module\n"
-	"      --vty_socket   Override vty socket path\n",
+	"      --vty_socket   Override vty socket path\n"
+	"      --moduledir    Override modules directory\n",
 	lo_always
 };
 
@@ -193,6 +197,7 @@ struct option_chain {
 	struct option_chain *next;
 	const char *arg;
 };
+
 static struct option_chain *modules = NULL, **modnext = &modules;
 static int errors = 0;
 
@@ -277,6 +282,14 @@ static int frr_opt(int opt)
 		}
 		di->vty_sock_path = optarg;
 		break;
+	case OPTION_MODULEDIR:
+		if (di->module_path) {
+			fprintf(stderr, "----moduledir option specified more than once!\n");
+			errors++;
+			break;
+		}
+		di->module_path = optarg;
+		break;
 	case 'u':
 		if (di->flags & FRR_NO_PRIVSEP)
 			return 1;
@@ -319,6 +332,8 @@ struct thread_master *frr_init(void)
 	struct option_chain *oc;
 	struct frrmod_runtime *module;
 	char moderr[256];
+	const char *dir;
+	dir = di->module_path ? di->module_path : frr_moduledir;
 
 	srandom(time(NULL));
 
@@ -331,7 +346,7 @@ struct thread_master *frr_init(void)
 	frrmod_init(di->module);
 	while (modules) {
 		modules = (oc = modules)->next;
-		module = frrmod_load(oc->arg, moderr, sizeof(moderr));
+		module = frrmod_load(oc->arg, dir, moderr, sizeof(moderr));
 		if (!module) {
 			fprintf(stderr, "%s\n", moderr);
 			exit(1);
diff --git a/lib/libfrr.h b/lib/libfrr.h
index a40fc34892..0cc7ad564b 100644
--- a/lib/libfrr.h
+++ b/lib/libfrr.h
@@ -52,6 +52,7 @@ struct frr_daemon_info {
 	const char *config_file;
 	const char *pid_file;
 	const char *vty_path;
+	const char *module_path;
 
 	const char *proghelp;
 	void (*printhelp)(FILE *target);
@@ -107,5 +108,6 @@ extern void frr_run(struct thread_master *master);
 extern char config_default[256];
 extern const char frr_sysconfdir[];
 extern const char frr_vtydir[];
+extern const char frr_moduledir[];
 
 #endif /* _ZEBRA_FRR_H */
diff --git a/lib/module.c b/lib/module.c
index 4ebe3c0da2..c7dd5538cd 100644
--- a/lib/module.c
+++ b/lib/module.c
@@ -69,7 +69,7 @@ void frrmod_init(struct frrmod_runtime *modinfo)
 }
 
 struct frrmod_runtime *frrmod_load(const char *spec,
-		char *err, size_t err_len)
+		const char *dir, char *err, size_t err_len)
 {
 	void *handle = NULL;
 	char name[PATH_MAX], fullpath[PATH_MAX], *args;
@@ -84,12 +84,12 @@ struct frrmod_runtime *frrmod_load(const char *spec,
 	if (!strchr(name, '/')) {
 		if (!handle && execname) {
 			snprintf(fullpath, sizeof(fullpath), "%s/%s_%s.so",
-					MODULE_PATH, execname, name);
+					dir, execname, name);
 			handle = dlopen(fullpath, RTLD_NOW | RTLD_GLOBAL);
 		}
 		if (!handle) {
 			snprintf(fullpath, sizeof(fullpath), "%s/%s.so",
-					MODULE_PATH, name);
+					dir, name);
 			handle = dlopen(fullpath, RTLD_NOW | RTLD_GLOBAL);
 		}
 	}
diff --git a/lib/module.h b/lib/module.h
index cb66e60976..01819f91e1 100644
--- a/lib/module.h
+++ b/lib/module.h
@@ -95,7 +95,7 @@ extern struct frrmod_runtime *frrmod_list;
 
 extern void frrmod_init(struct frrmod_runtime *modinfo);
 extern struct frrmod_runtime *frrmod_load(const char *spec,
-		char *err, size_t err_len);
+		const char *dir, char *err, size_t err_len);
 #if 0
 /* not implemented yet */
 extern void frrmod_unload(struct frrmod_runtime *module);

From d851b2fc0d872e461b3c90056f0a1b551c0e4cf6 Mon Sep 17 00:00:00 2001
From: Martin Winter <mwinter@opensourcerouting.org>
Date: Fri, 21 Apr 2017 02:49:39 -0700
Subject: [PATCH 06/19] snapcraft: Add FPM module with command to set mode or
 disable it

- New snap command:
    frr.set fpm {disable | protobuf | netlink}

Signed-off-by: Martin Winter <mwinter@opensourcerouting.org>
---
 snapcraft/README.usage.md       | 16 +++++++++++++
 snapcraft/scripts/Makefile      |  1 +
 snapcraft/scripts/set-options   | 40 +++++++++++++++++++++++++++++++++
 snapcraft/scripts/zebra-service |  8 +++++--
 snapcraft/snapcraft.yaml.in     |  5 +++++
 5 files changed, 68 insertions(+), 2 deletions(-)
 create mode 100755 snapcraft/scripts/set-options

diff --git a/snapcraft/README.usage.md b/snapcraft/README.usage.md
index c678c8805c..537147303a 100644
--- a/snapcraft/README.usage.md
+++ b/snapcraft/README.usage.md
@@ -30,6 +30,8 @@ Commands defined by this snap
 	options
 - `frr.readme`:
 	Returns this document `cat README_usage.md`
+- `frr.set`:
+	Allows to enable `FPM` module. See FPM section below
 
 and for debugging defined at this time (May get removed later - do not 
 depend on them). These are mainly intended to debug the Snap
@@ -86,6 +88,20 @@ are named `eth0`, `eth1` and `eth2`, then the additional lines in
 These settings require either a reboot or a manual configuration with
 `sysctl` as well.
 
+FPM Module
+----------
+The `frr.set` allows to turn FPM module on or off.
+
+    frr.set fpm {disable|protobuf|netlink}
+    
+    Disables FPM or enables FPM with selected mode
+
+By default, the FPM module is disabled, but installed with netlink and
+protobuf support. To enable the FPM module, use the `frr.set fpm protobuf`
+or `frr.set fpm netlink` command. The command will only enable the mode
+for the next restart of zebra. Please reboot or restart zebra after
+changing the mode to become effective.
+
 FAQ
 ---
 - frr.vtysh displays `--MORE--` on long output. How to suppress this?
diff --git a/snapcraft/scripts/Makefile b/snapcraft/scripts/Makefile
index 0f9a24b649..2a60d4d652 100644
--- a/snapcraft/scripts/Makefile
+++ b/snapcraft/scripts/Makefile
@@ -12,3 +12,4 @@ install:
 	install -D -m 0755 pimd-service $(DESTDIR)/bin/
 	install -D -m 0755 ldpd-service $(DESTDIR)/bin/
 	install -D -m 0755 nhrpd-service $(DESTDIR)/bin/
+	install -D -m 0755 set-options $(DESTDIR)/bin/
diff --git a/snapcraft/scripts/set-options b/snapcraft/scripts/set-options
new file mode 100755
index 0000000000..e76e321ef8
--- /dev/null
+++ b/snapcraft/scripts/set-options
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+set -e
+
+case $1 in
+    fpm) 
+        case $2 in
+            disable)
+                rm -f $SNAP_DATA/fpm.conf
+                echo "FPM module disabled. Please restart FRR"
+                ;;
+            protobuf)
+                echo "-M fpm:protobuf" > $SNAP_DATA/fpm.conf
+                echo "FPM enabled and set to protobuf mode. Please restart FRR"
+                ;;
+            netlink)
+                echo "-M fpm:netlink" > $SNAP_DATA/fpm.conf
+                echo "FPM enabled and set to netlink mode. Please restart FRR"
+                ;;
+            *)
+                echo "Usage:"
+                echo "    ${SNAP_NAME}.set fpm {disable|protobuf|netlink}"
+                echo ""
+                echo "    Disables FPM module or enables it with specified mode"
+                echo "    Mode will be saved for next restart of zebra, but zebra"
+                echo "    is not automatically restarted"               
+                exit 1
+                ;;
+        esac
+        ;;
+    *)
+        echo "Usage:"
+        echo "    ${SNAP_NAME}.set fpm {disable|protobuf|netlink}"
+        echo ""
+        echo "    Disables FPM or enables FPM with selected mode"
+        exit 1
+        ;;
+esac
+
+exit 0
diff --git a/snapcraft/scripts/zebra-service b/snapcraft/scripts/zebra-service
index 9119a4055d..2ee131f9e5 100644
--- a/snapcraft/scripts/zebra-service
+++ b/snapcraft/scripts/zebra-service
@@ -8,9 +8,13 @@ fi
 if ! [ -e $SNAP_DATA/vtysh.conf ]; then
     cp $SNAP/etc/frr/vtysh.conf.default $SNAP_DATA/vtysh.conf
 fi
+EXTRA_OPTIONS=""
+if [ -e $SNAP_DATA/fpm.conf ]; then
+	EXTRA_OPTIONS="`cat $SNAP_DATA/fpm.conf`"
+fi
 exec $SNAP/sbin/zebra \
     -f $SNAP_DATA/zebra.conf \
     --pid_file $SNAP_DATA/zebra.pid \
     --socket $SNAP_DATA/zsock \
-    --vty_socket $SNAP_DATA
-
+    --vty_socket $SNAP_DATA \
+    --moduledir $SNAP/lib/frr/modules $EXTRA_OPTIONS
diff --git a/snapcraft/snapcraft.yaml.in b/snapcraft/snapcraft.yaml.in
index 54a1cfd4f0..b5fb24a361 100644
--- a/snapcraft/snapcraft.yaml.in
+++ b/snapcraft/snapcraft.yaml.in
@@ -90,6 +90,8 @@ apps:
             - network
             - network-bind
             - network-control
+    set:
+        command: bin/set-options
     zebra-debug:
         command: sbin/zebra -f $SNAP_DATA/zebra.conf --pid_file $SNAP_DATA/zebra.pid --socket $SNAP_DATA/zsock --vty_socket $SNAP_DATA
         plugs:
@@ -177,6 +179,7 @@ parts:
            - bison
            - flex
            - python3-dev
+           - protobuf-c-compiler
         stage-packages:
            - coreutils
            - iproute2
@@ -208,6 +211,8 @@ parts:
             - --enable-group=root
             - --enable-pimd
             - --enable-ldpd
+            - --enable-fpm
+            - --enable-protobuf
             - --enable-configfile-mask=0640
             - --enable-logfile-mask=0640 
             - --localstatedir=/var/run

From 02f3b0512ad45f7519480fb0a32e9aded1f722ea Mon Sep 17 00:00:00 2001
From: Phil Huang <phil_huang@edge-core.com>
Date: Thu, 13 Apr 2017 18:11:28 +0800
Subject: [PATCH 07/19] Add user `frr` into group `frrvty`

Signed-off-by: Phil Huang <phil_huang@edge-core.com>
---
 doc/Building_FRR_on_Debian8.md    | 8 ++++----
 doc/Building_FRR_on_Ubuntu1204.md | 5 +++--
 doc/Building_FRR_on_Ubuntu1404.md | 5 +++--
 doc/Building_FRR_on_Ubuntu1604.md | 5 +++--
 4 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/doc/Building_FRR_on_Debian8.md b/doc/Building_FRR_on_Debian8.md
index b902033d5e..1e57c1bea1 100644
--- a/doc/Building_FRR_on_Debian8.md
+++ b/doc/Building_FRR_on_Debian8.md
@@ -31,9 +31,9 @@ any packages**
 
     sudo addgroup --system --gid 92 frr
     sudo addgroup --system --gid 85 frrvty
-    sudo adduser --system --ingroup frr --groups frrvty --home /var/run/frr/ \
-       --gecos "FRR FRRouting suite" --shell /bin/false frr
-    sudo usermode
+    sudo adduser --system --ingroup frr --home /var/run/frr/ \
+       --gecos "FRR suite" --shell /bin/false frr
+    sudo usermod -a -G frrvty frr
 
 ### Download Source, configure and compile it
 (You may prefer different options on configure statement. These are just
@@ -94,4 +94,4 @@ other settings)
     #  based on Router Advertisements for this host
     net.ipv6.conf.all.forwarding=1
 
-**Reboot** or use `sysctl` to apply the same config to the running system
+**Reboot** or use `sysctl -p` to apply the same config to the running system
diff --git a/doc/Building_FRR_on_Ubuntu1204.md b/doc/Building_FRR_on_Ubuntu1204.md
index 82404097c5..aa6cdcea7b 100644
--- a/doc/Building_FRR_on_Ubuntu1204.md
+++ b/doc/Building_FRR_on_Ubuntu1204.md
@@ -65,8 +65,9 @@ any packages**
 
     sudo groupadd -g 92 frr
     sudo groupadd -r -g 85 frrvty
-    sudo adduser --system --ingroup frr --groups frrvty --home /var/run/frr/ \
+    sudo adduser --system --ingroup frr --home /var/run/frr/ \
        --gecos "FRR suite" --shell /sbin/nologin frr
+    sudo usermod -a -G frrvty frr
 
 ### Download Source, configure and compile it
 (You may prefer different options on configure statement. These are just
@@ -131,4 +132,4 @@ other settings)
     #  based on Router Advertisements for this host
     net.ipv6.conf.all.forwarding=1
 
-**Reboot** or use `sysctl` to apply the same config to the running system
+**Reboot** or use `sysctl -p` to apply the same config to the running system
diff --git a/doc/Building_FRR_on_Ubuntu1404.md b/doc/Building_FRR_on_Ubuntu1404.md
index 0d7c6f76fe..1f1caf4a58 100644
--- a/doc/Building_FRR_on_Ubuntu1404.md
+++ b/doc/Building_FRR_on_Ubuntu1404.md
@@ -25,8 +25,9 @@ any packages**
 
     sudo groupadd -g 92 frr
     sudo groupadd -r -g 85 frrvty
-    sudo adduser --system --ingroup frr --groups frrvty --home /var/run/frr/ \
+    sudo adduser --system --ingroup frr --home /var/run/frr/ \
        --gecos "FRR suite" --shell /sbin/nologin frr
+    sudo usermod -a -G frrvty frr
 
 ### Download Source, configure and compile it
 (You may prefer different options on configure statement. These are just 
@@ -90,4 +91,4 @@ other settings)
     #  based on Router Advertisements for this host
     net.ipv6.conf.all.forwarding=1
 
-**Reboot** or use `sysctl` to apply the same config to the running system
+**Reboot** or use `sysctl -p` to apply the same config to the running system
diff --git a/doc/Building_FRR_on_Ubuntu1604.md b/doc/Building_FRR_on_Ubuntu1604.md
index b6e18088f0..ace5f2b7f7 100644
--- a/doc/Building_FRR_on_Ubuntu1604.md
+++ b/doc/Building_FRR_on_Ubuntu1604.md
@@ -26,8 +26,9 @@ any packages**
 
     sudo groupadd -g 92 frr
     sudo groupadd -r -g 85 frrvty
-    sudo adduser --system --ingroup frr --groups frrvty --home /var/run/frr/ \
+    sudo adduser --system --ingroup frr --home /var/run/frr/ \
        --gecos "FRR suite" --shell /sbin/nologin frr
+    sudo usermod -a -G frrvty frr
 
 ### Download Source, configure and compile it
 (You may prefer different options on configure statement. These are just 
@@ -111,4 +112,4 @@ Add the following lines to `/etc/modules-load.d/modules.conf`:
     mpls-router
     mpls-iptunnel
 
-**Reboot** or use `sysctl` to apply the same config to the running system
+**Reboot** or use `sysctl -p` to apply the same config to the running system

From aa1322f9f66c7e461267ec433823c602afb52d53 Mon Sep 17 00:00:00 2001
From: Phil Huang <phil_huang@edge-core.com>
Date: Thu, 13 Apr 2017 21:20:53 +0800
Subject: [PATCH 08/19] Clean installation guide with Debian/Ubuntu

Signed-off-by: Phil Huang <phil_huang@edge-core.com>
---
 doc/Building_FRR_on_Debian8.md    |  3 +++
 doc/Building_FRR_on_Ubuntu1204.md | 28 ++++++++++++----------------
 doc/Building_FRR_on_Ubuntu1404.md | 28 +++++++++++++---------------
 doc/Building_FRR_on_Ubuntu1604.md | 28 ++++++++++++----------------
 4 files changed, 40 insertions(+), 47 deletions(-)

diff --git a/doc/Building_FRR_on_Debian8.md b/doc/Building_FRR_on_Debian8.md
index 1e57c1bea1..635f9680b6 100644
--- a/doc/Building_FRR_on_Debian8.md
+++ b/doc/Building_FRR_on_Debian8.md
@@ -62,6 +62,7 @@ an example.)
         --enable-rtadv \
         --enable-tcp-zebra \
         --enable-fpm \
+        --enable-ldpd \
         --with-pkg-git-version \
         --with-pkg-extra-version=-MyOwnFRRVersion   
     make
@@ -69,6 +70,7 @@ an example.)
     sudo make install
 
 ### Create empty FRR configuration files
+
     sudo install -m 755 -o frr -g frr -d /var/log/frr
     sudo install -m 775 -o frr -g frrvty -d /etc/frr
     sudo install -m 640 -o frr -g frr /dev/null /etc/frr/zebra.conf
@@ -79,6 +81,7 @@ an example.)
     sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ripd.conf
     sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ripngd.conf
     sudo install -m 640 -o frr -g frr /dev/null /etc/frr/pimd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ldpd.conf
     sudo install -m 640 -o frr -g frrvty /dev/null /etc/frr/vtysh.conf
 
 ### Enable IP & IPv6 forwarding
diff --git a/doc/Building_FRR_on_Ubuntu1204.md b/doc/Building_FRR_on_Ubuntu1204.md
index aa6cdcea7b..cafed0fb31 100644
--- a/doc/Building_FRR_on_Ubuntu1204.md
+++ b/doc/Building_FRR_on_Ubuntu1204.md
@@ -102,22 +102,18 @@ an example.)
 
 ### Create empty FRR configuration files
 
-    sudo mkdir /var/log/frr
-    sudo chown frr:fee /var/log/frr
-    sudo mkdir /etc/frr
-    sudo touch /etc/frr/etc/zebra.conf
-    sudo touch /etc/frr/etc/bgpd.conf
-    sudo touch /etc/frr/etc/ospfd.conf
-    sudo touch /etc/frr/etc/ospf6d.conf
-    sudo touch /etc/frr/etc/isisd.conf
-    sudo touch /etc/frr/etc/ripd.conf
-    sudo touch /etc/frr/etc/ripngd.conf
-    sudo touch /etc/frr/etc/pimd.conf
-    sudo touch /etc/frr/etc/ldpd.conf
-    sudo chown frr:frr /etc/frr/
-    sudo touch /etc/frr/etc/vtysh.conf
-    sudo chown frr:frrvty /etc/frr/etc/vtysh.conf
-    sudo chmod 640 /etc/frr/*.conf
+    sudo install -m 755 -o frr -g frr -d /var/log/frr
+    sudo install -m 775 -o frr -g frrvty -d /etc/frr
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/zebra.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/bgpd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ospfd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ospf6d.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/isisd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ripd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ripngd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/pimd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ldpd.conf
+    sudo install -m 640 -o frr -g frrvty /dev/null /etc/frr/vtysh.conf
 
 ### Enable IP & IPv6 forwarding
 
diff --git a/doc/Building_FRR_on_Ubuntu1404.md b/doc/Building_FRR_on_Ubuntu1404.md
index 1f1caf4a58..1524dd7778 100644
--- a/doc/Building_FRR_on_Ubuntu1404.md
+++ b/doc/Building_FRR_on_Ubuntu1404.md
@@ -54,6 +54,7 @@ an example.)
         --enable-rtadv \
         --enable-tcp-zebra \
         --enable-fpm \
+        --enable-ldpd \
         --with-pkg-git-version \
         --with-pkg-extra-version=-MyOwnFRRVersion   
     make
@@ -62,21 +63,18 @@ an example.)
 
 ### Create empty FRR configuration files
 
-    sudo mkdir /var/log/frr
-    sudo chown frr:fee /var/log/frr
-    sudo mkdir /etc/frr
-    sudo touch /etc/frr/etc/zebra.conf
-    sudo touch /etc/frr/etc/bgpd.conf
-    sudo touch /etc/frr/etc/ospfd.conf
-    sudo touch /etc/frr/etc/ospf6d.conf
-    sudo touch /etc/frr/etc/isisd.conf
-    sudo touch /etc/frr/etc/ripd.conf
-    sudo touch /etc/frr/etc/ripngd.conf
-    sudo touch /etc/frr/etc/pimd.conf
-    sudo chown frr:frr /etc/frr/
-    sudo touch /etc/frr/etc/vtysh.conf
-    sudo chown frr:frrvty /etc/frr/etc/vtysh.conf
-    sudo chmod 640 /etc/frr/*.conf
+    sudo install -m 755 -o frr -g frr -d /var/log/frr
+    sudo install -m 775 -o frr -g frrvty -d /etc/frr
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/zebra.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/bgpd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ospfd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ospf6d.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/isisd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ripd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ripngd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/pimd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ldpd.conf
+    sudo install -m 640 -o frr -g frrvty /dev/null /etc/frr/vtysh.conf
 
 ### Enable IP & IPv6 forwarding
 
diff --git a/doc/Building_FRR_on_Ubuntu1604.md b/doc/Building_FRR_on_Ubuntu1604.md
index ace5f2b7f7..ed599a2551 100644
--- a/doc/Building_FRR_on_Ubuntu1604.md
+++ b/doc/Building_FRR_on_Ubuntu1604.md
@@ -63,22 +63,18 @@ an example.)
 
 ### Create empty FRR configuration files
 
-    sudo mkdir /var/log/frr
-    sudo chown frr:fee /var/log/frr
-    sudo mkdir /etc/frr
-    sudo touch /etc/frr/etc/zebra.conf
-    sudo touch /etc/frr/etc/bgpd.conf
-    sudo touch /etc/frr/etc/ospfd.conf
-    sudo touch /etc/frr/etc/ospf6d.conf
-    sudo touch /etc/frr/etc/isisd.conf
-    sudo touch /etc/frr/etc/ripd.conf
-    sudo touch /etc/frr/etc/ripngd.conf
-    sudo touch /etc/frr/etc/pimd.conf
-    sudo touch /etc/frr/etc/ldpd.conf
-    sudo chown frr:frr /etc/frr/
-    sudo touch /etc/frr/etc/vtysh.conf
-    sudo chown frr:frrvty /etc/frr/etc/vtysh.conf
-    sudo chmod 640 /etc/frr/*.conf
+    sudo install -m 755 -o frr -g frr -d /var/log/frr
+    sudo install -m 775 -o frr -g frrvty -d /etc/frr
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/zebra.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/bgpd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ospfd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ospf6d.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/isisd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ripd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ripngd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/pimd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ldpd.conf
+    sudo install -m 640 -o frr -g frrvty /dev/null /etc/frr/vtysh.conf
 
 ### Enable IP & IPv6 forwarding
 

From 97b02007935d00ad8f9126045922ccc8173296ee Mon Sep 17 00:00:00 2001
From: Martin Winter <mwinter@opensourcerouting.org>
Date: Fri, 21 Apr 2017 03:55:14 -0700
Subject: [PATCH 09/19] doc: Add nhrpd to relevant section in
 Building_FRR_on_xxxx doc

Signed-off-by: Martin Winter <mwinter@opensourcerouting.org>
---
 doc/Building_FRR_on_Debian8.md    | 1 +
 doc/Building_FRR_on_Fedora24.md   | 3 ++-
 doc/Building_FRR_on_OpenBSD6.md   | 1 +
 doc/Building_FRR_on_Ubuntu1204.md | 1 +
 doc/Building_FRR_on_Ubuntu1404.md | 1 +
 doc/Building_FRR_on_Ubuntu1604.md | 1 +
 6 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/doc/Building_FRR_on_Debian8.md b/doc/Building_FRR_on_Debian8.md
index 635f9680b6..a42ca6d781 100644
--- a/doc/Building_FRR_on_Debian8.md
+++ b/doc/Building_FRR_on_Debian8.md
@@ -82,6 +82,7 @@ an example.)
     sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ripngd.conf
     sudo install -m 640 -o frr -g frr /dev/null /etc/frr/pimd.conf
     sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ldpd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/nhrpd.conf
     sudo install -m 640 -o frr -g frrvty /dev/null /etc/frr/vtysh.conf
 
 ### Enable IP & IPv6 forwarding
diff --git a/doc/Building_FRR_on_Fedora24.md b/doc/Building_FRR_on_Fedora24.md
index 941126da42..1f5f12b9cc 100644
--- a/doc/Building_FRR_on_Fedora24.md
+++ b/doc/Building_FRR_on_Fedora24.md
@@ -70,6 +70,7 @@ them if you are not building on a x86_64 architecture
     sudo touch /etc/frr/ripngd.conf
     sudo touch /etc/frr/pimd.conf
     sudo touch /etc/frr/ldpd.conf
+    sudo touch /etc/frr/nhrpd.conf
     sudo chown -R frr:frr /etc/frr/
     sudo touch /etc/frr/vtysh.conf
     sudo chown frr:frrvt /etc/frr/vtysh.conf
@@ -111,7 +112,7 @@ Create a new file `/etc/modules-load.d/mpls.conf` with the following content:
     install -p -m 644 redhat/ospf6d.service /usr/lib/systemd/system/ospf6d.service
     install -p -m 644 redhat/ripngd.service /usr/lib/systemd/system/ripngd.service
     install -p -m 644 redhat/pimd.service /usr/lib/systemd/system/pimd.service
-    install -p -m 644 redhat/pimd.service /usr/lib/systemd/system/ldpd.service
+    install -p -m 644 redhat/ldpd.service /usr/lib/systemd/system/ldpd.service
     install -p -m 644 redhat/frr.sysconfig /etc/sysconfig/frr
     install -p -m 644 redhat/frr.logrotate /etc/logrotate.d/frr
 
diff --git a/doc/Building_FRR_on_OpenBSD6.md b/doc/Building_FRR_on_OpenBSD6.md
index a59452a72b..c1b583664d 100644
--- a/doc/Building_FRR_on_OpenBSD6.md
+++ b/doc/Building_FRR_on_OpenBSD6.md
@@ -75,6 +75,7 @@ an example)
     sudo touch /etc/frr/ripngd.conf
     sudo touch /etc/frr/pimd.conf
     sudo touch /etc/frr/ldpd.conf
+    sudo touch /etc/frr/nhrpd.conf
     sudo chown -R _frr:_frr /etc/frr
     sudo touch /etc/frr/vtysh.conf
     sudo chown -R _frr:_frrvty /etc/frr/vtysh.conf
diff --git a/doc/Building_FRR_on_Ubuntu1204.md b/doc/Building_FRR_on_Ubuntu1204.md
index cafed0fb31..2d6ccffd9f 100644
--- a/doc/Building_FRR_on_Ubuntu1204.md
+++ b/doc/Building_FRR_on_Ubuntu1204.md
@@ -113,6 +113,7 @@ an example.)
     sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ripngd.conf
     sudo install -m 640 -o frr -g frr /dev/null /etc/frr/pimd.conf
     sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ldpd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/nhrpd.conf
     sudo install -m 640 -o frr -g frrvty /dev/null /etc/frr/vtysh.conf
 
 ### Enable IP & IPv6 forwarding
diff --git a/doc/Building_FRR_on_Ubuntu1404.md b/doc/Building_FRR_on_Ubuntu1404.md
index 1524dd7778..5da423affc 100644
--- a/doc/Building_FRR_on_Ubuntu1404.md
+++ b/doc/Building_FRR_on_Ubuntu1404.md
@@ -74,6 +74,7 @@ an example.)
     sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ripngd.conf
     sudo install -m 640 -o frr -g frr /dev/null /etc/frr/pimd.conf
     sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ldpd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/nhrpd.conf    
     sudo install -m 640 -o frr -g frrvty /dev/null /etc/frr/vtysh.conf
 
 ### Enable IP & IPv6 forwarding
diff --git a/doc/Building_FRR_on_Ubuntu1604.md b/doc/Building_FRR_on_Ubuntu1604.md
index ed599a2551..70a8159cc1 100644
--- a/doc/Building_FRR_on_Ubuntu1604.md
+++ b/doc/Building_FRR_on_Ubuntu1604.md
@@ -74,6 +74,7 @@ an example.)
     sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ripngd.conf
     sudo install -m 640 -o frr -g frr /dev/null /etc/frr/pimd.conf
     sudo install -m 640 -o frr -g frr /dev/null /etc/frr/ldpd.conf
+    sudo install -m 640 -o frr -g frr /dev/null /etc/frr/nhrpd.conf    
     sudo install -m 640 -o frr -g frrvty /dev/null /etc/frr/vtysh.conf
 
 ### Enable IP & IPv6 forwarding

From 34288970c42bcfaebc6d101e500c0fdad6420ff5 Mon Sep 17 00:00:00 2001
From: Renato Westphal <renato@opensourcerouting.org>
Date: Sat, 22 Apr 2017 17:41:54 -0300
Subject: [PATCH 10/19] ospf6d: fix "no router ospf6"

The "no router ospf6" command wasn't working.

Regression introduced by commit 16cedbb.

Signed-off-by: Renato Westphal <renato@opensourcerouting.org>
---
 ospf6d/ospf6_top.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/ospf6d/ospf6_top.c b/ospf6d/ospf6_top.c
index f2a1c8c2ec..e592d3c4f9 100644
--- a/ospf6d/ospf6_top.c
+++ b/ospf6d/ospf6_top.c
@@ -315,6 +315,14 @@ DEFUN (no_router_ospf6,
        ROUTER_STR
        OSPF6_STR)
 {
+  if (ospf6 == NULL)
+    vty_out (vty, "OSPFv3 is not configured%s", VNL);
+  else
+    {
+      ospf6_delete (ospf6);
+      ospf6 = NULL;
+    }
+
   /* return to config node . */
   VTY_PUSH_CONTEXT_NULL(CONFIG_NODE);
 

From 27821f6a5a1d469e878dc4d35a87475c6643141f Mon Sep 17 00:00:00 2001
From: Daniel Walton <dwalton@cumulusnetworks.com>
Date: Mon, 24 Apr 2017 18:22:29 +0000
Subject: [PATCH 11/19] vtysh: "Command incomplete: log syslog"

Signed-off-by: Daniel Walton <dwalton@cumulusnetworks.com>

Before
======
root@spine-1[~]# cat /etc/frr/frr.conf
no log monitor
!
service integrated-vtysh-config
!
log syslog
!
log syslog informational
!
line vty
!
root@spine-1[~]#
root@spine-1[~]# vtysh -m -f /etc/frr/frr.conf
no log monitor
!
service integrated-vtysh-config
!
line 5: % Command incomplete: log syslog

root@spine-1[~]#

After
=====
root@spine-1[~]# vtysh -m -f /etc/frr/frr.conf
no log monitor
!
service integrated-vtysh-config
!
log syslog
!
log syslog informational
!
line vty
!
end
root@spine-1[~]#
---
 vtysh/vtysh.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vtysh/vtysh.c b/vtysh/vtysh.c
index d0038ea3cf..74f865b613 100644
--- a/vtysh/vtysh.c
+++ b/vtysh/vtysh.c
@@ -2165,7 +2165,7 @@ DEFUNSH (VTYSH_ALL,
 DEFUNSH (VTYSH_ALL,
 	 vtysh_log_syslog,
 	 vtysh_log_syslog_cmd,
-	 "log syslog <emergencies|alerts|critical|errors|warnings|notifications|informational|debugging>",
+	 "log syslog [<emergencies|alerts|critical|errors|warnings|notifications|informational|debugging>]",
 	 "Logging control\n"
 	 "Set syslog logging level\n"
          LOG_LEVEL_DESC)

From 0ca036b4569d695cfe668a0d9efbe7da94016673 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timo=20Ter=C3=A4s?= <timo.teras@iki.fi>
Date: Sat, 25 Mar 2017 17:27:24 +0200
Subject: [PATCH 12/19] nhrpd: implement 'show ip nhrp nhs'

---
 nhrpd/nhrp_nhs.c | 28 ++++++++-------
 nhrpd/nhrp_vty.c | 88 +++++++++++++++++++++++++++++++-----------------
 nhrpd/nhrpd.h    | 13 +++++++
 3 files changed, 86 insertions(+), 43 deletions(-)

diff --git a/nhrpd/nhrp_nhs.c b/nhrpd/nhrp_nhs.c
index 10245d308a..555c0d1de1 100644
--- a/nhrpd/nhrp_nhs.c
+++ b/nhrpd/nhrp_nhs.c
@@ -18,19 +18,6 @@ DEFINE_MTYPE_STATIC(NHRPD, NHRP_NHS, "NHRP next hop server")
 DEFINE_MTYPE_STATIC(NHRPD, NHRP_REGISTRATION, "NHRP registration entries")
 
 static int nhrp_nhs_resolve(struct thread *t);
-
-struct nhrp_registration {
-	struct list_head reglist_entry;
-	struct thread *t_register;
-	struct nhrp_nhs *nhs;
-	struct nhrp_reqid reqid;
-	unsigned int timeout;
-	unsigned mark : 1;
-	union sockunion proto_addr;
-	struct nhrp_peer *peer;
-	struct notifier_block peer_notifier;
-};
-
 static int nhrp_reg_send_req(struct thread *t);
 
 static void nhrp_reg_reply(struct nhrp_reqid *reqid, void *arg)
@@ -370,3 +357,18 @@ void nhrp_nhs_terminate(void)
 		}
 	}
 }
+
+void nhrp_nhs_foreach(struct interface *ifp, afi_t afi, void (*cb)(struct nhrp_nhs *, struct nhrp_registration *, void *), void *ctx)
+{
+	struct nhrp_interface *nifp = ifp->info;
+	struct nhrp_nhs *nhs;
+	struct nhrp_registration *reg;
+
+	list_for_each_entry(nhs, &nifp->afi[afi].nhslist_head, nhslist_entry) {
+		if (!list_empty(&nhs->reglist_head)) {
+			list_for_each_entry(reg, &nhs->reglist_head, reglist_entry)
+				cb(nhs, reg, ctx);
+		} else
+			cb(nhs, 0, ctx);
+	}
+}
diff --git a/nhrpd/nhrp_vty.c b/nhrpd/nhrp_vty.c
index 2e3164410c..f127d24dc8 100644
--- a/nhrpd/nhrp_vty.c
+++ b/nhrpd/nhrp_vty.c
@@ -592,6 +592,56 @@ static void show_ip_nhrp_cache(struct nhrp_cache *c, void *pctx)
 		VTY_NEWLINE);
 }
 
+static void show_ip_nhrp_nhs(struct nhrp_nhs *n, struct nhrp_registration *reg, void *pctx)
+{
+	struct info_ctx *ctx = pctx;
+	struct vty *vty = ctx->vty;
+	char buf[2][SU_ADDRSTRLEN];
+
+	if (!ctx->count) {
+		vty_out(vty, "%-8s %-24s %-16s %-16s%s",
+			"Iface",
+			"FQDN",
+			"NBMA",
+			"Protocol",
+			VTY_NEWLINE);
+	}
+	ctx->count++;
+
+	vty_out(vty, "%-8s %-24s %-16s %-16s%s",
+		n->ifp->name,
+		n->nbma_fqdn,
+		(reg && reg->peer) ? sockunion2str(&reg->peer->vc->remote.nbma, buf[0], sizeof buf[0]) : "-",
+		sockunion2str(reg ? &reg->proto_addr : &n->proto_addr, buf[1], sizeof buf[1]),
+		VTY_NEWLINE);
+}
+
+static void show_ip_nhrp_shortcut(struct nhrp_shortcut *s, void *pctx)
+{
+	struct info_ctx *ctx = pctx;
+	struct nhrp_cache *c;
+	struct vty *vty = ctx->vty;
+	char buf1[PREFIX_STRLEN], buf2[SU_ADDRSTRLEN];
+
+	if (!ctx->count) {
+		vty_out(vty, "%-8s %-24s %-24s %s%s",
+			"Type",
+			"Prefix",
+			"Via",
+			"Identity",
+			VTY_NEWLINE);
+	}
+	ctx->count++;
+
+	c = s->cache;
+	vty_out(ctx->vty, "%-8s %-24s %-24s %s%s",
+		nhrp_cache_type_str[s->type],
+		prefix2str(s->p, buf1, sizeof buf1),
+		c ? sockunion2str(&c->remote_addr, buf2, sizeof buf2) : "",
+		(c && c->cur.peer) ? c->cur.peer->vc->remote.id : "",
+		VTY_NEWLINE);
+}
+
 static void show_ip_opennhrp_cache(struct nhrp_cache *c, void *pctx)
 {
 	struct info_ctx *ctx = pctx;
@@ -631,38 +681,13 @@ static void show_ip_opennhrp_cache(struct nhrp_cache *c, void *pctx)
 	vty_out(ctx->vty, "%s", VTY_NEWLINE);
 }
 
-static void show_ip_nhrp_shortcut(struct nhrp_shortcut *s, void *pctx)
-{
-	struct info_ctx *ctx = pctx;
-	struct nhrp_cache *c;
-	struct vty *vty = ctx->vty;
-	char buf1[PREFIX_STRLEN], buf2[SU_ADDRSTRLEN];
-
-	if (!ctx->count) {
-		vty_out(vty, "%-8s %-24s %-24s %s%s",
-			"Type",
-			"Prefix",
-			"Via",
-			"Identity",
-			VTY_NEWLINE);
-	}
-	ctx->count++;
-
-	c = s->cache;
-	vty_out(ctx->vty, "%-8s %-24s %-24s %s%s",
-		nhrp_cache_type_str[s->type],
-		prefix2str(s->p, buf1, sizeof buf1),
-		c ? sockunion2str(&c->remote_addr, buf2, sizeof buf2) : "",
-		(c && c->cur.peer) ? c->cur.peer->vc->remote.id : "",
-		VTY_NEWLINE);
-}
-
 DEFUN(show_ip_nhrp, show_ip_nhrp_cmd,
-	"show " AFI_CMD " nhrp [cache|shortcut|opennhrp]",
+	"show " AFI_CMD " nhrp [cache|nhs|shortcut|opennhrp]",
 	SHOW_STR
 	AFI_STR
 	"NHRP information\n"
 	"Forwarding cache information\n"
+	"Next hop server information\n"
 	"Shortcut information\n"
 	"opennhrpctl style cache dump\n")
 {
@@ -676,13 +701,16 @@ DEFUN(show_ip_nhrp, show_ip_nhrp_cmd,
 	if (argc <= 3 || argv[3]->text[0] == 'c') {
 		for (ALL_LIST_ELEMENTS_RO(vrf_iflist(VRF_DEFAULT), node, ifp))
 			nhrp_cache_foreach(ifp, show_ip_nhrp_cache, &ctx);
-	} else if (argv[3]->text[0] == 'o') {
+	} else if (argv[3]->text[0] == 'n') {
+		for (ALL_LIST_ELEMENTS_RO(vrf_iflist(VRF_DEFAULT), node, ifp))
+			nhrp_nhs_foreach(ifp, ctx.afi, show_ip_nhrp_nhs, &ctx);
+	} else if (argv[3]->text[0] == 's') {
+		nhrp_shortcut_foreach(ctx.afi, show_ip_nhrp_shortcut, &ctx);
+	} else {
 		vty_out(vty, "Status: ok%s%s", VTY_NEWLINE, VTY_NEWLINE);
 		ctx.count++;
 		for (ALL_LIST_ELEMENTS_RO(vrf_iflist(VRF_DEFAULT), node, ifp))
 			nhrp_cache_foreach(ifp, show_ip_opennhrp_cache, &ctx);
-	} else {
-		nhrp_shortcut_foreach(ctx.afi, show_ip_nhrp_shortcut, &ctx);
 	}
 
 	if (!ctx.count) {
diff --git a/nhrpd/nhrpd.h b/nhrpd/nhrpd.h
index cd2b0d5580..9a4f26d577 100644
--- a/nhrpd/nhrpd.h
+++ b/nhrpd/nhrpd.h
@@ -254,6 +254,18 @@ struct nhrp_nhs {
 	struct list_head reglist_head;
 };
 
+struct nhrp_registration {
+	struct list_head reglist_entry;
+	struct thread *t_register;
+	struct nhrp_nhs *nhs;
+	struct nhrp_reqid reqid;
+	unsigned int timeout;
+	unsigned mark : 1;
+	union sockunion proto_addr;
+	struct nhrp_peer *peer;
+	struct notifier_block peer_notifier;
+};
+
 #define NHRP_IFF_SHORTCUT		0x0001
 #define NHRP_IFF_REDIRECT		0x0002
 #define NHRP_IFF_REG_NO_UNIQUE		0x0100
@@ -311,6 +323,7 @@ int nhrp_nhs_add(struct interface *ifp, afi_t afi, union sockunion *proto_addr,
 int nhrp_nhs_del(struct interface *ifp, afi_t afi, union sockunion *proto_addr, const char *nbma_fqdn);
 int nhrp_nhs_free(struct nhrp_nhs *nhs);
 void nhrp_nhs_terminate(void);
+void nhrp_nhs_foreach(struct interface *ifp, afi_t afi, void (*cb)(struct nhrp_nhs *, struct nhrp_registration *, void *), void *ctx);
 
 void nhrp_route_update_nhrp(const struct prefix *p, struct interface *ifp);
 void nhrp_route_announce(int add, enum nhrp_cache_type type, const struct prefix *p, struct interface *ifp, const union sockunion *nexthop, uint32_t mtu);

From 2d4eab226953e4754a38f179a2733a165e176c16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timo=20Ter=C3=A4s?= <timo.teras@iki.fi>
Date: Fri, 21 Apr 2017 13:37:07 +0300
Subject: [PATCH 13/19] nhrp: implement 'no ip nhrp map' command

was accidentally not implemented earlier
---
 nhrpd/nhrp_vty.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/nhrpd/nhrp_vty.c b/nhrpd/nhrp_vty.c
index f127d24dc8..3d1d29ecb9 100644
--- a/nhrpd/nhrp_vty.c
+++ b/nhrpd/nhrp_vty.c
@@ -504,6 +504,32 @@ DEFUN(if_nhrp_map, if_nhrp_map_cmd,
 	return CMD_SUCCESS;
 }
 
+DEFUN(if_no_nhrp_map, if_no_nhrp_map_cmd,
+	"no " AFI_CMD " nhrp map (A.B.C.D|X:X::X:X)",
+	NO_STR
+	AFI_STR
+	NHRP_STR
+	"Nexthop Server configuration\n"
+	"IPv4 protocol address\n"
+	"IPv6 protocol address\n")
+{
+	VTY_DECLVAR_CONTEXT(interface,ifp);
+	afi_t afi = cmd_to_afi(argv[1]);
+	union sockunion proto_addr;
+	struct nhrp_cache *c;
+
+	if (str2sockunion(argv[4]->arg, &proto_addr) < 0 ||
+	    afi2family(afi) != sockunion_family(&proto_addr))
+		return nhrp_vty_return(vty, NHRP_ERR_PROTOCOL_ADDRESS_MISMATCH);
+
+	c = nhrp_cache_get(ifp, &proto_addr, 0);
+	if (!c || !c->map)
+		return nhrp_vty_return(vty, NHRP_ERR_ENTRY_NOT_FOUND);
+
+	nhrp_cache_update_binding(c, c->cur.type, -1, NULL, 0, NULL);
+	return CMD_SUCCESS;
+}
+
 DEFUN(if_nhrp_nhs, if_nhrp_nhs_cmd,
 	AFI_CMD " nhrp nhs <A.B.C.D|X:X::X:X|dynamic> nbma <A.B.C.D|FQDN>",
 	AFI_STR
@@ -947,6 +973,7 @@ void nhrp_config_init(void)
 	install_element(INTERFACE_NODE, &if_nhrp_reg_flags_cmd);
 	install_element(INTERFACE_NODE, &if_no_nhrp_reg_flags_cmd);
 	install_element(INTERFACE_NODE, &if_nhrp_map_cmd);
+	install_element(INTERFACE_NODE, &if_no_nhrp_map_cmd);
 	install_element(INTERFACE_NODE, &if_nhrp_nhs_cmd);
 	install_element(INTERFACE_NODE, &if_no_nhrp_nhs_cmd);
 }

From 8c01a3bd14f1ed70963159e5e0866dad412020c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timo=20Ter=C3=A4s?= <timo.teras@iki.fi>
Date: Thu, 20 Apr 2017 16:24:14 +0300
Subject: [PATCH 14/19] nhrp: fix protocol address family parsing on receive

See bugzilla #948
---
 nhrpd/nhrp_peer.c | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/nhrpd/nhrp_peer.c b/nhrpd/nhrp_peer.c
index 4c1d97a489..1f624eba0b 100644
--- a/nhrpd/nhrp_peer.c
+++ b/nhrpd/nhrp_peer.c
@@ -730,6 +730,15 @@ static void nhrp_packet_debug(struct zbuf *zb, const char *dir)
 		reply ? buf[0] : buf[1]);
 }
 
+static int proto2afi(uint16_t proto)
+{
+	switch (proto) {
+	case ETH_P_IP: return AFI_IP;
+	case ETH_P_IPV6: return AFI_IP6;
+	}
+	return AF_UNSPEC;
+}
+
 struct nhrp_route_info {
 	int local;
 	struct interface *ifp;
@@ -749,7 +758,7 @@ void nhrp_peer_recv(struct nhrp_peer *p, struct zbuf *zb)
 	const char *info = NULL;
 	union sockunion *target_addr;
 	unsigned paylen, extoff, extlen, realsize;
-	afi_t afi;
+	afi_t nbma_afi, proto_afi;
 
 	debugf(NHRP_DEBUG_KERNEL, "PACKET: Recv %s -> %s",
 		sockunion2str(&vc->remote.nbma, buf[0], sizeof buf[0]),
@@ -777,20 +786,21 @@ void nhrp_peer_recv(struct nhrp_peer *p, struct zbuf *zb)
 	pp.hdr = hdr;
 	pp.peer = p;
 
-	afi = htons(hdr->afnum);
+	nbma_afi = htons(hdr->afnum);
+	proto_afi = proto2afi(htons(hdr->protocol_type));
 	if (hdr->type > ZEBRA_NUM_OF(packet_types) ||
 	    hdr->version != NHRP_VERSION_RFC2332 ||
-	    afi >= AFI_MAX ||
+	    nbma_afi >= AFI_MAX || proto_afi == AF_UNSPEC ||
 	    packet_types[hdr->type].type == PACKET_UNKNOWN ||
 	    htons(hdr->packet_size) > realsize) {
-		zlog_info("From %s: error: packet type %d, version %d, AFI %d, size %d (real size %d)",
+		zlog_info("From %s: error: packet type %d, version %d, AFI %d, proto %x, size %d (real size %d)",
 			   sockunion2str(&vc->remote.nbma, buf[0], sizeof buf[0]),
-			   (int) hdr->type, (int) hdr->version, (int) afi,
-			   (int) htons(hdr->packet_size),
-			   (int) realsize);
+			   (int) hdr->type, (int) hdr->version,
+			   (int) nbma_afi, (int) htons(hdr->protocol_type),
+			   (int) htons(hdr->packet_size), (int) realsize);
 		goto drop;
 	}
-	pp.if_ad = &((struct nhrp_interface *)ifp->info)->afi[afi];
+	pp.if_ad = &((struct nhrp_interface *)ifp->info)->afi[proto_afi];
 
 	extoff = htons(hdr->extension_offset);
 	if (extoff) {
@@ -806,7 +816,7 @@ void nhrp_peer_recv(struct nhrp_peer *p, struct zbuf *zb)
 	extlen = zbuf_used(zb);
 	zbuf_init(&pp.extensions, zbuf_pulln(zb, extlen), extlen, extlen);
 
-	if (!nifp->afi[afi].network_id) {
+	if (!nifp->afi[proto_afi].network_id) {
 		info = "nhrp not enabled";
 		goto drop;
 	}

From 3d3813b22976a1092a03ac7d94885f9d2ed5e56d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timo=20Ter=C3=A4s?= <timo.teras@iki.fi>
Date: Fri, 21 Apr 2017 13:57:28 +0300
Subject: [PATCH 15/19] nhrp: explicitly cast ints to size_t for
 vici_request_vc va_list handling

---
 nhrpd/vici.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/nhrpd/vici.c b/nhrpd/vici.c
index 507dd14a9c..12409f90b9 100644
--- a/nhrpd/vici.c
+++ b/nhrpd/vici.c
@@ -449,9 +449,9 @@ void vici_request_vc(const char *profile, union sockunion *src, union sockunion
 	vici_submit_request(
 		vici, "initiate",
 		VICI_KEY_VALUE, "child", strlen(profile), profile,
-		VICI_KEY_VALUE, "timeout", 2, "-1",
-		VICI_KEY_VALUE, "async", 1, "1",
-		VICI_KEY_VALUE, "init-limits", 1, prio ? "0" : "1",
+		VICI_KEY_VALUE, "timeout", (size_t) 2, "-1",
+		VICI_KEY_VALUE, "async", (size_t) 1, "1",
+		VICI_KEY_VALUE, "init-limits", (size_t) 1, prio ? "0" : "1",
 		VICI_KEY_VALUE, "my-host", strlen(buf[0]), buf[0],
 		VICI_KEY_VALUE, "other-host", strlen(buf[1]), buf[1],
 		VICI_END);

From 111aec1e42fd5f3130518a90e56a0dd3d4e48375 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timo=20Ter=C3=A4s?= <timo.teras@iki.fi>
Date: Fri, 21 Apr 2017 14:56:45 +0300
Subject: [PATCH 16/19] nhrp: notify 'tunnel protection' changes

to triggers refresh of IKE SAs immediately on this command
---
 nhrpd/nhrp_interface.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/nhrpd/nhrp_interface.c b/nhrpd/nhrp_interface.c
index bbaa630cd5..58ad167549 100644
--- a/nhrpd/nhrp_interface.c
+++ b/nhrpd/nhrp_interface.c
@@ -393,6 +393,8 @@ void nhrp_interface_set_protection(struct interface *ifp, const char *profile, c
 
 	if (nifp->ipsec_fallback_profile) free(nifp->ipsec_fallback_profile);
 	nifp->ipsec_fallback_profile = fallback_profile ? strdup(fallback_profile) : NULL;
+
+	notifier_call(&nifp->notifier_list, NOTIFY_INTERFACE_ADDRESS_CHANGED);
 }
 
 void nhrp_interface_set_source(struct interface *ifp, const char *ifname)

From 8ec0c3c125d2a6abda67eef2d990fa3ea15b94ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timo=20Ter=C3=A4s?= <timo.teras@iki.fi>
Date: Fri, 21 Apr 2017 14:57:57 +0300
Subject: [PATCH 17/19] nhrp: fix potential crash when vici profile name is not
 configured

---
 nhrpd/nhrp_peer.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/nhrpd/nhrp_peer.c b/nhrpd/nhrp_peer.c
index 1f624eba0b..3cc91a9083 100644
--- a/nhrpd/nhrp_peer.c
+++ b/nhrpd/nhrp_peer.c
@@ -250,6 +250,8 @@ int nhrp_peer_check(struct nhrp_peer *p, int establish)
 		return 0;
 	if (p->requested)
 		return 0;
+	if (!nifp->ipsec_profile)
+		return 0;
 	if (sockunion_family(&vc->local.nbma) == AF_UNSPEC)
 		return 0;
 

From d139786ae8be106a0ec7d9f16dd5480fc6cb1193 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timo=20Ter=C3=A4s?= <timo.teras@iki.fi>
Date: Fri, 21 Apr 2017 14:58:17 +0300
Subject: [PATCH 18/19] nhrp: parse and log command response errors from
 strongSwan

helps to debug configuration problems
---
 nhrpd/vici.c | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/nhrpd/vici.c b/nhrpd/vici.c
index 12409f90b9..5491bacf7c 100644
--- a/nhrpd/vici.c
+++ b/nhrpd/vici.c
@@ -220,6 +220,23 @@ static void parse_sa_message(
 	}
 }
 
+static void parse_cmd_response(
+	struct vici_message_ctx *ctx,
+	enum vici_type_t msgtype,
+	const struct blob *key, const struct blob *val)
+{
+	char buf[512];
+
+	switch (msgtype) {
+	case VICI_KEY_VALUE:
+		if (blob_equal(key, "errmsg") && blob2buf(val, buf, sizeof(buf)))
+			zlog_err("VICI: strongSwan: %s", buf);
+		break;
+	default:
+		break;
+	}
+}
+
 static void vici_recv_sa(struct vici_conn *vici, struct zbuf *msg, int event)
 {
 	char buf[32];
@@ -265,11 +282,14 @@ static void vici_recv_message(struct vici_conn *vici, struct zbuf *msg)
 		else if (blob_equal(&name, "child-state-destroying"))
 			vici_recv_sa(vici, msg, 2);
 		break;
+	case VICI_CMD_RESPONSE:
+		vici_parse_message(vici, msg, parse_cmd_response, 0);
+		break;
 	case VICI_EVENT_UNKNOWN:
+	case VICI_CMD_UNKNOWN:
 		zlog_err("VICI: StrongSwan does not support mandatory events (unpatched?)");
 		break;
 	case VICI_EVENT_CONFIRM:
-	case VICI_CMD_RESPONSE:
 		break;
 	default:
 		zlog_notice("VICI: Unrecognized message type %d", msgtype);

From c1c17a0ff78fbd37dd587fe2c08435cfe414fca1 Mon Sep 17 00:00:00 2001
From: Donald Sharp <sharpd@cumulusnetworks.com>
Date: Tue, 25 Apr 2017 09:07:05 -0400
Subject: [PATCH 19/19] nhrpd: Fix cli changes missed

Signed-off-by: Donald Sharp <sharpd@cumulusnetworks.com>
---
 nhrpd/nhrp_vty.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nhrpd/nhrp_vty.c b/nhrpd/nhrp_vty.c
index 3d1d29ecb9..9b8463fb54 100644
--- a/nhrpd/nhrp_vty.c
+++ b/nhrpd/nhrp_vty.c
@@ -505,7 +505,7 @@ DEFUN(if_nhrp_map, if_nhrp_map_cmd,
 }
 
 DEFUN(if_no_nhrp_map, if_no_nhrp_map_cmd,
-	"no " AFI_CMD " nhrp map (A.B.C.D|X:X::X:X)",
+	"no " AFI_CMD " nhrp map <A.B.C.D|X:X::X:X>",
 	NO_STR
 	AFI_STR
 	NHRP_STR