From 4bf3dda597cea57745e1aab906d52fc808f088e2 Mon Sep 17 00:00:00 2001 From: Satheesh Kumar K Date: Sun, 25 Aug 2019 22:31:47 -0700 Subject: [PATCH 1/6] pimd : Add support for MLAG Register & Un-register when ever a FRR Client wants to send any data to another node using MLAG Channel, uses below mechanisam. 1. sends a MLAG Registration to zebra with interested messages that it is intended to receive from peer. 2. In response to this request, Zebra opens communication channel with MLAG. and also in Rx. diretion zebra forwards only those messages which client shown interest during registration 3. when client is no-longer interested in communicating with MLAG, client posts De-register to Zebra 4. if this is the last client which is interested for MLAG Communication, zebra closes the channel. why PIM Needs MLAG Communication ================================ 1. In general on LAN Networks elecetd DR will send the Join towards Multicast RP in case of a LHR and Register in case of FHR. 2. But in case DR Goes down, traffic will be re-converged only after the New DR is elected, but this can take time based on Hold Timer to detect the DR down. 3. this can be optimised by using MLAG Mecganisam. 4. and also Traffic can be forwarded more efficiently by knowing the cost towards RP using MLAG Signed-off-by: Satheesh Kumar K --- lib/mlag.h | 21 +++++ lib/zclient.c | 33 ++++++++ lib/zclient.h | 9 +++ pimd/pim_cmd.c | 21 ++++- pimd/pim_cmd.h | 1 + pimd/pim_instance.c | 5 ++ pimd/pim_instance.h | 14 ++++ pimd/pim_main.c | 2 + pimd/pim_mlag.c | 187 ++++++++++++++++++++++++++++++++++++++++++++ pimd/pim_mlag.h | 42 ++++++++++ pimd/pim_zebra.c | 2 +- pimd/pimd.h | 4 + pimd/subdir.am | 2 + 13 files changed, 340 insertions(+), 3 deletions(-) create mode 100644 pimd/pim_mlag.c create mode 100644 pimd/pim_mlag.h diff --git a/lib/mlag.h b/lib/mlag.h index 2b904d44f4..d461fdc44e 100644 --- a/lib/mlag.h +++ b/lib/mlag.h @@ -26,12 +26,33 @@ extern "C" { #endif +#define MLAG_BUF_LIMIT 2048 + enum mlag_role { MLAG_ROLE_NONE, MLAG_ROLE_PRIMARY, MLAG_ROLE_SECONDARY }; +/* + * This message definition should match mlag.proto + * Beacuse mesasge registartion is based on this + */ +enum mlag_msg_type { + MLAG_MSG_NONE = 0, + MLAG_REGISTER = 1, + MLAG_DEREGISTER = 2, + MLAG_STATUS_UPDATE = 3, + MLAG_MROUTE_ADD = 4, + MLAG_MROUTE_DEL = 5, + MLAG_DUMP = 6, + MLAG_MROUTE_ADD_BULK = 7, + MLAG_MROUTE_DEL_BULK = 8, + MLAG_PIM_CFG_DUMP = 10, + MLAG_VXLAN_UPDATE = 11, + MLAG_PEER_FRR_STATUS = 12, +}; + extern char *mlag_role2str(enum mlag_role role, char *buf, size_t size); #ifdef __cplusplus diff --git a/lib/zclient.c b/lib/zclient.c index f809704f86..a51e35fd98 100644 --- a/lib/zclient.c +++ b/lib/zclient.c @@ -2536,6 +2536,39 @@ stream_failure: return; } +void zclient_send_mlag_register(struct zclient *client, uint32_t bit_map) +{ + struct stream *s; + + s = client->obuf; + stream_reset(s); + + zclient_create_header(s, ZEBRA_MLAG_CLIENT_REGISTER, VRF_DEFAULT); + stream_putl(s, bit_map); + + stream_putw_at(s, 0, stream_get_endp(s)); + zclient_send_message(client); +} + +void zclient_send_mlag_deregister(struct zclient *client) +{ + zebra_message_send(client, ZEBRA_MLAG_CLIENT_UNREGISTER, VRF_DEFAULT); +} + +void zclient_send_mlag_data(struct zclient *client, struct stream *client_s) +{ + struct stream *s; + + s = client->obuf; + stream_reset(s); + + zclient_create_header(s, ZEBRA_MLAG_FORWARD_MSG, VRF_DEFAULT); + stream_put(s, client_s->data, client_s->endp); + + stream_putw_at(s, 0, stream_get_endp(s)); + zclient_send_message(client); +} + /* Zebra client message read function. */ static int zclient_read(struct thread *thread) { diff --git a/lib/zclient.h b/lib/zclient.h index 81e454d192..1735495fcc 100644 --- a/lib/zclient.h +++ b/lib/zclient.h @@ -177,6 +177,9 @@ typedef enum { ZEBRA_VXLAN_SG_ADD, ZEBRA_VXLAN_SG_DEL, ZEBRA_VXLAN_SG_REPLAY, + ZEBRA_MLAG_CLIENT_REGISTER, + ZEBRA_MLAG_CLIENT_UNREGISTER, + ZEBRA_MLAG_FORWARD_MSG, } zebra_message_types_t; struct redist_proto { @@ -670,5 +673,11 @@ static inline void zapi_route_set_blackhole(struct zapi_route *api, SET_FLAG(api->message, ZAPI_MESSAGE_NEXTHOP); }; +extern void zclient_send_mlag_register(struct zclient *client, + uint32_t bit_map); +extern void zclient_send_mlag_deregister(struct zclient *client); + +extern void zclient_send_mlag_data(struct zclient *client, + struct stream *client_s); #endif /* _ZEBRA_ZCLIENT_H */ diff --git a/pimd/pim_cmd.c b/pimd/pim_cmd.c index 28b4af9457..2c2b1c9586 100644 --- a/pimd/pim_cmd.c +++ b/pimd/pim_cmd.c @@ -61,6 +61,7 @@ #include "pim_nht.h" #include "pim_bfd.h" #include "pim_vxlan.h" +#include "pim_mlag.h" #include "bfd.h" #include "pim_bsm.h" @@ -7462,9 +7463,9 @@ DEFPY_HIDDEN (interface_ip_pim_activeactive, pim_ifp = ifp->info; if (no) - pim_ifp->activeactive = false; + pim_if_unconfigure_mlag_dualactive(pim_ifp); else - pim_ifp->activeactive = true; + pim_if_configure_mlag_dualactive(pim_ifp); return CMD_SUCCESS; } @@ -8382,6 +8383,20 @@ DEFUN (no_debug_pim_zebra, return CMD_SUCCESS; } +DEFUN(debug_pim_mlag, debug_pim_mlag_cmd, "debug pim mlag", + DEBUG_STR DEBUG_PIM_STR DEBUG_PIM_MLAG_STR) +{ + PIM_DO_DEBUG_MLAG; + return CMD_SUCCESS; +} + +DEFUN(no_debug_pim_mlag, no_debug_pim_mlag_cmd, "no debug pim mlag", + NO_STR DEBUG_STR DEBUG_PIM_STR DEBUG_PIM_MLAG_STR) +{ + PIM_DONT_DEBUG_MLAG; + return CMD_SUCCESS; +} + DEFUN (debug_pim_vxlan, debug_pim_vxlan_cmd, "debug pim vxlan", @@ -10408,6 +10423,8 @@ void pim_cmd_init(void) install_element(ENABLE_NODE, &no_debug_ssmpingd_cmd); install_element(ENABLE_NODE, &debug_pim_zebra_cmd); install_element(ENABLE_NODE, &no_debug_pim_zebra_cmd); + install_element(ENABLE_NODE, &debug_pim_mlag_cmd); + install_element(ENABLE_NODE, &no_debug_pim_mlag_cmd); install_element(ENABLE_NODE, &debug_pim_vxlan_cmd); install_element(ENABLE_NODE, &no_debug_pim_vxlan_cmd); install_element(ENABLE_NODE, &debug_msdp_cmd); diff --git a/pimd/pim_cmd.h b/pimd/pim_cmd.h index f5bb316a7a..9f05bc74e5 100644 --- a/pimd/pim_cmd.h +++ b/pimd/pim_cmd.h @@ -54,6 +54,7 @@ #define DEBUG_PIM_PACKETDUMP_RECV_STR "Dump received packets\n" #define DEBUG_PIM_TRACE_STR "PIM internal daemon activity\n" #define DEBUG_PIM_ZEBRA_STR "ZEBRA protocol activity\n" +#define DEBUG_PIM_MLAG_STR "PIM Mlag activity\n" #define DEBUG_PIM_VXLAN_STR "PIM VxLAN events\n" #define DEBUG_SSMPINGD_STR "ssmpingd activity\n" #define CLEAR_IP_IGMP_STR "IGMP clear commands\n" diff --git a/pimd/pim_instance.c b/pimd/pim_instance.c index 6848d2dabb..955ad14b01 100644 --- a/pimd/pim_instance.c +++ b/pimd/pim_instance.c @@ -34,6 +34,7 @@ #include "pim_ssmpingd.h" #include "pim_vty.h" #include "pim_bsm.h" +#include "pim_mlag.h" static void pim_instance_terminate(struct pim_instance *pim) { @@ -47,6 +48,8 @@ static void pim_instance_terminate(struct pim_instance *pim) if (pim->static_routes) list_delete(&pim->static_routes); + pim_instance_mlag_terminate(pim); + pim_upstream_terminate(pim); pim_rp_free(pim); @@ -115,6 +118,8 @@ static struct pim_instance *pim_instance_init(struct vrf *vrf) pim_upstream_init(pim); + pim_instance_mlag_init(pim); + pim->last_route_change_time = -1; return pim; } diff --git a/pimd/pim_instance.h b/pimd/pim_instance.h index 06d41c4b53..dd3ac8fcb0 100644 --- a/pimd/pim_instance.h +++ b/pimd/pim_instance.h @@ -64,6 +64,17 @@ struct pim_router { vrf_id_t vrf_id; enum mlag_role role; + uint32_t pim_mlag_intf_cnt; + /* if true we have registered with MLAG */ + bool mlag_process_register; + /* if true local MLAG process reported that it is connected + * with the peer MLAG process + */ + bool connected_to_mlag; + /* Holds the client data(unencoded) that need to be pushed to MCLAGD*/ + struct stream_fifo *mlag_fifo; + struct stream *mlag_stream; + struct thread *zpthread_mlag_write; }; /* Per VRF PIM DB */ @@ -122,6 +133,9 @@ struct pim_instance { bool ecmp_enable; bool ecmp_rebalance_enable; + /* No. of Dual active I/fs in pim_instance */ + uint32_t inst_mlag_intf_cnt; + /* Bsm related */ struct bsm_scope global_scope; uint64_t bsm_rcvd; diff --git a/pimd/pim_main.c b/pimd/pim_main.c index 5a8991c4c0..c57151b900 100644 --- a/pimd/pim_main.c +++ b/pimd/pim_main.c @@ -47,6 +47,7 @@ #include "pim_msdp.h" #include "pim_iface.h" #include "pim_bfd.h" +#include "pim_mlag.h" #include "pim_errors.h" extern struct host host; @@ -129,6 +130,7 @@ int main(int argc, char **argv, char **envp) */ pim_zebra_init(); pim_bfd_init(); + pim_mlag_init(); frr_config_fork(); diff --git a/pimd/pim_mlag.c b/pimd/pim_mlag.c new file mode 100644 index 0000000000..eaec0c716a --- /dev/null +++ b/pimd/pim_mlag.c @@ -0,0 +1,187 @@ +/* PIM Mlag Code. + * Copyright (C) 2018 Cumulus Networks, Inc. + * Donald Sharp + * + * This file is part of FRR. + * + * FRR is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2, or (at your option) any + * later version. + * + * FRR is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with FRR; see the file COPYING. If not, write to the Free + * Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA + * 02111-1307, USA. + */ +#include + +#include "pimd.h" +#include "pim_mlag.h" + +extern struct zclient *zclient; + +static int pim_mlag_register_handler(struct thread *thread) +{ + uint32_t bit_mask = 0; + + if (!zclient) + return -1; + + SET_FLAG(bit_mask, (1 << MLAG_STATUS_UPDATE)); + SET_FLAG(bit_mask, (1 << MLAG_MROUTE_ADD)); + SET_FLAG(bit_mask, (1 << MLAG_MROUTE_DEL)); + SET_FLAG(bit_mask, (1 << MLAG_DUMP)); + SET_FLAG(bit_mask, (1 << MLAG_MROUTE_ADD_BULK)); + SET_FLAG(bit_mask, (1 << MLAG_MROUTE_DEL_BULK)); + SET_FLAG(bit_mask, (1 << MLAG_PIM_CFG_DUMP)); + SET_FLAG(bit_mask, (1 << MLAG_VXLAN_UPDATE)); + SET_FLAG(bit_mask, (1 << MLAG_PEER_FRR_STATUS)); + + if (PIM_DEBUG_MLAG) + zlog_debug("%s: Posting Client Register to MLAG mask:0x%x", + __func__, bit_mask); + + zclient_send_mlag_register(zclient, bit_mask); + return 0; +} + +void pim_mlag_register(void) +{ + if (router->mlag_process_register) + return; + + router->mlag_process_register = true; + + thread_add_event(router->master, pim_mlag_register_handler, NULL, 0, + NULL); +} + +static int pim_mlag_deregister_handler(struct thread *thread) +{ + if (!zclient) + return -1; + + if (PIM_DEBUG_MLAG) + zlog_debug("%s: Posting Client De-Register to MLAG from PIM", + __func__); + router->connected_to_mlag = false; + zclient_send_mlag_deregister(zclient); + return 0; +} + +void pim_mlag_deregister(void) +{ + /* if somebody still interested in the MLAG channel skip de-reg */ + if (router->pim_mlag_intf_cnt) + return; + + /* not registered; nothing do */ + if (!router->mlag_process_register) + return; + + router->mlag_process_register = false; + + thread_add_event(router->master, pim_mlag_deregister_handler, NULL, 0, + NULL); +} + +void pim_if_configure_mlag_dualactive(struct pim_interface *pim_ifp) +{ + if (!pim_ifp || !pim_ifp->pim || pim_ifp->activeactive == true) + return; + + if (PIM_DEBUG_MLAG) + zlog_debug("%s: Configuring active-active on Interface: %s", + __func__, "NULL"); + + pim_ifp->activeactive = true; + if (pim_ifp->pim) + pim_ifp->pim->inst_mlag_intf_cnt++; + + router->pim_mlag_intf_cnt++; + if (PIM_DEBUG_MLAG) + zlog_debug( + "%s: Total MLAG configured Interfaces on router: %d, Inst:%d", + __func__, router->pim_mlag_intf_cnt, + pim_ifp->pim->inst_mlag_intf_cnt); + + if (router->pim_mlag_intf_cnt == 1) { + /* + * atleast one Interface is configured for MLAG, send register + * to Zebra for receiving MLAG Updates + */ + pim_mlag_register(); + } +} + +void pim_if_unconfigure_mlag_dualactive(struct pim_interface *pim_ifp) +{ + if (!pim_ifp || !pim_ifp->pim || pim_ifp->activeactive == false) + return; + + if (PIM_DEBUG_MLAG) + zlog_debug("%s: UnConfiguring active-active on Interface: %s", + __func__, "NULL"); + + pim_ifp->activeactive = false; + if (pim_ifp->pim) + pim_ifp->pim->inst_mlag_intf_cnt--; + + router->pim_mlag_intf_cnt--; + if (PIM_DEBUG_MLAG) + zlog_debug( + "%s: Total MLAG configured Interfaces on router: %d, Inst:%d", + __func__, router->pim_mlag_intf_cnt, + pim_ifp->pim->inst_mlag_intf_cnt); + + if (router->pim_mlag_intf_cnt == 0) { + /* + * all the Interfaces are MLAG un-configured, post MLAG + * De-register to Zebra + */ + pim_mlag_deregister(); + } +} + + +void pim_instance_mlag_init(struct pim_instance *pim) +{ + if (!pim) + return; + + pim->inst_mlag_intf_cnt = 0; +} + + +void pim_instance_mlag_terminate(struct pim_instance *pim) +{ + struct interface *ifp; + + if (!pim) + return; + + FOR_ALL_INTERFACES (pim->vrf, ifp) { + struct pim_interface *pim_ifp = ifp->info; + + if (!pim_ifp || pim_ifp->activeactive == false) + continue; + + pim_if_unconfigure_mlag_dualactive(pim_ifp); + } + pim->inst_mlag_intf_cnt = 0; +} + +void pim_mlag_init(void) +{ + router->pim_mlag_intf_cnt = 0; + router->connected_to_mlag = false; + router->mlag_fifo = stream_fifo_new(); + router->zpthread_mlag_write = NULL; + router->mlag_stream = stream_new(MLAG_BUF_LIMIT); +} diff --git a/pimd/pim_mlag.h b/pimd/pim_mlag.h new file mode 100644 index 0000000000..03fbfccb42 --- /dev/null +++ b/pimd/pim_mlag.h @@ -0,0 +1,42 @@ +/* PIM mlag header. + * Copyright (C) 2018 Cumulus Networks, Inc. + * Donald Sharp + * + * This file is part of FRR. + * + * FRR is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2, or (at your option) any + * later version. + * + * FRR is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with FRR; see the file COPYING. If not, write to the Free + * Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA + * 02111-1307, USA. + */ +#ifndef __PIM_MLAG_H__ +#define __PIM_MLAG_H__ + +#include "mlag.h" +#include "pim_iface.h" + +extern void pim_mlag_init(void); + +extern void pim_instance_mlag_init(struct pim_instance *pim); + +extern void pim_instance_mlag_terminate(struct pim_instance *pim); + +extern void pim_if_configure_mlag_dualactive(struct pim_interface *pim_ifp); + +extern void pim_if_unconfigure_mlag_dualactive(struct pim_interface *pim_ifp); + +extern void pim_mlag_register(void); + +extern void pim_mlag_deregister(void); + +#endif diff --git a/pimd/pim_zebra.c b/pimd/pim_zebra.c index b0db23f54a..e9cfc7e520 100644 --- a/pimd/pim_zebra.c +++ b/pimd/pim_zebra.c @@ -50,7 +50,7 @@ #undef PIM_DEBUG_IFADDR_DUMP #define PIM_DEBUG_IFADDR_DUMP -static struct zclient *zclient = NULL; +struct zclient *zclient; /* Router-id update message from zebra. */ diff --git a/pimd/pimd.h b/pimd/pimd.h index 3b83d3b6c7..b7e6d916a8 100644 --- a/pimd/pimd.h +++ b/pimd/pimd.h @@ -115,6 +115,7 @@ #define PIM_MASK_MTRACE (1 << 25) #define PIM_MASK_VXLAN (1 << 26) #define PIM_MASK_BSM_PROC (1 << 27) +#define PIM_MASK_MLAG (1 << 28) /* Remember 32 bits!!! */ /* PIM error codes */ @@ -171,6 +172,7 @@ extern uint8_t qpim_ecmp_rebalance_enable; #define PIM_DEBUG_IGMP_TRACE_DETAIL \ (router->debugs & (PIM_MASK_IGMP_TRACE_DETAIL | PIM_MASK_IGMP_TRACE)) #define PIM_DEBUG_ZEBRA (router->debugs & PIM_MASK_ZEBRA) +#define PIM_DEBUG_MLAG (router->debugs & PIM_MASK_MLAG) #define PIM_DEBUG_SSMPINGD (router->debugs & PIM_MASK_SSMPINGD) #define PIM_DEBUG_MROUTE (router->debugs & PIM_MASK_MROUTE) #define PIM_DEBUG_MROUTE_DETAIL \ @@ -217,6 +219,7 @@ extern uint8_t qpim_ecmp_rebalance_enable; #define PIM_DO_DEBUG_IGMP_TRACE_DETAIL \ (router->debugs |= PIM_MASK_IGMP_TRACE_DETAIL) #define PIM_DO_DEBUG_ZEBRA (router->debugs |= PIM_MASK_ZEBRA) +#define PIM_DO_DEBUG_MLAG (router->debugs |= PIM_MASK_MLAG) #define PIM_DO_DEBUG_SSMPINGD (router->debugs |= PIM_MASK_SSMPINGD) #define PIM_DO_DEBUG_MROUTE (router->debugs |= PIM_MASK_MROUTE) #define PIM_DO_DEBUG_MROUTE_DETAIL (router->debugs |= PIM_MASK_MROUTE_DETAIL) @@ -248,6 +251,7 @@ extern uint8_t qpim_ecmp_rebalance_enable; #define PIM_DONT_DEBUG_IGMP_TRACE_DETAIL \ (router->debugs &= ~PIM_MASK_IGMP_TRACE_DETAIL) #define PIM_DONT_DEBUG_ZEBRA (router->debugs &= ~PIM_MASK_ZEBRA) +#define PIM_DONT_DEBUG_MLAG (router->debugs &= ~PIM_MASK_MLAG) #define PIM_DONT_DEBUG_SSMPINGD (router->debugs &= ~PIM_MASK_SSMPINGD) #define PIM_DONT_DEBUG_MROUTE (router->debugs &= ~PIM_MASK_MROUTE) #define PIM_DONT_DEBUG_MROUTE_DETAIL (router->debugs &= ~PIM_MASK_MROUTE_DETAIL) diff --git a/pimd/subdir.am b/pimd/subdir.am index 240b62804f..5407e566a5 100644 --- a/pimd/subdir.am +++ b/pimd/subdir.am @@ -62,6 +62,7 @@ pimd_libpim_a_SOURCES = \ pimd/pim_zebra.c \ pimd/pim_zlookup.c \ pimd/pim_vxlan.c \ + pimd/pim_mlag.c \ pimd/pimd.c \ # end @@ -114,6 +115,7 @@ noinst_HEADERS += \ pimd/pim_zebra.h \ pimd/pim_zlookup.h \ pimd/pim_vxlan.h \ + pimd/pim_mlag.h \ pimd/pim_vxlan_instance.h \ pimd/pimd.h \ pimd/mtracebis_netlink.h \ From 5fdca4ded5794dcf97235fa3f2d1f0e7e0969c29 Mon Sep 17 00:00:00 2001 From: Satheesh Kumar K Date: Mon, 26 Aug 2019 02:18:05 -0700 Subject: [PATCH 2/6] pimd, lib: adding support for MLAG Message processing at PIM This includes: 1. Defining message formats 2. Stream Decoding after receiving the message at PIM 3. Handling MLAG UP & Down Notifications Signed-off-by: Satheesh Kumar K --- lib/mlag.c | 130 +++++++++++++++++++++++++++++++++++++++ lib/mlag.h | 84 ++++++++++++++++++++++++- lib/zclient.c | 33 ++++++++++ lib/zclient.h | 5 ++ pimd/pim_mlag.c | 157 +++++++++++++++++++++++++++++++++++++++++++++++ pimd/pim_mlag.h | 6 ++ pimd/pim_zebra.c | 4 ++ 7 files changed, 418 insertions(+), 1 deletion(-) diff --git a/lib/mlag.c b/lib/mlag.c index acdc662924..7aac571da6 100644 --- a/lib/mlag.c +++ b/lib/mlag.c @@ -39,3 +39,133 @@ char *mlag_role2str(enum mlag_role role, char *buf, size_t size) return buf; } + +char *zebra_mlag_lib_msgid_to_str(enum mlag_msg_type msg_type, char *buf, + size_t size) +{ + switch (msg_type) { + case MLAG_REGISTER: + snprintf(buf, size, "Register"); + break; + case MLAG_DEREGISTER: + snprintf(buf, size, "De-Register"); + break; + case MLAG_MROUTE_ADD: + snprintf(buf, size, "Mroute add"); + break; + case MLAG_MROUTE_DEL: + snprintf(buf, size, "Mroute del"); + break; + case MLAG_DUMP: + snprintf(buf, size, "Mlag Replay"); + break; + case MLAG_MROUTE_ADD_BULK: + snprintf(buf, size, "Mroute Add Batch"); + break; + case MLAG_MROUTE_DEL_BULK: + snprintf(buf, size, "Mroute Del Batch"); + break; + case MLAG_STATUS_UPDATE: + snprintf(buf, size, "Mlag Status"); + break; + case MLAG_VXLAN_UPDATE: + snprintf(buf, size, "Mlag vxlan update"); + break; + case MLAG_PEER_FRR_STATUS: + snprintf(buf, size, "Mlag Peer FRR Status"); + break; + default: + snprintf(buf, size, "Unknown %d", msg_type); + break; + } + return buf; +} + + +int zebra_mlag_lib_decode_mlag_hdr(struct stream *s, struct mlag_msg *msg) +{ + if (s == NULL || msg == NULL) + return -1; + + STREAM_GETL(s, msg->msg_type); + STREAM_GETW(s, msg->data_len); + STREAM_GETW(s, msg->msg_cnt); + return 0; +stream_failure: + return -1; +} + +int zebra_mlag_lib_decode_mroute_add(struct stream *s, + struct mlag_mroute_add *msg) +{ + if (s == NULL || msg == NULL) + return -1; + + STREAM_GET(msg->vrf_name, s, VRF_NAMSIZ); + STREAM_GETL(s, msg->source_ip); + STREAM_GETL(s, msg->group_ip); + STREAM_GETL(s, msg->cost_to_rp); + STREAM_GETL(s, msg->owner_id); + STREAM_GETC(s, msg->am_i_dr); + STREAM_GETC(s, msg->am_i_dual_active); + STREAM_GETL(s, msg->vrf_id); + STREAM_GET(msg->intf_name, s, INTERFACE_NAMSIZ); + return 0; +stream_failure: + return -1; +} + +int zebra_mlag_lib_decode_mroute_del(struct stream *s, + struct mlag_mroute_del *msg) +{ + if (s == NULL || msg == NULL) + return -1; + + STREAM_GET(msg->vrf_name, s, VRF_NAMSIZ); + STREAM_GETL(s, msg->source_ip); + STREAM_GETL(s, msg->group_ip); + STREAM_GETL(s, msg->owner_id); + STREAM_GETL(s, msg->vrf_id); + STREAM_GET(msg->intf_name, s, INTERFACE_NAMSIZ); + return 0; +stream_failure: + return -1; +} + +int zebra_mlag_lib_decode_mlag_status(struct stream *s, struct mlag_status *msg) +{ + if (s == NULL || msg == NULL) + return -1; + + STREAM_GET(msg->peerlink_rif, s, INTERFACE_NAMSIZ); + STREAM_GETL(s, msg->my_role); + STREAM_GETL(s, msg->peer_state); + return 0; +stream_failure: + return -1; +} + +int zebra_mlag_lib_decode_vxlan_update(struct stream *s, struct mlag_vxlan *msg) +{ + if (s == NULL || msg == NULL) + return -1; + + STREAM_GETL(s, msg->anycast_ip); + STREAM_GETL(s, msg->local_ip); + return 0; + +stream_failure: + return -1; +} + +int zebra_mlag_lib_decode_frr_status(struct stream *s, + struct mlag_frr_status *msg) +{ + if (s == NULL || msg == NULL) + return -1; + + STREAM_GETL(s, msg->frr_state); + return 0; +stream_failure: + return -1; +} diff --git a/lib/mlag.h b/lib/mlag.h index d461fdc44e..b93d5b7497 100644 --- a/lib/mlag.h +++ b/lib/mlag.h @@ -26,6 +26,10 @@ extern "C" { #endif +#include "lib/if.h" +#include "lib/vrf.h" +#include "lib/stream.h" + #define MLAG_BUF_LIMIT 2048 enum mlag_role { @@ -34,9 +38,26 @@ enum mlag_role { MLAG_ROLE_SECONDARY }; +enum mlag_state { + MLAG_STATE_DOWN, + MLAG_STATE_RUNNING, +}; + +enum mlag_frr_state { + MLAG_FRR_STATE_NONE, + MLAG_FRR_STATE_DOWN, + MLAG_FRR_STATE_UP, +}; + +enum mlag_owner { + MLAG_OWNER_NONE, + MLAG_OWNER_INTERFACE, + MLAG_OWNER_VXLAN, +}; + /* * This message definition should match mlag.proto - * Beacuse mesasge registartion is based on this + * Because message registration is based on this */ enum mlag_msg_type { MLAG_MSG_NONE = 0, @@ -53,7 +74,68 @@ enum mlag_msg_type { MLAG_PEER_FRR_STATUS = 12, }; +struct mlag_frr_status { + enum mlag_frr_state frr_state; +}; + +struct mlag_status { + char peerlink_rif[INTERFACE_NAMSIZ]; + enum mlag_role my_role; + enum mlag_state peer_state; +}; + +#define MLAG_ROLE_STRSIZE 16 + +struct mlag_vxlan { + uint32_t anycast_ip; + uint32_t local_ip; +}; + +struct mlag_mroute_add { + char vrf_name[VRF_NAMSIZ]; + uint32_t source_ip; + uint32_t group_ip; + uint32_t cost_to_rp; + enum mlag_owner owner_id; + uint8_t am_i_dr; + uint8_t am_i_dual_active; + uint32_t vrf_id; + char intf_name[INTERFACE_NAMSIZ]; +}; + +struct mlag_mroute_del { + char vrf_name[VRF_NAMSIZ]; + uint32_t source_ip; + uint32_t group_ip; + enum mlag_owner owner_id; + uint32_t vrf_id; + char intf_name[INTERFACE_NAMSIZ]; +}; + +struct mlag_msg { + enum mlag_msg_type msg_type; + uint16_t data_len; + uint16_t msg_cnt; + uint8_t data[0]; +}; + + extern char *mlag_role2str(enum mlag_role role, char *buf, size_t size); +extern char *zebra_mlag_lib_msgid_to_str(enum mlag_msg_type msg_type, char *buf, + size_t size); +extern int zebra_mlag_lib_decode_mlag_hdr(struct stream *s, + struct mlag_msg *msg); +extern int zebra_mlag_lib_decode_mroute_add(struct stream *s, + struct mlag_mroute_add *msg); +extern int zebra_mlag_lib_decode_mroute_del(struct stream *s, + struct mlag_mroute_del *msg); +extern int zebra_mlag_lib_decode_mlag_status(struct stream *s, + struct mlag_status *msg); +extern int zebra_mlag_lib_decode_vxlan_update(struct stream *s, + struct mlag_vxlan *msg); + +extern int zebra_mlag_lib_decode_frr_status(struct stream *s, + struct mlag_frr_status *msg); #ifdef __cplusplus } diff --git a/lib/zclient.c b/lib/zclient.c index a51e35fd98..dad0ffa945 100644 --- a/lib/zclient.c +++ b/lib/zclient.c @@ -2569,6 +2569,30 @@ void zclient_send_mlag_data(struct zclient *client, struct stream *client_s) zclient_send_message(client); } +static void zclient_mlag_process_up(int command, struct zclient *zclient, + zebra_size_t length, vrf_id_t vrf_id) +{ + if (zclient->mlag_process_up) + (*zclient->mlag_process_up)(); + +} + +static void zclient_mlag_process_down(int command, struct zclient *zclient, + zebra_size_t length, vrf_id_t vrf_id) +{ + if (zclient->mlag_process_down) + (*zclient->mlag_process_down)(); + +} + +static void zclient_mlag_handle_msg(int command, struct zclient *zclient, + zebra_size_t length, vrf_id_t vrf_id) +{ + if (zclient->mlag_handle_msg) + (*zclient->mlag_handle_msg)(zclient->ibuf, length); + +} + /* Zebra client message read function. */ static int zclient_read(struct thread *thread) { @@ -2871,6 +2895,15 @@ static int zclient_read(struct thread *thread) (*zclient->vxlan_sg_del)(command, zclient, length, vrf_id); break; + case ZEBRA_MLAG_PROCESS_UP: + zclient_mlag_process_up(command, zclient, length, vrf_id); + break; + case ZEBRA_MLAG_PROCESS_DOWN: + zclient_mlag_process_down(command, zclient, length, vrf_id); + break; + case ZEBRA_MLAG_FORWARD_MSG: + zclient_mlag_handle_msg(command, zclient, length, vrf_id); + break; default: break; } diff --git a/lib/zclient.h b/lib/zclient.h index 1735495fcc..f7c579a4d6 100644 --- a/lib/zclient.h +++ b/lib/zclient.h @@ -177,6 +177,8 @@ typedef enum { ZEBRA_VXLAN_SG_ADD, ZEBRA_VXLAN_SG_DEL, ZEBRA_VXLAN_SG_REPLAY, + ZEBRA_MLAG_PROCESS_UP, + ZEBRA_MLAG_PROCESS_DOWN, ZEBRA_MLAG_CLIENT_REGISTER, ZEBRA_MLAG_CLIENT_UNREGISTER, ZEBRA_MLAG_FORWARD_MSG, @@ -278,6 +280,9 @@ struct zclient { int (*iptable_notify_owner)(ZAPI_CALLBACK_ARGS); int (*vxlan_sg_add)(ZAPI_CALLBACK_ARGS); int (*vxlan_sg_del)(ZAPI_CALLBACK_ARGS); + int (*mlag_process_up)(void); + int (*mlag_process_down)(void); + int (*mlag_handle_msg)(struct stream *msg, int len); }; /* Zebra API message flag. */ diff --git a/pimd/pim_mlag.c b/pimd/pim_mlag.c index eaec0c716a..e39af9b941 100644 --- a/pimd/pim_mlag.c +++ b/pimd/pim_mlag.c @@ -26,6 +26,163 @@ extern struct zclient *zclient; + +/********************API to process PIM MLAG Data ************************/ + +static void pim_mlag_process_mlagd_state_change(struct mlag_status msg) +{ + char buf[80]; + + if (PIM_DEBUG_MLAG) + zlog_debug("%s: msg dump: my_role:%s, peer_state:%s", __func__, + mlag_role2str(msg.my_role, buf, sizeof(buf)), + (msg.peer_state == MLAG_STATE_RUNNING ? "RUNNING" + : "DOWN")); +} + +static void pim_mlag_process_peer_frr_state_change(struct mlag_frr_status msg) +{ + if (PIM_DEBUG_MLAG) + zlog_debug( + "%s: msg dump: peer_frr_state:%s", __func__, + (msg.frr_state == MLAG_FRR_STATE_UP ? "UP" : "DOWN")); +} + +static void pim_mlag_process_vxlan_update(struct mlag_vxlan *msg) +{ +} + +static void pim_mlag_process_mroute_add(struct mlag_mroute_add msg) +{ + if (PIM_DEBUG_MLAG) { + zlog_debug( + "%s: msg dump: vrf_name:%s, s.ip:0x%x, g.ip:0x%x cost:%u", + __func__, msg.vrf_name, msg.source_ip, msg.group_ip, + msg.cost_to_rp); + zlog_debug( + "owner_id:%d, DR:%d, Dual active:%d, vrf_id:0x%x intf_name:%s", + msg.owner_id, msg.am_i_dr, msg.am_i_dual_active, + msg.vrf_id, msg.intf_name); + } +} + +static void pim_mlag_process_mroute_del(struct mlag_mroute_del msg) +{ + if (PIM_DEBUG_MLAG) { + zlog_debug("%s: msg dump: vrf_name:%s, s.ip:0x%x, g.ip:0x%x ", + __func__, msg.vrf_name, msg.source_ip, msg.group_ip); + zlog_debug("owner_id:%d, vrf_id:0x%x intf_name:%s", + msg.owner_id, msg.vrf_id, msg.intf_name); + } +} + + +int pim_zebra_mlag_handle_msg(struct stream *s, int len) +{ + struct mlag_msg mlag_msg; + char buf[80]; + int rc = 0; + + rc = zebra_mlag_lib_decode_mlag_hdr(s, &mlag_msg); + if (rc) + return (rc); + + if (PIM_DEBUG_MLAG) + zlog_debug( + "%s: Received msg type:%s length:%d, bulk_cnt:%d", + __func__, + zebra_mlag_lib_msgid_to_str(mlag_msg.msg_type, buf, 80), + mlag_msg.data_len, mlag_msg.msg_cnt); + + switch (mlag_msg.msg_type) { + case MLAG_STATUS_UPDATE: { + struct mlag_status msg; + + rc = zebra_mlag_lib_decode_mlag_status(s, &msg); + if (rc) + return (rc); + pim_mlag_process_mlagd_state_change(msg); + } break; + case MLAG_PEER_FRR_STATUS: { + struct mlag_frr_status msg; + + rc = zebra_mlag_lib_decode_frr_status(s, &msg); + if (rc) + return (rc); + pim_mlag_process_peer_frr_state_change(msg); + } break; + case MLAG_VXLAN_UPDATE: { + struct mlag_vxlan msg; + + rc = zebra_mlag_lib_decode_vxlan_update(s, &msg); + if (rc) + return rc; + pim_mlag_process_vxlan_update(&msg); + } break; + case MLAG_MROUTE_ADD: { + struct mlag_mroute_add msg; + + rc = zebra_mlag_lib_decode_mroute_add(s, &msg); + if (rc) + return (rc); + pim_mlag_process_mroute_add(msg); + } break; + case MLAG_MROUTE_DEL: { + struct mlag_mroute_del msg; + + rc = zebra_mlag_lib_decode_mroute_del(s, &msg); + if (rc) + return (rc); + pim_mlag_process_mroute_del(msg); + } break; + case MLAG_MROUTE_ADD_BULK: { + struct mlag_mroute_add msg; + int i = 0; + + for (i = 0; i < mlag_msg.msg_cnt; i++) { + + rc = zebra_mlag_lib_decode_mroute_add(s, &msg); + if (rc) + return (rc); + pim_mlag_process_mroute_add(msg); + } + } break; + case MLAG_MROUTE_DEL_BULK: { + struct mlag_mroute_del msg; + int i = 0; + + for (i = 0; i < mlag_msg.msg_cnt; i++) { + + rc = zebra_mlag_lib_decode_mroute_del(s, &msg); + if (rc) + return (rc); + pim_mlag_process_mroute_del(msg); + } + } break; + default: + break; + } + return 0; +} + +/****************End of PIM Mesasge processing handler********************/ + +int pim_zebra_mlag_process_up(void) +{ + if (PIM_DEBUG_MLAG) + zlog_debug("%s: Received Process-Up from Mlag", __func__); + + return 0; +} + +int pim_zebra_mlag_process_down(void) +{ + if (PIM_DEBUG_MLAG) + zlog_debug("%s: Received Process-Down from Mlag", __func__); + + return 0; +} + static int pim_mlag_register_handler(struct thread *thread) { uint32_t bit_mask = 0; diff --git a/pimd/pim_mlag.h b/pimd/pim_mlag.h index 03fbfccb42..f35124e485 100644 --- a/pimd/pim_mlag.h +++ b/pimd/pim_mlag.h @@ -39,4 +39,10 @@ extern void pim_mlag_register(void); extern void pim_mlag_deregister(void); +extern int pim_zebra_mlag_process_up(void); + +extern int pim_zebra_mlag_process_down(void); + +extern int pim_zebra_mlag_handle_msg(struct stream *msg, int len); + #endif diff --git a/pimd/pim_zebra.c b/pimd/pim_zebra.c index e9cfc7e520..073943c695 100644 --- a/pimd/pim_zebra.c +++ b/pimd/pim_zebra.c @@ -46,6 +46,7 @@ #include "pim_nht.h" #include "pim_ssm.h" #include "pim_vxlan.h" +#include "pim_mlag.h" #undef PIM_DEBUG_IFADDR_DUMP #define PIM_DEBUG_IFADDR_DUMP @@ -803,6 +804,9 @@ void pim_zebra_init(void) zclient->nexthop_update = pim_parse_nexthop_update; zclient->vxlan_sg_add = pim_zebra_vxlan_sg_proc; zclient->vxlan_sg_del = pim_zebra_vxlan_sg_proc; + zclient->mlag_process_up = pim_zebra_mlag_process_up; + zclient->mlag_process_down = pim_zebra_mlag_process_down; + zclient->mlag_handle_msg = pim_zebra_mlag_handle_msg; zclient_init(zclient, ZEBRA_ROUTE_PIM, 0, &pimd_privs); if (PIM_DEBUG_PIM_TRACE) { From 40d9d1cc444bc77847a45629c58c65044ce48efa Mon Sep 17 00:00:00 2001 From: Satheesh Kumar K Date: Mon, 26 Aug 2019 04:35:37 -0700 Subject: [PATCH 3/6] Zebra: adding support for Zebra MLAG Functionality This includes: 1. Processing client Registrations for MLAG 2. storing client Interests for MLAG updates 3. Opening communication channel to MLAG with First client reg 4. Closing Communication channel with last client De-reg 5. Spawning a new thread for handling MLAG updates peocessing 6. adding Test code 7. advertising MLAG Updates to clients based on their interests Signed-off-by: Satheesh Kumar K --- lib/mlag.h | 2 + zebra/subdir.am | 1 + zebra/zapi_msg.c | 3 + zebra/zebra_mlag.c | 845 ++++++++++++++++++++++++++++++++++++- zebra/zebra_mlag.h | 51 ++- zebra/zebra_mlag_private.c | 298 +++++++++++++ zebra/zebra_mlag_private.h | 40 ++ zebra/zebra_router.h | 29 ++ zebra/zserv.h | 7 + 9 files changed, 1260 insertions(+), 16 deletions(-) create mode 100644 zebra/zebra_mlag_private.c create mode 100644 zebra/zebra_mlag_private.h diff --git a/lib/mlag.h b/lib/mlag.h index b93d5b7497..a88db8b35e 100644 --- a/lib/mlag.h +++ b/lib/mlag.h @@ -30,6 +30,8 @@ extern "C" { #include "lib/vrf.h" #include "lib/stream.h" +#define MLAG_MSG_NULL_PAYLOAD 0 +#define MLAG_MSG_NO_BATCH 1 #define MLAG_BUF_LIMIT 2048 enum mlag_role { diff --git a/zebra/subdir.am b/zebra/subdir.am index 25040a2717..5016ca1f28 100644 --- a/zebra/subdir.am +++ b/zebra/subdir.am @@ -66,6 +66,7 @@ zebra_zebra_SOURCES = \ zebra/rule_netlink.c \ zebra/rule_socket.c \ zebra/zebra_mlag.c \ + zebra/zebra_mlag_private.c \ zebra/zebra_l2.c \ zebra/zebra_memory.c \ zebra/zebra_dplane.c \ diff --git a/zebra/zapi_msg.c b/zebra/zapi_msg.c index fa6a2f62ec..6c9fff03ef 100644 --- a/zebra/zapi_msg.c +++ b/zebra/zapi_msg.c @@ -2491,6 +2491,9 @@ void (*zserv_handlers[])(ZAPI_HANDLER_ARGS) = { [ZEBRA_IPTABLE_DELETE] = zread_iptable, [ZEBRA_VXLAN_FLOOD_CONTROL] = zebra_vxlan_flood_control, [ZEBRA_VXLAN_SG_REPLAY] = zebra_vxlan_sg_replay, + [ZEBRA_MLAG_CLIENT_REGISTER] = zebra_mlag_client_register, + [ZEBRA_MLAG_CLIENT_UNREGISTER] = zebra_mlag_client_unregister, + [ZEBRA_MLAG_FORWARD_MSG] = zebra_mlag_forward_client_msg, }; #if defined(HANDLE_ZAPI_FUZZING) diff --git a/zebra/zebra_mlag.c b/zebra/zebra_mlag.c index 5012cc2a49..90af959d27 100644 --- a/zebra/zebra_mlag.c +++ b/zebra/zebra_mlag.c @@ -23,9 +23,13 @@ #include "command.h" #include "hook.h" +#include "frr_pthread.h" +#include "mlag.h" #include "zebra/zebra_mlag.h" +#include "zebra/zebra_mlag_private.h" #include "zebra/zebra_router.h" +#include "zebra/zebra_memory.h" #include "zebra/zapi_msg.h" #include "zebra/debug.h" @@ -33,6 +37,545 @@ #include "zebra/zebra_mlag_clippy.c" #endif +#define ZEBRA_MLAG_METADATA_LEN 4 +#define ZEBRA_MLAG_MSG_BCAST 0xFFFFFFFF + +uint8_t mlag_wr_buffer[ZEBRA_MLAG_BUF_LIMIT]; +uint8_t mlag_rd_buffer[ZEBRA_MLAG_BUF_LIMIT]; +uint32_t mlag_rd_buf_offset; + +static bool test_mlag_in_progress; + +static int zebra_mlag_signal_write_thread(void); +static int zebra_mlag_terminate_pthread(struct thread *event); +static int zebra_mlag_post_data_from_main_thread(struct thread *thread); +static void zebra_mlag_publish_process_state(struct zserv *client, + zebra_message_types_t msg_type); + +/**********************MLAG Interaction***************************************/ + +/* + * API to post the Registartion to MLAGD + * MLAG will not process any messages with out the registration + */ +void zebra_mlag_send_register(void) +{ + struct stream *s = NULL; + + s = stream_new(sizeof(struct mlag_msg)); + if (!s) + return; + + stream_putl(s, MLAG_REGISTER); + stream_putw(s, MLAG_MSG_NULL_PAYLOAD); + stream_putw(s, MLAG_MSG_NO_BATCH); + stream_fifo_push_safe(zrouter.mlag_info.mlag_fifo, s); + zebra_mlag_signal_write_thread(); + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("%s: Enqueued MLAG Register to MLAG Thread ", + __func__); +} + +/* + * API to post the De-Registartion to MLAGD + * MLAG will not process any messages after the de-registration + */ +void zebra_mlag_send_deregister(void) +{ + struct stream *s = NULL; + + s = stream_new(sizeof(struct mlag_msg)); + if (!s) + return; + + stream_putl(s, MLAG_DEREGISTER); + stream_putw(s, MLAG_MSG_NULL_PAYLOAD); + stream_putw(s, MLAG_MSG_NO_BATCH); + stream_fifo_push_safe(zrouter.mlag_info.mlag_fifo, s); + zebra_mlag_signal_write_thread(); + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("%s: Enqueued MLAG De-Register to MLAG Thread ", + __func__); +} + +/* + * API To handle MLAG Received data + * Decodes teh data using protobuf and enqueue to main thread + * main thread publish this to clients based on client subscrption + */ +void zebra_mlag_process_mlag_data(uint8_t *data, uint32_t len) +{ + struct stream *s = NULL; + struct stream *s1 = NULL; + int msg_type = 0; + + s = stream_new(ZEBRA_MAX_PACKET_SIZ); + if (s) + msg_type = zebra_mlag_protobuf_decode_message(&s, data, len); + + if (msg_type <= 0) { + /* Something went wrong in decoding */ + stream_free(s); + zlog_err("%s: failed to process mlag data-%d, %u", __func__, + msg_type, len); + return; + } + + /* + * additional four bytes are for mesasge type + */ + s1 = stream_new(stream_get_endp(s) + ZEBRA_MLAG_METADATA_LEN); + stream_putl(s1, msg_type); + stream_put(s1, s->data, stream_get_endp(s)); + thread_add_event(zrouter.master, zebra_mlag_post_data_from_main_thread, + s1, 0, NULL); + stream_free(s); +} + +/**********************End of MLAG Interaction********************************/ + +/************************MLAG Thread Processing*******************************/ + +/* + * after posting every 1000 packets, MLAG Thread wll be yielded to give CPU + * for other threads + */ +#define ZEBRA_MLAG_POST_LIMIT 100 + +/* + * Thsi thread reads the clients data from the Gloabl queue and encodes with + * protobuf and pass on to the MLAG socket. + */ +static int zebra_mlag_thread_handler(struct thread *event) +{ + struct stream *s; + uint32_t wr_count = 0; + uint32_t msg_type = 0; + int len = 0; + + wr_count = stream_fifo_count_safe(zrouter.mlag_info.mlag_fifo); + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug(":%s: Processing MLAG write, %d messages in queue", + __func__, wr_count); + + zrouter.mlag_info.t_write = NULL; + for (wr_count = 0; wr_count < ZEBRA_MLAG_POST_LIMIT; wr_count++) { + /* FIFO is empty,wait for teh message to be add */ + if (stream_fifo_count_safe(zrouter.mlag_info.mlag_fifo) == 0) + break; + + s = stream_fifo_pop_safe(zrouter.mlag_info.mlag_fifo); + if (!s) { + zlog_debug(":%s: Got a NULL Messages, some thing wrong", + __func__); + break; + } + + zebra_mlag_reset_write_buffer(); + /* + * Encode the data now + */ + len = zebra_mlag_protobuf_encode_client_data(s, &msg_type); + + /* + * write to MCLAGD + */ + if (len > 0) + zebra_mlag_private_write_data(mlag_wr_buffer, len); + + /* + * If mesasge type is De-register, send a signal to main thread, + * sothat necessary cleanup will be done by main thread. + */ + if (msg_type == MLAG_DEREGISTER) { + thread_add_event(zrouter.master, + zebra_mlag_terminate_pthread, NULL, 0, + NULL); + } + + stream_free(s); + } + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug(":%s: Posted %d messages to MLAGD", __func__, + wr_count); + /* + * Currently there is only message write task is enqueued to this + * thread, yielding was added for future purpose, sothat this thread can + * server other tasks also and in case FIFO is empty, this task will be + * schedule when main thread adds some messages + */ + if (wr_count >= ZEBRA_MLAG_POST_LIMIT) + zebra_mlag_signal_write_thread(); + return 0; +} + +/* + * API to handle teh process state. + * In case of Down, Zebra keep monitoring the MLAG state. + * all the state Notifications will be published to clients + */ +void zebra_mlag_handle_process_state(enum zebra_mlag_state state) +{ + if (state == MLAG_UP) { + zrouter.mlag_info.connected = true; + zebra_mlag_publish_process_state(NULL, ZEBRA_MLAG_PROCESS_UP); + zebra_mlag_send_register(); + } else if (state == MLAG_DOWN) { + zrouter.mlag_info.connected = false; + zebra_mlag_publish_process_state(NULL, ZEBRA_MLAG_PROCESS_DOWN); + zebra_mlag_private_monitor_state(); + } +} + +/***********************End of MLAG Thread processing*************************/ + +/*************************Multi-entratnt Api's********************************/ + +/* + * Provider api to signal that work/events are available + * for the Zebra MLAG Write pthread. + * This API is called from 2 pthreads.. + * 1) by main thread when client posts a MLAG Message + * 2) by MLAG Thread, in case of yield + * though this api, is called from two threads we don't need any locking + * because Thread task enqueue is thread safe means internally it had + * necessary protection + */ +static int zebra_mlag_signal_write_thread(void) +{ + if (zrouter.mlag_info.zebra_pth_mlag) { + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug(":%s: Scheduling MLAG write", __func__); + thread_add_event(zrouter.mlag_info.th_master, + zebra_mlag_thread_handler, NULL, 0, + &zrouter.mlag_info.t_write); + } + return 0; +} + +/* + * API will be used to publish the MLAG state to interested clients + * In case client is passed, state is posted only for that client, + * otherwise to all interested clients + * this api can be called from two threads. + * 1) from main thread: when client is passed + * 2) from MLAG Thread: when client is NULL + * + * In second case, to avoid global data access data will be post to Main + * thread, so that actual posting to cleints will happen from Main thread. + */ +static void zebra_mlag_publish_process_state(struct zserv *client, + zebra_message_types_t msg_type) +{ + struct stream *s = NULL; + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("%s: Publishing MLAG process state:%s to %s Client", + __func__, + (msg_type == ZEBRA_MLAG_PROCESS_UP) ? "UP" : "DOWN", + (client) ? "one" : "all"); + + if (client) { + s = stream_new(ZEBRA_HEADER_SIZE); + zclient_create_header(s, msg_type, VRF_DEFAULT); + zserv_send_message(client, s); + return; + } + + + /* + * additional four bytes are for mesasge type + */ + s = stream_new(ZEBRA_HEADER_SIZE + ZEBRA_MLAG_METADATA_LEN); + stream_putl(s, ZEBRA_MLAG_MSG_BCAST); + zclient_create_header(s, msg_type, VRF_DEFAULT); + thread_add_event(zrouter.master, zebra_mlag_post_data_from_main_thread, + s, 0, NULL); +} + +/**************************End of Multi-entrant Apis**************************/ + +/***********************Zebra Main thread processing**************************/ + +/* + * To avoid data corruption, messages will be post to clients only from + * main thread, beacuse for that access was needed for clients list. + * so instaed of forcing the locks, messages will be posted from main thread. + */ +static int zebra_mlag_post_data_from_main_thread(struct thread *thread) +{ + struct stream *s = THREAD_ARG(thread); + struct stream *zebra_s = NULL; + struct listnode *node; + struct zserv *client; + uint32_t msg_type = 0; + uint32_t msg_len = 0; + + if (!s) + return -1; + + STREAM_GETL(s, msg_type); + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug( + "%s: Posting MLAG data for msg_type:0x%x to interested cleints", + __func__, msg_type); + + msg_len = s->endp - ZEBRA_MLAG_METADATA_LEN; + for (ALL_LIST_ELEMENTS_RO(zrouter.client_list, node, client)) { + if (client->mlag_updates_interested == true) { + if (msg_type != ZEBRA_MLAG_MSG_BCAST + && !CHECK_FLAG(client->mlag_reg_mask1, + (1 << msg_type))) { + continue; + } + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug( + "%s: Posting MLAG data of length-%d to client:%d ", + __func__, msg_len, client->proto); + + zebra_s = stream_new(msg_len); + STREAM_GET(zebra_s->data, s, msg_len); + zebra_s->endp = msg_len; + stream_putw_at(zebra_s, 0, msg_len); + + /* + * This stream will be enqueued to client_obuf, it will + * be freed after posting to client socket. + */ + zserv_send_message(client, zebra_s); + zebra_s = NULL; + } + } + + stream_free(s); + return 0; +stream_failure: + stream_free(s); + if (zebra_s) + stream_free(zebra_s); + return 0; +} + +/* + * Start the MLAG Thread, this will be used to write client data on to + * MLAG Process and to read the data from MLAG and post to cleints. + * when all clients are un-registered, this Thread will be + * suspended. + */ +static void zebra_mlag_spawn_pthread(void) +{ + /* Start MLAG write pthread */ + + struct frr_pthread_attr pattr = {.start = + frr_pthread_attr_default.start, + .stop = frr_pthread_attr_default.stop}; + + zrouter.mlag_info.zebra_pth_mlag = + frr_pthread_new(&pattr, "Zebra MLAG thread", "Zebra MLAG"); + + zrouter.mlag_info.th_master = zrouter.mlag_info.zebra_pth_mlag->master; + + + /* Enqueue an initial event for the dataplane pthread */ + zebra_mlag_signal_write_thread(); + + frr_pthread_run(zrouter.mlag_info.zebra_pth_mlag, NULL); +} + +/* + * all clients are un-registered for MLAG Updates, terminate the + * MLAG write thread + */ +static int zebra_mlag_terminate_pthread(struct thread *event) +{ + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("Zebra MLAG write thread terminate calleid"); + + if (zrouter.mlag_info.clients_interested_cnt) { + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug( + "Zebra MLAG: still some clients are interested"); + return 0; + } + + frr_pthread_stop(zrouter.mlag_info.zebra_pth_mlag, NULL); + + /* Destroy pthread */ + frr_pthread_destroy(zrouter.mlag_info.zebra_pth_mlag); + zrouter.mlag_info.zebra_pth_mlag = NULL; + zrouter.mlag_info.th_master = NULL; + zrouter.mlag_info.t_read = NULL; + zrouter.mlag_info.t_write = NULL; + + /* + * Send Notification to clean private data + */ + zebra_mlag_private_cleanup_data(); + return 0; +} + +/* + * API to register zebra client for MLAG Updates + */ +void zebra_mlag_client_register(ZAPI_HANDLER_ARGS) +{ + struct stream *s; + uint32_t reg_mask = 0; + int rc = 0; + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("Received MLAG Registration from client-proto:%d", + client->proto); + + + /* Get input stream. */ + s = msg; + + /* Get data. */ + STREAM_GETL(s, reg_mask); + + if (client->mlag_updates_interested == true) { + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug( + "Client is registered, existing mask: 0x%x, new mask: 0x%x", + client->mlag_reg_mask1, reg_mask); + if (client->mlag_reg_mask1 != reg_mask) + client->mlag_reg_mask1 = reg_mask; + /* + * Client might missed MLAG-UP Notification, post-it again + */ + zebra_mlag_publish_process_state(client, ZEBRA_MLAG_PROCESS_UP); + return; + } + + + client->mlag_updates_interested = true; + client->mlag_reg_mask1 = reg_mask; + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("Registering for MLAG Upadtes with mask: 0x%x, ", + client->mlag_reg_mask1); + + zrouter.mlag_info.clients_interested_cnt++; + + if (zrouter.mlag_info.clients_interested_cnt == 1) { + /* + * First-client for MLAG Updates,open the communication channel + * with MLAG + */ + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug( + "First client, opening the channel with MLAG"); + + zebra_mlag_spawn_pthread(); + rc = zebra_mlag_private_open_channel(); + if (rc < 0) { + /* + * For some reason, zebra not able to open the + * comm-channel with MLAG, so post MLAG-DOWN to client. + * later when the channel is open, zebra will send + * MLAG-UP + */ + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug( + "Fail to open channel with MLAG,rc:%d, post Proto-down", + rc); + } + } + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("Client Registered successfully for MLAG Updates"); + + if (zrouter.mlag_info.connected == true) + zebra_mlag_publish_process_state(client, ZEBRA_MLAG_PROCESS_UP); +stream_failure: + return; +} + +/* + * API to un-register for MLAG Updates + */ +void zebra_mlag_client_unregister(ZAPI_HANDLER_ARGS) +{ + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("Received MLAG De-Registration from client-proto:%d", + client->proto); + + if (client->mlag_updates_interested == false) + /* Unexpected */ + return; + + client->mlag_updates_interested = false; + client->mlag_reg_mask1 = 0; + zrouter.mlag_info.clients_interested_cnt--; + + if (zrouter.mlag_info.clients_interested_cnt == 0) { + /* + * No-client is interested for MLAG Updates,close the + * communication channel with MLAG + */ + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("Last client for MLAG, close the channel "); + + /* + * Clean up flow: + * ============= + * 1) main thread calls socket close which posts De-register + * to MLAG write thread + * 2) after MLAG write thread posts De-register it sends a + * signal back to main thread to do the thread cleanup + * this was mainly to make sure De-register is posted to MCLAGD. + */ + zebra_mlag_private_close_channel(); + } + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug( + "Client De-Registered successfully for MLAG Updates"); +} + +/* + * Does following things. + * 1) allocated new local stream, and copies teh client data and enqueue + * to MLAG Thread + * 2) MLAG Thread after dequeing, encode the client data using protobuf + * and write on to MLAG + */ +void zebra_mlag_forward_client_msg(ZAPI_HANDLER_ARGS) +{ + struct stream *zebra_s; + struct stream *mlag_s; + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("Received Client MLAG Data from client-proto:%d", + client->proto); + + /* Get input stream. */ + zebra_s = msg; + mlag_s = stream_new(zebra_s->endp); + if (!mlag_s) + return; + + /* + * Client data is | Zebra Header + MLAG Data | + * we need to enqueue only the MLAG data, skipping Zebra Header + */ + stream_put(mlag_s, zebra_s->data + zebra_s->getp, + zebra_s->endp - zebra_s->getp); + stream_fifo_push_safe(zrouter.mlag_info.mlag_fifo, mlag_s); + zebra_mlag_signal_write_thread(); + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("%s: Enqueued Client:%d data to MLAG Thread ", + __func__, client->proto); +} + +/***********************End of Zebra Main thread processing*************/ + enum mlag_role zebra_mlag_get_role(void) { return zrouter.mlag_info.role; @@ -53,15 +596,198 @@ DEFUN_HIDDEN (show_mlag, return CMD_SUCCESS; } -DEFPY_HIDDEN (test_mlag, - test_mlag_cmd, - "test zebra mlag ", - "Test code\n" - ZEBRA_STR - "Modify the Mlag state\n" - "Mlag is not setup on the machine\n" - "Mlag is setup to be primary\n" - "Mlag is setup to be the secondary\n") +static void test_mlag_post_mroute_add(void) +{ + struct stream *s = NULL; + char vrf_temp[20]; + char intf_temp[20]; + + s = stream_new(ZEBRA_MAX_PACKET_SIZ); + if (!s) + return; + + memset(vrf_temp, 0, 20); + memset(intf_temp, 0, 20); + + strlcpy(vrf_temp, "test", 20); + strlcpy(intf_temp, "br0.11", 20); + + stream_putl(s, MLAG_MROUTE_ADD); + stream_putw(s, sizeof(struct mlag_mroute_add)); + stream_putw(s, MLAG_MSG_NO_BATCH); + + /* payload*/ + stream_put(s, vrf_temp, VRF_NAMSIZ); + stream_putl(s, 0x01010101); /*source_ip*/ + stream_putl(s, 0xE4000001); /*group_ip*/ + stream_putl(s, 10); /*cost_to_rp*/ + stream_putl(s, 5); /*vni_id */ + stream_putc(s, 1); /*am_i_dr */ + stream_putc(s, 1); /*dual_active */ + stream_putl(s, 0x1004); /*vrf_id*/ + stream_put(s, intf_temp, INTERFACE_NAMSIZ); + stream_fifo_push_safe(zrouter.mlag_info.mlag_fifo, s); + zebra_mlag_signal_write_thread(); + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("%s: Enqueued MLAG Mroute-add to MLAG Thread ", + __func__); +} + +static void test_mlag_post_mroute_del(void) +{ + struct stream *s = NULL; + char vrf_temp[20]; + char intf_temp[20]; + + + s = stream_new(ZEBRA_MAX_PACKET_SIZ); + if (!s) + return; + + memset(vrf_temp, 0, 20); + memset(intf_temp, 0, 20); + + strlcpy(vrf_temp, "test", 20); + strlcpy(intf_temp, "br0.11", 20); + + stream_putl(s, MLAG_MROUTE_DEL); + stream_putw(s, sizeof(struct mlag_mroute_del)); + stream_putw(s, MLAG_MSG_NO_BATCH); + + /* payload*/ + stream_put(s, vrf_temp, VRF_NAMSIZ); + stream_putl(s, 0x01010101); /*source_ip*/ + stream_putl(s, 0xE4000001); /*group_ip*/ + stream_putl(s, 5); /*vni_id */ + stream_putl(s, 0x1004); /*vrf_id*/ + stream_put(s, intf_temp, INTERFACE_NAMSIZ); + stream_fifo_push_safe(zrouter.mlag_info.mlag_fifo, s); + zebra_mlag_signal_write_thread(); + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("%s: Enqueued MLAG Mroute-Del to MLAG Thread ", + __func__); +} + +static void test_mlag_post_mroute_bulk_add(void) +{ + struct stream *s = NULL; + char vrf_temp[20]; + char intf_temp[20]; + + s = stream_new(ZEBRA_MAX_PACKET_SIZ); + if (!s) + return; + + memset(vrf_temp, 0, 20); + memset(intf_temp, 0, 20); + + strlcpy(vrf_temp, "test", 20); + strlcpy(intf_temp, "br0.11", 20); + + stream_putl(s, MLAG_MROUTE_ADD_BULK); + stream_putw(s, 3 * sizeof(struct mlag_mroute_add)); + stream_putw(s, 3); + + /* payload-1*/ + stream_put(s, vrf_temp, VRF_NAMSIZ); + stream_putl(s, 0x01010101); /*source_ip*/ + stream_putl(s, 0xE4000001); /*group_ip*/ + stream_putl(s, 10); /*cost_to_rp*/ + stream_putl(s, 5); /*vni_id */ + stream_putc(s, 1); /*am_i_dr */ + stream_putc(s, 1); /*dual_active */ + stream_putl(s, 0x1004); /*vrf_id*/ + stream_put(s, intf_temp, INTERFACE_NAMSIZ); + + /* payload-2*/ + stream_put(s, vrf_temp, VRF_NAMSIZ); + stream_putl(s, 0x0); /*source_ip*/ + stream_putl(s, 0xE9000001); /*group_ip*/ + stream_putl(s, 10); /*cost_to_rp*/ + stream_putl(s, 5); /*vni_id */ + stream_putc(s, 1); /*am_i_dr */ + stream_putc(s, 1); /*dual_active */ + stream_putl(s, 0x1004); /*vrf_id*/ + stream_put(s, intf_temp, INTERFACE_NAMSIZ); + + /* payload-3*/ + stream_put(s, vrf_temp, VRF_NAMSIZ); + stream_putl(s, 0x01010101); /*source_ip*/ + stream_putl(s, 0xE5000001); /*group_ip*/ + stream_putl(s, 10); /*cost_to_rp*/ + stream_putl(s, 5); /*vni_id */ + stream_putc(s, 1); /*am_i_dr */ + stream_putc(s, 1); /*dual_active */ + stream_putl(s, 0x1004); /*vrf_id*/ + stream_put(s, intf_temp, INTERFACE_NAMSIZ); + stream_fifo_push_safe(zrouter.mlag_info.mlag_fifo, s); + zebra_mlag_signal_write_thread(); + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("%s: Enqueued MLAG Mroute-Bulk to MLAG Thread ", + __func__); +} + +static void test_mlag_post_mroute_bulk_del(void) +{ + struct stream *s = NULL; + char vrf_temp[20]; + char intf_temp[20]; + + s = stream_new(ZEBRA_MAX_PACKET_SIZ); + if (!s) + return; + + memset(vrf_temp, 0, 20); + memset(intf_temp, 0, 20); + + strlcpy(vrf_temp, "test", 20); + strlcpy(intf_temp, "br0.11", 20); + + stream_putl(s, MLAG_MROUTE_DEL_BULK); + stream_putw(s, 2 * sizeof(struct mlag_mroute_del)); + stream_putw(s, 2); + + /* payload-1*/ + stream_put(s, vrf_temp, VRF_NAMSIZ); + stream_putl(s, 0x01010101); /*source_ip*/ + stream_putl(s, 0xE4000001); /*group_ip*/ + stream_putl(s, 5); /*vni_id */ + stream_putl(s, 0x1004); /*vrf_id*/ + stream_put(s, intf_temp, INTERFACE_NAMSIZ); + + /* payload-2*/ + stream_put(s, vrf_temp, VRF_NAMSIZ); + stream_putl(s, 0x0); /*source_ip*/ + stream_putl(s, 0xE9000001); /*group_ip*/ + stream_putl(s, 5); /*vni_id */ + stream_putl(s, 0x1004); /*vrf_id*/ + stream_put(s, intf_temp, INTERFACE_NAMSIZ); + + /* payload-3*/ + stream_put(s, vrf_temp, VRF_NAMSIZ); + stream_putl(s, 0x01010101); /*source_ip*/ + stream_putl(s, 0xE5000001); /*group_ip*/ + stream_putl(s, 5); /*vni_id */ + stream_putl(s, 0x1004); /*vrf_id*/ + stream_put(s, intf_temp, INTERFACE_NAMSIZ); + stream_fifo_push_safe(zrouter.mlag_info.mlag_fifo, s); + zebra_mlag_signal_write_thread(); + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("%s: Enqueued MLAG Mroute-Bulk to MLAG Thread ", + __func__); +} + +DEFPY(test_mlag, test_mlag_cmd, + "test zebra mlag ", + "Test code\n" ZEBRA_STR + "Modify the Mlag state\n" + "Mlag is not setup on the machine\n" + "Mlag is setup to be primary\n" + "Mlag is setup to be the secondary\n") { enum mlag_role orig = zrouter.mlag_info.role; char buf1[80], buf2[80]; @@ -78,8 +804,72 @@ DEFPY_HIDDEN (test_mlag, mlag_role2str(orig, buf1, sizeof(buf1)), mlag_role2str(orig, buf2, sizeof(buf2))); - if (orig != zrouter.mlag_info.role) + if (orig != zrouter.mlag_info.role) { zsend_capabilities_all_clients(); + if (zrouter.mlag_info.role != MLAG_ROLE_NONE) { + if (zrouter.mlag_info.clients_interested_cnt == 0 + && test_mlag_in_progress == false) { + if (zrouter.mlag_info.zebra_pth_mlag == NULL) + zebra_mlag_spawn_pthread(); + zrouter.mlag_info.clients_interested_cnt++; + test_mlag_in_progress = true; + zebra_mlag_private_open_channel(); + } + } else { + if (test_mlag_in_progress == true) { + test_mlag_in_progress = false; + zrouter.mlag_info.clients_interested_cnt--; + zebra_mlag_private_close_channel(); + } + } + } + + return CMD_SUCCESS; +} + +DEFPY(test_mlag_route, test_mlag_route_cmd, + "test zebra mlag route ", + "Test code\n" ZEBRA_STR + "Modify the Mlag state\n" + "Post Route Action to Mlag\n" + "Posting Route-add\n" + "Posting Route-del\n") +{ + + if (zrouter.mlag_info.connected == false) { + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("Test: Not connected to MLAG"); + return CMD_SUCCESS; + } + + if (add) + test_mlag_post_mroute_add(); + if (del) + test_mlag_post_mroute_del(); + + return CMD_SUCCESS; +} + +DEFPY(test_mlag_route_bulk, test_mlag_route_bulk_cmd, + "test zebra mlag route bulk ", + "Test code\n" ZEBRA_STR + "Modify the Mlag state\n" + "Post Route Action to Mlag\n" + "Posting Route-bulk\n" + "Posting Route-add\n" + "Posting Route-del\n") +{ + + if (zrouter.mlag_info.connected == false) { + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("Test: Not connected to MLAG"); + return CMD_SUCCESS; + } + + if (add) + test_mlag_post_mroute_bulk_add(); + if (del) + test_mlag_post_mroute_bulk_del(); return CMD_SUCCESS; } @@ -88,8 +878,43 @@ void zebra_mlag_init(void) { install_element(VIEW_NODE, &show_mlag_cmd); install_element(ENABLE_NODE, &test_mlag_cmd); + install_element(ENABLE_NODE, &test_mlag_route_cmd); + install_element(ENABLE_NODE, &test_mlag_route_bulk_cmd); + + /* + * Intialiaze teh MLAG Global variableis + * write thread will be craeted during actual registration with MCLAG + */ + zrouter.mlag_info.clients_interested_cnt = 0; + zrouter.mlag_info.connected = false; + zrouter.mlag_info.timer_running = false; + zrouter.mlag_info.mlag_fifo = stream_fifo_new(); + zrouter.mlag_info.zebra_pth_mlag = NULL; + zrouter.mlag_info.th_master = NULL; + zrouter.mlag_info.t_read = NULL; + zrouter.mlag_info.t_write = NULL; + test_mlag_in_progress = false; + zebra_mlag_reset_write_buffer(); + zebra_mlag_reset_read_buffer(); } void zebra_mlag_terminate(void) { } + + +/* + * + * ProtoBuf Encoding APIs + */ + +int zebra_mlag_protobuf_encode_client_data(struct stream *s, uint32_t *msg_type) +{ + return 0; +} + +int zebra_mlag_protobuf_decode_message(struct stream **s, uint8_t *data, + uint32_t len) +{ + return 0; +} diff --git a/zebra/zebra_mlag.h b/zebra/zebra_mlag.h index 90a5a41fa4..a145753d88 100644 --- a/zebra/zebra_mlag.h +++ b/zebra/zebra_mlag.h @@ -23,18 +23,57 @@ #define __ZEBRA_MLAG_H__ #include "mlag.h" +#include "zclient.h" +#include "zebra/zserv.h" -#ifdef __cplusplus -extern "C" { -#endif +#define ZEBRA_MLAG_BUF_LIMIT 2048 +#define ZEBRA_MLAG_LEN_SIZE 4 + +extern uint8_t mlag_wr_buffer[ZEBRA_MLAG_BUF_LIMIT]; +extern uint8_t mlag_rd_buffer[ZEBRA_MLAG_BUF_LIMIT]; +extern uint32_t mlag_rd_buf_offset; + +static inline void zebra_mlag_reset_write_buffer(void) +{ + memset(mlag_wr_buffer, 0, ZEBRA_MLAG_BUF_LIMIT); +} + +static inline void zebra_mlag_reset_read_buffer(void) +{ + memset(mlag_rd_buffer, 0, ZEBRA_MLAG_BUF_LIMIT); + mlag_rd_buf_offset = 0; +} + +enum zebra_mlag_state { + MLAG_UP = 1, + MLAG_DOWN = 2, +}; void zebra_mlag_init(void); void zebra_mlag_terminate(void); enum mlag_role zebra_mlag_get_role(void); -#ifdef __cplusplus -} -#endif +void zebra_mlag_client_register(ZAPI_HANDLER_ARGS); + +void zebra_mlag_client_unregister(ZAPI_HANDLER_ARGS); + +void zebra_mlag_forward_client_msg(ZAPI_HANDLER_ARGS); + +void zebra_mlag_send_register(void); + +void zebra_mlag_send_deregister(void); + +void zebra_mlag_handle_process_state(enum zebra_mlag_state state); + +void zebra_mlag_process_mlag_data(uint8_t *data, uint32_t len); + +/* + * ProtoBuffer Api's + */ +int zebra_mlag_protobuf_encode_client_data(struct stream *s, + uint32_t *msg_type); +int zebra_mlag_protobuf_decode_message(struct stream **s, uint8_t *data, + uint32_t len); #endif diff --git a/zebra/zebra_mlag_private.c b/zebra/zebra_mlag_private.c new file mode 100644 index 0000000000..a5637c67ae --- /dev/null +++ b/zebra/zebra_mlag_private.c @@ -0,0 +1,298 @@ +/* Zebra Mlag Code. + * Copyright (C) 2018 Cumulus Networks, Inc. + * Donald Sharp + * + * This file is part of FRR. + * + * FRR is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2, or (at your option) any + * later version. + * + * FRR is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with FRR; see the file COPYING. If not, write to the Free + * Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA + * 02111-1307, USA. + */ +#include "zebra.h" + +#include "hook.h" +#include "module.h" +#include "thread.h" +#include "libfrr.h" +#include "version.h" +#include "network.h" + +#include "lib/stream.h" + +#include "zebra/debug.h" +#include "zebra/zebra_router.h" +#include "zebra/zebra_mlag.h" +#include "zebra/zebra_mlag_private.h" + +#include + + +/* + * This file will have platform specific apis to communicate with MCLAG. + * + */ + +#ifdef HAVE_CUMULUS + +static struct thread_master *zmlag_master; +static int mlag_socket; + +static int zebra_mlag_connect(struct thread *thread); +static int zebra_mlag_read(struct thread *thread); + +/* + * Write teh data to MLAGD + */ +int zebra_mlag_private_write_data(uint8_t *data, uint32_t len) +{ + int rc = 0; + + if (IS_ZEBRA_DEBUG_MLAG) { + zlog_debug("%s: Writing %d length Data to clag", __func__, len); + zlog_hexdump(data, len); + } + rc = write(mlag_socket, data, len); + return rc; +} + +static void zebra_mlag_sched_read(void) +{ + thread_add_read(zmlag_master, zebra_mlag_read, NULL, mlag_socket, + &zrouter.mlag_info.t_read); +} + +static int zebra_mlag_read(struct thread *thread) +{ + uint32_t *msglen; + uint32_t h_msglen; + uint32_t tot_len, curr_len = mlag_rd_buf_offset; + + zrouter.mlag_info.t_read = NULL; + + /* + * Received message in sock_stream looks like below + * | len-1 (4 Bytes) | payload-1 (len-1) | + * len-2 (4 Bytes) | payload-2 (len-2) | .. + * + * Idea is read one message completely, then process, until message is + * read completely, keep on reading from the socket + */ + if (curr_len < ZEBRA_MLAG_LEN_SIZE) { + ssize_t data_len; + + data_len = read(mlag_socket, mlag_rd_buffer + curr_len, + ZEBRA_MLAG_LEN_SIZE - curr_len); + if (data_len == 0 || data_len == -1) { + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("MLAG connection closed socket : %d", + mlag_socket); + close(mlag_socket); + zebra_mlag_handle_process_state(MLAG_DOWN); + return -1; + } + if (data_len != (ssize_t)ZEBRA_MLAG_LEN_SIZE - curr_len) { + /* Try again later */ + zebra_mlag_sched_read(); + return 0; + } + curr_len = ZEBRA_MLAG_LEN_SIZE; + } + + /* Get the actual packet length */ + msglen = (uint32_t *)mlag_rd_buffer; + h_msglen = ntohl(*msglen); + + /* This will be the actual length of the packet */ + tot_len = h_msglen + ZEBRA_MLAG_LEN_SIZE; + + if (curr_len < tot_len) { + ssize_t data_len; + + data_len = read(mlag_socket, mlag_rd_buffer + curr_len, + tot_len - curr_len); + if (data_len == 0 || data_len == -1) { + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("MLAG connection closed socket : %d", + mlag_socket); + close(mlag_socket); + zebra_mlag_handle_process_state(MLAG_DOWN); + return -1; + } + if (data_len != (ssize_t)tot_len - curr_len) { + /* Try again later */ + zebra_mlag_sched_read(); + return 0; + } + } + + if (IS_ZEBRA_DEBUG_MLAG) { + zlog_debug("Received a MLAG Message from socket: %d, len:%u ", + mlag_socket, tot_len); + zlog_hexdump(mlag_rd_buffer, tot_len); + } + + tot_len -= ZEBRA_MLAG_LEN_SIZE; + + /* Process the packet */ + zebra_mlag_process_mlag_data(mlag_rd_buffer + ZEBRA_MLAG_LEN_SIZE, + tot_len); + + /* Register read thread. */ + zebra_mlag_reset_read_buffer(); + zebra_mlag_sched_read(); + return 0; +} + +static int zebra_mlag_connect(struct thread *thread) +{ + struct sockaddr_un svr; + struct ucred ucred; + socklen_t len = 0; + + /* Reset the Timer-running flag */ + zrouter.mlag_info.timer_running = false; + + zrouter.mlag_info.t_read = NULL; + memset(&svr, 0, sizeof(svr)); + svr.sun_family = AF_UNIX; +#define MLAG_SOCK_NAME "/var/run/clag-zebra.socket" + strlcpy(svr.sun_path, MLAG_SOCK_NAME, sizeof(MLAG_SOCK_NAME) + 1); + + mlag_socket = socket(svr.sun_family, SOCK_STREAM, 0); + if (mlag_socket < 0) + return -1; + + if (connect(mlag_socket, (struct sockaddr *)&svr, sizeof(svr)) == -1) { + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug( + "Unable to connect to %s try again in 10 secs", + svr.sun_path); + close(mlag_socket); + zrouter.mlag_info.timer_running = true; + thread_add_timer(zmlag_master, zebra_mlag_connect, NULL, 10, + &zrouter.mlag_info.t_read); + return 0; + } + len = sizeof(struct ucred); + ucred.pid = getpid(); + + set_nonblocking(mlag_socket); + setsockopt(mlag_socket, SOL_SOCKET, SO_PEERCRED, &ucred, len); + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("%s: Connection with MLAG is established ", + __func__); + + thread_add_read(zmlag_master, zebra_mlag_read, NULL, mlag_socket, + &zrouter.mlag_info.t_read); + /* + * Connection is established with MLAGD, post to clients + */ + zebra_mlag_handle_process_state(MLAG_UP); + return 0; +} + +/* + * Currently we are doing polling later we will look for better options + */ +void zebra_mlag_private_monitor_state(void) +{ + thread_add_event(zmlag_master, zebra_mlag_connect, NULL, 0, + &zrouter.mlag_info.t_read); +} + +int zebra_mlag_private_open_channel(void) +{ + zmlag_master = zrouter.mlag_info.th_master; + + if (zrouter.mlag_info.connected == true) { + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("%s: Zebra already connected to MLAGD", + __func__); + return 0; + } + + if (zrouter.mlag_info.timer_running == true) { + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug( + "%s: Connection retry is in progress for MLAGD", + __func__); + return 0; + } + + if (zrouter.mlag_info.clients_interested_cnt) { + /* + * Connect only if any clients are showing interest + */ + thread_add_event(zmlag_master, zebra_mlag_connect, NULL, 0, + &zrouter.mlag_info.t_read); + } + return 0; +} + +int zebra_mlag_private_close_channel(void) +{ + if (zmlag_master == NULL) + return -1; + + if (zrouter.mlag_info.clients_interested_cnt) { + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("%s: still %d clients are connected, skip", + __func__, + zrouter.mlag_info.clients_interested_cnt); + return -1; + } + + /* + * Post the De-register to MLAG, so that it can do necesasry cleanup + */ + zebra_mlag_send_deregister(); + + return 0; +} + +void zebra_mlag_private_cleanup_data(void) +{ + zmlag_master = NULL; + zrouter.mlag_info.connected = false; + zrouter.mlag_info.timer_running = false; + + close(mlag_socket); +} + +#else /*HAVE_CUMULUS */ + +int zebra_mlag_private_write_data(uint8_t *data, uint32_t len) +{ + return 0; +} + +void zebra_mlag_private_monitor_state(void) +{ +} + +int zebra_mlag_private_open_channel(void) +{ + return 0; +} + +int zebra_mlag_private_close_channel(void) +{ + return 0; +} + +void zebra_mlag_private_cleanup_data(void) +{ +} +#endif /*HAVE_CUMULUS*/ diff --git a/zebra/zebra_mlag_private.h b/zebra/zebra_mlag_private.h new file mode 100644 index 0000000000..02797429a8 --- /dev/null +++ b/zebra/zebra_mlag_private.h @@ -0,0 +1,40 @@ +/* Zebra mlag header. + * Copyright (C) 2018 Cumulus Networks, Inc. + * Donald Sharp + * + * This file is part of FRR. + * + * FRR is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2, or (at your option) any + * later version. + * + * FRR is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with FRR; see the file COPYING. If not, write to the Free + * Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA + * 02111-1307, USA. + */ +#ifndef __ZEBRA_MLAG_PRIVATE_H__ +#define __ZEBRA_MLAG_PRIVATE_H__ + + +/* + * all the platform specific API's + */ + +int zebra_mlag_private_open_channel(void); + +int zebra_mlag_private_close_channel(void); + +void zebra_mlag_private_monitor_state(void); + +int zebra_mlag_private_write_data(uint8_t *data, uint32_t len); + +void zebra_mlag_private_cleanup_data(void); + +#endif diff --git a/zebra/zebra_router.h b/zebra/zebra_router.h index e50f8a1186..141b2579e2 100644 --- a/zebra/zebra_router.h +++ b/zebra/zebra_router.h @@ -71,6 +71,35 @@ struct zebra_mlag_info { /* The system mac being used */ struct ethaddr mac; + /* + * Zebra will open the communication channel with MLAGD only if any + * clients are interested and it is controlled dynamically based on + * client registers & un-registers. + */ + uint32_t clients_interested_cnt; + + /* coomunication channel with MLAGD is established */ + bool connected; + + /* connection retry timer is running */ + bool timer_running; + + /* Holds the client data(unencoded) that need to be pushed to MCLAGD*/ + struct stream_fifo *mlag_fifo; + + /* + * A new Kernel thread will be created to post the data to MCLAGD. + * where as, read will be performed from the zebra main thread, because + * read involves accessing client registartion data structures. + */ + struct frr_pthread *zebra_pth_mlag; + + /* MLAG Thread context 'master' */ + struct thread_master *th_master; + + /* Threads for read/write. */ + struct thread *t_read; + struct thread *t_write; }; struct zebra_router { diff --git a/zebra/zserv.h b/zebra/zserv.h index 708ff1e226..ccc8d92aa2 100644 --- a/zebra/zserv.h +++ b/zebra/zserv.h @@ -99,6 +99,13 @@ struct zserv { uint8_t proto; uint16_t instance; + /* + * Interested for MLAG Updates, and also stores the client + * interested message mask + */ + bool mlag_updates_interested; + uint32_t mlag_reg_mask1; + /* Statistics */ uint32_t redist_v4_add_cnt; uint32_t redist_v4_del_cnt; From 9918b45e636c3142fe85ef8482ec6de3de6e5145 Mon Sep 17 00:00:00 2001 From: Satheesh Kumar K Date: Mon, 26 Aug 2019 21:11:32 -0700 Subject: [PATCH 4/6] lib,mlag : Defining MLAG Proto file Defining the message structures between Zebra & MLAG Signed-off-by: Satheesh Kumar K --- Makefile.am | 1 + mlag/mlag.proto | 182 ++++++++++++++++++++++++++++++++++++++++++++++++ mlag/subdir.am | 19 +++++ 3 files changed, 202 insertions(+) create mode 100644 mlag/mlag.proto create mode 100644 mlag/subdir.am diff --git a/Makefile.am b/Makefile.am index 851cefc85c..ada715dbca 100644 --- a/Makefile.am +++ b/Makefile.am @@ -129,6 +129,7 @@ include zebra/subdir.am include watchfrr/subdir.am include qpb/subdir.am include fpm/subdir.am +include mlag/subdir.am include grpc/subdir.am include tools/subdir.am include solaris/subdir.am diff --git a/mlag/mlag.proto b/mlag/mlag.proto new file mode 100644 index 0000000000..3f402bab49 --- /dev/null +++ b/mlag/mlag.proto @@ -0,0 +1,182 @@ +// See README.txt for information and build instructions. +// +// Note: START and END tags are used in comments to define sections used in +// tutorials. They are not part of the syntax for Protocol Buffers. +// +// To get an in-depth walkthrough of this file and the related examples, see: +// https://developers.google.com/protocol-buffers/docs/tutorials + +// [START declaration] +syntax = "proto3"; +//package tutorial; + +/* + * This Contains the Mesadge structurtes used for PIM MLAG Active-Active support. + * Mainly there were two types of messages + * + * 1. Messages sent from PIM (Node-1) to PIM (Node-2) + * 2. Messages sent from MCLAGD to PIM (status Messages) + * + * ProtoBuf supports maximum 32 fileds, so to make it more generic message + * encoding is like below. + * __________________________________________ + * | | | + * | Header | bytes | + * ___________________________________________ + * + * + * Header carries Information about + * 1) what Message it is carrying + * 2) Bytes carries teh actual payload encoded with protobuf + * + * + * Limitations + *============= + * Since message-type is 32-bit, there were no real limitations on number of + * messages Infra can support, but each message can carry only 32 fileds. + * + */ + + +// [START messages] +message ZebraMlag_Header { + enum MessageType { + ZEBRA_MLAG_NONE = 0; //Invalid message-type + ZEBRA_MLAG_REGISTER = 1; + ZEBRA_MLAG_DEREGISTER = 2; + ZEBRA_MLAG_STATUS_UPDATE = 3; + ZEBRA_MLAG_MROUTE_ADD = 4; + ZEBRA_MLAG_MROUTE_DEL = 5; + ZEBRA_MLAG_DUMP = 6; + ZEBRA_MLAG_MROUTE_ADD_BULK = 7; + ZEBRA_MLAG_MROUTE_DEL_BULK = 8; + ZEBRA_MLAG_PIM_CFG_DUMP = 10; + ZEBRA_MLAG_VXLAN_UPDATE = 11; + ZEBRA_MLAG_ZEBRA_STATUS_UPDATE = 12; + } + + /* + * tells what type of message this payload carries + */ + MessageType type = 1; + + /* + * Length of payload + */ + uint32 len = 2; + + /* + * Actual Encoded payload + */ + bytes data = 3; +} + + +/* + * ZEBRA_MLAG_REGISTER & ZEBRA_MLAG_DEREGISTER + * + * After the MLAGD is up, First Zebra has to register to send any data, + * otherwise MLAGD will not accept any data from the client. + * De-register will be used for the Data cleanup at MLAGD + * These are NULL payload message currently + */ + +/* + * ZEBRA_MLAG_STATUS_UPDATE + * + * This message will be posted by CLAGD(an external control plane manager + * which monitors MCLAG failures) to inform peerlink/CLAG Failure + * to zebra, after the failure Notification Node with primary role will + * forward the Traffic and Node with standby will drop the traffic + */ + +message ZebraMlagStatusUpdate { + enum ClagState { + CLAG_STATE_DOWN = 0; + CLAG_STATE_RUNNING = 1; + } + + enum ClagRole { + CLAG_ROLE_NONE = 0; + CLAG_ROLE_PRIMAY = 1; + CLAG_ROLE_SECONDARY = 2; + } + + string peerlink = 1; + ClagRole my_role = 2; + ClagState peer_state = 3; +} + +/* + * ZEBRA_MLAG_VXLAN_UPDATE + * + * This message will be posted by CLAGD(an external control plane Manager + * which is responsible for MCLAG) to inform zebra obout anycast/local + * ip updates. + */ +message ZebraMlagVxlanUpdate { + uint32 anycast_ip = 1; + uint32 local_ip = 2; +} + +/* + * ZebraMlagZebraStatusUpdate + * + * This message will be posted by CLAGD to advertise FRR state + * Change Information to peer + */ + +message ZebraMlagZebraStatusUpdate{ + enum FrrState { + FRR_STATE_NONE = 0; + FRR_STATE_DOWN = 1; + FRR_STATE_UP = 2; + } + + FrrState peer_frrstate = 1; +} + +/* + * ZEBRA_MLAG_MROUTE_ADD & ZEBRA_MLAG_MROUTE_DEL + * + * These meesages will be sent from PIM (Node-1) to PIM (Node-2) to perform + * DF Election for each Mcast flow. Elected DF will forward the tarffic + * towards the host and loser will keep the OIL as empty, sothat only single + * copy will be sent to host + * This message will be posted with any chnage in the params. + * + * ZEBRA_MLAG_MROUTE_DEL is mainly to delete the record at MLAGD when the + * mcast flow is deleted. + * key for the MLAGD lookup is (vrf_id, source_ip & group_ip) + */ + +message ZebraMlagMrouteAdd { + string vrf_name = 1; + uint32 source_ip = 2; + uint32 group_ip = 3; + uint32 cost_to_rp = 4; + uint32 owner_id = 5; + bool am_i_DR = 6; + bool am_i_Dual_active = 7; + uint32 vrf_id = 8; + string intf_name = 9; +} + +message ZebraMlagMrouteDel { + string vrf_name = 1; + uint32 source_ip = 2; + uint32 group_ip = 3; + uint32 owner_id = 4; + uint32 vrf_id = 5; + string intf_name = 6; +} + +message ZebraMlagMrouteAddBulk { + repeated ZebraMlagMrouteAdd mroute_add = 1; +} + +message ZebraMlagMrouteDelBulk { + repeated ZebraMlagMrouteDel mroute_del = 1; +} + +// [END messages] diff --git a/mlag/subdir.am b/mlag/subdir.am new file mode 100644 index 0000000000..9fab662860 --- /dev/null +++ b/mlag/subdir.am @@ -0,0 +1,19 @@ +if HAVE_PROTOBUF +lib_LTLIBRARIES += mlag/libmlag_pb.la +endif + +mlag_libmlag_pb_la_LDFLAGS = -version-info 0:0:0 +mlag_libmlag_pb_la_CPPFLAGS = $(AM_CPPFLAGS) $(PROTOBUF_C_CFLAGS) +mlag_libmlag_pb_la_SOURCES = \ + # end + +nodist_mlag_libmlag_pb_la_SOURCES = \ + mlag/mlag.pb-c.c \ + # end + +CLEANFILES += \ + mlag/mlag.pb-c.c \ + mlag/mlag.pb-c.h \ + # end + +EXTRA_DIST += mlag/mlag.proto From d5167496339ba462e0b6e8d69363b393f6c6ffa0 Mon Sep 17 00:00:00 2001 From: Satheesh Kumar K Date: Mon, 26 Aug 2019 21:24:19 -0700 Subject: [PATCH 5/6] Zebra: ADD Protobuf Encoding & Decoding for MLAG Messages 1. add the Mlag ProtoBuf Lib to Zebra Compilation 2. Encode the messages with protobuf before writing to MLAG 3. Decode the MLAG Messages using protobuf and write to clients based on their subscrption. Signed-off-by: Satheesh Kumar K --- Makefile.am | 2 +- zebra/subdir.am | 4 + zebra/zebra_mlag.c | 514 ++++++++++++++++++++++++++++++++++++++++++++- zebra/zebra_mlag.h | 4 + 4 files changed, 522 insertions(+), 2 deletions(-) diff --git a/Makefile.am b/Makefile.am index ada715dbca..34f112bf01 100644 --- a/Makefile.am +++ b/Makefile.am @@ -125,11 +125,11 @@ include doc/manpages/subdir.am include doc/developer/subdir.am include include/subdir.am include lib/subdir.am +include mlag/subdir.am include zebra/subdir.am include watchfrr/subdir.am include qpb/subdir.am include fpm/subdir.am -include mlag/subdir.am include grpc/subdir.am include tools/subdir.am include solaris/subdir.am diff --git a/zebra/subdir.am b/zebra/subdir.am index 5016ca1f28..859c8c0836 100644 --- a/zebra/subdir.am +++ b/zebra/subdir.am @@ -38,6 +38,9 @@ man8 += $(MANBUILD)/zebra.8 endif zebra_zebra_LDADD = lib/libfrr.la $(LIBCAP) +if HAVE_PROTOBUF +zebra_zebra_LDADD += mlag/libmlag_pb.la $(PROTOBUF_C_LIBS) +endif zebra_zebra_SOURCES = \ zebra/connected.c \ zebra/debug.c \ @@ -131,6 +134,7 @@ noinst_HEADERS += \ zebra/rtadv.h \ zebra/rule_netlink.h \ zebra/zebra_mlag.h \ + zebra/zebra_mlag_private.h \ zebra/zebra_fpm_private.h \ zebra/zebra_l2.h \ zebra/zebra_dplane.h \ diff --git a/zebra/zebra_mlag.c b/zebra/zebra_mlag.c index 90af959d27..4d5145f570 100644 --- a/zebra/zebra_mlag.c +++ b/zebra/zebra_mlag.c @@ -380,7 +380,7 @@ static void zebra_mlag_spawn_pthread(void) zrouter.mlag_info.th_master = zrouter.mlag_info.zebra_pth_mlag->master; - /* Enqueue an initial event for the dataplane pthread */ + /* Enqueue an initial event to the Newly spawn MLAG pthread */ zebra_mlag_signal_write_thread(); frr_pthread_run(zrouter.mlag_info.zebra_pth_mlag, NULL); @@ -908,6 +908,517 @@ void zebra_mlag_terminate(void) * ProtoBuf Encoding APIs */ +#ifdef HAVE_PROTOBUF + +DEFINE_MTYPE_STATIC(ZEBRA, MLAG_PBUF, "ZEBRA MLAG PROTOBUF") + +int zebra_mlag_protobuf_encode_client_data(struct stream *s, uint32_t *msg_type) +{ + ZebraMlagHeader hdr = ZEBRA_MLAG__HEADER__INIT; + struct mlag_msg mlag_msg; + uint8_t tmp_buf[ZEBRA_MLAG_BUF_LIMIT]; + int len = 0; + int n_len = 0; + int rc = 0; + char buf[80]; + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("%s: Entering..", __func__); + + rc = zebra_mlag_lib_decode_mlag_hdr(s, &mlag_msg); + if (rc) + return rc; + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("%s: Decoded msg length:%d..", __func__, + mlag_msg.data_len); + + memset(tmp_buf, 0, ZEBRA_MLAG_BUF_LIMIT); + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("%s: Mlag ProtoBuf encoding of message:%s", __func__, + zebra_mlag_lib_msgid_to_str(mlag_msg.msg_type, buf, + 80)); + *msg_type = mlag_msg.msg_type; + switch (mlag_msg.msg_type) { + case MLAG_MROUTE_ADD: { + struct mlag_mroute_add msg; + ZebraMlagMrouteAdd pay_load = ZEBRA_MLAG_MROUTE_ADD__INIT; + uint32_t vrf_name_len = 0; + + rc = zebra_mlag_lib_decode_mroute_add(s, &msg); + if (rc) + return rc; + vrf_name_len = strlen(msg.vrf_name) + 1; + pay_load.vrf_name = XMALLOC(MTYPE_MLAG_PBUF, vrf_name_len); + strlcpy(pay_load.vrf_name, msg.vrf_name, vrf_name_len); + pay_load.source_ip = msg.source_ip; + pay_load.group_ip = msg.group_ip; + pay_load.cost_to_rp = msg.cost_to_rp; + pay_load.owner_id = msg.owner_id; + pay_load.am_i_dr = msg.am_i_dr; + pay_load.am_i_dual_active = msg.am_i_dual_active; + pay_load.vrf_id = msg.vrf_id; + + if (msg.owner_id == MLAG_OWNER_INTERFACE) { + vrf_name_len = strlen(msg.intf_name) + 1; + pay_load.intf_name = + XMALLOC(MTYPE_MLAG_PBUF, vrf_name_len); + strlcpy(pay_load.intf_name, msg.intf_name, + vrf_name_len); + } + + len = zebra_mlag_mroute_add__pack(&pay_load, tmp_buf); + XFREE(MTYPE_MLAG_PBUF, pay_load.vrf_name); + if (msg.owner_id == MLAG_OWNER_INTERFACE) + XFREE(MTYPE_MLAG_PBUF, pay_load.intf_name); + } break; + case MLAG_MROUTE_DEL: { + struct mlag_mroute_del msg; + ZebraMlagMrouteDel pay_load = ZEBRA_MLAG_MROUTE_DEL__INIT; + uint32_t vrf_name_len = 0; + + rc = zebra_mlag_lib_decode_mroute_del(s, &msg); + if (rc) + return rc; + vrf_name_len = strlen(msg.vrf_name) + 1; + pay_load.vrf_name = XMALLOC(MTYPE_MLAG_PBUF, vrf_name_len); + strlcpy(pay_load.vrf_name, msg.vrf_name, vrf_name_len); + pay_load.source_ip = msg.source_ip; + pay_load.group_ip = msg.group_ip; + pay_load.owner_id = msg.owner_id; + pay_load.vrf_id = msg.vrf_id; + + if (msg.owner_id == MLAG_OWNER_INTERFACE) { + vrf_name_len = strlen(msg.intf_name) + 1; + pay_load.intf_name = + XMALLOC(MTYPE_MLAG_PBUF, vrf_name_len); + strlcpy(pay_load.intf_name, msg.intf_name, + vrf_name_len); + } + + len = zebra_mlag_mroute_del__pack(&pay_load, tmp_buf); + XFREE(MTYPE_MLAG_PBUF, pay_load.vrf_name); + if (msg.owner_id == MLAG_OWNER_INTERFACE) + XFREE(MTYPE_MLAG_PBUF, pay_load.intf_name); + } break; + case MLAG_MROUTE_ADD_BULK: { + struct mlag_mroute_add msg; + ZebraMlagMrouteAddBulk Bulk_msg = + ZEBRA_MLAG_MROUTE_ADD_BULK__INIT; + ZebraMlagMrouteAdd **pay_load = NULL; + int i = 0; + bool cleanup = false; + + Bulk_msg.n_mroute_add = mlag_msg.msg_cnt; + pay_load = XMALLOC(MTYPE_MLAG_PBUF, + sizeof(ZebraMlagMrouteAdd *) + * Bulk_msg.n_mroute_add); + + for (i = 0; i < mlag_msg.msg_cnt; i++) { + + uint32_t vrf_name_len = 0; + + rc = zebra_mlag_lib_decode_mroute_add(s, &msg); + if (rc) { + cleanup = true; + break; + } + pay_load[i] = XMALLOC(MTYPE_MLAG_PBUF, + sizeof(ZebraMlagMrouteAdd)); + zebra_mlag_mroute_add__init(pay_load[i]); + + vrf_name_len = strlen(msg.vrf_name) + 1; + pay_load[i]->vrf_name = + XMALLOC(MTYPE_MLAG_PBUF, vrf_name_len); + strlcpy(pay_load[i]->vrf_name, msg.vrf_name, + vrf_name_len); + pay_load[i]->source_ip = msg.source_ip; + pay_load[i]->group_ip = msg.group_ip; + pay_load[i]->cost_to_rp = msg.cost_to_rp; + pay_load[i]->owner_id = msg.owner_id; + pay_load[i]->am_i_dr = msg.am_i_dr; + pay_load[i]->am_i_dual_active = msg.am_i_dual_active; + pay_load[i]->vrf_id = msg.vrf_id; + if (msg.owner_id == MLAG_OWNER_INTERFACE) { + vrf_name_len = strlen(msg.intf_name) + 1; + pay_load[i]->intf_name = + XMALLOC(MTYPE_MLAG_PBUF, vrf_name_len); + + strlcpy(pay_load[i]->intf_name, msg.intf_name, + vrf_name_len); + } + } + if (cleanup == false) { + Bulk_msg.mroute_add = pay_load; + len = zebra_mlag_mroute_add_bulk__pack(&Bulk_msg, + tmp_buf); + } + + for (i = 0; i < mlag_msg.msg_cnt; i++) { + if (pay_load[i]->vrf_name) + XFREE(MTYPE_MLAG_PBUF, pay_load[i]->vrf_name); + if (pay_load[i]->owner_id == MLAG_OWNER_INTERFACE + && pay_load[i]->intf_name) + XFREE(MTYPE_MLAG_PBUF, pay_load[i]->intf_name); + if (pay_load[i]) + XFREE(MTYPE_MLAG_PBUF, pay_load[i]); + } + XFREE(MTYPE_MLAG_PBUF, pay_load); + if (cleanup == true) + return -1; + } break; + case MLAG_MROUTE_DEL_BULK: { + struct mlag_mroute_del msg; + ZebraMlagMrouteDelBulk Bulk_msg = + ZEBRA_MLAG_MROUTE_DEL_BULK__INIT; + ZebraMlagMrouteDel **pay_load = NULL; + int i = 0; + bool cleanup = false; + + Bulk_msg.n_mroute_del = mlag_msg.msg_cnt; + pay_load = XMALLOC(MTYPE_MLAG_PBUF, + sizeof(ZebraMlagMrouteDel *) + * Bulk_msg.n_mroute_del); + + for (i = 0; i < mlag_msg.msg_cnt; i++) { + + uint32_t vrf_name_len = 0; + + rc = zebra_mlag_lib_decode_mroute_del(s, &msg); + if (rc) { + cleanup = true; + break; + } + + pay_load[i] = XMALLOC(MTYPE_MLAG_PBUF, + sizeof(ZebraMlagMrouteDel)); + zebra_mlag_mroute_del__init(pay_load[i]); + + vrf_name_len = strlen(msg.vrf_name) + 1; + pay_load[i]->vrf_name = + XMALLOC(MTYPE_MLAG_PBUF, vrf_name_len); + + strlcpy(pay_load[i]->vrf_name, msg.vrf_name, + vrf_name_len); + pay_load[i]->source_ip = msg.source_ip; + pay_load[i]->group_ip = msg.group_ip; + pay_load[i]->owner_id = msg.owner_id; + pay_load[i]->vrf_id = msg.vrf_id; + if (msg.owner_id == MLAG_OWNER_INTERFACE) { + vrf_name_len = strlen(msg.intf_name) + 1; + pay_load[i]->intf_name = + XMALLOC(MTYPE_MLAG_PBUF, vrf_name_len); + + strlcpy(pay_load[i]->intf_name, msg.intf_name, + vrf_name_len); + } + } + if (cleanup == false) { + Bulk_msg.mroute_del = pay_load; + len = zebra_mlag_mroute_del_bulk__pack(&Bulk_msg, + tmp_buf); + } + + for (i = 0; i < mlag_msg.msg_cnt; i++) { + if (pay_load[i]->vrf_name) + XFREE(MTYPE_MLAG_PBUF, pay_load[i]->vrf_name); + if (pay_load[i]->owner_id == MLAG_OWNER_INTERFACE + && pay_load[i]->intf_name) + XFREE(MTYPE_MLAG_PBUF, pay_load[i]->intf_name); + if (pay_load[i]) + XFREE(MTYPE_MLAG_PBUF, pay_load[i]); + } + XFREE(MTYPE_MLAG_PBUF, pay_load); + if (cleanup == true) + return -1; + } break; + default: + break; + } + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug( + "%s: length of Mlag ProtoBuf encoded message:%s, %d", + __func__, + zebra_mlag_lib_msgid_to_str(mlag_msg.msg_type, buf, 80), + len); + hdr.type = (ZebraMlagHeader__MessageType)mlag_msg.msg_type; + if (len != 0) { + hdr.data.len = len; + hdr.data.data = XMALLOC(MTYPE_MLAG_PBUF, len); + if (hdr.data.data == NULL) + return -1; + memcpy(hdr.data.data, tmp_buf, len); + } + + /* + * ProtoBuf Infra will not support to demarc the pointers whem multiple + * mesasges are posted inside a single Buffer. + * 2 -sloutions exist to solve this + * 1. add Unenoced length at the beginning of every message, this will + * be used to point to next mesasge in the buffer + * 2. another solution is defining all messages insides another message + * But this will permit only 32 messages. this can be extended with + * multiple levels. + * for simplicity we are going with solution-1. + */ + len = zebra_mlag__header__pack(&hdr, + (mlag_wr_buffer + ZEBRA_MLAG_LEN_SIZE)); + n_len = htonl(len); + memcpy(mlag_wr_buffer, &n_len, ZEBRA_MLAG_LEN_SIZE); + len += ZEBRA_MLAG_LEN_SIZE; + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug( + "%s: length of Mlag ProtoBuf message:%s with Header %d", + __func__, + zebra_mlag_lib_msgid_to_str(mlag_msg.msg_type, buf, 80), + len); + if (hdr.data.data) + XFREE(MTYPE_MLAG_PBUF, hdr.data.data); + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("%s: Exiting..", __func__); + return len; +} + +int zebra_mlag_protobuf_decode_message(struct stream **s, uint8_t *data, + uint32_t len) +{ + uint32_t msg_type; + ZebraMlagHeader *hdr = NULL; + char buf[80]; + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("%s: Entering..", __func__); + hdr = zebra_mlag__header__unpack(NULL, len, data); + if (hdr == NULL) + return -1; + + /* + * ADD The MLAG Header + */ + zclient_create_header(*s, ZEBRA_MLAG_FORWARD_MSG, VRF_DEFAULT); + + msg_type = hdr->type; + + if (IS_ZEBRA_DEBUG_MLAG) + zlog_debug("%s: Mlag ProtoBuf decoding of message:%s", __func__, + zebra_mlag_lib_msgid_to_str(msg_type, buf, 80)); + + /* + * Internal MLAG Message-types & MLAG.proto message types should + * always match, otherwise there can be decoding errors + * To avoid exposing clients with Protobuf flags, using internal + * message-types + */ + stream_putl(*s, hdr->type); + + if (hdr->data.len == 0) { + /* NULL Payload */ + stream_putw(*s, MLAG_MSG_NULL_PAYLOAD); + /* No Batching */ + stream_putw(*s, MLAG_MSG_NO_BATCH); + } else { + switch (msg_type) { + case ZEBRA_MLAG__HEADER__MESSAGE_TYPE__ZEBRA_MLAG_STATUS_UPDATE: { + ZebraMlagStatusUpdate *msg = NULL; + + msg = zebra_mlag_status_update__unpack( + NULL, hdr->data.len, hdr->data.data); + if (msg == NULL) { + zebra_mlag__header__free_unpacked(hdr, NULL); + return -1; + } + /* Payload len */ + stream_putw(*s, sizeof(struct mlag_status)); + /* No Batching */ + stream_putw(*s, MLAG_MSG_NO_BATCH); + /* Actual Data */ + stream_put(*s, msg->peerlink, INTERFACE_NAMSIZ); + stream_putl(*s, msg->my_role); + stream_putl(*s, msg->peer_state); + zebra_mlag_status_update__free_unpacked(msg, NULL); + } break; + case ZEBRA_MLAG__HEADER__MESSAGE_TYPE__ZEBRA_MLAG_VXLAN_UPDATE: { + ZebraMlagVxlanUpdate *msg = NULL; + + msg = zebra_mlag_vxlan_update__unpack( + NULL, hdr->data.len, hdr->data.data); + if (msg == NULL) { + zebra_mlag__header__free_unpacked(hdr, NULL); + return -1; + } + /* Payload len */ + stream_putw(*s, sizeof(struct mlag_vxlan)); + /* No Batching */ + stream_putw(*s, MLAG_MSG_NO_BATCH); + /* Actual Data */ + stream_putl(*s, msg->anycast_ip); + stream_putl(*s, msg->local_ip); + zebra_mlag_vxlan_update__free_unpacked(msg, NULL); + } break; + case ZEBRA_MLAG__HEADER__MESSAGE_TYPE__ZEBRA_MLAG_MROUTE_ADD: { + ZebraMlagMrouteAdd *msg = NULL; + + msg = zebra_mlag_mroute_add__unpack(NULL, hdr->data.len, + hdr->data.data); + if (msg == NULL) { + zebra_mlag__header__free_unpacked(hdr, NULL); + return -1; + } + /* Payload len */ + stream_putw(*s, sizeof(struct mlag_mroute_add)); + /* No Batching */ + stream_putw(*s, MLAG_MSG_NO_BATCH); + /* Actual Data */ + stream_put(*s, msg->vrf_name, VRF_NAMSIZ); + + stream_putl(*s, msg->source_ip); + stream_putl(*s, msg->group_ip); + stream_putl(*s, msg->cost_to_rp); + stream_putl(*s, msg->owner_id); + stream_putc(*s, msg->am_i_dr); + stream_putc(*s, msg->am_i_dual_active); + stream_putl(*s, msg->vrf_id); + if (msg->owner_id == MLAG_OWNER_INTERFACE) + stream_put(*s, msg->intf_name, + INTERFACE_NAMSIZ); + else + stream_put(*s, NULL, INTERFACE_NAMSIZ); + zebra_mlag_mroute_add__free_unpacked(msg, NULL); + } break; + case ZEBRA_MLAG__HEADER__MESSAGE_TYPE__ZEBRA_MLAG_MROUTE_DEL: { + ZebraMlagMrouteDel *msg = NULL; + + msg = zebra_mlag_mroute_del__unpack(NULL, hdr->data.len, + hdr->data.data); + if (msg == NULL) { + zebra_mlag__header__free_unpacked(hdr, NULL); + return -1; + } + /* Payload len */ + stream_putw(*s, sizeof(struct mlag_mroute_del)); + /* No Batching */ + stream_putw(*s, MLAG_MSG_NO_BATCH); + /* Actual Data */ + stream_put(*s, msg->vrf_name, VRF_NAMSIZ); + + stream_putl(*s, msg->source_ip); + stream_putl(*s, msg->group_ip); + stream_putl(*s, msg->group_ip); + stream_putl(*s, msg->owner_id); + stream_putl(*s, msg->vrf_id); + if (msg->owner_id == MLAG_OWNER_INTERFACE) + stream_put(*s, msg->intf_name, + INTERFACE_NAMSIZ); + else + stream_put(*s, NULL, INTERFACE_NAMSIZ); + zebra_mlag_mroute_del__free_unpacked(msg, NULL); + } break; + case ZEBRA_MLAG__HEADER__MESSAGE_TYPE__ZEBRA_MLAG_MROUTE_ADD_BULK: { + ZebraMlagMrouteAddBulk *Bulk_msg = NULL; + ZebraMlagMrouteAdd *msg = NULL; + size_t i = 0; + + Bulk_msg = zebra_mlag_mroute_add_bulk__unpack( + NULL, hdr->data.len, hdr->data.data); + if (Bulk_msg == NULL) { + zebra_mlag__header__free_unpacked(hdr, NULL); + return -1; + } + /* Payload len */ + stream_putw(*s, (Bulk_msg->n_mroute_add + * sizeof(struct mlag_mroute_add))); + /* No. of msgs in Batch */ + stream_putw(*s, Bulk_msg->n_mroute_add); + + /* Actual Data */ + for (i = 0; i < Bulk_msg->n_mroute_add; i++) { + + msg = Bulk_msg->mroute_add[i]; + + stream_put(*s, msg->vrf_name, VRF_NAMSIZ); + stream_putl(*s, msg->source_ip); + stream_putl(*s, msg->group_ip); + stream_putl(*s, msg->cost_to_rp); + stream_putl(*s, msg->owner_id); + stream_putc(*s, msg->am_i_dr); + stream_putc(*s, msg->am_i_dual_active); + stream_putl(*s, msg->vrf_id); + if (msg->owner_id == MLAG_OWNER_INTERFACE) + stream_put(*s, msg->intf_name, + INTERFACE_NAMSIZ); + else + stream_put(*s, NULL, INTERFACE_NAMSIZ); + } + zebra_mlag_mroute_add_bulk__free_unpacked(Bulk_msg, + NULL); + } break; + case ZEBRA_MLAG__HEADER__MESSAGE_TYPE__ZEBRA_MLAG_MROUTE_DEL_BULK: { + ZebraMlagMrouteDelBulk *Bulk_msg = NULL; + ZebraMlagMrouteDel *msg = NULL; + size_t i = 0; + + Bulk_msg = zebra_mlag_mroute_del_bulk__unpack( + NULL, hdr->data.len, hdr->data.data); + if (Bulk_msg == NULL) { + zebra_mlag__header__free_unpacked(hdr, NULL); + return -1; + } + /* Payload len */ + stream_putw(*s, (Bulk_msg->n_mroute_del + * sizeof(struct mlag_mroute_del))); + /* No. of msgs in Batch */ + stream_putw(*s, Bulk_msg->n_mroute_del); + + /* Actual Data */ + for (i = 0; i < Bulk_msg->n_mroute_del; i++) { + + msg = Bulk_msg->mroute_del[i]; + + stream_put(*s, msg->vrf_name, VRF_NAMSIZ); + stream_putl(*s, msg->source_ip); + stream_putl(*s, msg->group_ip); + stream_putl(*s, msg->owner_id); + stream_putl(*s, msg->vrf_id); + if (msg->owner_id == MLAG_OWNER_INTERFACE) + stream_put(*s, msg->intf_name, + INTERFACE_NAMSIZ); + else + stream_put(*s, NULL, INTERFACE_NAMSIZ); + } + zebra_mlag_mroute_del_bulk__free_unpacked(Bulk_msg, + NULL); + } break; + case ZEBRA_MLAG__HEADER__MESSAGE_TYPE__ZEBRA_MLAG_ZEBRA_STATUS_UPDATE: { + ZebraMlagZebraStatusUpdate *msg = NULL; + + msg = zebra_mlag_zebra_status_update__unpack( + NULL, hdr->data.len, hdr->data.data); + if (msg == NULL) { + zebra_mlag__header__free_unpacked(hdr, NULL); + return -1; + } + /* Payload len */ + stream_putw(*s, sizeof(struct mlag_frr_status)); + /* No Batching */ + stream_putw(*s, MLAG_MSG_NO_BATCH); + /* Actual Data */ + stream_putl(*s, msg->peer_frrstate); + zebra_mlag_zebra_status_update__free_unpacked(msg, + NULL); + } break; + default: + break; + } + } + zebra_mlag__header__free_unpacked(hdr, NULL); + return msg_type; +} + +#else int zebra_mlag_protobuf_encode_client_data(struct stream *s, uint32_t *msg_type) { return 0; @@ -918,3 +1429,4 @@ int zebra_mlag_protobuf_decode_message(struct stream **s, uint8_t *data, { return 0; } +#endif diff --git a/zebra/zebra_mlag.h b/zebra/zebra_mlag.h index a145753d88..01459cd29c 100644 --- a/zebra/zebra_mlag.h +++ b/zebra/zebra_mlag.h @@ -26,6 +26,10 @@ #include "zclient.h" #include "zebra/zserv.h" +#ifdef HAVE_PROTOBUF +#include "mlag/mlag.pb-c.h" +#endif + #define ZEBRA_MLAG_BUF_LIMIT 2048 #define ZEBRA_MLAG_LEN_SIZE 4 From 40e79e94110c772bfb71b97b1d97e8ad35901b88 Mon Sep 17 00:00:00 2001 From: Satheesh Kumar K Date: Tue, 24 Sep 2019 02:29:15 -0700 Subject: [PATCH 6/6] Zebra: Fixing Review comments in Zebra_MLAG Signed-off-by: Satheesh Kumar K --- mlag/mlag.proto | 4 +++ pimd/pim_mlag.c | 2 +- pimd/pim_mlag.h | 2 +- zebra/zebra_mlag.c | 68 ++++++++++++++++++++------------------ zebra/zebra_mlag.h | 8 +---- zebra/zebra_mlag_private.c | 6 +++- zebra/zebra_mlag_private.h | 2 +- zebra/zebra_router.h | 1 + 8 files changed, 50 insertions(+), 43 deletions(-) diff --git a/mlag/mlag.proto b/mlag/mlag.proto index 3f402bab49..6991015a3d 100644 --- a/mlag/mlag.proto +++ b/mlag/mlag.proto @@ -154,6 +154,10 @@ message ZebraMlagMrouteAdd { string vrf_name = 1; uint32 source_ip = 2; uint32 group_ip = 3; + /* + * This is the IGP Cost to reach Configured RP in case of (*,G) or + * Cost to the source in case of (S,G) entry + */ uint32 cost_to_rp = 4; uint32 owner_id = 5; bool am_i_DR = 6; diff --git a/pimd/pim_mlag.c b/pimd/pim_mlag.c index e39af9b941..1fa5c49bd5 100644 --- a/pimd/pim_mlag.c +++ b/pimd/pim_mlag.c @@ -1,5 +1,5 @@ /* PIM Mlag Code. - * Copyright (C) 2018 Cumulus Networks, Inc. + * Copyright (C) 2019 Cumulus Networks, Inc. * Donald Sharp * * This file is part of FRR. diff --git a/pimd/pim_mlag.h b/pimd/pim_mlag.h index f35124e485..87a57ca29a 100644 --- a/pimd/pim_mlag.h +++ b/pimd/pim_mlag.h @@ -1,5 +1,5 @@ /* PIM mlag header. - * Copyright (C) 2018 Cumulus Networks, Inc. + * Copyright (C) 2019 Cumulus Networks, Inc. * Donald Sharp * * This file is part of FRR. diff --git a/zebra/zebra_mlag.c b/zebra/zebra_mlag.c index 4d5145f570..633ebe95cb 100644 --- a/zebra/zebra_mlag.c +++ b/zebra/zebra_mlag.c @@ -1,5 +1,5 @@ /* Zebra Mlag Code. - * Copyright (C) 2018 Cumulus Networks, Inc. + * Copyright (C) 2019 Cumulus Networks, Inc. * Donald Sharp * * This file is part of FRR. @@ -39,6 +39,7 @@ #define ZEBRA_MLAG_METADATA_LEN 4 #define ZEBRA_MLAG_MSG_BCAST 0xFFFFFFFF +#define MAXCH_LEN 80 uint8_t mlag_wr_buffer[ZEBRA_MLAG_BUF_LIMIT]; uint8_t mlag_rd_buffer[ZEBRA_MLAG_BUF_LIMIT]; @@ -153,6 +154,7 @@ static int zebra_mlag_thread_handler(struct thread *event) struct stream *s; uint32_t wr_count = 0; uint32_t msg_type = 0; + uint32_t max_count = 0; int len = 0; wr_count = stream_fifo_count_safe(zrouter.mlag_info.mlag_fifo); @@ -160,12 +162,13 @@ static int zebra_mlag_thread_handler(struct thread *event) zlog_debug(":%s: Processing MLAG write, %d messages in queue", __func__, wr_count); + pthread_mutex_lock(&zrouter.mlag_info.mlag_th_mtx); zrouter.mlag_info.t_write = NULL; - for (wr_count = 0; wr_count < ZEBRA_MLAG_POST_LIMIT; wr_count++) { - /* FIFO is empty,wait for teh message to be add */ - if (stream_fifo_count_safe(zrouter.mlag_info.mlag_fifo) == 0) - break; + pthread_mutex_unlock(&zrouter.mlag_info.mlag_th_mtx); + max_count = MIN(wr_count, ZEBRA_MLAG_POST_LIMIT); + + for (wr_count = 0; wr_count < max_count; wr_count++) { s = stream_fifo_pop_safe(zrouter.mlag_info.mlag_fifo); if (!s) { zlog_debug(":%s: Got a NULL Messages, some thing wrong", @@ -173,7 +176,6 @@ static int zebra_mlag_thread_handler(struct thread *event) break; } - zebra_mlag_reset_write_buffer(); /* * Encode the data now */ @@ -182,17 +184,19 @@ static int zebra_mlag_thread_handler(struct thread *event) /* * write to MCLAGD */ - if (len > 0) + if (len > 0) { zebra_mlag_private_write_data(mlag_wr_buffer, len); - /* - * If mesasge type is De-register, send a signal to main thread, - * sothat necessary cleanup will be done by main thread. - */ - if (msg_type == MLAG_DEREGISTER) { - thread_add_event(zrouter.master, - zebra_mlag_terminate_pthread, NULL, 0, - NULL); + /* + * If mesasge type is De-register, send a signal to main + * thread, sothat necessary cleanup will be done by main + * thread. + */ + if (msg_type == MLAG_DEREGISTER) { + thread_add_event(zrouter.master, + zebra_mlag_terminate_pthread, + NULL, 0, NULL); + } } stream_free(s); @@ -246,6 +250,7 @@ void zebra_mlag_handle_process_state(enum zebra_mlag_state state) */ static int zebra_mlag_signal_write_thread(void) { + pthread_mutex_lock(&zrouter.mlag_info.mlag_th_mtx); if (zrouter.mlag_info.zebra_pth_mlag) { if (IS_ZEBRA_DEBUG_MLAG) zlog_debug(":%s: Scheduling MLAG write", __func__); @@ -253,6 +258,7 @@ static int zebra_mlag_signal_write_thread(void) zebra_mlag_thread_handler, NULL, 0, &zrouter.mlag_info.t_write); } + pthread_mutex_unlock(&zrouter.mlag_info.mlag_th_mtx); return 0; } @@ -588,7 +594,7 @@ DEFUN_HIDDEN (show_mlag, ZEBRA_STR "The mlag role on this machine\n") { - char buf[80]; + char buf[MAXCH_LEN]; vty_out(vty, "MLag is configured to: %s\n", mlag_role2str(zrouter.mlag_info.role, buf, sizeof(buf))); @@ -790,7 +796,7 @@ DEFPY(test_mlag, test_mlag_cmd, "Mlag is setup to be the secondary\n") { enum mlag_role orig = zrouter.mlag_info.role; - char buf1[80], buf2[80]; + char buf1[MAXCH_LEN], buf2[MAXCH_LEN]; if (none) zrouter.mlag_info.role = MLAG_ROLE_NONE; @@ -894,8 +900,8 @@ void zebra_mlag_init(void) zrouter.mlag_info.t_read = NULL; zrouter.mlag_info.t_write = NULL; test_mlag_in_progress = false; - zebra_mlag_reset_write_buffer(); zebra_mlag_reset_read_buffer(); + pthread_mutex_init(&zrouter.mlag_info.mlag_th_mtx, NULL); } void zebra_mlag_terminate(void) @@ -920,7 +926,7 @@ int zebra_mlag_protobuf_encode_client_data(struct stream *s, uint32_t *msg_type) int len = 0; int n_len = 0; int rc = 0; - char buf[80]; + char buf[MAXCH_LEN]; if (IS_ZEBRA_DEBUG_MLAG) zlog_debug("%s: Entering..", __func__); @@ -933,12 +939,10 @@ int zebra_mlag_protobuf_encode_client_data(struct stream *s, uint32_t *msg_type) zlog_debug("%s: Decoded msg length:%d..", __func__, mlag_msg.data_len); - memset(tmp_buf, 0, ZEBRA_MLAG_BUF_LIMIT); - if (IS_ZEBRA_DEBUG_MLAG) zlog_debug("%s: Mlag ProtoBuf encoding of message:%s", __func__, zebra_mlag_lib_msgid_to_str(mlag_msg.msg_type, buf, - 80)); + sizeof(buf))); *msg_type = mlag_msg.msg_type; switch (mlag_msg.msg_type) { case MLAG_MROUTE_ADD: { @@ -1138,17 +1142,15 @@ int zebra_mlag_protobuf_encode_client_data(struct stream *s, uint32_t *msg_type) } if (IS_ZEBRA_DEBUG_MLAG) - zlog_debug( - "%s: length of Mlag ProtoBuf encoded message:%s, %d", - __func__, - zebra_mlag_lib_msgid_to_str(mlag_msg.msg_type, buf, 80), - len); + zlog_debug("%s: length of Mlag ProtoBuf encoded message:%s, %d", + __func__, + zebra_mlag_lib_msgid_to_str(mlag_msg.msg_type, buf, + sizeof(buf)), + len); hdr.type = (ZebraMlagHeader__MessageType)mlag_msg.msg_type; if (len != 0) { hdr.data.len = len; hdr.data.data = XMALLOC(MTYPE_MLAG_PBUF, len); - if (hdr.data.data == NULL) - return -1; memcpy(hdr.data.data, tmp_buf, len); } @@ -1173,7 +1175,8 @@ int zebra_mlag_protobuf_encode_client_data(struct stream *s, uint32_t *msg_type) zlog_debug( "%s: length of Mlag ProtoBuf message:%s with Header %d", __func__, - zebra_mlag_lib_msgid_to_str(mlag_msg.msg_type, buf, 80), + zebra_mlag_lib_msgid_to_str(mlag_msg.msg_type, buf, + sizeof(buf)), len); if (hdr.data.data) XFREE(MTYPE_MLAG_PBUF, hdr.data.data); @@ -1188,7 +1191,7 @@ int zebra_mlag_protobuf_decode_message(struct stream **s, uint8_t *data, { uint32_t msg_type; ZebraMlagHeader *hdr = NULL; - char buf[80]; + char buf[MAXCH_LEN]; if (IS_ZEBRA_DEBUG_MLAG) zlog_debug("%s: Entering..", __func__); @@ -1205,7 +1208,8 @@ int zebra_mlag_protobuf_decode_message(struct stream **s, uint8_t *data, if (IS_ZEBRA_DEBUG_MLAG) zlog_debug("%s: Mlag ProtoBuf decoding of message:%s", __func__, - zebra_mlag_lib_msgid_to_str(msg_type, buf, 80)); + zebra_mlag_lib_msgid_to_str(msg_type, buf, + sizeof(buf))); /* * Internal MLAG Message-types & MLAG.proto message types should diff --git a/zebra/zebra_mlag.h b/zebra/zebra_mlag.h index 01459cd29c..015c94bf5c 100644 --- a/zebra/zebra_mlag.h +++ b/zebra/zebra_mlag.h @@ -1,5 +1,5 @@ /* Zebra mlag header. - * Copyright (C) 2018 Cumulus Networks, Inc. + * Copyright (C) 2019 Cumulus Networks, Inc. * Donald Sharp * * This file is part of FRR. @@ -37,14 +37,8 @@ extern uint8_t mlag_wr_buffer[ZEBRA_MLAG_BUF_LIMIT]; extern uint8_t mlag_rd_buffer[ZEBRA_MLAG_BUF_LIMIT]; extern uint32_t mlag_rd_buf_offset; -static inline void zebra_mlag_reset_write_buffer(void) -{ - memset(mlag_wr_buffer, 0, ZEBRA_MLAG_BUF_LIMIT); -} - static inline void zebra_mlag_reset_read_buffer(void) { - memset(mlag_rd_buffer, 0, ZEBRA_MLAG_BUF_LIMIT); mlag_rd_buf_offset = 0; } diff --git a/zebra/zebra_mlag_private.c b/zebra/zebra_mlag_private.c index a5637c67ae..896e78ca0a 100644 --- a/zebra/zebra_mlag_private.c +++ b/zebra/zebra_mlag_private.c @@ -1,5 +1,5 @@ /* Zebra Mlag Code. - * Copyright (C) 2018 Cumulus Networks, Inc. + * Copyright (C) 2019 Cumulus Networks, Inc. * Donald Sharp * * This file is part of FRR. @@ -68,8 +68,10 @@ int zebra_mlag_private_write_data(uint8_t *data, uint32_t len) static void zebra_mlag_sched_read(void) { + pthread_mutex_lock(&zrouter.mlag_info.mlag_th_mtx); thread_add_read(zmlag_master, zebra_mlag_read, NULL, mlag_socket, &zrouter.mlag_info.t_read); + pthread_mutex_unlock(&zrouter.mlag_info.mlag_th_mtx); } static int zebra_mlag_read(struct thread *thread) @@ -78,7 +80,9 @@ static int zebra_mlag_read(struct thread *thread) uint32_t h_msglen; uint32_t tot_len, curr_len = mlag_rd_buf_offset; + pthread_mutex_lock(&zrouter.mlag_info.mlag_th_mtx); zrouter.mlag_info.t_read = NULL; + pthread_mutex_unlock(&zrouter.mlag_info.mlag_th_mtx); /* * Received message in sock_stream looks like below diff --git a/zebra/zebra_mlag_private.h b/zebra/zebra_mlag_private.h index 02797429a8..2ae7e39325 100644 --- a/zebra/zebra_mlag_private.h +++ b/zebra/zebra_mlag_private.h @@ -1,5 +1,5 @@ /* Zebra mlag header. - * Copyright (C) 2018 Cumulus Networks, Inc. + * Copyright (C) 2019 Cumulus Networks, Inc. * Donald Sharp * * This file is part of FRR. diff --git a/zebra/zebra_router.h b/zebra/zebra_router.h index 141b2579e2..a12e34a495 100644 --- a/zebra/zebra_router.h +++ b/zebra/zebra_router.h @@ -100,6 +100,7 @@ struct zebra_mlag_info { /* Threads for read/write. */ struct thread *t_read; struct thread *t_write; + pthread_mutex_t mlag_th_mtx; }; struct zebra_router {