From 5bb7ca5da5e75952105d85e21d3bac9ec587834e Mon Sep 17 00:00:00 2001 From: Christine Caulfield Date: Mon, 13 Oct 2008 14:27:41 +0000 Subject: [PATCH] This is an initial pass at a top-level quorum system. As it stands, this module doesn't provide quorum itself, merely a framework for setting and querying it. I envisage YKD plugging into this rather than straight into sync() eventually. I've plugged this into the sync() routines rather than replacing them so that quorum is itself a VSF, rather than a replacement - I'm not sure if that is best or not. Opinions are welcome. I've added an extra enum member to the service_handler so that we can send IPC messages when the cluster isn't quorate. This will default to NO (as now) but allows us to query and set quorum when we don't have it .. a useful feature ! git-svn-id: http://svn.fedorahosted.org/svn/corosync/trunk@1674 fd59a12c-fef9-0310-b244-a6a79926bd2f --- exec/Makefile | 15 +- exec/ipc.c | 4 +- exec/quorum.h | 75 +++++ exec/vsf_quorum.c | 389 +++++++++++++++++++++++ include/corosync/engine/coroapi.h | 6 + include/corosync/ipc_gen.h | 3 +- include/corosync/ipc_quorum.h | 73 +++++ include/corosync/quorum.h | 118 +++++++ lib/Makefile | 14 +- lib/libquorum.versions | 27 ++ lib/quorum.c | 493 ++++++++++++++++++++++++++++++ test/Makefile | 7 +- test/testquorum.c | 58 ++++ 13 files changed, 1271 insertions(+), 11 deletions(-) create mode 100644 exec/quorum.h create mode 100644 exec/vsf_quorum.c create mode 100644 include/corosync/ipc_quorum.h create mode 100644 include/corosync/quorum.h create mode 100644 lib/libquorum.versions create mode 100644 lib/quorum.c create mode 100644 test/testquorum.c diff --git a/exec/Makefile b/exec/Makefile index 8e42d87a..307d5ffe 100644 --- a/exec/Makefile +++ b/exec/Makefile @@ -59,8 +59,8 @@ LOGSYS_OBJS = wthread.o logsys.o EXEC_LIBS = libtotem_pg.a liblogsys.a # LCR objects -LCR_SRC = vsf_ykd.c objdb.c coroparse.c -LCR_OBJS = vsf_ykd.o objdb.o coroparse.o +LCR_SRC = vsf_ykd.c objdb.c coroparse.c vsf_quorum.c +LCR_OBJS = vsf_ykd.o objdb.o coroparse.o vsf_quorum.o # main executive objects MAIN_SRC = main.c mempool.c util.c sync.c apidef.c service.c ipc.c flow.c \ @@ -75,7 +75,7 @@ override CFLAGS += -fPIC all:libtotem_pg.a libtotem_pg.so.2.0.0 liblogsys.a liblogsys.so.2.0.0 \ ../lcr/lcr_ifact.o corosync_ \ - objdb.lcrso vsf_ykd.lcrso coroparse.lcrso + objdb.lcrso vsf_ykd.lcrso coroparse.lcrso vsf_quorum.lcrso else EXEC_OBJS = $(MAIN_OBJS) $(LCR_OBJS) all: libtotem_pg.a liblogsys.a corosync @@ -90,6 +90,9 @@ objdb.lcrso: objdb.o vsf_ykd.lcrso: vsf_ykd.o $(CC) $(LDFLAGS) -bundle $(LDFLAGS) -bundle_loader ./corosync -bind_at_load vsf_ykd.o -o $@ +vsf_quorum.lcrso: vsf_quorum.o + $(CC) $(LDFLAGS) -bundle $(LDFLAGS) -bundle_loader ./corosync -bind_at_load vsf_quorum.o -o $@ + coroparse.lcrso: coroparse.o $(CC) -bundle -bundle_loader ./corosync -bind_at_load coroparse.o -o $@ @@ -98,6 +101,9 @@ else vsf_ykd.lcrso: vsf_ykd.o $(CC) -shared -Wl,-soname,vsf_ykd.lcrso vsf_ykd.o -o $@ +vsf_quorum.lcrso: vsf_quorum.o + $(CC) -shared -Wl,-soname,vsf_quorum.lcrso vsf_quorum.o -o $@ + objdb.lcrso: objdb.o $(CC) -shared -Wl,-soname,objdb.lcrso objdb.o -o $@ @@ -145,6 +151,9 @@ depend: vsf_ykd.o: vsf_ykd.c $(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $< +vsf_quorum.o: vsf_quorum.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $< + objdb.o: objdb.c $(CC) $(CFLAGS) -c -o $@ $< diff --git a/exec/ipc.c b/exec/ipc.c index 04712478..4222b471 100644 --- a/exec/ipc.c +++ b/exec/ipc.c @@ -897,11 +897,11 @@ retry_recv: &send_ok_joined_iovec, 1); send_ok = - (sync_primary_designated() == 1) && ( + (sync_primary_designated() == 1 || ais_service[service]->allow_inquorate == COROSYNC_LIB_ALLOW_INQUORATE) && ( (ais_service[service]->lib_engine[header->id].flow_control == COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED) || ((ais_service[service]->lib_engine[header->id].flow_control == COROSYNC_LIB_FLOW_CONTROL_REQUIRED) && (send_ok_joined) && - (sync_in_process() == 0))); + (sync_in_process() == 0))); if (send_ok) { ais_service[service]->lib_engine[header->id].lib_handler_fn(conn_info, header); diff --git a/exec/quorum.h b/exec/quorum.h new file mode 100644 index 00000000..def280fb --- /dev/null +++ b/exec/quorum.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2008 Red Hat, Inc. + * + * All rights reserved. + * + * Author: Christine Caulfield (ccaulfie@redhat.com) + * + * This software licensed under BSD license, the text of which follows: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * - Neither the name of the MontaVista Software, Inc. nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef QUORUM_H_DEFINED +#define QUORUM_H_DEFINED + +struct quorum_services_api_ver1 { + void (*quorum_api_set_quorum) (unsigned int *,int, + int, struct memb_ring_id *); + }; + +static inline struct quorum_services_api_ver1 * +quorum_services_api_reference ( + struct corosync_api_v1 *coroapi, + unsigned int *handle) +{ + static void *quorum_services_api_p; + struct quorum_services_api_ver1 *return_api; + unsigned int res; + + res = coroapi->plugin_interface_reference ( + handle, + "quorum_services_api", + 0, + &quorum_services_api_p, + 0); + if (res == -1) { + return (NULL); + } + return_api = (struct quorum_services_api_ver1 *)quorum_services_api_p; + return (return_api); +} + +static int inline quorum_services_api_release ( + struct corosync_api_v1 *coroapi, + unsigned int handle) +{ + unsigned int res; + + res = coroapi->plugin_interface_release (handle); + return (res); +} + +#endif /* QUORUM_H_DEFINED */ diff --git a/exec/vsf_quorum.c b/exec/vsf_quorum.c new file mode 100644 index 00000000..b129ff78 --- /dev/null +++ b/exec/vsf_quorum.c @@ -0,0 +1,389 @@ +/* + * Copyright (c) 2008 Red Hat, Inc. + * + * All rights reserved. + * + * Author: Christine Caulfield (ccaulfie@redhat.com) + * + * This software licensed under BSD license, the text of which follows: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * - Neither the name of the MontaVista Software, Inc. nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "vsf.h" +#include "quorum.h" + +LOGSYS_DECLARE_SUBSYS ("QUORUM", LOG_INFO); + +struct quorum_pd { + unsigned char track_flags; + int tracking_enabled; + struct list_head list; + void *conn; +}; + +static void message_handler_req_lib_quorum_getquorate (void *conn, void *msg); +static void message_handler_req_lib_quorum_trackstart (void *conn, void *msg); +static void message_handler_req_lib_quorum_trackstop (void *conn, void *msg); +static int send_quorum_notification(void *conn); +static int quorum_exec_init_fn (struct corosync_api_v1 *api); +static int quorum_lib_init_fn (void *conn); +static int quorum_lib_exit_fn (void *conn); + +static int primary_designated = 0; +static struct corosync_api_v1 *corosync_api; +static struct list_head trackers_list; +static struct memb_ring_id quorum_ring_id; +static int quorum_view_list_entries = 0; +static int quorum_view_list[PROCESSOR_COUNT_MAX]; + +static void (*quorum_primary_callback_fn) ( + unsigned int *view_list, + int view_list_entries, + int primary_designated, + struct memb_ring_id *ring_id); + +/* Internal quorum API function */ +static void quorum_api_set_quorum(unsigned int *view_list, + int view_list_entries, + int quorum, struct memb_ring_id *ring_id) +{ + primary_designated = quorum; + memcpy(&quorum_ring_id, &ring_id, sizeof (quorum_ring_id)); + + quorum_view_list_entries = view_list_entries; + memcpy(quorum_view_list, view_list, sizeof(unsigned int)*view_list_entries); + + /* Tell sync() */ + quorum_primary_callback_fn(view_list, view_list_entries, + primary_designated, &quorum_ring_id); + + /* Tell IPC listeners */ + send_quorum_notification(NULL); +} + +static int quorum_init ( + void (*primary_callback_fn) ( + unsigned int *view_list, + int view_list_entries, + int primary_designated, + struct memb_ring_id *ring_id)) +{ + quorum_primary_callback_fn = primary_callback_fn; + + return (0); +} + +/* + * Returns 1 if this processor is in the primary (has quorum) + */ +static int quorum_primary (void) +{ + return (primary_designated); +} + +/* + * lcrso object definition + */ +static struct corosync_vsf_iface_ver0 vsf_quorum_iface_ver0 = { + .init = quorum_init, + .primary = quorum_primary +}; + +static struct quorum_services_api_ver1 quorum_service_api_v1 = { + .quorum_api_set_quorum = quorum_api_set_quorum +}; + +static struct corosync_lib_handler quorum_lib_service[] = +{ + { /* 0 */ + .lib_handler_fn = message_handler_req_lib_quorum_getquorate, + .response_size = sizeof (struct res_lib_quorum_getquorate), + .response_id = MESSAGE_RES_QUORUM_GETQUORATE, + .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED + }, + { /* 1 */ + .lib_handler_fn = message_handler_req_lib_quorum_trackstart, + .response_size = sizeof (mar_res_header_t), + .response_id = MESSAGE_RES_QUORUM_NOTIFICATION, + .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED + }, + { /* 2 */ + .lib_handler_fn = message_handler_req_lib_quorum_trackstop, + .response_size = sizeof (mar_res_header_t), + .response_id = MESSAGE_RES_QUORUM_TRACKSTOP, + .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED + } +}; + +static struct corosync_service_engine quorum_service_handler = { + .name = "corosync cluster quorum service v0.1", + .id = QUORUM_SERVICE, + .private_data_size = sizeof (struct quorum_pd), + .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED, + .allow_inquorate = COROSYNC_LIB_ALLOW_INQUORATE, + .lib_init_fn = quorum_lib_init_fn, + .lib_exit_fn = quorum_lib_exit_fn, + .lib_engine = quorum_lib_service, + .exec_init_fn = quorum_exec_init_fn, + .lib_engine_count = sizeof (quorum_lib_service) / sizeof (struct corosync_lib_handler), +}; + +static struct lcr_iface corosync_vsf_quorum_ver0[3] = { + { /* the VSF handler */ + .name = "corosync_vsf_quorum", + .version = 0, + .versions_replace = 0, + .versions_replace_count = 0, + .dependencies = 0, + .dependency_count = 0, + .constructor = NULL, + .destructor = NULL, + .interfaces = (void **)(void *)&vsf_quorum_iface_ver0, + }, + { /* API for quorum users to call */ + .name = "corosync_quorum_api", + .version = 0, + .versions_replace = 0, + .versions_replace_count = 0, + .dependencies = 0, + .dependency_count = 0, + .constructor = NULL, + .destructor = NULL, + .interfaces = NULL + }, + { /* Library calls */ + .name = "corosync_quorum", + .version = 0, + .versions_replace = 0, + .versions_replace_count = 0, + .dependencies = 0, + .dependency_count = 0, + .constructor = NULL, + .destructor = NULL, + .interfaces = NULL, + }, + +}; + +static struct corosync_service_engine *quorum_get_service_handler_ver0 (void) +{ + return (&quorum_service_handler); +} + +static struct lcr_comp vsf_quorum_comp_ver0 = { + .iface_count = 3, + .ifaces = corosync_vsf_quorum_ver0 +}; + +static struct corosync_service_engine_iface_ver0 quorum_service_handler_iface = { + .corosync_get_service_engine_ver0 = quorum_get_service_handler_ver0 +}; + +__attribute__ ((constructor)) static void vsf_quorum_comp_register (void) { + lcr_component_register (&vsf_quorum_comp_ver0); + lcr_interfaces_set (&corosync_vsf_quorum_ver0[0], &vsf_quorum_iface_ver0); + lcr_interfaces_set (&corosync_vsf_quorum_ver0[1], &quorum_service_api_v1); + lcr_interfaces_set (&corosync_vsf_quorum_ver0[2], &quorum_service_handler_iface); +} + +/* -------------------------------------------------- */ + +static int quorum_exec_init_fn (struct corosync_api_v1 *api) +{ + corosync_api = api; + list_init (&trackers_list); + return (0); +} + +static int quorum_lib_init_fn (void *conn) +{ + struct quorum_pd *pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn); + + log_printf(LOG_LEVEL_DEBUG, "lib_init_fn: conn=%p\n", conn); + + list_init (&pd->list); + pd->conn = conn; + + return (0); +} + +static int quorum_lib_exit_fn (void *conn) +{ + struct quorum_pd *quorum_pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn); + + log_printf(LOG_LEVEL_DEBUG, "lib_exit_fn: conn=%p\n", conn); + + if (quorum_pd->tracking_enabled) { + list_del (&quorum_pd->list); + list_init (&quorum_pd->list); + } + return (0); +} + +static int send_quorum_notification(void *conn) +{ + int size = sizeof(struct res_lib_quorum_notification) + sizeof(unsigned int)*quorum_view_list_entries; + char *buf[size]; + struct res_lib_quorum_notification *res_lib_quorum_notification = (struct res_lib_quorum_notification *)buf; + struct list_head *tmp; + int i; + + log_printf(LOG_LEVEL_DEBUG, "sending quorum notification to %p, length = %d\n", conn, size); + + res_lib_quorum_notification->quorate = primary_designated; + res_lib_quorum_notification->ring_seq = quorum_ring_id.seq; + res_lib_quorum_notification->view_list_entries = quorum_view_list_entries; + for (i=0; iview_list[i] = quorum_view_list[i]; + } + + res_lib_quorum_notification->header.id = MESSAGE_RES_QUORUM_NOTIFICATION; + res_lib_quorum_notification->header.size = size; + res_lib_quorum_notification->header.error = SA_AIS_OK; + + /* Send it to all interested parties */ + if (conn) { + return corosync_api->ipc_conn_send_response(conn, res_lib_quorum_notification, size); + } + else { + struct quorum_pd *qpd; + + for (tmp = trackers_list.next; tmp != &trackers_list; tmp = tmp->next) { + + qpd = list_entry(tmp, struct quorum_pd, list); + + corosync_api->ipc_conn_send_response(corosync_api->ipc_conn_partner_get(qpd->conn), + res_lib_quorum_notification, size); + } + } + return (0); +} + +static void message_handler_req_lib_quorum_getquorate (void *conn, void *msg) +{ + struct res_lib_quorum_getquorate res_lib_quorum_getquorate; + + log_printf(LOG_LEVEL_DEBUG, "got quorate request on %p\n", conn); + + /* send status */ + res_lib_quorum_getquorate.quorate = primary_designated; + res_lib_quorum_getquorate.header.size = sizeof(res_lib_quorum_getquorate); + res_lib_quorum_getquorate.header.id = MESSAGE_RES_QUORUM_GETQUORATE; + res_lib_quorum_getquorate.header.error = SA_AIS_OK; + corosync_api->ipc_conn_send_response(conn, &res_lib_quorum_getquorate, sizeof(res_lib_quorum_getquorate)); +} + + +static void message_handler_req_lib_quorum_trackstart (void *conn, void *msg) +{ + struct req_lib_quorum_trackstart *req_lib_quorum_trackstart = (struct req_lib_quorum_trackstart *)msg; + mar_res_header_t res; + struct quorum_pd *quorum_pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn); + + log_printf(LOG_LEVEL_DEBUG, "got trackstart request on %p\n", conn); + + /* + * If an immediate listing of the current cluster membership + * is requested, generate membership list + */ + if (req_lib_quorum_trackstart->track_flags & SA_TRACK_CURRENT || + req_lib_quorum_trackstart->track_flags & SA_TRACK_CHANGES) { + log_printf(LOG_LEVEL_DEBUG, "sending initial status to %p\n", conn); + send_quorum_notification(corosync_api->ipc_conn_partner_get (conn)); + } + + /* + * Record requests for tracking + */ + if (req_lib_quorum_trackstart->track_flags & SA_TRACK_CHANGES || + req_lib_quorum_trackstart->track_flags & SA_TRACK_CHANGES_ONLY) { + + quorum_pd->track_flags = req_lib_quorum_trackstart->track_flags; + quorum_pd->tracking_enabled = 1; + + list_add (&quorum_pd->list, &trackers_list); + } + + /* send status */ + res.size = sizeof(res); + res.id = MESSAGE_RES_QUORUM_TRACKSTART; + res.error = SA_AIS_OK; + corosync_api->ipc_conn_send_response(conn, &res, sizeof(mar_res_header_t)); +} + +static void message_handler_req_lib_quorum_trackstop (void *conn, void *msg) +{ + mar_res_header_t res; + struct quorum_pd *quorum_pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn); + + log_printf(LOG_LEVEL_DEBUG, "got trackstop request on %p\n", conn); + + if (quorum_pd->tracking_enabled) { + res.error = SA_AIS_OK; + quorum_pd->tracking_enabled = 0; + list_del (&quorum_pd->list); + list_init (&quorum_pd->list); + } else { + res.error = SA_AIS_ERR_NOT_EXIST; + } + + /* send status */ + res.size = sizeof(res); + res.id = MESSAGE_RES_QUORUM_TRACKSTOP; + res.error = SA_AIS_OK; + corosync_api->ipc_conn_send_response(conn, &res, sizeof(mar_res_header_t)); +} + diff --git a/include/corosync/engine/coroapi.h b/include/corosync/engine/coroapi.h index d2abd12d..b285c966 100644 --- a/include/corosync/engine/coroapi.h +++ b/include/corosync/engine/coroapi.h @@ -93,6 +93,11 @@ enum corosync_lib_flow_control { COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED = 2 }; +enum corosync_lib_allow_inquorate { + COROSYNC_LIB_DISALLOW_INQUORATE = 0, /* default */ + COROSYNC_LIB_ALLOW_INQUORATE = 1 +}; + #if !defined (COROSYNC_FLOW_CONTROL_STATE) enum corosync_flow_control_state { COROSYNC_FLOW_CONTROL_STATE_DISABLED, @@ -513,6 +518,7 @@ struct corosync_service_engine { unsigned short id; unsigned int private_data_size; enum corosync_lib_flow_control flow_control; + enum corosync_lib_allow_inquorate allow_inquorate; int (*exec_init_fn) (struct corosync_api_v1 *); int (*exec_exit_fn) (void); void (*exec_dump_fn) (void); diff --git a/include/corosync/ipc_gen.h b/include/corosync/ipc_gen.h index 97407d0c..27056a50 100644 --- a/include/corosync/ipc_gen.h +++ b/include/corosync/ipc_gen.h @@ -46,7 +46,8 @@ enum service_types { MSG_SERVICE = 6, CFG_SERVICE = 7, CPG_SERVICE = 8, - CONFDB_SERVICE = 10 + CONFDB_SERVICE = 10, + QUORUM_SERVICE = 11, }; enum req_init_types { diff --git a/include/corosync/ipc_quorum.h b/include/corosync/ipc_quorum.h new file mode 100644 index 00000000..89915047 --- /dev/null +++ b/include/corosync/ipc_quorum.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2008 Red Hat, Inc. + * + * All rights reserved. + * + * Author: Christine Caulfield (ccaulfie@redhat.com) + * + * This software licensed under BSD license, the text of which follows: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * - Neither the name of the MontaVista Software, Inc. nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef IPC_QUORUM_H_DEFINED +#define IPC_QUORUM_H_DEFINED + +#include +#include "saAis.h" +#include "corosync/ipc_gen.h" + +enum req_quorum_types { + MESSAGE_REQ_QUORUM_GETQUORATE = 0, + MESSAGE_REQ_QUORUM_TRACKSTART, + MESSAGE_REQ_QUORUM_TRACKSTOP +}; + +enum res_quorum_types { + MESSAGE_RES_QUORUM_GETQUORATE = 0, + MESSAGE_RES_QUORUM_TRACKSTART, + MESSAGE_RES_QUORUM_TRACKSTOP, + MESSAGE_RES_QUORUM_NOTIFICATION +}; + +struct req_lib_quorum_trackstart { + mar_req_header_t header __attribute__((aligned(8))); + unsigned int track_flags; +}; + + +struct res_lib_quorum_getquorate { + mar_res_header_t header __attribute__((aligned(8))); + mar_uint32_t quorate; +}; + +struct res_lib_quorum_notification { + mar_res_header_t header __attribute__((aligned(8))); + mar_int32_t quorate __attribute__((aligned(8))); + mar_uint64_t ring_seq __attribute__((aligned(8))); + mar_uint32_t view_list_entries __attribute__((aligned(8))); + mar_uint32_t view_list[]; +}; + +#endif diff --git a/include/corosync/quorum.h b/include/corosync/quorum.h new file mode 100644 index 00000000..816f5e6e --- /dev/null +++ b/include/corosync/quorum.h @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2008 Red Hat, Inc. + * + * All rights reserved. + * + * Author: Christine Caulfield (ccaulfi@redhat.com) + * + * This software licensed under BSD license, the text of which follows: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * - Neither the name of the MontaVista Software, Inc. nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef COROSYNC_QUORUM_H_DEFINED +#define COROSYNC_QUORUM_H_DEFINED + +typedef uint64_t quorum_handle_t; + + +typedef enum { + QUORUM_OK = 1, + QUORUM_ERR_LIBRARY = 2, + QUORUM_ERR_TIMEOUT = 5, + QUORUM_ERR_TRY_AGAIN = 6, + QUORUM_ERR_INVALID_PARAM = 7, + QUORUM_ERR_NO_MEMORY = 8, + QUORUM_ERR_BAD_HANDLE = 9, + QUORUM_ERR_ACCESS = 11, + QUORUM_ERR_NOT_EXIST = 12, + QUORUM_ERR_EXIST = 14, + QUORUM_ERR_NOT_SUPPORTED = 20, + QUORUM_ERR_SECURITY = 29 +} quorum_error_t; + +typedef enum { + QUORUM_DISPATCH_ONE, + QUORUM_DISPATCH_ALL, + QUORUM_DISPATCH_BLOCKING +} quorum_dispatch_t; + +typedef struct { + uint32_t nodeid; + uint32_t state; +} quorum_node_t; + + +typedef void (*quorum_notification_fn_t) ( + quorum_handle_t handle, + uint32_t quorate, + uint64_t ring_seq, + uint32_t view_list_entries, + uint32_t *view_list + ); + +typedef struct { + quorum_notification_fn_t quorum_notify_fn; +} quorum_callbacks_t; + + +/* + * Create a new quorum connection + */ +quorum_error_t quorum_initialize ( + quorum_handle_t *handle, + quorum_callbacks_t *callbacks); + +/* + * Close the quorum handle + */ +quorum_error_t quorum_finalize ( + quorum_handle_t handle); + + +/* + * Dispatch messages and configuration changes + */ +quorum_error_t quorum_dispatch ( + quorum_handle_t handle, + quorum_dispatch_t dispatch_types); + + +/* + * Get quorum information. + */ +quorum_error_t quorum_getquorate ( + quorum_handle_t handle, + int *quorate); + +/* Track node and quorum changes */ +quorum_error_t quorum_trackstart ( + quorum_handle_t handle, + unsigned int flags ); + +quorum_error_t quorum_trackstop ( + quorum_handle_t handle); + + +#endif /* COROSYNC_QUORUM_H_DEFINED */ diff --git a/lib/Makefile b/lib/Makefile index 2dba110b..7f623dc1 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -41,6 +41,7 @@ all: libcpg.a libcpg.so.2.0.0 \ libconfdb.a libconfdb.so.2.0.0 \ libevs.a libevs.so.2.0.0 \ libcfg.a libcfg.so.2.0.0 \ + libquorum.a libquorum.so.2.0.0 \ libcoroutil.a libcoroutil.so.2.0.0 libcoroutil.a: util.o @@ -58,15 +59,15 @@ libevs.so.2.0.0: util.o evs.o libcpg.so.2.0.0: util.o cpg.o $(CC) $(DARWIN_OPTS) util.o cpg.o -o $@ +libquorum.so.2.0.0: util.o quorum.o + $(CC) $(DARWIN_OPTS) util.o quorum.o -o $@ + libconfdb.so.2.0.0: util.o confdb.o sa-confdb.o $(CC) $(LDFLAGS) $(DARWIN_OPTS) util.o confdb.o sa-confdb.o ../lcr/lcr_ifact.o -o $@ libcfg.so.2.0.0: util.o cfg.o $(CC) $(DARWIN_OPTS) util.o cfg.o -o $@ -libcpg.so.2.0.0: util.o cpg.o - $(CC) $(DARWIN_OPTS) util.o cpg.o -o $@ - else libcoroutil.so.2.0.0: util.o @@ -78,6 +79,9 @@ libevs.so.2.0.0: util.o evs.o libcpg.so.2.0.0: util.o cpg.o $(CC) -shared -Wl,-soname,libcpg.so.2,-version-script=$(srcdir)$(subdir)libcpg.versions util.o cpg.o -o $@ +libquorum.so.2.0.0: util.o quorum.o + $(CC) -shared -Wl,-soname,libquorum.so.2,-version-script=$(srcdir)$(subdir)libquorum.versions util.o quorum.o -o $@ + libconfdb.so.2.0.0: util.o confdb.o sa-confdb.o $(CC) $(LDFLAGS) -shared -Wl,-soname,libconfdb.so.2,-version-script=$(srcdir)$(subdir)libconfdb.versions util.o confdb.o sa-confdb.o ../lcr/lcr_ifact.o -o $@ @@ -92,6 +96,9 @@ libevs.a: util.o evs.o libcpg.a: util.o cpg.o $(AR) -rc libcpg.a util.o cpg.o +libquorum.a: util.o quorum.o + $(AR) -rc libquorum.a util.o quorum.o + libconfdb.a: util.o confdb.o sa-confdb.o $(AR) -rc libconfdb.a util.o confdb.o sa-confdb.o ../lcr/lcr_ifact.o @@ -101,6 +108,7 @@ libcfg.a: util.o cfg.o clean: rm -f *.o libcfg.so* libcoroutil.so* libcoroutil.a \ libevs.so* libevs.a libcpg.so* libcpg.a libcfg.a libconfdb.so* \ + libquorum.so* libquorum.a \ libconfdb.a libconfdb.a \ *.da *.bb *.bbg # -fPIC rules required for all libraries diff --git a/lib/libquorum.versions b/lib/libquorum.versions new file mode 100644 index 00000000..0242f9cb --- /dev/null +++ b/lib/libquorum.versions @@ -0,0 +1,27 @@ +# Version and symbol export for libcpg.so + +COROSYNC_QUORUM_1.0 { + global: + quorum_initialize; + quorum_finalize; + quorum_fd_get; + quorum_dispatch; + quorum_get_quorate; + quorum_context_get; + quorum_context_set; + + local: + saHandleCreate; + saHandleDestroy; + saHandleInstanceGet; + saHandleInstancePut; + saRecvRetry; + saSelectRetry; + saSendMsgReceiveReply; + saSendMsgRetry; + saSendReceiveReply; + saSendRetry; + saServiceConnect; + saVersionVerify; + clustTimeNow; +}; diff --git a/lib/quorum.c b/lib/quorum.c new file mode 100644 index 00000000..a342ac5a --- /dev/null +++ b/lib/quorum.c @@ -0,0 +1,493 @@ +/* + * Copyright (c) 2008 Red Hat, Inc. + * + * All rights reserved. + * + * Author: Christine Caulfield (ccaulfie@redhat.com) + * + * This software licensed under BSD license, the text of which follows: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * - Neither the name of the MontaVista Software, Inc. nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ +/* + * Provides a quorum API using the corosync executive + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include "corosync/quorum.h" +#include "corosync/ipc_quorum.h" + +struct quorum_inst { + int response_fd; + int dispatch_fd; + int finalize; + void *context; + quorum_callbacks_t callbacks; + pthread_mutex_t response_mutex; + pthread_mutex_t dispatch_mutex; +}; + +static void quorum_instance_destructor (void *instance); + +static struct saHandleDatabase quorum_handle_t_db = { + .handleCount = 0, + .handles = 0, + .mutex = PTHREAD_MUTEX_INITIALIZER, + .handleInstanceDestructor = quorum_instance_destructor +}; + +/* + * Clean up function for a quorum instance (quorum_initialize) handle + */ +static void quorum_instance_destructor (void *instance) +{ + struct quorum_inst *quorum_inst = instance; + + pthread_mutex_destroy (&quorum_inst->response_mutex); +} + +quorum_error_t quorum_initialize ( + quorum_handle_t *handle, + quorum_callbacks_t *callbacks) +{ + SaAisErrorT error; + struct quorum_inst *quorum_inst; + + error = saHandleCreate (&quorum_handle_t_db, sizeof (struct quorum_inst), handle); + if (error != SA_AIS_OK) { + goto error_no_destroy; + } + + error = saHandleInstanceGet (&quorum_handle_t_db, *handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + goto error_destroy; + } + + error = saServiceConnect (&quorum_inst->dispatch_fd, + &quorum_inst->response_fd, + QUORUM_SERVICE); + if (error != SA_AIS_OK) { + goto error_put_destroy; + } + + pthread_mutex_init (&quorum_inst->response_mutex, NULL); + pthread_mutex_init (&quorum_inst->dispatch_mutex, NULL); + if (callbacks) + memcpy(&quorum_inst->callbacks, callbacks, sizeof (callbacks)); + else + memset(&quorum_inst->callbacks, 0, sizeof (callbacks)); + + saHandleInstancePut (&quorum_handle_t_db, *handle); + + return (SA_AIS_OK); + +error_put_destroy: + saHandleInstancePut (&quorum_handle_t_db, *handle); +error_destroy: + saHandleDestroy (&quorum_handle_t_db, *handle); +error_no_destroy: + return (error); +} + +quorum_error_t quorum_finalize ( + quorum_handle_t handle) +{ + struct quorum_inst *quorum_inst; + SaAisErrorT error; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + pthread_mutex_lock (&quorum_inst->response_mutex); + + /* + * Another thread has already started finalizing + */ + if (quorum_inst->finalize) { + pthread_mutex_unlock (&quorum_inst->response_mutex); + saHandleInstancePut (&quorum_handle_t_db, handle); + return (QUORUM_ERR_BAD_HANDLE); + } + + quorum_inst->finalize = 1; + + pthread_mutex_unlock (&quorum_inst->response_mutex); + + saHandleDestroy (&quorum_handle_t_db, handle); + + /* + * Disconnect from the server + */ + if (quorum_inst->response_fd != -1) { + shutdown(quorum_inst->response_fd, 0); + close(quorum_inst->response_fd); + } + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (QUORUM_OK); +} + +quorum_error_t quorum_getquorate ( + quorum_handle_t handle, + int *quorate) +{ + quorum_error_t error; + struct quorum_inst *quorum_inst; + struct iovec iov[2]; + mar_req_header_t req; + struct res_lib_quorum_getquorate res_lib_quorum_getquorate; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + pthread_mutex_lock (&quorum_inst->response_mutex); + + req.size = sizeof (req); + req.id = MESSAGE_REQ_QUORUM_GETQUORATE; + + iov[0].iov_base = (char *)&req; + iov[0].iov_len = sizeof (req); + + error = saSendMsgReceiveReply (quorum_inst->response_fd, iov, 1, + &res_lib_quorum_getquorate, sizeof (struct res_lib_quorum_getquorate)); + + pthread_mutex_unlock (&quorum_inst->response_mutex); + + if (error != SA_AIS_OK) { + goto error_exit; + } + + error = res_lib_quorum_getquorate.header.error; + + *quorate = res_lib_quorum_getquorate.quorate; + +error_exit: + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (error); +} + +quorum_error_t quorum_fd_get ( + quorum_handle_t handle, + int *fd) +{ + SaAisErrorT error; + struct quorum_inst *quorum_inst; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + *fd = quorum_inst->dispatch_fd; + + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (SA_AIS_OK); +} + + +quorum_error_t quorum_context_get ( + quorum_handle_t handle, + void **context) +{ + SaAisErrorT error; + struct quorum_inst *quorum_inst; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + *context = quorum_inst->context; + + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (SA_AIS_OK); +} + +quorum_error_t quorum_context_set ( + quorum_handle_t handle, + void *context) +{ + SaAisErrorT error; + struct quorum_inst *quorum_inst; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + quorum_inst->context = context; + + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (SA_AIS_OK); +} + + +quorum_error_t quorum_trackstart ( + quorum_handle_t handle, + unsigned int flags ) +{ + quorum_error_t error; + struct quorum_inst *quorum_inst; + struct iovec iov[2]; + struct req_lib_quorum_trackstart req_lib_quorum_trackstart; + mar_res_header_t res; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + pthread_mutex_lock (&quorum_inst->response_mutex); + + req_lib_quorum_trackstart.header.size = sizeof (struct req_lib_quorum_trackstart); + req_lib_quorum_trackstart.header.id = MESSAGE_REQ_QUORUM_TRACKSTART; + req_lib_quorum_trackstart.track_flags = flags; + + iov[0].iov_base = (char *)&req_lib_quorum_trackstart; + iov[0].iov_len = sizeof (struct req_lib_quorum_trackstart); + + error = saSendMsgReceiveReply (quorum_inst->response_fd, iov, 1, + &res, sizeof (res)); + + pthread_mutex_unlock (&quorum_inst->response_mutex); + + if (error != SA_AIS_OK) { + goto error_exit; + } + + error = res.error; + +error_exit: + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (error); +} + +quorum_error_t quorum_trackstop ( + quorum_handle_t handle) +{ + quorum_error_t error; + struct quorum_inst *quorum_inst; + struct iovec iov[2]; + mar_req_header_t req; + mar_res_header_t res; + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + pthread_mutex_lock (&quorum_inst->response_mutex); + + req.size = sizeof (req); + req.id = MESSAGE_REQ_QUORUM_TRACKSTOP; + + iov[0].iov_base = (char *)&req; + iov[0].iov_len = sizeof (req); + + error = saSendMsgReceiveReply (quorum_inst->response_fd, iov, 1, + &res, sizeof (res)); + + pthread_mutex_unlock (&quorum_inst->response_mutex); + + if (error != SA_AIS_OK) { + goto error_exit; + } + + error = res.error; + +error_exit: + saHandleInstancePut (&quorum_handle_t_db, handle); + + return (error); +} + +struct res_overlay { + mar_res_header_t header __attribute__((aligned(8))); + char data[512000]; +}; + +quorum_error_t quorum_dispatch ( + quorum_handle_t handle, + quorum_dispatch_t dispatch_types) +{ + struct pollfd ufds; + int timeout = -1; + SaAisErrorT error; + int cont = 1; /* always continue do loop except when set to 0 */ + int dispatch_avail; + struct quorum_inst *quorum_inst; + quorum_callbacks_t callbacks; + struct res_overlay dispatch_data; + struct res_lib_quorum_notification *res_lib_quorum_notification; + + if (dispatch_types != SA_DISPATCH_ONE && + dispatch_types != SA_DISPATCH_ALL && + dispatch_types != SA_DISPATCH_BLOCKING) { + + return (SA_AIS_ERR_INVALID_PARAM); + } + + error = saHandleInstanceGet (&quorum_handle_t_db, handle, + (void *)&quorum_inst); + if (error != SA_AIS_OK) { + return (error); + } + + /* + * Timeout instantly for SA_DISPATCH_ONE or SA_DISPATCH_ALL and + * wait indefinately for SA_DISPATCH_BLOCKING + */ + if (dispatch_types == SA_DISPATCH_ALL) { + timeout = 0; + } + + do { + ufds.fd = quorum_inst->dispatch_fd; + ufds.events = POLLIN; + ufds.revents = 0; + + pthread_mutex_lock (&quorum_inst->dispatch_mutex); + + error = saPollRetry (&ufds, 1, timeout); + if (error != SA_AIS_OK) { + goto error_unlock; + } + + /* + * Handle has been finalized in another thread + */ + if (quorum_inst->finalize == 1) { + error = SA_AIS_OK; + goto error_unlock; + } + + if ((ufds.revents & (POLLERR|POLLHUP|POLLNVAL)) != 0) { + error = SA_AIS_ERR_BAD_HANDLE; + goto error_unlock; + } + + dispatch_avail = ufds.revents & POLLIN; + if (dispatch_avail == 0 && dispatch_types == SA_DISPATCH_ALL) { + pthread_mutex_unlock (&quorum_inst->dispatch_mutex); + break; /* exit do while cont is 1 loop */ + } else + if (dispatch_avail == 0) { + pthread_mutex_unlock (&quorum_inst->dispatch_mutex); + continue; /* next poll */ + } + + if (ufds.revents & POLLIN) { + error = saRecvRetry (quorum_inst->dispatch_fd, &dispatch_data.header, + sizeof (mar_res_header_t)); + if (error != SA_AIS_OK) { + goto error_unlock; + } + if (dispatch_data.header.size > sizeof (mar_res_header_t)) { + error = saRecvRetry (quorum_inst->dispatch_fd, &dispatch_data.data, + dispatch_data.header.size - sizeof (mar_res_header_t)); + if (error != SA_AIS_OK) { + goto error_unlock; + } + } + } else { + pthread_mutex_unlock (&quorum_inst->dispatch_mutex); + continue; + } + + /* + * Make copy of callbacks, message data, unlock instance, and call callback + * A risk of this dispatch method is that the callback routines may + * operate at the same time that quorum_finalize has been called in another thread. + */ + memcpy (&callbacks, &quorum_inst->callbacks, sizeof (quorum_callbacks_t)); + pthread_mutex_unlock (&quorum_inst->dispatch_mutex); + + /* + * Dispatch incoming message + */ + switch (dispatch_data.header.id) { + + case MESSAGE_RES_QUORUM_NOTIFICATION: + if (callbacks.quorum_notify_fn == NULL) { + continue; + } + res_lib_quorum_notification = (struct res_lib_quorum_notification *)&dispatch_data; + + callbacks.quorum_notify_fn ( handle, + res_lib_quorum_notification->quorate, + res_lib_quorum_notification->ring_seq, + res_lib_quorum_notification->view_list_entries, + res_lib_quorum_notification->view_list); + break; + + default: + error = SA_AIS_ERR_LIBRARY; + goto error_put; + break; + } + + /* + * Determine if more messages should be processed + * */ + switch (dispatch_types) { + case QUORUM_DISPATCH_ONE: + cont = 0; + break; + case QUORUM_DISPATCH_ALL: + break; + case QUORUM_DISPATCH_BLOCKING: + break; + } + } while (cont); + + goto error_put; + +error_unlock: + pthread_mutex_unlock (&quorum_inst->dispatch_mutex); + +error_put: + saHandleInstancePut (&quorum_handle_t_db, handle); + return (error); +} diff --git a/test/Makefile b/test/Makefile index 8199bfc3..91779320 100644 --- a/test/Makefile +++ b/test/Makefile @@ -42,9 +42,9 @@ ifeq (${COROSYNC_COMPAT}, SOLARIS) override LDFLAGS += -lnsl -lsocket -lrt endif -LIBRARIES= ../lib/libevs.a ../lib/libcpg.a ../lib/libcfg.a ../lib/libconfdb.a +LIBRARIES= ../lib/libevs.a ../lib/libcpg.a ../lib/libcfg.a ../lib/libconfdb.a ../lib/libquorum.a LIBS = $(LIBRARIES) -BINARIES= testevs evsbench evsverify testcpg testcpg2 cpgbench testconfdb +BINARIES= testevs evsbench evsverify testcpg testcpg2 cpgbench testconfdb testquorum override CFLAGS += -I../include override LDFLAGS += -L../lib @@ -75,6 +75,9 @@ testcpg: testcpg.o $(LIBRARIES) testcpg2: testcpg2.o $(LIBRARIES) $(CC) $(LDFLAGS) -o testcpg2 testcpg2.o $(LIBS) +testquorum: testquorum.o $(LIBRARIES) + $(CC) $(LDFLAGS) -o testquorum testquorum.o $(LIBS) + cpgbench: cpgbench.o $(LIBRARIES) $(CC) $(LDFLAGS) -o cpgbench cpgbench.o $(LIBS) diff --git a/test/testquorum.c b/test/testquorum.c new file mode 100644 index 00000000..30277f7d --- /dev/null +++ b/test/testquorum.c @@ -0,0 +1,58 @@ +#include +#include +#include +#include +#include +#include +#include "../include/corosync/quorum.h" + +static quorum_handle_t handle; + +static void quorum_notification_fn( + quorum_handle_t handle, + uint32_t quorate, + uint64_t ring_id, + uint32_t view_list_entries, + uint32_t *view_list) +{ + int i; + + printf("quorum notification called \n"); + printf(" quorate = %d\n", quorate); + printf(" ring id = %lld\n", ring_id); + printf(" num nodes = %d ", view_list_entries); + + for (i=0; i