From cfc7da3573edd8bf05bbd4dc0d4879a8c3defb56 Mon Sep 17 00:00:00 2001 From: Christine Caulfield Date: Mon, 8 Dec 2008 15:55:41 +0000 Subject: [PATCH] quorum is now an optional loadable module (though I've put it into the defaults in services.c) and can load another module to do the quorum work (eg YKD which I've made more compliant too). All the quorum code has been removed from sync.c. quorum.c is simply a shim later for the coroapi, the main module is in vsf_quorum.c There are coroapi calls to query quorate status and also to get notifications when it changes. I've included the testquorum.lcrso module in this patch because I think it's really helpful for testing. It sets the quorum state based on an objdb variable, this can be set or cleared using corosync-cfgtool git-svn-id: http://svn.fedorahosted.org/svn/corosync/trunk@1704 fd59a12c-fef9-0310-b244-a6a79926bd2f --- exec/Makefile | 10 +- exec/apidef.c | 5 + exec/ipc.c | 3 +- exec/main.c | 6 +- exec/quorum.h | 57 ++++--- exec/service.c | 4 + exec/sync.c | 69 +-------- exec/sync.h | 11 +- exec/vsf.h | 2 + exec/vsf_quorum.c | 239 +++++++++++++++++++----------- exec/vsf_ykd.c | 56 +++---- include/corosync/engine/coroapi.h | 33 ++++- include/corosync/quorum.h | 2 +- lib/libquorum.versions | 32 ++++ services/Makefile | 8 +- services/confdb.c | 1 + test/Makefile | 5 + 17 files changed, 320 insertions(+), 223 deletions(-) diff --git a/exec/Makefile b/exec/Makefile index 1d138f26..156af946 100644 --- a/exec/Makefile +++ b/exec/Makefile @@ -64,9 +64,9 @@ LCR_OBJS = vsf_ykd.o objdb.o coroparse.o vsf_quorum.o # main executive objects MAIN_SRC = main.c mempool.c util.c sync.c apidef.c service.c ipc.c flow.c \ - timer.c totemconfig.c mainconfig.c + quorum.c timer.c totemconfig.c mainconfig.c MAIN_OBJS = main.o mempool.o util.o sync.o apidef.o service.o ipc.o flow.o \ - timer.o totemconfig.o mainconfig.o ../lcr/lcr_ifact.o + quorum.o timer.o totemconfig.o mainconfig.o ../lcr/lcr_ifact.o ifeq (${BUILD_DYNAMIC}, 1) #EXEC_OBJS = $(TOTEM_OBJS) $(LOGSYS_OBJS) $(MAIN_OBJS) @@ -107,6 +107,9 @@ vsf_quorum.lcrso: vsf_quorum.o objdb.lcrso: objdb.o $(CC) -shared -Wl,-soname,objdb.lcrso objdb.o -o $@ +testquorum.lcrso: testquorum.o + $(CC) -shared -Wl,-soname,testquorum.lcrso objdb.o -o $@ + coroparse.lcrso: coroparse.o $(CC) -shared -Wl,-soname,coroparse.lcrso coroparse.o -o $@ endif @@ -160,6 +163,9 @@ vsf_quorum.o: vsf_quorum.c objdb.o: objdb.c $(CC) $(CFLAGS) -c -o $@ $< +testquorum.o: testquorum.c + $(CC) $(CFLAGS) -c -o $@ $< + coroparse.o: coroparse.c $(CC) $(CFLAGS) -c -o $@ $< diff --git a/exec/apidef.c b/exec/apidef.c index 0ada397d..9ab6db43 100644 --- a/exec/apidef.c +++ b/exec/apidef.c @@ -46,6 +46,7 @@ #include "main.h" #include "ipc.h" #include "sync.h" +#include "quorum.h" #include #include "service.h" #include @@ -101,6 +102,10 @@ static struct corosync_api_v1 apidef_corosync_api_v1 = { .tpg_groups_mcast = (typedef_tpg_groups_mcast)totempg_groups_mcast_groups, .tpg_groups_send_ok = (typedef_tpg_groups_send_ok)totempg_groups_send_ok_groups, .sync_request = sync_request, + .quorum_is_quorate = corosync_quorum_is_quorate, + .quorum_register_callback = corosync_quorum_register_callback, + .quorum_unregister_callback = corosync_quorum_unregister_callback, + .quorum_initialize = corosync_quorum_initialize, .service_link_and_init = corosync_service_link_and_init, .service_unlink_and_exit = corosync_service_unlink_and_exit, .plugin_interface_reference = lcr_ifact_reference, diff --git a/exec/ipc.c b/exec/ipc.c index 91de75b4..238bba82 100644 --- a/exec/ipc.c +++ b/exec/ipc.c @@ -72,6 +72,7 @@ #include #include +#include "quorum.h" #include "poll.h" #include "totemsrp.h" #include "mempool.h" @@ -899,7 +900,7 @@ retry_recv: &send_ok_joined_iovec, 1); send_ok = - (sync_primary_designated() == 1 || ais_service[service]->allow_inquorate == CS_LIB_ALLOW_INQUORATE) && ( + (corosync_quorum_is_quorate() == 1 || ais_service[service]->allow_inquorate == CS_LIB_ALLOW_INQUORATE) && ( (ais_service[service]->lib_engine[header->id].flow_control == CS_LIB_FLOW_CONTROL_NOT_REQUIRED) || ((ais_service[service]->lib_engine[header->id].flow_control == CS_LIB_FLOW_CONTROL_REQUIRED) && (send_ok_joined) && diff --git a/exec/main.c b/exec/main.c index 45267cf0..e16bad68 100644 --- a/exec/main.c +++ b/exec/main.c @@ -66,6 +66,7 @@ #include #include +#include "quorum.h" #include "totemsrp.h" #include "mempool.h" #include "mainconfig.h" @@ -411,7 +412,6 @@ static void aisexec_mlockall (void) #endif } - static void deliver_fn ( unsigned int nodeid, struct iovec *iovec, @@ -711,9 +711,7 @@ int main (int argc, char **argv) } - sync_register (corosync_sync_callbacks_retrieve, corosync_sync_completed, - totem_config.vsf_type); - + sync_register (corosync_sync_callbacks_retrieve, corosync_sync_completed); res = cs_flow_control_initialize (); diff --git a/exec/quorum.h b/exec/quorum.h index def280fb..a59df04d 100644 --- a/exec/quorum.h +++ b/exec/quorum.h @@ -15,7 +15,7 @@ * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * - Neither the name of the MontaVista Software, Inc. nor the names of its + * - Neither the name of the Red Hat, Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * @@ -35,41 +35,34 @@ #ifndef QUORUM_H_DEFINED #define QUORUM_H_DEFINED -struct quorum_services_api_ver1 { - void (*quorum_api_set_quorum) (unsigned int *,int, - int, struct memb_ring_id *); - }; +struct memb_ring_id; -static inline struct quorum_services_api_ver1 * -quorum_services_api_reference ( - struct corosync_api_v1 *coroapi, - unsigned int *handle) +typedef void (*quorum_callback_fn_t) (int quorate, void *context); + +typedef void (*sync_callback_fn_t) ( + unsigned int *view_list, + int view_list_entries, + int primary_designated, + struct memb_ring_id *ring_id); + +struct quorum_callin_functions { - static void *quorum_services_api_p; - struct quorum_services_api_ver1 *return_api; - unsigned int res; + int (*quorate) (void); + int (*register_callback) (quorum_callback_fn_t, void*); + int (*unregister_callback) (quorum_callback_fn_t, void*); +}; - res = coroapi->plugin_interface_reference ( - handle, - "quorum_services_api", - 0, - &quorum_services_api_p, - 0); - if (res == -1) { - return (NULL); - } - return_api = (struct quorum_services_api_ver1 *)quorum_services_api_p; - return (return_api); -} +extern int corosync_quorum_is_quorate (void); -static int inline quorum_services_api_release ( - struct corosync_api_v1 *coroapi, - unsigned int handle) -{ - unsigned int res; +extern int corosync_quorum_register_callback (quorum_callback_fn_t fn, void *context); + +extern int corosync_quorum_unregister_callback (quorum_callback_fn_t fn, void *context); + +extern int corosync_quorum_initialize (struct quorum_callin_functions *fns, + sync_callback_fn_t *sync_callback_fn); + + +extern int quorum_none(void); - res = coroapi->plugin_interface_release (handle); - return (res); -} #endif /* QUORUM_H_DEFINED */ diff --git a/exec/service.c b/exec/service.c index 1735a1dd..c3b8aebe 100644 --- a/exec/service.c +++ b/exec/service.c @@ -82,6 +82,10 @@ static struct default_service default_services[] = { .name = "corosync_pload", .ver = 0, }, + { + .name = "corosync_quorum", + .ver = 0, + }, }; struct corosync_service_engine *ais_service[SERVICE_HANDLER_MAXIMUM_COUNT]; diff --git a/exec/sync.c b/exec/sync.c index a5e279b5..7979e8b6 100644 --- a/exec/sync.c +++ b/exec/sync.c @@ -55,10 +55,11 @@ #include #include #include +#include "quorum.h" #include "main.h" #include "sync.h" -#include "vsf.h" + LOGSYS_DECLARE_SUBSYS ("SYNC", LOG_INFO); @@ -72,8 +73,6 @@ struct barrier_data { static struct memb_ring_id *sync_ring_id; -static int vsf_none = 0; - static int (*sync_callbacks_retrieve) (int sync_id, struct sync_callbacks *callack); static struct sync_callbacks sync_callbacks; @@ -93,8 +92,6 @@ static int barrier_data_confchg_entries; static struct barrier_data barrier_data_process[PROCESSOR_COUNT_MAX]; -static struct corosync_vsf_iface_ver0 *vsf_iface; - static int sync_barrier_send (struct memb_ring_id *ring_id); static int sync_start_process (enum totem_callback_token_type type, void *data); @@ -116,12 +113,6 @@ static void sync_confchg_fn ( unsigned int *joined_list, int joined_list_entries, struct memb_ring_id *ring_id); -static void sync_primary_callback_fn ( - unsigned int *view_list, - int view_list_entries, - int primary_designated, - struct memb_ring_id *ring_id); - static struct totempg_group sync_group = { .group = "sync", .group_len = 4 @@ -266,13 +257,10 @@ static int sync_service_process (enum totem_callback_token_type type, void *data int sync_register ( int (*callbacks_retrieve) (int sync_id, struct sync_callbacks *callack), - void (*synchronization_completed) (void), - char *vsf_type) + void (*synchronization_completed) (void)) + { unsigned int res; - unsigned int vsf_handle; - void *vsf_iface_p; - char corosync_vsf_type[1024]; res = totempg_groups_initialize ( &sync_group_handle, @@ -292,42 +280,13 @@ int sync_register ( log_printf (LOG_LEVEL_ERROR, "Couldn't join group.\n"); return (-1); } - - if (strcmp (vsf_type, "none") == 0) { - log_printf (LOG_LEVEL_NOTICE, - "Not using a virtual synchrony filter.\n"); - vsf_none = 1; - } else { - vsf_none = 0; - - sprintf (corosync_vsf_type, "corosync_vsf_%s", vsf_type); - res = lcr_ifact_reference ( - &vsf_handle, - corosync_vsf_type, - 0, - &vsf_iface_p, - 0); - - if (res == -1) { - log_printf (LOG_LEVEL_NOTICE, - "Couldn't load virtual synchrony filter %s\n", - vsf_type); - return (-1); - } - - log_printf (LOG_LEVEL_NOTICE, - "Using virtual synchrony filter %s\n", corosync_vsf_type); - - vsf_iface = (struct corosync_vsf_iface_ver0 *)vsf_iface_p; - vsf_iface->init (sync_primary_callback_fn); - } sync_callbacks_retrieve = callbacks_retrieve; sync_synchronization_completed = synchronization_completed; return (0); } -static void sync_primary_callback_fn ( +void sync_primary_callback_fn ( unsigned int *view_list, int view_list_entries, int primary_designated, @@ -335,13 +294,6 @@ static void sync_primary_callback_fn ( { int i; - if (primary_designated) { - log_printf (LOG_LEVEL_NOTICE, "This node is within the primary component and will provide service.\n"); - } else { - log_printf (LOG_LEVEL_NOTICE, "This node is within the non-primary component and will NOT provide any services.\n"); - return; - } - /* * Execute configuration change for synchronization service */ @@ -521,7 +473,7 @@ static void sync_confchg_fn ( * If no virtual synchrony filter configured, then start * synchronization process */ - if (vsf_none == 1) { + if (quorum_none() == 1) { sync_primary_callback_fn ( member_list, member_list_entries, @@ -589,15 +541,6 @@ int sync_in_process (void) return (sync_processing); } -int sync_primary_designated (void) -{ - if (vsf_none == 1) { - return (1); - } else { - return (vsf_iface->primary()); - } -} - /** * Execute synchronization upon request for the named service * @param name diff --git a/exec/sync.h b/exec/sync.h index 0ab22c48..1a89a914 100644 --- a/exec/sync.h +++ b/exec/sync.h @@ -47,10 +47,10 @@ struct sync_callbacks { char *name; }; +struct corosync_api_v1; int sync_register ( int (*sync_callbacks_retrieve) (int sync_id, struct sync_callbacks *callbacks), - void (*synchronization_completed) (void), - char *vsf_type); + void (*synchronization_completed) (void)); int sync_in_process (void); @@ -64,4 +64,11 @@ int sync_primary_designated (void); */ extern int sync_request (char *name); +extern void sync_primary_callback_fn ( + unsigned int *view_list, + int view_list_entries, + int primary_designated, + struct memb_ring_id *ring_id); + + #endif /* SYNC_H_DEFINED */ diff --git a/exec/vsf.h b/exec/vsf.h index b90e80a3..61583ba3 100644 --- a/exec/vsf.h +++ b/exec/vsf.h @@ -34,12 +34,14 @@ #ifndef VSF_H_DEFINED #define VSF_H_DEFINED +struct corosync_api_v1; struct corosync_vsf_iface_ver0 { /* * Executes a callback whenever component changes */ int (*init) ( + struct corosync_api_v1 *api, void (*primary_callback_fn) ( unsigned int *view_list, int view_list_entries, diff --git a/exec/vsf_quorum.c b/exec/vsf_quorum.c index b7b95238..e08ff5c1 100644 --- a/exec/vsf_quorum.c +++ b/exec/vsf_quorum.c @@ -15,7 +15,7 @@ * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * - Neither the name of the MontaVista Software, Inc. nor the names of its + * - Neither the name of Red Hat Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * @@ -60,10 +60,9 @@ #include #include #include +#include #include - -#include "vsf.h" -#include "quorum.h" +#include LOGSYS_DECLARE_SUBSYS ("QUORUM", LOG_INFO); @@ -74,22 +73,31 @@ struct quorum_pd { void *conn; }; +struct internal_callback_pd { + struct list_head list; + quorum_callback_fn_t callback; + void *context; +}; + static void message_handler_req_lib_quorum_getquorate (void *conn, void *msg); static void message_handler_req_lib_quorum_trackstart (void *conn, void *msg); static void message_handler_req_lib_quorum_trackstop (void *conn, void *msg); -static int send_quorum_notification(void *conn); +static void send_library_notification(void *conn); +static void send_internal_notification(void); static int quorum_exec_init_fn (struct corosync_api_v1 *api); static int quorum_lib_init_fn (void *conn); static int quorum_lib_exit_fn (void *conn); static int primary_designated = 0; static struct corosync_api_v1 *corosync_api; -static struct list_head trackers_list; +static struct list_head lib_trackers_list; +static struct list_head internal_trackers_list; static struct memb_ring_id quorum_ring_id; static int quorum_view_list_entries = 0; static int quorum_view_list[PROCESSOR_COUNT_MAX]; +struct quorum_services_api_ver1 *quorum_iface = NULL; -static void (*quorum_primary_callback_fn) ( +static void (*sync_primary_callback_fn) ( unsigned int *view_list, int view_list_entries, int primary_designated, @@ -101,51 +109,29 @@ static void quorum_api_set_quorum(unsigned int *view_list, int quorum, struct memb_ring_id *ring_id) { primary_designated = quorum; - memcpy(&quorum_ring_id, &ring_id, sizeof (quorum_ring_id)); + + if (primary_designated) { + log_printf (LOG_LEVEL_NOTICE, "This node is within the primary component and will provide service.\n"); + } else { + log_printf (LOG_LEVEL_NOTICE, "This node is within the non-primary component and will NOT provide any services.\n"); + } + + memcpy(&quorum_ring_id, ring_id, sizeof (quorum_ring_id)); quorum_view_list_entries = view_list_entries; memcpy(quorum_view_list, view_list, sizeof(unsigned int)*view_list_entries); /* Tell sync() */ - quorum_primary_callback_fn(view_list, view_list_entries, - primary_designated, &quorum_ring_id); + sync_primary_callback_fn(view_list, view_list_entries, + primary_designated, &quorum_ring_id); + + /* Tell internal listeners */ + send_internal_notification(); /* Tell IPC listeners */ - send_quorum_notification(NULL); + send_library_notification(NULL); } -static int quorum_init ( - void (*primary_callback_fn) ( - unsigned int *view_list, - int view_list_entries, - int primary_designated, - struct memb_ring_id *ring_id)) -{ - quorum_primary_callback_fn = primary_callback_fn; - - return (0); -} - -/* - * Returns 1 if this processor is in the primary (has quorum) - */ -static int quorum_primary (void) -{ - return (primary_designated); -} - -/* - * lcrso object definition - */ -static struct corosync_vsf_iface_ver0 vsf_quorum_iface_ver0 = { - .init = quorum_init, - .primary = quorum_primary -}; - -static struct quorum_services_api_ver1 quorum_service_api_v1 = { - .quorum_api_set_quorum = quorum_api_set_quorum -}; - static struct corosync_lib_handler quorum_lib_service[] = { { /* 0 */ @@ -181,30 +167,8 @@ static struct corosync_service_engine quorum_service_handler = { .lib_engine_count = sizeof (quorum_lib_service) / sizeof (struct corosync_lib_handler), }; -static struct lcr_iface corosync_vsf_quorum_ver0[3] = { - { /* the VSF handler */ - .name = "corosync_vsf_quorum", - .version = 0, - .versions_replace = 0, - .versions_replace_count = 0, - .dependencies = 0, - .dependency_count = 0, - .constructor = NULL, - .destructor = NULL, - .interfaces = (void **)(void *)&vsf_quorum_iface_ver0, - }, - { /* API for quorum users to call */ - .name = "corosync_quorum_api", - .version = 0, - .versions_replace = 0, - .versions_replace_count = 0, - .dependencies = 0, - .dependency_count = 0, - .constructor = NULL, - .destructor = NULL, - .interfaces = NULL - }, - { /* Library calls */ +static struct lcr_iface corosync_quorum_ver0[1] = { + { .name = "corosync_quorum", .version = 0, .versions_replace = 0, @@ -215,7 +179,6 @@ static struct lcr_iface corosync_vsf_quorum_ver0[3] = { .destructor = NULL, .interfaces = NULL, }, - }; static struct corosync_service_engine *quorum_get_service_handler_ver0 (void) @@ -223,28 +186,124 @@ static struct corosync_service_engine *quorum_get_service_handler_ver0 (void) return (&quorum_service_handler); } -static struct lcr_comp vsf_quorum_comp_ver0 = { - .iface_count = 3, - .ifaces = corosync_vsf_quorum_ver0 +static struct lcr_comp quorum_comp_ver0 = { + .iface_count = 1, + .ifaces = corosync_quorum_ver0 }; static struct corosync_service_engine_iface_ver0 quorum_service_handler_iface = { .corosync_get_service_engine_ver0 = quorum_get_service_handler_ver0 }; -__attribute__ ((constructor)) static void vsf_quorum_comp_register (void) { - lcr_component_register (&vsf_quorum_comp_ver0); - lcr_interfaces_set (&corosync_vsf_quorum_ver0[0], &vsf_quorum_iface_ver0); - lcr_interfaces_set (&corosync_vsf_quorum_ver0[1], &quorum_service_api_v1); - lcr_interfaces_set (&corosync_vsf_quorum_ver0[2], &quorum_service_handler_iface); +__attribute__ ((constructor)) static void quorum_comp_register (void) { + lcr_component_register (&quorum_comp_ver0); + lcr_interfaces_set (&corosync_quorum_ver0[0], &quorum_service_handler_iface); } /* -------------------------------------------------- */ + +/* + * Internal API functions for corosync + */ + +static int quorum_quorate(void) +{ + return primary_designated; +} + + +static int quorum_register_callback(quorum_callback_fn_t function, void *context) +{ + struct internal_callback_pd *pd = malloc(sizeof(struct internal_callback_pd)); + if (!pd) + return -1; + + pd->context = context; + pd->callback = function; + list_add (&pd->list, &internal_trackers_list); + + return 0; +} + +static int quorum_unregister_callback(quorum_callback_fn_t function, void *context) +{ + struct internal_callback_pd *pd; + struct list_head *tmp; + + for (tmp = internal_trackers_list.next; tmp != &internal_trackers_list; tmp = tmp->next) { + + pd = list_entry(tmp, struct internal_callback_pd, list); + if (pd->callback == function && pd->context == context) { + list_del(&pd->list); + return 0; + } + } + return -1; +} + +static struct quorum_callin_functions callins = { + .quorate = quorum_quorate, + .register_callback = quorum_register_callback, + .unregister_callback = quorum_unregister_callback +}; + +/* --------------------------------------------------------------------- */ + static int quorum_exec_init_fn (struct corosync_api_v1 *api) { + unsigned int find_handle; + unsigned int quorum_handle = 0; + unsigned int q_handle; + char *quorum_module; + int res; + void *quorum_iface_p; + corosync_api = api; - list_init (&trackers_list); + list_init (&lib_trackers_list); + list_init (&internal_trackers_list); + + /* + * Tell corosync we have a quorum engine. + */ + api->quorum_initialize(&callins, &sync_primary_callback_fn); + + /* + * Look for a quorum provider + */ + api->object_find_create(OBJECT_PARENT_HANDLE, "quorum", strlen("quorum"), &find_handle); + api->object_find_next(find_handle, &quorum_handle); + api->object_find_destroy(find_handle); + + if (quorum_handle) { + if ( !(res = api->object_key_get(quorum_handle, + "provider", + strlen("provider"), + (void *)&quorum_module, + NULL))) { + + res = lcr_ifact_reference ( + &q_handle, + quorum_module, + 0, + &quorum_iface_p, + 0); + + if (res == -1) { + log_printf (LOG_LEVEL_NOTICE, + "Couldn't load quorum provider %s\n", + quorum_module); + return (-1); + } + + log_printf (LOG_LEVEL_NOTICE, + "Using quorum provider %s\n", quorum_module); + + quorum_iface = (struct quorum_services_api_ver1 *)quorum_iface_p; + quorum_iface->init (api, quorum_api_set_quorum); + } + } + return (0); } @@ -273,7 +332,21 @@ static int quorum_lib_exit_fn (void *conn) return (0); } -static int send_quorum_notification(void *conn) + +static void send_internal_notification(void) +{ + struct list_head *tmp; + struct internal_callback_pd *pd; + + for (tmp = internal_trackers_list.next; tmp != &internal_trackers_list; tmp = tmp->next) { + + pd = list_entry(tmp, struct internal_callback_pd, list); + + pd->callback(primary_designated, pd->context); + } +} + +static void send_library_notification(void *conn) { int size = sizeof(struct res_lib_quorum_notification) + sizeof(unsigned int)*quorum_view_list_entries; char buf[size]; @@ -296,12 +369,12 @@ static int send_quorum_notification(void *conn) /* Send it to all interested parties */ if (conn) { - return corosync_api->ipc_conn_send_response(conn, res_lib_quorum_notification, size); + corosync_api->ipc_conn_send_response(conn, res_lib_quorum_notification, size); } else { struct quorum_pd *qpd; - for (tmp = trackers_list.next; tmp != &trackers_list; tmp = tmp->next) { + for (tmp = lib_trackers_list.next; tmp != &lib_trackers_list; tmp = tmp->next) { qpd = list_entry(tmp, struct quorum_pd, list); @@ -309,7 +382,7 @@ static int send_quorum_notification(void *conn) res_lib_quorum_notification, size); } } - return (0); + return; } static void message_handler_req_lib_quorum_getquorate (void *conn, void *msg) @@ -342,7 +415,7 @@ static void message_handler_req_lib_quorum_trackstart (void *conn, void *msg) if (req_lib_quorum_trackstart->track_flags & CS_TRACK_CURRENT || req_lib_quorum_trackstart->track_flags & CS_TRACK_CHANGES) { log_printf(LOG_LEVEL_DEBUG, "sending initial status to %p\n", conn); - send_quorum_notification(corosync_api->ipc_conn_partner_get (conn)); + send_library_notification(corosync_api->ipc_conn_partner_get (conn)); } /* @@ -354,7 +427,7 @@ static void message_handler_req_lib_quorum_trackstart (void *conn, void *msg) quorum_pd->track_flags = req_lib_quorum_trackstart->track_flags; quorum_pd->tracking_enabled = 1; - list_add (&quorum_pd->list, &trackers_list); + list_add (&quorum_pd->list, &lib_trackers_list); } /* send status */ diff --git a/exec/vsf_ykd.c b/exec/vsf_ykd.c index e771a9fa..5445318f 100644 --- a/exec/vsf_ykd.c +++ b/exec/vsf_ykd.c @@ -56,12 +56,12 @@ #include #include +#include +#include +#include #include #include -#include "main.h" -#include "vsf.h" - LOGSYS_DECLARE_SUBSYS ("YKD", LOG_INFO); #define YKD_PROCESSOR_COUNT_MAX 32 @@ -108,7 +108,7 @@ struct state_received { struct ykd_state ykd_state; -static totempg_groups_handle ykd_group_handle; +static cs_tpg_handle ykd_group_handle; static struct state_received state_received_confchg[YKD_PROCESSOR_COUNT_MAX]; @@ -140,6 +140,8 @@ static void *ykd_attempt_send_callback_token_handle = 0; static void *ykd_state_send_callback_token_handle = 0; +static struct corosync_api_v1 *api; + static void (*ykd_primary_callback_fn) ( unsigned int *view_list, int view_list_entries, @@ -168,15 +170,15 @@ static int ykd_state_send_msg (enum totem_callback_token_type type, void *contex iovec[1].iov_base = (char *)&ykd_state; iovec[1].iov_len = sizeof (struct ykd_state); - res = totempg_groups_mcast_joined (ykd_group_handle, iovec, 2, - TOTEMPG_AGREED); + res = api->tpg_joined_mcast (ykd_group_handle, iovec, 2, + TOTEM_AGREED); return (res); } static void ykd_state_send (void) { - totempg_callback_token_create ( + api->totem_callback_token_create ( &ykd_state_send_callback_token_handle, TOTEM_CALLBACK_TOKEN_SENT, 1, /* delete after callback */ @@ -195,15 +197,15 @@ static int ykd_attempt_send_msg (enum totem_callback_token_type type, void *cont iovec.iov_base = (char *)&header; iovec.iov_len = sizeof (struct ykd_header); - res = totempg_groups_mcast_joined (ykd_group_handle, &iovec, 1, - TOTEMPG_AGREED); + res = api->tpg_joined_mcast (ykd_group_handle, &iovec, 1, + TOTEM_AGREED); return (res); } static void ykd_attempt_send (void) { - totempg_callback_token_create ( + api->totem_callback_token_create ( &ykd_attempt_send_callback_token_handle, TOTEM_CALLBACK_TOKEN_SENT, 1, /* delete after callback */ @@ -460,7 +462,7 @@ static void ykd_confchg_fn ( memcpy (&ykd_ring_id, ring_id, sizeof (struct memb_ring_id)); if (first_run) { - ykd_state.last_primary.member_list[0] = totempg_my_nodeid_get(); + ykd_state.last_primary.member_list[0] = api->totem_nodeid_get(); ykd_state.last_primary.member_list_entries = 1; ykd_state.last_primary.session_id = 0; first_run = 0; @@ -493,53 +495,41 @@ static void ykd_confchg_fn ( ykd_state_send (); } -struct totempg_group ykd_group = { +struct corosync_tpg_group ykd_group = { .group = "ykd", .group_len = 3 }; -static int ykd_init ( - void (*primary_callback_fn) ( - unsigned int *view_list, - int view_list_entries, - int primary_designated, - struct memb_ring_id *ring_id)) +static void ykd_init ( + struct corosync_api_v1 *corosync_api, + quorum_set_quorate_fn_t set_primary) { - ykd_primary_callback_fn = primary_callback_fn; + ykd_primary_callback_fn = set_primary; + api = corosync_api; - totempg_groups_initialize ( + api->tpg_init ( &ykd_group_handle, ykd_deliver_fn, ykd_confchg_fn); - totempg_groups_join ( + api->tpg_join ( ykd_group_handle, &ykd_group, 1); ykd_state_init (); - - return (0); -} - -/* - * Returns 1 if this processor is in the primary - */ -static int ykd_primary (void) { - return (primary_designated); } /* * lcrso object definition */ -static struct corosync_vsf_iface_ver0 vsf_ykd_iface_ver0 = { +static struct quorum_services_api_ver1 vsf_ykd_iface_ver0 = { .init = ykd_init, - .primary = ykd_primary }; static struct lcr_iface corosync_vsf_ykd_ver0[1] = { { - .name = "corosync_vsf_ykd", + .name = "corosync_quorum_ykd", .version = 0, .versions_replace = 0, .versions_replace_count = 0, diff --git a/include/corosync/engine/coroapi.h b/include/corosync/engine/coroapi.h index 6dd4d28b..e83c256c 100644 --- a/include/corosync/engine/coroapi.h +++ b/include/corosync/engine/coroapi.h @@ -188,9 +188,27 @@ typedef void (*object_notify_callback_fn_t)(unsigned int object_handle, typedef void (*object_reload_notify_fn_t) (objdb_reload_notify_type_t, int flush, void *priv_data_pt); - #endif /* OBJECT_PARENT_HANDLE_DEFINED */ +#ifndef QUORUM_H_DEFINED +typedef void (*quorum_callback_fn_t) (int quorate, void *context); + +struct quorum_callin_functions +{ + int (*quorate) (void); + int (*register_callback) (quorum_callback_fn_t callback_fn, void *context); + int (*unregister_callback) (quorum_callback_fn_t callback_fn, void *context); +}; + +typedef void (*sync_callback_fn_t) ( + unsigned int *view_list, + int view_list_entries, + int primary_designated, + struct memb_ring_id *ring_id); + +#endif /* QUORUM_H_DEFINED */ + + struct corosync_api_v1 { /* * Object and configuration APIs @@ -495,6 +513,19 @@ struct corosync_api_v1 { int (*sync_request) ( char *service_name); + /* + * User plugin-callable functions for quorum + */ + int (*quorum_is_quorate) (void); + int (*quorum_register_callback) (quorum_callback_fn_t callback_fn, void *context); + int (*quorum_unregister_callback) (quorum_callback_fn_t callback_fn, void *context); + + /* + * This one is for the quorum management plugin's use + */ + int (*quorum_initialize)(struct quorum_callin_functions *fns, + sync_callback_fn_t *sync_callback_fn); + /* * Plugin loading and unloading */ diff --git a/include/corosync/quorum.h b/include/corosync/quorum.h index 5380dc5b..291a69c6 100644 --- a/include/corosync/quorum.h +++ b/include/corosync/quorum.h @@ -15,7 +15,7 @@ * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * - Neither the name of the MontaVista Software, Inc. nor the names of its + * - Neither the name of the Red Hat, Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * diff --git a/lib/libquorum.versions b/lib/libquorum.versions index 0242f9cb..d6d53ffd 100644 --- a/lib/libquorum.versions +++ b/lib/libquorum.versions @@ -1,3 +1,35 @@ +# Version and symbol export for libquorum.so + +OPENAIS_QUORUM_1.0 { + global: + quorum_initialize; + quorum_finalize; + quorum_getinfo; + quorum_setexpected; + quorum_setvotes; + quorum_qdisk_register; + quorum_qdisk_unregister; + quorum_qdisk_poll; + quorum_qdisk_getinfo; + quorum_setdirty; + quorum_killnode; + quorum_leaving; + + local: + saHandleCreate; + saHandleDestroy; + saHandleInstanceGet; + saHandleInstancePut; + saRecvRetry; + saSelectRetry; + saSendMsgReceiveReply; + saSendMsgRetry; + saSendReceiveReply; + saSendRetry; + saServiceConnect; + saVersionVerify; + clustTimeNow; +}; # Version and symbol export for libcpg.so COROSYNC_QUORUM_1.0 { diff --git a/services/Makefile b/services/Makefile index 9a586faa..a51cd6cb 100644 --- a/services/Makefile +++ b/services/Makefile @@ -55,7 +55,7 @@ LCR_OBJS = evs.o cfg.o cpg.o confdb.o $(AMF_OBJS) pload.o override CFLAGS += -fPIC -all: service_evs.lcrso service_cfg.lcrso service_cpg.lcrso service_confdb.lcrso service_pload.lcrso +all: service_evs.lcrso service_cfg.lcrso service_cpg.lcrso service_confdb.lcrso service_pload.lcrso testquorum.lcrso ifeq (${COROSYNC_COMPAT}, DARWIN) @@ -91,6 +91,9 @@ service_cpg.lcrso: cpg.o service_pload.lcrso: pload.o $(CC) -shared -Wl,-soname,service_pload.lcrso pload.o -o $@ +testquorum.lcrso: testquorum.o + $(CC) -shared -Wl,-soname,testquorum.lcrso testquorum.o -o $@ + endif clean: @@ -113,3 +116,6 @@ confdb.o: confdb.c cpg.o: cpg.c $(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $< + +testquorum.o: testquorum.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $< diff --git a/services/confdb.c b/services/confdb.c index 497540ad..28843fd3 100644 --- a/services/confdb.c +++ b/services/confdb.c @@ -212,6 +212,7 @@ struct corosync_service_engine confdb_service_engine = { .id = CONFDB_SERVICE, .private_data_size = 0, .flow_control = CS_LIB_FLOW_CONTROL_NOT_REQUIRED, + .allow_inquorate = CS_LIB_ALLOW_INQUORATE, .lib_init_fn = confdb_lib_init_fn, .lib_exit_fn = confdb_lib_exit_fn, .lib_engine = confdb_lib_engine, diff --git a/test/Makefile b/test/Makefile index d7d49dcd..1ee7b1cc 100644 --- a/test/Makefile +++ b/test/Makefile @@ -90,6 +90,11 @@ logsysbench: logsysbench.o ../exec/liblogsys.a logsysrec: logsysrec.o ../exec/liblogsys.a $(CC) -o logsysrec logsysrec.o ../exec/liblogsys.a $(LDFLAGS) +testquorum1: testquorum1.o $(LIBRARIES) + $(CC) $(LDFLAGS) -o testquorum1 testquorum1.o $(LIBS) +testquorum2: testquorum2.o $(LIBRARIES) + $(CC) $(LDFLAGS) -o testquorum2 testquorum2.o $(LIBS) + logsys_s: logsys_s.o logsys_s1.o logsys_s2.o ../exec/liblogsys.a $(CC) -o logsys_s logsys_s.o logsys_s1.o logsys_s2.o ../exec/liblogsys.a $(LDFLAGS)