From c0772557023a75c6ac971793db3708bc0837ed5d Mon Sep 17 00:00:00 2001 From: Steven Dake Date: Thu, 19 Feb 2009 02:23:58 +0000 Subject: [PATCH] Whitetank IPC Forward Port. git-svn-id: http://svn.fedorahosted.org/svn/corosync/trunk@1766 fd59a12c-fef9-0310-b244-a6a79926bd2f --- Makefile | 5 +- exec/Makefile | 4 +- exec/apidef.c | 15 +- exec/coropoll.c | 2 +- exec/flow.c | 468 ----- exec/flow.h | 74 - exec/ipc.c | 1794 +++++++++----------- exec/ipc.h | 58 +- exec/main.c | 15 +- exec/vsf_quorum.c | 14 +- include/corosync/{ais_util.h => coroipc.h} | 86 +- include/corosync/engine/coroapi.h | 28 +- include/corosync/ipc_cfg.h | 6 +- include/corosync/ipc_cpg.h | 2 - include/corosync/ipc_gen.h | 42 +- lib/Makefile | 98 +- lib/cfg.c | 216 +-- lib/confdb.c | 320 ++-- lib/coroipc.c | 867 ++++++++++ lib/cpg.c | 316 +--- lib/evs.c | 207 +-- lib/libcfg.versions | 16 +- lib/libconfdb.versions | 15 +- lib/libcoroipc.versions | 16 + lib/libcoroutil.versions | 18 - lib/libcpg.versions | 16 +- lib/libevs.versions | 16 +- lib/libpload.versions | 16 +- lib/libquorum.versions | 40 +- lib/libvotequorum.versions | 52 +- lib/pload.c | 37 +- lib/quorum.c | 105 +- lib/sa-confdb.c | 2 +- lib/util.c | 788 --------- lib/votequorum.c | 248 +-- services/cfg.c | 32 +- services/confdb.c | 69 +- services/cpg.c | 145 +- services/evs.c | 30 +- services/votequorum.c | 28 +- test/cpgbench.c | 5 +- test/evsbench.c | 18 +- test/evsverify.c | 10 +- test/testcpg.c | 34 - test/testevs.c | 4 +- 45 files changed, 2602 insertions(+), 3795 deletions(-) delete mode 100644 exec/flow.c delete mode 100644 exec/flow.h rename include/corosync/{ais_util.h => coroipc.h} (75%) create mode 100644 lib/coroipc.c create mode 100644 lib/libcoroipc.versions delete mode 100644 lib/libcoroutil.versions delete mode 100644 lib/util.c diff --git a/Makefile b/Makefile index 3bdbeba8..a6b028ab 100644 --- a/Makefile +++ b/Makefile @@ -109,10 +109,10 @@ lint: (cd $(builddir)lib; echo ==== `pwd` ===; $(call sub_make,lib,lint)); (cd $(builddir)tools; echo ==== `pwd` ===; $(call sub_make,tools,lint)); -COROSYNC_LIBS = evs cpg cfg coroutil confdb quorum votequorum +COROSYNC_LIBS = evs cpg cfg coroipc confdb quorum votequorum COROSYNC_HEADERS = cpg.h cfg.h evs.h ipc_gen.h mar_gen.h swab.h \ - ais_util.h confdb.h quorum.h list.h corotypes.h votequorum.h + coroipc.h confdb.h quorum.h list.h corotypes.h votequorum.h EXEC_LIBS = totem_pg logsys @@ -162,6 +162,7 @@ install: all fi \ ) \ done + install -m 755 lib/libcoroipc.a $(DESTDIR)$(LIBDIR) echo $(LIBDIR) > "$(DESTDIR)$(ETCDIR)/ld.so.conf.d/corosync-$(ARCH).conf" diff --git a/exec/Makefile b/exec/Makefile index 156af946..deeaa315 100644 --- a/exec/Makefile +++ b/exec/Makefile @@ -63,9 +63,9 @@ LCR_SRC = vsf_ykd.c objdb.c coroparse.c vsf_quorum.c LCR_OBJS = vsf_ykd.o objdb.o coroparse.o vsf_quorum.o # main executive objects -MAIN_SRC = main.c mempool.c util.c sync.c apidef.c service.c ipc.c flow.c \ +MAIN_SRC = main.c mempool.c util.c sync.c apidef.c service.c ipc.c \ quorum.c timer.c totemconfig.c mainconfig.c -MAIN_OBJS = main.o mempool.o util.o sync.o apidef.o service.o ipc.o flow.o \ +MAIN_OBJS = main.o mempool.o util.o sync.o apidef.o service.o ipc.o \ quorum.o timer.o totemconfig.o mainconfig.o ../lcr/lcr_ifact.o ifeq (${BUILD_DYNAMIC}, 1) diff --git a/exec/apidef.c b/exec/apidef.c index 3d2dc203..fb49c7ce 100644 --- a/exec/apidef.c +++ b/exec/apidef.c @@ -76,15 +76,12 @@ static struct corosync_api_v1 apidef_corosync_api_v1 = { .ipc_source_set = message_source_set, .ipc_source_is_local = message_source_is_local, .ipc_private_data_get = cs_conn_private_data_get, - .ipc_response_send = NULL, - .ipc_response_no_fcc = cs_conn_send_response_no_fcc, - .ipc_dispatch_send = NULL, - .ipc_conn_send_response = cs_conn_send_response, - .ipc_conn_partner_get = cs_conn_partner_get, - .ipc_refcnt_inc = cs_ipc_flow_control_local_increment, - .ipc_refcnt_dec = cs_ipc_flow_control_local_decrement, - .ipc_fc_create = cs_ipc_flow_control_create, - .ipc_fc_destroy = cs_ipc_flow_control_destroy, + .ipc_response_iov_send = cs_response_iov_send, + .ipc_response_send = cs_response_send, + .ipc_dispatch_send = cs_dispatch_send, + .ipc_dispatch_iov_send = cs_dispatch_iov_send, + .ipc_refcnt_inc = cs_conn_refcount_inc, + .ipc_refcnt_dec = cs_conn_refcount_dec, .totem_nodeid_get = totempg_my_nodeid_get, .totem_family_get = totempg_my_family_get, .totem_ring_reenable = totempg_ring_reenable, diff --git a/exec/coropoll.c b/exec/coropoll.c index 77a80986..ec56a9c3 100644 --- a/exec/coropoll.c +++ b/exec/coropoll.c @@ -294,7 +294,7 @@ int poll_timer_add ( int res = 0; if (timer_handle_out == NULL) { - res -ENOENT; + res = -ENOENT; goto error_exit; } diff --git a/exec/flow.c b/exec/flow.c deleted file mode 100644 index fd84d53c..00000000 --- a/exec/flow.c +++ /dev/null @@ -1,468 +0,0 @@ -/* - * Copyright (c) 2006 Red Hat, Inc. - * - * All rights reserved. - * - * Author: Steven Dake (sdake@redhat.com) - * - * This software licensed under BSD license, the text of which follows: - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - Neither the name of the MontaVista Software, Inc. nor the names of its - * contributors may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -/* - * New messages are allowed from the library ONLY when the processor has not - * received a CS_FLOW_CONTROL_STATE_ENABLED from any processor. If a - * CS_FLOW_CONTROL_STATE_ENABLED message is sent, it must later be - * cancelled by a CS_FLOW_CONTROL_STATE_DISABLED message. A configuration - * change with the flow controlled processor leaving the configuration will - * also cancel flow control. - */ - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "flow.h" - -LOGSYS_DECLARE_SUBSYS ("FLOW", LOG_INFO); - -struct flow_control_instance { - struct list_head list_head; - unsigned int service; -}; - -DECLARE_LIST_INIT (flow_control_service_list_head); - -struct flow_control_message { - unsigned int service __attribute__((aligned(8))); - char id[1024] __attribute__((aligned(8))); - unsigned int id_len __attribute__((aligned(8))); - enum cs_flow_control_state flow_control_state __attribute__((aligned(8))); -}; - -struct flow_control_node_state { - unsigned int nodeid; - enum cs_flow_control_state flow_control_state; -}; - -struct flow_control_service { - struct flow_control_node_state flow_control_node_state[PROCESSOR_COUNT_MAX]; - unsigned int service; - char id[1024]; - unsigned int id_len; - void (*flow_control_state_set_fn) (void *context, enum cs_flow_control_state flow_control_state); - void *context; - unsigned int processor_count; - enum cs_flow_control_state flow_control_state; - struct list_head list; - struct list_head list_all; -}; - -static struct totempg_group flow_control_group = { - .group = "flowcontrol", - .group_len = 12 -}; - -static totempg_groups_handle flow_control_handle; - -static struct hdb_handle_database flow_control_hdb = { - .handle_count = 0, - .handles = NULL, - .iterator = 0, - .mutex = PTHREAD_MUTEX_INITIALIZER -}; - -static unsigned int flow_control_member_list[PROCESSOR_COUNT_MAX]; -static unsigned int flow_control_member_list_entries; - -static inline int flow_control_xmit ( - struct flow_control_service *flow_control_service, - enum cs_flow_control_state flow_control_state) -{ - struct flow_control_message flow_control_message; - struct iovec iovec; - unsigned int res; - - flow_control_message.service = flow_control_service->service; - flow_control_message.flow_control_state = flow_control_state; - memcpy (&flow_control_message.id, flow_control_service->id, - flow_control_service->id_len); - flow_control_message.id_len = flow_control_service->id_len; - - iovec.iov_base = (char *)&flow_control_message; - iovec.iov_len = sizeof (flow_control_message); - - res = totempg_groups_mcast_joined (flow_control_handle, &iovec, 1, - TOTEMPG_AGREED); - - flow_control_service->flow_control_state_set_fn ( - flow_control_service->context, - flow_control_service->flow_control_state); - - return (res); -} - -static void flow_control_deliver_fn ( - unsigned int nodeid, - struct iovec *iovec, - int iov_len, - int endian_conversion_required) -{ - struct flow_control_message *flow_control_message = (struct flow_control_message *)iovec[0].iov_base; - struct flow_control_service *flow_control_service; - struct list_head *list; - unsigned int i; - - for (list = flow_control_service_list_head.next; - list != &flow_control_service_list_head; - list = list->next) { - - flow_control_service = list_entry (list, struct flow_control_service, list_all); - /* - * Find this nodeid in the flow control service and set the message - * enabled or disabled flag - */ - for (i = 0; i < flow_control_service->processor_count; i++) { - if (nodeid == flow_control_service->flow_control_node_state[i].nodeid) { - flow_control_service->flow_control_node_state[i].flow_control_state = - flow_control_message->flow_control_state; - break; - } - } - - /* - * Determine if any flow control is enabled on any nodes and set - * the internal variable appropriately - */ - flow_control_service->flow_control_state = CS_FLOW_CONTROL_STATE_DISABLED; - flow_control_service->flow_control_state_set_fn (flow_control_service->context, flow_control_service->flow_control_state); - for (i = 0; i < flow_control_service->processor_count; i++) { - if (flow_control_service->flow_control_node_state[i].flow_control_state == CS_FLOW_CONTROL_STATE_ENABLED) { - flow_control_service->flow_control_state = CS_FLOW_CONTROL_STATE_ENABLED; - flow_control_service->flow_control_state_set_fn (flow_control_service->context, flow_control_service->flow_control_state); - } - } - } /* for list iteration */ -} - -static void flow_control_confchg_fn ( - enum totem_configuration_type configuration_type, - unsigned int *member_list, int member_list_entries, - unsigned int *left_list, int left_list_entries, - unsigned int *joined_list, int joined_list_entries, - struct memb_ring_id *ring_id) -{ - unsigned int i; - unsigned int j; - struct flow_control_service *flow_control_service; - struct list_head *list; - struct flow_control_node_state flow_control_node_state_temp[PROCESSOR_COUNT_MAX]; - - memcpy (flow_control_member_list, member_list, - sizeof (unsigned int) * member_list_entries); - flow_control_member_list_entries = member_list_entries; - - for (list = flow_control_service_list_head.next; - list != &flow_control_service_list_head; - list = list->next) { - - flow_control_service = list_entry (list, struct flow_control_service, list_all); - - /* - * Generate temporary flow control node state information - */ - for (i = 0; i < member_list_entries; i++) { - flow_control_node_state_temp[i].nodeid = member_list[i]; - flow_control_node_state_temp[i].flow_control_state = CS_FLOW_CONTROL_STATE_DISABLED; - - /* - * Determine if previous state was set for this processor - * if so keep that setting - */ - for (j = 0; j < flow_control_service->processor_count; j++) { - if (flow_control_service->flow_control_node_state[j].nodeid == member_list[i]) { - flow_control_node_state_temp[i].flow_control_state = - flow_control_service->flow_control_node_state[j].flow_control_state; - break; /* from for */ - } - } - } - - /* - * Copy temporary node state information to node state information - */ - memcpy (flow_control_service->flow_control_node_state, - flow_control_node_state_temp, - sizeof (struct flow_control_node_state) * member_list_entries); - - /* - * Set all of the node ids after a configuration change - * Turn on all flow control after a configuration change - */ - flow_control_service->processor_count = flow_control_member_list_entries; - flow_control_service->flow_control_state = CS_FLOW_CONTROL_STATE_DISABLED; - for (i = 0; i < member_list_entries; i++) { - if (flow_control_service->flow_control_node_state[i].flow_control_state == CS_FLOW_CONTROL_STATE_ENABLED) { - flow_control_service->flow_control_state = CS_FLOW_CONTROL_STATE_ENABLED; - flow_control_service->flow_control_state_set_fn (flow_control_service->context, flow_control_service->flow_control_state); - } - } - - } -} -/* - * External API - */ -unsigned int cs_flow_control_initialize (void) -{ - unsigned int res; - - res = totempg_groups_initialize ( - &flow_control_handle, - flow_control_deliver_fn, - flow_control_confchg_fn); - - if (res == -1) { - log_printf (LOG_LEVEL_ERROR, - "Couldn't initialize flow control interface.\n"); - return (-1); - } - res = totempg_groups_join ( - flow_control_handle, - &flow_control_group, - 1); - - if (res == -1) { - log_printf (LOG_LEVEL_ERROR, "Couldn't join flow control group.\n"); - return (-1); - } - - return (0); -} - -unsigned int cs_flow_control_ipc_init ( - unsigned int *flow_control_handle, - unsigned int service) -{ - struct flow_control_instance *instance; - unsigned int res; - - res = hdb_handle_create (&flow_control_hdb, - sizeof (struct flow_control_instance), flow_control_handle); - if (res != 0) { - goto error_exit; - } - res = hdb_handle_get (&flow_control_hdb, *flow_control_handle, - (void *)&instance); - if (res != 0) { - goto error_destroy; - } - instance->service = service; - - list_init (&instance->list_head); - - return (0); - -error_destroy: - hdb_handle_destroy (&flow_control_hdb, *flow_control_handle); -error_exit: - return (-1); - -} - -unsigned int cs_flow_control_ipc_exit ( - unsigned int flow_control_handle) -{ - hdb_handle_destroy (&flow_control_hdb, flow_control_handle); - return (0); -} - -unsigned int cs_flow_control_create ( - unsigned int flow_control_handle, - unsigned int service, - void *id, - unsigned int id_len, - void (*flow_control_state_set_fn) (void *context, enum cs_flow_control_state flow_control_state), - void *context) -{ - struct flow_control_service *flow_control_service; - struct flow_control_instance *instance; - unsigned int res; - unsigned int i; - - res = hdb_handle_get (&flow_control_hdb, flow_control_handle, - (void *)&instance); - if (res != 0) { - goto error_exit; - } - - flow_control_service = malloc (sizeof (struct flow_control_service)); - if (flow_control_service == NULL) { - goto error_put; - } - - /* - * Add new service to flow control system - */ - memset (flow_control_service, 0, sizeof (struct flow_control_service)); - - flow_control_service->flow_control_state = CS_FLOW_CONTROL_STATE_DISABLED; - flow_control_service->service = service; - memcpy (flow_control_service->id, id, id_len); - flow_control_service->id_len = id_len; - flow_control_service->flow_control_state_set_fn = flow_control_state_set_fn; - flow_control_service->context = context; - - list_init (&flow_control_service->list); - list_add_tail (&instance->list_head, - &flow_control_service->list); - - list_init (&flow_control_service->list_all); - list_add_tail (&flow_control_service_list_head, - &flow_control_service->list_all); - - for (i = 0; i < flow_control_member_list_entries; i++) { - flow_control_service->flow_control_node_state[i].nodeid = flow_control_member_list[i]; - flow_control_service->processor_count = flow_control_member_list_entries; - } -error_put: - hdb_handle_put (&flow_control_hdb, flow_control_handle); - -error_exit: - return (res); -} - -unsigned int cs_flow_control_destroy ( - unsigned int flow_control_identifier, - unsigned int service, - unsigned char *id, - unsigned int id_len) -{ - struct flow_control_service *flow_control_service; - struct flow_control_instance *instance; - struct list_head *list; - unsigned int res; - - res = hdb_handle_get (&flow_control_hdb, flow_control_handle, - (void *)&instance); - if (res != 0) { - goto error_exit; - } - - for (list = flow_control_service_list_head.next; - list != &flow_control_service_list_head; - list = list->next) { - - flow_control_service = list_entry (list, struct flow_control_service, list_all); - - if ((flow_control_service->id_len == id_len) && - (memcmp (flow_control_service->id, id, id_len) == 0)) { - flow_control_xmit (flow_control_service, - CS_FLOW_CONTROL_STATE_DISABLED); - list_del (&flow_control_service->list); - list_del (&flow_control_service->list_all); - free (flow_control_service); - break; /* done - no delete-safe for loop needed */ - } - } - hdb_handle_put (&flow_control_hdb, flow_control_handle); - -error_exit: - return (res); -} - -/* - * Disable the ability for new messages to be sent for this service - * with the handle id of length id_len - */ -unsigned int cs_flow_control_disable ( - unsigned int flow_control_handle) -{ - struct flow_control_instance *instance; - struct flow_control_service *flow_control_service; - struct list_head *list; - unsigned int res; - - res = hdb_handle_get (&flow_control_hdb, flow_control_handle, - (void *)&instance); - if (res != 0) { - goto error_exit; - } - - for (list = instance->list_head.next; - list != &instance->list_head; - list = list->next) { - - flow_control_service = list_entry (list, struct flow_control_service, list); - flow_control_service->flow_control_state = CS_FLOW_CONTROL_STATE_DISABLED; - flow_control_xmit (flow_control_service, CS_FLOW_CONTROL_STATE_DISABLED); - } - hdb_handle_put (&flow_control_hdb, flow_control_handle); - -error_exit: - return (res); -} - -/* - * Enable the ability for new messagess to be sent for this service - * with the handle id of length id_len - */ -unsigned int cs_flow_control_enable ( - unsigned int flow_control_handle) -{ - struct flow_control_instance *instance; - struct flow_control_service *flow_control_service; - struct list_head *list; - unsigned int res; - - res = hdb_handle_get (&flow_control_hdb, flow_control_handle, - (void *)&instance); - if (res != 0) { - goto error_exit; - } - - for (list = instance->list_head.next; - list != &instance->list_head; - list = list->next) { - - - flow_control_service = list_entry (list, struct flow_control_service, list); - flow_control_service->flow_control_state = CS_FLOW_CONTROL_STATE_ENABLED; - flow_control_xmit (flow_control_service, CS_FLOW_CONTROL_STATE_ENABLED); - } - hdb_handle_put (&flow_control_hdb, flow_control_handle); - -error_exit: - return (res); -} diff --git a/exec/flow.h b/exec/flow.h deleted file mode 100644 index f843606e..00000000 --- a/exec/flow.h +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2006 Red Hat, Inc. - * - * - * All rights reserved. - * - * Author: Steven Dake (sdake@redhat.com) - * - * This software licensed under BSD license, the text of which follows: - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - Neither the name of the MontaVista Software, Inc. nor the names of its - * contributors may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef FLOW_H_DEFINED -#define FLOW_H_DEFINED - -#define COROSYNC_FLOW_CONTROL_STATE -enum cs_flow_control_state { - CS_FLOW_CONTROL_STATE_DISABLED, - CS_FLOW_CONTROL_STATE_ENABLED -}; - -unsigned int cs_flow_control_initialize (void); - -unsigned int cs_flow_control_ipc_init ( - unsigned int *flow_control_identifier, - unsigned int service); - -unsigned int cs_flow_control_ipc_exit ( - unsigned int flow_control_identifier); - -unsigned int cs_flow_control_create ( - unsigned int flow_control_handle, - unsigned int service, - void *id, - unsigned int id_len, - void (*flow_control_state_set_fn) (void *context, enum cs_flow_control_state flow_control_state), - void *context); - -unsigned int cs_flow_control_destroy ( - unsigned int flow_control_identifier, - unsigned int service, - unsigned char *id, - unsigned int id_len); - -unsigned int cs_flow_control_disable ( - unsigned int flow_control_identifier); - -unsigned int cs_flow_control_enable ( - unsigned int flow_control_identifier); - -#endif /* FLOW_H_DEFINED */ diff --git a/exec/ipc.c b/exec/ipc.c index c2238118..5c096c87 100644 --- a/exec/ipc.c +++ b/exec/ipc.c @@ -1,6 +1,5 @@ /* - * Copyright (c) 2002-2006 MontaVista Software, Inc. - * Copyright (c) 2006-2007 Red Hat, Inc. + * Copyright (c) 2006-2009 Red Hat, Inc. * * All rights reserved. * @@ -57,10 +56,13 @@ #include #include #include -#if defined(COROSYNC_SOLARIS) && defined(HAVE_GETPEERUCRED) +#if defined(HAVE_GETPEERUCRED) #include #endif +#include +#include + #include #include #include @@ -79,10 +81,8 @@ #include "mainconfig.h" #include "totemconfig.h" #include "main.h" -#include "flow.h" #include "tlist.h" #include "ipc.h" -#include "flow.h" #include "sync.h" #include #include "service.h" @@ -91,504 +91,520 @@ LOGSYS_DECLARE_SUBSYS ("IPC", LOG_INFO); -#ifdef COROSYNC_SOLARIS +#ifdef CS_SOLARIS #define MSG_NOSIGNAL 0 #endif #define SERVER_BACKLOG 5 -/* - * When there are this many entries left in a queue, turn on flow control - */ -#ifndef FLOW_CONTROL_ENTRIES_ENABLE -#define FLOW_CONTROL_ENTRIES_ENABLE 400 -#endif /* FLOW_CONTROL_ENTRIES_ENABLE */ - -/* - * When there are this many entries in a queue, turn off flow control - */ -#define FLOW_CONTROL_ENTRIES_DISABLE 64 - +#define MSG_SEND_LOCKED 0 +#define MSG_SEND_UNLOCKED 1 static unsigned int g_gid_valid = 0; -static unsigned int dont_call_flow_control = 0; - -static totempg_groups_handle ipc_handle; - -DECLARE_LIST_INIT (conn_info_list_head); - static void (*ipc_serialize_lock_fn) (void); static void (*ipc_serialize_unlock_fn) (void); +DECLARE_LIST_INIT (conn_info_list_head); + struct outq_item { void *msg; size_t mlen; -}; - -enum conn_state { - CONN_STATE_ACTIVE, - CONN_STATE_SECURITY, - CONN_STATE_REQUESTED, - CONN_STATE_CLOSED, - CONN_STATE_DISCONNECTED -}; - -struct conn_info { - int fd; /* File descriptor */ - unsigned int events; /* events polled for by file descriptor */ - enum conn_state state; /* State of this connection */ - pthread_t thread; /* thread identifier */ - pthread_attr_t thread_attr; /* thread attribute */ - char *inb; /* Input buffer for non-blocking reads */ - int inb_nextheader; /* Next message header starts here */ - int inb_start; /* Start location of input buffer */ - int inb_inuse; /* Bytes currently stored in input buffer */ - struct queue outq; /* Circular queue for outgoing requests */ - int byte_start; /* Byte to start sending from in head of queue */ - enum service_types service;/* Type of service so dispatch knows how to route message */ - int authenticated; /* Is this connection authenticated? */ - void *private_data; /* library connection private data */ - struct conn_info *conn_info_partner; /* partner connection dispatch<->response */ - unsigned int flow_control_handle; /* flow control identifier */ - unsigned int flow_control_enabled; /* flow control enabled bit */ - unsigned int flow_control_local_count; /* flow control local count */ - enum cs_lib_flow_control flow_control; /* Does this service use IPC flow control */ - pthread_mutex_t flow_control_mutex; - int (*lib_exit_fn) (void *conn); - struct timerlist timerlist; - pthread_mutex_t mutex; - pthread_mutex_t *shared_mutex; struct list_head list; }; -static void *prioritized_poll_thread (void *conn); -static int conn_info_outq_flush (struct conn_info *conn_info); -static void libais_deliver (struct conn_info *conn_info); -static void ipc_flow_control (struct conn_info *conn_info); - - /* - * IPC Initializers - */ - -static int response_init_send_response ( - struct conn_info *conn_info, - void *message); -static int dispatch_init_send_response ( - struct conn_info *conn_info, - void *message); - -static int (*ais_init_service[]) (struct conn_info *conn_info, void *message) = { - response_init_send_response, - dispatch_init_send_response +#if defined(_SEM_SEMUN_UNDEFINED) +union semun { + int val; + struct semid_ds *buf; + unsigned short int *array; + struct seminfo *__buf; }; - -static void libais_disconnect_security (struct conn_info *conn_info) -{ - conn_info->state = CONN_STATE_SECURITY; - close (conn_info->fd); -} - -static int response_init_send_response ( - struct conn_info *conn_info, - void *message) -{ - cs_error_t error = CS_ERR_ACCESS; - uintptr_t cinfo = (uintptr_t)conn_info; - mar_req_lib_response_init_t *req_lib_response_init = (mar_req_lib_response_init_t *)message; - mar_res_lib_response_init_t res_lib_response_init; - - if (conn_info->authenticated) { - conn_info->service = req_lib_response_init->resdis_header.service; - error = CS_OK; - } - res_lib_response_init.header.size = sizeof (mar_res_lib_response_init_t); - res_lib_response_init.header.id = MESSAGE_RES_INIT; - res_lib_response_init.header.error = error; - res_lib_response_init.conn_info = (mar_uint64_t)cinfo; - - cs_conn_send_response ( - conn_info, - &res_lib_response_init, - sizeof (res_lib_response_init)); - - if (error == CS_ERR_ACCESS) { - libais_disconnect_security (conn_info); - return (-1); - } - return (0); -} - -static int dispatch_init_send_response ( - struct conn_info *conn_info, - void *message) -{ - cs_error_t error = CS_ERR_ACCESS; - uintptr_t cinfo; - mar_req_lib_dispatch_init_t *req_lib_dispatch_init = (mar_req_lib_dispatch_init_t *)message; - mar_res_lib_dispatch_init_t res_lib_dispatch_init; - struct conn_info *msg_conn_info; - - if (conn_info->authenticated) { - conn_info->service = req_lib_dispatch_init->resdis_header.service; - if (!ais_service[req_lib_dispatch_init->resdis_header.service]) - error = CS_ERR_NOT_SUPPORTED; - else - error = CS_OK; - - cinfo = (uintptr_t)req_lib_dispatch_init->conn_info; - conn_info->conn_info_partner = (struct conn_info *)cinfo; - - /* temporary fix for memory leak - */ - pthread_mutex_destroy (conn_info->conn_info_partner->shared_mutex); - free (conn_info->conn_info_partner->shared_mutex); - - conn_info->conn_info_partner->shared_mutex = conn_info->shared_mutex; - - list_add (&conn_info_list_head, &conn_info->list); - list_add (&conn_info_list_head, &conn_info->conn_info_partner->list); - - msg_conn_info = (struct conn_info *)cinfo; - msg_conn_info->conn_info_partner = conn_info; - - if (error == CS_OK) { - int private_data_size; - - private_data_size = ais_service[req_lib_dispatch_init->resdis_header.service]->private_data_size; - if (private_data_size) { - conn_info->private_data = malloc (private_data_size); - - conn_info->conn_info_partner->private_data = conn_info->private_data; - if (conn_info->private_data == NULL) { - error = CS_ERR_NO_MEMORY; - } else { - memset (conn_info->private_data, 0, private_data_size); - } - } else { - conn_info->private_data = NULL; - conn_info->conn_info_partner->private_data = NULL; - } - } - } - - res_lib_dispatch_init.header.size = sizeof (mar_res_lib_dispatch_init_t); - res_lib_dispatch_init.header.id = MESSAGE_RES_INIT; - res_lib_dispatch_init.header.error = error; - - cs_conn_send_response ( - conn_info, - &res_lib_dispatch_init, - sizeof (res_lib_dispatch_init)); - - if (error == CS_ERR_ACCESS) { - libais_disconnect_security (conn_info); - return (-1); - } - if (error != CS_OK) { - return (-1); - } - - conn_info->state = CONN_STATE_ACTIVE; - conn_info->conn_info_partner->state = CONN_STATE_ACTIVE; - conn_info->lib_exit_fn = ais_service[conn_info->service]->lib_exit_fn; - ais_service[conn_info->service]->lib_init_fn (conn_info); - - conn_info->flow_control = ais_service[conn_info->service]->flow_control; - conn_info->conn_info_partner->flow_control = ais_service[conn_info->service]->flow_control; - if (ais_service[conn_info->service]->flow_control == CS_LIB_FLOW_CONTROL_REQUIRED) { - cs_flow_control_ipc_init ( - &conn_info->flow_control_handle, - conn_info->service); - - } - return (0); -} - -/* - * Create a connection data structure - */ -static inline unsigned int conn_info_create (int fd) { - struct conn_info *conn_info; - int res; - - conn_info = malloc (sizeof (struct conn_info)); - if (conn_info == 0) { - return (ENOMEM); - } - - memset (conn_info, 0, sizeof (struct conn_info)); - - res = queue_init (&conn_info->outq, SIZEQUEUE, - sizeof (struct outq_item)); - if (res != 0) { - free (conn_info); - return (ENOMEM); - } - conn_info->inb = malloc (sizeof (char) * SIZEINB); - if (conn_info->inb == NULL) { - queue_free (&conn_info->outq); - free (conn_info); - return (ENOMEM); - } - conn_info->shared_mutex = malloc (sizeof (pthread_mutex_t)); - if (conn_info->shared_mutex == NULL) { - free (conn_info->inb); - queue_free (&conn_info->outq); - free (conn_info); - return (ENOMEM); - } - - pthread_mutex_init (&conn_info->mutex, NULL); - pthread_mutex_init (&conn_info->flow_control_mutex, NULL); - pthread_mutex_init (conn_info->shared_mutex, NULL); - - list_init (&conn_info->list); - conn_info->state = CONN_STATE_ACTIVE; - conn_info->fd = fd; - conn_info->events = POLLIN|POLLNVAL; - conn_info->service = SOCKET_SERVICE_INIT; - - pthread_attr_init (&conn_info->thread_attr); -/* - * IA64 needs more stack space then other arches - */ -#if defined(__ia64__) - pthread_attr_setstacksize (&conn_info->thread_attr, 400000); -#else - pthread_attr_setstacksize (&conn_info->thread_attr, 200000); #endif - pthread_attr_setdetachstate (&conn_info->thread_attr, PTHREAD_CREATE_DETACHED); - res = pthread_create (&conn_info->thread, &conn_info->thread_attr, - prioritized_poll_thread, conn_info); - return (res); -} +struct conn_info { + int fd; + pthread_t thread; + pthread_attr_t thread_attr; + unsigned int service; + int destroyed; + int disconnect_requested; + int notify_flow_control_enabled; + int refcount; + key_t shmkey; + key_t semkey; + int shmid; + int semid; + unsigned int pending_semops; + pthread_mutex_t mutex; + struct shared_memory *mem; + struct list_head outq_head; + void *private_data; + int (*lib_exit_fn) (void *conn); + struct list_head list; + char setup_msg[sizeof (mar_req_setup_t)]; + unsigned int setup_bytes_read; +}; -static void conn_info_destroy (struct conn_info *conn_info) +static int shared_mem_dispatch_bytes_left (struct conn_info *conn_info); + +static void outq_flush (struct conn_info *conn_info); + +static int priv_change (struct conn_info *conn_info); + +static void ipc_disconnect (struct conn_info *conn_info); + +static inline int conn_info_destroy (struct conn_info *conn_info) { - struct outq_item *outq_item; - - /* - * Free the outq queued items - */ - while (!queue_is_empty (&conn_info->outq)) { - outq_item = queue_item_get (&conn_info->outq); - free (outq_item->msg); - queue_item_remove (&conn_info->outq); - } - - queue_free (&conn_info->outq); - free (conn_info->inb); - if (conn_info->conn_info_partner) { - conn_info->conn_info_partner->conn_info_partner = NULL; - } - - pthread_attr_destroy (&conn_info->thread_attr); - pthread_mutex_destroy (&conn_info->mutex); - pthread_mutex_destroy (&conn_info->flow_control_mutex); + unsigned int res; list_del (&conn_info->list); - free (conn_info); -} + list_init (&conn_info->list); -int libais_connection_active (struct conn_info *conn_info); - -int libais_connection_active (struct conn_info *conn_info) -{ - return (conn_info->state == CONN_STATE_ACTIVE); -} - -static void libais_disconnect_request (struct conn_info *conn_info) -{ - if (conn_info->state == CONN_STATE_ACTIVE) { - conn_info->state = CONN_STATE_REQUESTED; - conn_info->conn_info_partner->state = CONN_STATE_REQUESTED; - } -} - -static int libais_disconnect (struct conn_info *conn_info) -{ - int res = 0; - - assert (conn_info->state != CONN_STATE_ACTIVE); - - if (conn_info->state == CONN_STATE_DISCONNECTED) { - assert (0); - } - - /* - * Close active connections - */ - if (conn_info->state == CONN_STATE_ACTIVE || conn_info->state == CONN_STATE_REQUESTED) { + if (conn_info->service == SOCKET_SERVICE_INIT) { + list_del (&conn_info->list); close (conn_info->fd); - conn_info->state = CONN_STATE_CLOSED; - close (conn_info->conn_info_partner->fd); - conn_info->conn_info_partner->state = CONN_STATE_CLOSED; + free (conn_info); + return (0); + } + /* + * Destroy shared memory segment and semaphore + */ + if (conn_info->destroyed == 0) { + cs_conn_refcount_dec (conn_info); + shmdt (conn_info->mem); + res = shmctl (conn_info->shmid, IPC_RMID, NULL); + semctl (conn_info->semid, 0, IPC_RMID); + conn_info->destroyed = 1; + } + + pthread_mutex_lock (&conn_info->mutex); + if (conn_info->refcount > 0) { + pthread_mutex_unlock (&conn_info->mutex); + return (-1); + } + pthread_mutex_unlock (&conn_info->mutex); + + /* + * Retry library exit function if busy + */ + res = ais_service[conn_info->service]->lib_exit_fn (conn_info); + if (res == -1) { + return (-1); } /* - * Note we will only call the close operation once on the first time - * one of the connections is closed - */ - if (conn_info->state == CONN_STATE_CLOSED) { - if (conn_info->lib_exit_fn) { - res = conn_info->lib_exit_fn (conn_info); - } - if (res == -1) { - return (-1); - } - if (conn_info->conn_info_partner->lib_exit_fn) { - res = conn_info->conn_info_partner->lib_exit_fn (conn_info); - } - if (res == -1) { - return (-1); - } - } - conn_info->state = CONN_STATE_DISCONNECTED; - conn_info->conn_info_partner->state = CONN_STATE_DISCONNECTED; - if (conn_info->flow_control_enabled == 1) { - cs_flow_control_disable (conn_info->flow_control_handle); + * Free allocated data needed to retry exiting library IPC connection + */ + if (conn_info->private_data) { + free (conn_info->private_data); } + close (conn_info->fd); + list_del (&conn_info->list); + free (conn_info); return (0); } -static inline void conn_info_mutex_lock ( - struct conn_info *conn_info, - unsigned int service) -{ - if (service == SOCKET_SERVICE_INIT) { - pthread_mutex_lock (&conn_info->mutex); - } else { - pthread_mutex_lock (conn_info->shared_mutex); - } -} -static inline void conn_info_mutex_unlock ( - struct conn_info *conn_info, - unsigned int service) -{ - if (service == SOCKET_SERVICE_INIT) { - pthread_mutex_unlock (&conn_info->mutex); - } else { - pthread_mutex_unlock (conn_info->shared_mutex); - } -} +struct res_overlay { + mar_res_header_t header __attribute__((aligned(8))); + char buf[4096]; +}; -/* - * This thread runs in a specific thread priority mode to handle - * I/O requests from the library - */ -static void *prioritized_poll_thread (void *conn) +static void *pthread_ipc_consumer (void *conn) { struct conn_info *conn_info = (struct conn_info *)conn; - struct pollfd ufd; - int fds; - struct sched_param sched_param; + struct sembuf sop; int res; - pthread_mutex_t *rel_mutex; - pthread_mutex_t *rel2_mutex; - unsigned int service; - struct conn_info *cinfo_partner; - void *private_data; + mar_req_header_t *header; + struct res_overlay res_overlay; + struct iovec send_ok_joined_iovec; + int send_ok = 0; + int send_ok_joined = 0; - sched_param.sched_priority = 1; - res = pthread_setschedparam (conn_info->thread, SCHED_RR, &sched_param); - - ufd.fd = conn_info->fd; for (;;) { -retry_poll: - service = conn_info->service; - ufd.events = conn_info->events; - ufd.revents = 0; - fds = poll (&ufd, 1, -1); - - conn_info_mutex_lock (conn_info, service); - - switch (conn_info->state) { - case CONN_STATE_SECURITY: - conn_info_mutex_unlock (conn_info, service); - pthread_mutex_destroy (conn_info->shared_mutex); - free (conn_info->shared_mutex); - conn_info_destroy (conn); - pthread_exit (0); - break; - - case CONN_STATE_REQUESTED: - case CONN_STATE_CLOSED: - res = libais_disconnect (conn); - if (res != 0) { - conn_info_mutex_unlock (conn_info, service); - goto retry_poll; - } - break; - - case CONN_STATE_DISCONNECTED: - rel_mutex = conn_info->shared_mutex; - rel2_mutex = &conn_info->mutex; - private_data = conn_info->private_data; - cinfo_partner = conn_info->conn_info_partner; - conn_info_destroy (conn); - if (service == SOCKET_SERVICE_INIT) { - pthread_mutex_unlock (rel2_mutex); - } else { - pthread_mutex_unlock (rel_mutex); - } - if (cinfo_partner == NULL) { - pthread_mutex_destroy (rel_mutex); - free (rel_mutex); - free (private_data); - } - pthread_exit (0); - /* - * !! NOTE !! this is the exit point for this thread - */ - break; - - default: + sop.sem_num = 0; + sop.sem_op = -1; + sop.sem_flg = 0; +retry_semop: + res = semop (conn_info->semid, &sop, 1); + if ((res == -1) && (errno == EINTR || errno == EAGAIN)) { + goto retry_semop; + } else + if ((res == -1) && (errno == EINVAL || errno == EIDRM)) { + cs_conn_refcount_dec (conn); + return (0); + } + if (conn_info->destroyed || conn_info->disconnect_requested) { break; } - if (fds == -1) { - conn_info_mutex_unlock (conn_info, service); - goto retry_poll; - } + header = (mar_req_header_t *)conn_info->mem->req_buffer; ipc_serialize_lock_fn (); - if (fds == 1 && ufd.revents) { - if (ufd.revents & (POLLERR|POLLHUP)) { + send_ok_joined_iovec.iov_base = (char *)header; + send_ok_joined_iovec.iov_len = header->size; + send_ok_joined = totempg_groups_send_ok_joined (corosync_group_handle, + &send_ok_joined_iovec, 1); - libais_disconnect_request (conn_info); - - conn_info_mutex_unlock (conn_info, service); - ipc_serialize_unlock_fn (); - continue; - } - - if (ufd.revents & POLLOUT) { - conn_info_outq_flush (conn_info); - } - - if ((ufd.revents & POLLIN) == POLLIN) { - libais_deliver (conn_info); - } - - ipc_flow_control (conn_info); + send_ok = + (corosync_quorum_is_quorate() == 1 || ais_service[conn_info->service]->allow_inquorate == CS_LIB_ALLOW_INQUORATE) && ( + (ais_service[conn_info->service]->lib_engine[header->id].flow_control == CS_LIB_FLOW_CONTROL_NOT_REQUIRED) || + ((ais_service[conn_info->service]->lib_engine[header->id].flow_control == CS_LIB_FLOW_CONTROL_REQUIRED) && + (send_ok_joined) && + (sync_in_process() == 0))); + if (send_ok) { + ais_service[conn_info->service]->lib_engine[header->id].lib_handler_fn (conn_info, header); + } else { + /* + * Overload, tell library to retry + */ + res_overlay.header.size = + ais_service[conn_info->service]->lib_engine[header->id].response_size; + res_overlay.header.id = + ais_service[conn_info->service]->lib_engine[header->id].response_id; + res_overlay.header.error = CS_ERR_TRY_AGAIN; + cs_response_send (conn_info, &res_overlay, + res_overlay.header.size); } ipc_serialize_unlock_fn (); - conn_info_mutex_unlock (conn_info, service); } + cs_conn_refcount_dec (conn); + return (NULL); +} - /* - * This code never reached - */ +static int +req_setup_send ( + struct conn_info *conn_info, + int error) +{ + mar_res_setup_t res_setup; + res_setup.error = error; + unsigned int res; + +retry_send: + res = send (conn_info->fd, &res_setup, sizeof (mar_res_setup_t), MSG_WAITALL); + if (res == -1 && errno == EINTR) { + goto retry_send; + } else + if (res == -1 && errno == EAGAIN) { + goto retry_send; + } return (0); } -#if defined(COROSYNC_LINUX) || defined(COROSYNC_SOLARIS) +static int +req_setup_recv ( + struct conn_info *conn_info) +{ + int res; + struct msghdr msg_recv; + struct iovec iov_recv; +#ifdef COROSYNC_LINUX + struct cmsghdr *cmsg; + char cmsg_cred[CMSG_SPACE (sizeof (struct ucred))]; + struct ucred *cred; + int off = 0; + int on = 1; +#endif + + msg_recv.msg_iov = &iov_recv; + msg_recv.msg_iovlen = 1; + msg_recv.msg_name = 0; + msg_recv.msg_namelen = 0; +#ifdef COROSYNC_LINUX + msg_recv.msg_control = (void *)cmsg_cred; + msg_recv.msg_controllen = sizeof (cmsg_cred); +#endif + +#ifdef PORTABILITY_WORK_TODO +#ifdef CS_SOLARIS + msg_recv.msg_flags = 0; + uid_t euid; + gid_t egid; + + euid = -1; + egid = -1; + if (getpeereid(conn_info->fd, &euid, &egid) != -1 && + (euid == 0 || egid == g_gid_valid)) { + if (conn_info->state == CONN_IO_STATE_INITIALIZING) { + log_printf (LOG_LEVEL_SECURITY, "Connection not authenticated because gid is %d, expecting %d\n", egid, g_gid_valid); + return (-1); + } + } + msg_recv.msg_accrights = 0; + msg_recv.msg_accrightslen = 0; +#else /* CS_SOLARIS */ + +#ifdef HAVE_GETPEERUCRED + ucred_t *uc; + uid_t euid = -1; + gid_t egid = -1; + + if (getpeerucred (conn_info->fd, &uc) == 0) { + euid = ucred_geteuid (uc); + egid = ucred_getegid (uc); + if ((euid == 0) || (egid == g_gid_valid)) { + conn_info->authenticated = 1; + } + ucred_free(uc); + } + if (conn_info->authenticated == 0) { + log_printf (LOG_LEVEL_SECURITY, "Connection not authenticated because gid is %d, expecting %d\n", (int)egid, g_gid_valid); + } +#else /* HAVE_GETPEERUCRED */ + log_printf (LOG_LEVEL_SECURITY, "Connection not authenticated " + "because platform does not support " + "authentication with sockets, continuing " + "with a fake authentication\n"); +#endif /* HAVE_GETPEERUCRED */ +#endif /* CS_SOLARIS */ + +#endif + +#ifdef COROSYNC_LINUX + iov_recv.iov_base = &conn_info->setup_msg[conn_info->setup_bytes_read]; + iov_recv.iov_len = sizeof (mar_req_setup_t) - conn_info->setup_bytes_read; + setsockopt(conn_info->fd, SOL_SOCKET, SO_PASSCRED, &on, sizeof (on)); +#endif + +retry_recv: + res = recvmsg (conn_info->fd, &msg_recv, MSG_NOSIGNAL); + if (res == -1 && errno == EINTR) { + goto retry_recv; + } else + if (res == -1 && errno != EAGAIN) { + return (0); + } else + if (res == 0) { +#if defined(CS_SOLARIS) || defined(CS_BSD) || defined(CS_DARWIN) + /* On many OS poll never return POLLHUP or POLLERR. + * EOF is detected when recvmsg return 0. + */ + ipc_disconnect (conn_info); +#endif + return (-1); + } + conn_info->setup_bytes_read += res; + +#ifdef COROSYNC_LINUX + + cmsg = CMSG_FIRSTHDR (&msg_recv); + assert (cmsg); + cred = (struct ucred *)CMSG_DATA (cmsg); + if (cred) { + if (cred->uid == 0 || cred->gid == g_gid_valid) { + } else { + ipc_disconnect (conn_info); + log_printf (LOG_LEVEL_SECURITY, + "Connection not authenticated because gid is %d, expecting %d\n", + cred->gid, g_gid_valid); + return (-1); + } + } +#endif + if (conn_info->setup_bytes_read == sizeof (mar_req_setup_t)) { +#ifdef COROSYNC_LINUX + setsockopt(conn_info->fd, SOL_SOCKET, SO_PASSCRED, + &off, sizeof (off)); +#endif + return (1); + } + return (0); +} + +static int poll_handler_connection_destroy( + struct conn_info *conn_info) +{ + int res; + res = conn_info_destroy (conn_info); + if (res == -1) { + return (0); + } else { + return (-1); + } +} + +static int poll_handler_connection ( + poll_handle handle, + int fd, + int revent, + void *data) +{ + mar_req_setup_t *req_setup; + struct conn_info *conn_info = (struct conn_info *)data; + int res; + char buf; + + + /* + * If an error occurs, try to exit if possible + */ + if ((conn_info->disconnect_requested) || (revent & (POLLERR|POLLHUP))) { + return poll_handler_connection_destroy (conn_info); + } + + /* + * Read the header and process it + */ + if (conn_info->service == SOCKET_SERVICE_INIT && (revent & POLLIN)) { + /* + * Receive in a nonblocking fashion the request + * IF security invalid, send TRY_AGAIN, otherwise + * send OK + */ + res = req_setup_recv (conn_info); + if (res == -1) { + req_setup_send (conn_info, CS_ERR_TRY_AGAIN); + } + if (res != 1) { + return (0); + } + req_setup_send (conn_info, CS_OK); + + pthread_mutex_init (&conn_info->mutex, NULL); + req_setup = (mar_req_setup_t *)conn_info->setup_msg; + conn_info->shmkey = req_setup->shmkey; + conn_info->semkey = req_setup->semkey; + conn_info->service = req_setup->service; + conn_info->destroyed = 0; + conn_info->disconnect_requested = 0; + conn_info->refcount = 0; + conn_info->notify_flow_control_enabled = 0; + conn_info->setup_bytes_read = 0; + + conn_info->shmid = shmget (conn_info->shmkey, + sizeof (struct shared_memory), 0600); + conn_info->mem = shmat (conn_info->shmid, NULL, 0); + conn_info->semid = semget (conn_info->semkey, 3, 0600); + conn_info->pending_semops = 0; + conn_info->refcount = 1; + cs_conn_refcount_inc (conn_info); + + conn_info->private_data = malloc (ais_service[conn_info->service]->private_data_size); + memset (conn_info->private_data, 0, + ais_service[conn_info->service]->private_data_size); + ais_service[conn_info->service]->lib_init_fn (conn_info); + + + pthread_attr_init (&conn_info->thread_attr); + /* + * IA64 needs more stack space then other arches + */ + #if defined(__ia64__) + pthread_attr_setstacksize (&conn_info->thread_attr, 400000); + #else + pthread_attr_setstacksize (&conn_info->thread_attr, 200000); + #endif + + pthread_attr_setdetachstate (&conn_info->thread_attr, PTHREAD_CREATE_DETACHED); + res = pthread_create (&conn_info->thread, + &conn_info->thread_attr, + pthread_ipc_consumer, + conn_info); + + /* + * Security check - disallow multiple configurations of + * the ipc connection + */ + if (conn_info->service == SOCKET_SERVICE_INIT) { + conn_info->service = -1; + } + } else + if (revent & POLLIN) { + res = recv (fd, &buf, 1, MSG_NOSIGNAL); + if (res == 1) { + switch (buf) { + case MESSAGE_REQ_OUTQ_FLUSH: + outq_flush (conn_info); + break; + case MESSAGE_REQ_CHANGE_EUID: + if (priv_change (conn_info) == -1) { + return poll_handler_connection_destroy (conn_info); + } + break; + default: + res = 0; + break; + } + } +#if defined(CS_SOLARIS) || defined(CS_BSD) || defined(CS_DARWIN) + /* On many OS poll never return POLLHUP or POLLERR. + * EOF is detected when recvmsg return 0. + */ + if (res == 0) { + return poll_handler_connection_destroy (conn_info); + } +#endif + } + + pthread_mutex_lock (&conn_info->mutex); + if ((conn_info->disconnect_requested == 0) && (revent & POLLOUT)) { + buf = !list_empty (&conn_info->outq_head); + for (; conn_info->pending_semops;) { + res = send (conn_info->fd, &buf, 1, MSG_NOSIGNAL); + if (res == 1) { + conn_info->pending_semops--; + } else { + break; + } + } + if (conn_info->notify_flow_control_enabled) { + buf = 2; + res = send (conn_info->fd, &buf, 1, MSG_NOSIGNAL); + if (res == 1) { + conn_info->notify_flow_control_enabled = 0; + } + } + if (conn_info->notify_flow_control_enabled == 0 && + conn_info->pending_semops == 0) { + + poll_dispatch_modify (aisexec_poll_handle, + conn_info->fd, POLLIN|POLLNVAL, + poll_handler_connection); + } + } + pthread_mutex_unlock (&conn_info->mutex); + + return (0); +} + +static void ipc_disconnect (struct conn_info *conn_info) +{ + pthread_mutex_lock (&conn_info->mutex); + conn_info->disconnect_requested = 1; + pthread_mutex_unlock (&conn_info->mutex); + + poll_dispatch_modify (aisexec_poll_handle, + conn_info->fd, POLLOUT|POLLNVAL, + poll_handler_connection); +} + +static int conn_info_create (int fd) +{ + struct conn_info *conn_info; + + conn_info = malloc (sizeof (struct conn_info)); + if (conn_info == NULL) { + return (-1); + } + memset (conn_info, 0, sizeof (struct conn_info)); + + conn_info->fd = fd; + conn_info->service = SOCKET_SERVICE_INIT; + list_init (&conn_info->outq_head); + list_init (&conn_info->list); + list_add (&conn_info->list, &conn_info_list_head); + + poll_dispatch_add (aisexec_poll_handle, fd, POLLIN|POLLNVAL, + conn_info, poll_handler_connection); + return (0); +} + +#if defined(COROSYNC_LINUX) || defined(CS_SOLARIS) /* SUN_LEN is broken for abstract namespace */ #define AIS_SUN_LEN(a) sizeof(*(a)) @@ -597,359 +613,12 @@ retry_poll: #endif #if defined(COROSYNC_LINUX) -char *socketname = "libcorosync.socket"; +char *socketname = "libais.socket"; #else -char *socketname = "/var/run/libcorosync.socket"; +char *socketname = "/var/run/libais.socket"; #endif - -static void ipc_flow_control (struct conn_info *conn_info) -{ - unsigned int entries_used; - unsigned int entries_usedhw; - unsigned int flow_control_local_count; - unsigned int fcc; - - /* - * Determine FCC variable and printing variables - */ - entries_used = queue_used (&conn_info->outq); - if (conn_info->conn_info_partner && - queue_used (&conn_info->conn_info_partner->outq) > entries_used) { - entries_used = queue_used (&conn_info->conn_info_partner->outq); - } - entries_usedhw = queue_usedhw (&conn_info->outq); - if (conn_info->conn_info_partner && - queue_usedhw (&conn_info->conn_info_partner->outq) > entries_used) { - entries_usedhw = queue_usedhw (&conn_info->conn_info_partner->outq); - } - flow_control_local_count = conn_info->flow_control_local_count; - if (conn_info->conn_info_partner && - conn_info->conn_info_partner->flow_control_local_count > flow_control_local_count) { - flow_control_local_count = conn_info->conn_info_partner->flow_control_local_count; - } - - fcc = entries_used; - if (flow_control_local_count > fcc) { - fcc = flow_control_local_count; - } - /* - * IPC group-wide flow control - */ - if (conn_info->flow_control == CS_LIB_FLOW_CONTROL_REQUIRED) { - if (conn_info->flow_control_enabled == 0 && - ((fcc + FLOW_CONTROL_ENTRIES_ENABLE) > SIZEQUEUE)) { - - log_printf (LOG_LEVEL_NOTICE, "Enabling flow control [%d/%d] - [%d].\n", - entries_usedhw, SIZEQUEUE, - flow_control_local_count); - cs_flow_control_enable (conn_info->flow_control_handle); - conn_info->flow_control_enabled = 1; - if (conn_info->conn_info_partner) { - conn_info->conn_info_partner->flow_control_enabled = 1; - } - } - if (conn_info->flow_control_enabled == 1 && - fcc <= FLOW_CONTROL_ENTRIES_DISABLE) { - - log_printf (LOG_LEVEL_NOTICE, "Disabling flow control [%d/%d] - [%d].\n", - entries_usedhw, SIZEQUEUE, - flow_control_local_count); - cs_flow_control_disable (conn_info->flow_control_handle); - conn_info->flow_control_enabled = 0; - if (conn_info->conn_info_partner) { - conn_info->conn_info_partner->flow_control_enabled = 0; - } - } - } -} - -static int conn_info_outq_flush (struct conn_info *conn_info) { - struct queue *outq; - ssize_t res = 0; - struct outq_item *queue_item; - struct msghdr msg_send; - struct iovec iov_send; - char *msg_addr; - - if (!libais_connection_active (conn_info)) { - return (-1); - } - outq = &conn_info->outq; - - msg_send.msg_iov = &iov_send; - msg_send.msg_name = 0; - msg_send.msg_namelen = 0; - msg_send.msg_iovlen = 1; -#ifndef COROSYNC_SOLARIS - msg_send.msg_control = 0; - msg_send.msg_controllen = 0; - msg_send.msg_flags = 0; -#else - msg_send.msg_accrights = 0; - msg_send.msg_accrightslen = 0; -#endif - - while (!queue_is_empty (outq)) { - queue_item = queue_item_get (outq); - msg_addr = (char *)queue_item->msg; - msg_addr = &msg_addr[conn_info->byte_start]; - - iov_send.iov_base = msg_addr; - iov_send.iov_len = queue_item->mlen - conn_info->byte_start; - -retry_sendmsg: - res = sendmsg (conn_info->fd, &msg_send, MSG_NOSIGNAL); - if (res == -1 && errno == EINTR) { - goto retry_sendmsg; - } - if (res == -1 && errno == EAGAIN) { - return (0); - } - if (res == -1 && errno == EPIPE) { - libais_disconnect_request (conn_info); - return (0); - } - if (res == -1) { - printf ("ERRNO is %d\n", errno); - assert (0); /* some other unhandled error here */ - } - if (res + conn_info->byte_start != queue_item->mlen) { - conn_info->byte_start += res; - - return (0); - } - - /* - * Message sent, try sending another message - */ - queue_item_remove (outq); - conn_info->byte_start = 0; - free (queue_item->msg); - } /* while queue not empty */ - - if (queue_is_empty (outq)) { - conn_info->events = POLLIN|POLLNVAL; - } - - return (0); -} - - - -struct ipc_res_overlay { - mar_res_header_t header __attribute((aligned(8))); - char buf[4096]; -}; - -static void libais_deliver (struct conn_info *conn_info) -{ - ssize_t res; - int dispatch_res; - mar_req_header_t *header; - int service; - struct msghdr msg_recv; - struct iovec iov_recv; -#ifdef COROSYNC_LINUX - struct cmsghdr *cmsg; - char cmsg_cred[CMSG_SPACE (sizeof (struct ucred))]; - struct ucred *cred; - int on = 0; -#endif - int send_ok = 0; - int send_ok_joined = 0; - struct iovec send_ok_joined_iovec; - struct ipc_res_overlay res_overlay; - - msg_recv.msg_iov = &iov_recv; - msg_recv.msg_iovlen = 1; - msg_recv.msg_name = 0; - msg_recv.msg_namelen = 0; -#ifndef COROSYNC_SOLARIS - msg_recv.msg_flags = 0; - - if (conn_info->authenticated) { - msg_recv.msg_control = 0; - msg_recv.msg_controllen = 0; - } else { -#ifdef COROSYNC_LINUX - msg_recv.msg_control = (void *)cmsg_cred; - msg_recv.msg_controllen = sizeof (cmsg_cred); -#else - uid_t euid = -1; - gid_t egid = -1; - if (getpeereid(conn_info->fd, &euid, &egid) != -1 && - (euid == 0 || egid == g_gid_valid)) { - conn_info->authenticated = 1; - } - if (conn_info->authenticated == 0) { - log_printf (LOG_LEVEL_SECURITY, "Connection not authenticated because gid is %d, expecting %d\n", egid, g_gid_valid); - } -#endif - } - -#else /* COROSYNC_SOLARIS */ - msg_recv.msg_accrights = 0; - msg_recv.msg_accrightslen = 0; - - if (! conn_info->authenticated) { -#ifdef HAVE_GETPEERUCRED - ucred_t *uc; - uid_t euid = -1; - gid_t egid = -1; - if (getpeerucred(conn_info->fd, &uc) == 0) { - euid = ucred_geteuid(uc); - egid = ucred_getegid(uc); - if ((euid == 0) || (egid == g_gid_valid)) { - conn_info->authenticated = 1; - } - ucred_free(uc); - } - if (conn_info->authenticated == 0) { - log_printf (LOG_LEVEL_SECURITY, "Connection not authenticated because gid is %d, expecting %d\n", (int)egid, g_gid_valid); - } -#else - log_printf (LOG_LEVEL_SECURITY, "Connection not authenticated " - "because platform does not support " - "authentication with sockets, continuing " - "with a fake authentication\n"); - conn_info->authenticated = 1; -#endif - } -#endif - - iov_recv.iov_base = &conn_info->inb[conn_info->inb_start]; - iov_recv.iov_len = (SIZEINB) - conn_info->inb_start; - if (conn_info->inb_inuse == SIZEINB) { - return; - } - -retry_recv: - res = recvmsg (conn_info->fd, &msg_recv, MSG_NOSIGNAL); - if (res == -1 && errno == EINTR) { - goto retry_recv; - } else - if (res == -1 && errno != EAGAIN) { - return; - } else - if (res == 0) { -#if defined(COROSYNC_SOLARIS) || defined(COROSYNC_BSD) || defined(COROSYNC_DARWIN) - /* On many OS poll never return POLLHUP or POLLERR. - * EOF is detected when recvmsg return 0. - */ - libais_disconnect_request (conn_info); -#endif - return; - } - - /* - * Authenticate if this connection has not been authenticated - */ -#ifdef COROSYNC_LINUX - if (conn_info->authenticated == 0) { - cmsg = CMSG_FIRSTHDR (&msg_recv); - assert (cmsg != NULL); - cred = (struct ucred *)CMSG_DATA (cmsg); - if (cred) { - if (cred->uid == 0 || cred->gid == g_gid_valid) { - setsockopt(conn_info->fd, SOL_SOCKET, SO_PASSCRED, &on, sizeof (on)); - conn_info->authenticated = 1; - } - } - if (conn_info->authenticated == 0) { - log_printf (LOG_LEVEL_SECURITY, "Connection not authenticated because gid is %d, expecting %d\n", cred->gid, g_gid_valid); - } - } -#endif - /* - * Dispatch all messages received in recvmsg that can be dispatched - * sizeof (mar_req_header_t) needed at minimum to do any processing - */ - conn_info->inb_inuse += res; - conn_info->inb_start += res; - - dispatch_res = 0; - while (conn_info->inb_inuse >= sizeof (mar_req_header_t) && dispatch_res != -1) { - header = (mar_req_header_t *)&conn_info->inb[conn_info->inb_start - conn_info->inb_inuse]; - - if (header->size > conn_info->inb_inuse) { - break; - } - service = conn_info->service; - - /* - * If this service is in init phase, initialize service - * else handle message using service service - */ - if (service == SOCKET_SERVICE_INIT) { - dispatch_res = ais_init_service[header->id] (conn_info, header); - } else { - /* - * Not an init service, but a standard service - */ - if (header->id < 0 || header->id > ais_service[service]->lib_engine_count) { - log_printf (LOG_LEVEL_SECURITY, "Invalid header id is %d min 0 max %d\n", - header->id, ais_service[service]->lib_engine_count); - return ; - } - - /* - * If flow control is required of the library handle, determine that - * corosync is not in synchronization and that totempg has room available - * to queue a message, otherwise tell the library we are busy and to - * try again later - */ - send_ok_joined_iovec.iov_base = (char *)header; - send_ok_joined_iovec.iov_len = header->size; - send_ok_joined = totempg_groups_send_ok_joined (corosync_group_handle, - &send_ok_joined_iovec, 1); - - send_ok = - (corosync_quorum_is_quorate() == 1 || ais_service[service]->allow_inquorate == CS_LIB_ALLOW_INQUORATE) && ( - (ais_service[service]->lib_engine[header->id].flow_control == CS_LIB_FLOW_CONTROL_NOT_REQUIRED) || - ((ais_service[service]->lib_engine[header->id].flow_control == CS_LIB_FLOW_CONTROL_REQUIRED) && - (send_ok_joined) && - (sync_in_process() == 0))); - - if (send_ok) { - ais_service[service]->lib_engine[header->id].lib_handler_fn(conn_info, header); - } else { - - /* - * Overload, tell library to retry - */ - res_overlay.header.size = - ais_service[service]->lib_engine[header->id].response_size; - res_overlay.header.id = - ais_service[service]->lib_engine[header->id].response_id; - res_overlay.header.error = CS_ERR_TRY_AGAIN; - cs_conn_send_response ( - conn_info, - &res_overlay, - res_overlay.header.size); - } - } - conn_info->inb_inuse -= header->size; - } /* while */ - - if (conn_info->inb_inuse == 0) { - conn_info->inb_start = 0; - } else -// BUG if (connections[conn_info->fd].inb_start + connections[conn_info->fd].inb_inuse >= SIZEINB) { - if (conn_info->inb_start >= SIZEINB) { - /* - * If in buffer is full, move it back to start - */ - memmove (conn_info->inb, - &conn_info->inb[conn_info->inb_start - conn_info->inb_inuse], - sizeof (char) * conn_info->inb_inuse); - conn_info->inb_start = conn_info->inb_inuse; - } - - return; -} - -static int poll_handler_libais_accept ( +static int poll_handler_accept ( poll_handle handle, int fd, int revent, @@ -1029,16 +698,7 @@ void message_source_set ( source->conn = conn; } -static void ipc_confchg_fn ( - enum totem_configuration_type configuration_type, - unsigned int *member_list, int member_list_entries, - unsigned int *left_list, int left_list_entries, - unsigned int *joined_list, int joined_list_entries, - struct memb_ring_id *ring_id) -{ -} - -void cs_ipc_init ( +extern void cs_ipc_init ( void (*serialize_lock_fn) (void), void (*serialize_unlock_fn) (void), unsigned int gid_valid) @@ -1048,9 +708,9 @@ void cs_ipc_init ( int res; ipc_serialize_lock_fn = serialize_lock_fn; - ipc_serialize_unlock_fn = serialize_unlock_fn; + /* * Create socket for libais clients, name socket, listen for connections */ @@ -1060,7 +720,7 @@ void cs_ipc_init ( corosync_exit_error (AIS_DONE_LIBAIS_SOCKET); }; - totemip_nosigpipe(libais_server_fd); + totemip_nosigpipe (libais_server_fd); res = fcntl (libais_server_fd, F_SETFL, O_NONBLOCK); if (res == -1) { log_printf (LOG_LEVEL_ERROR, "Could not set non-blocking operation on server socket: %s\n", strerror (errno)); @@ -1072,7 +732,7 @@ void cs_ipc_init ( #endif memset (&un_addr, 0, sizeof (struct sockaddr_un)); un_addr.sun_family = AF_UNIX; -#if defined(COROSYNC_BSD) || defined(COROSYNC_DARWIN) +#if defined(CS_BSD) || defined(CS_DARWIN) un_addr.sun_len = sizeof(struct sockaddr_un); #endif #if defined(COROSYNC_LINUX) @@ -1092,20 +752,29 @@ void cs_ipc_init ( * Setup libais connection dispatch routine */ poll_dispatch_add (aisexec_poll_handle, libais_server_fd, - POLLIN, 0, poll_handler_libais_accept); + POLLIN|POLLNVAL, 0, poll_handler_accept); g_gid_valid = gid_valid; - - /* - * Reset internal state of flow control when - * configuration change occurs - */ - res = totempg_groups_initialize ( - &ipc_handle, - NULL, - ipc_confchg_fn); } +void cs_ipc_exit (void) +{ + struct list_head *list; + struct conn_info *conn_info; + + for (list = conn_info_list_head.next; list != &conn_info_list_head; + list = list->next) { + + conn_info = list_entry (list, struct conn_info, list); + + shmdt (conn_info->mem); + shmctl (conn_info->shmid, IPC_RMID, NULL); + semctl (conn_info->semid, 0, IPC_RMID); + conn_info->destroyed = 1; + + pthread_kill (conn_info->thread, SIGUSR1); + } +} /* * Get the conn info private data @@ -1114,239 +783,306 @@ void *cs_conn_private_data_get (void *conn) { struct conn_info *conn_info = (struct conn_info *)conn; - if (conn != NULL) { - return ((void *)conn_info->private_data); - } else { - return NULL; - } + return (conn_info->private_data); } -/* - * Get the conn info partner connection - */ -void *cs_conn_partner_get (void *conn) +int cs_response_send (void *conn, void *msg, int mlen) { struct conn_info *conn_info = (struct conn_info *)conn; + struct sembuf sop; + int res; - if (conn != NULL) { - return ((void *)conn_info->conn_info_partner); - } else { - return NULL; - } -} + memcpy (conn_info->mem->res_buffer, msg, mlen); + sop.sem_num = 1; + sop.sem_op = 1; + sop.sem_flg = 0; -int cs_conn_send_response_no_fcc ( - void *conn, - void *msg, - int mlen) -{ - int ret; - dont_call_flow_control = 1; - ret = cs_conn_send_response ( - conn, msg, mlen); - dont_call_flow_control = 0; - return ret; -} - -int cs_conn_send_response ( - void *conn, - void *msg, - int mlen) -{ - struct queue *outq; - char *cmsg; - ssize_t res = 0; - int queue_empty; - struct outq_item *queue_item; - struct outq_item queue_item_out; - struct msghdr msg_send; - struct iovec iov_send; - char *msg_addr; - struct conn_info *conn_info = (struct conn_info *)conn; - - if (conn_info == NULL) { - return -1; - } - - if (!libais_connection_active (conn_info)) { - return (-1); - } - - if (dont_call_flow_control == 0) { - ipc_flow_control (conn_info); - } - - outq = &conn_info->outq; - - msg_send.msg_iov = &iov_send; - msg_send.msg_name = 0; - msg_send.msg_namelen = 0; - msg_send.msg_iovlen = 1; -#ifndef COROSYNC_SOLARIS - msg_send.msg_control = 0; - msg_send.msg_controllen = 0; - msg_send.msg_flags = 0; -#else - msg_send.msg_accrights = 0; - msg_send.msg_accrightslen = 0; -#endif - - if (queue_is_full (outq)) { - /* - * Start a disconnect if we have not already started one - * and report that the outgoing queue is full - */ - log_printf (LOG_LEVEL_ERROR, "Library queue is full, disconnecting library connection.\n"); - libais_disconnect_request (conn_info); - return (-1); - } - while (!queue_is_empty (outq)) { - queue_item = queue_item_get (outq); - msg_addr = (char *)queue_item->msg; - msg_addr = &msg_addr[conn_info->byte_start]; - - iov_send.iov_base = msg_addr; - iov_send.iov_len = queue_item->mlen - conn_info->byte_start; - -retry_sendmsg: - res = sendmsg (conn_info->fd, &msg_send, MSG_NOSIGNAL); - if (res == -1 && errno == EINTR) { - goto retry_sendmsg; - } - if (res == -1 && errno == EAGAIN) { - break; /* outgoing kernel queue full */ - } - if (res == -1 && errno == EPIPE) { - libais_disconnect_request (conn_info); - return (0); - } - if (res == -1) { - assert (0); - break; /* some other error, stop trying to send message */ - } - if (res + conn_info->byte_start != queue_item->mlen) { - conn_info->byte_start += res; - break; - } - - /* - * Message sent, try sending another message - */ - queue_item_remove (outq); - conn_info->byte_start = 0; - free (queue_item->msg); - } /* while queue not empty */ - - res = -1; - - queue_empty = queue_is_empty (outq); - /* - * Send request message - */ - if (queue_empty) { - - iov_send.iov_base = msg; - iov_send.iov_len = mlen; -retry_sendmsg_two: - res = sendmsg (conn_info->fd, &msg_send, MSG_NOSIGNAL); - if (res == -1 && errno == EINTR) { - goto retry_sendmsg_two; - } - if (res == -1 && errno == EAGAIN) { - conn_info->byte_start = 0; - conn_info->events = POLLIN|POLLNVAL; - } - if (res != -1) { - if (res != mlen) { - conn_info->byte_start += res; - res = -1; - } else { - conn_info->byte_start = 0; - conn_info->events = POLLIN|POLLNVAL; - } - } - } - - /* - * If res == -1 , errrno == EAGAIN which means kernel queue full - */ - if (res == -1) { - cmsg = malloc (mlen); - if (cmsg == 0) { - log_printf (LOG_LEVEL_ERROR, "Library queue couldn't allocate a message, disconnecting library connection.\n"); - libais_disconnect_request (conn_info); - return (-1); - } - queue_item_out.msg = cmsg; - queue_item_out.mlen = mlen; - memcpy (cmsg, msg, mlen); - queue_item_add (outq, &queue_item_out); - - /* - * Send a pthread_kill to interrupt the poll syscall - * and start a new poll operation in the thread - */ - conn_info->events = POLLIN|POLLOUT|POLLNVAL; - pthread_kill (conn_info->thread, SIGUSR1); +retry_semop: + res = semop (conn_info->semid, &sop, 1); + if ((res == -1) && (errno == EINTR || errno == EAGAIN)) { + goto retry_semop; + } else + if ((res == -1) && (errno == EINVAL || errno == EIDRM)) { + return (0); } return (0); } -void cs_ipc_flow_control_create ( - void *conn, - unsigned int service, - char *id, - int id_len, - void (*flow_control_state_set_fn) (void *conn, enum cs_flow_control_state), - void *context) +int cs_response_iov_send (void *conn, struct iovec *iov, int iov_len) +{ + struct conn_info *conn_info = (struct conn_info *)conn; + struct sembuf sop; + int res; + int write_idx = 0; + int i; + + for (i = 0; i < iov_len; i++) { + memcpy (&conn_info->mem->res_buffer[write_idx], iov[i].iov_base, iov[i].iov_len); + write_idx += iov[i].iov_len; + } + + sop.sem_num = 1; + sop.sem_op = 1; + sop.sem_flg = 0; + +retry_semop: + res = semop (conn_info->semid, &sop, 1); + if ((res == -1) && (errno == EINTR || errno == EAGAIN)) { + goto retry_semop; + } else + if ((res == -1) && (errno == EINVAL || errno == EIDRM)) { + return (0); + } + return (0); +} + +static int shared_mem_dispatch_bytes_left (struct conn_info *conn_info) +{ + unsigned int read; + unsigned int write; + unsigned int bytes_left; + + read = conn_info->mem->read; + write = conn_info->mem->write; + + if (read <= write) { + bytes_left = DISPATCH_SIZE - write + read; + } else { + bytes_left = read - write; + } + return (bytes_left); +} + +int memcpy_dwrap (struct conn_info *conn_info, void *msg, int len) +{ + char *dest_char = (char *)conn_info->mem->dispatch_buffer; + char *src_char = (char *)msg; + unsigned int first_write; + unsigned int second_write; + + first_write = len; + second_write = 0; + if (len + conn_info->mem->write >= DISPATCH_SIZE) { + first_write = DISPATCH_SIZE - conn_info->mem->write; + second_write = len - first_write; + } + memcpy (&dest_char[conn_info->mem->write], src_char, first_write); + if (second_write) { + memcpy (dest_char, &src_char[first_write], second_write); + } + conn_info->mem->write = (conn_info->mem->write + len) % DISPATCH_SIZE; + return (0); +} + +void msg_send (void *conn, struct iovec *iov, int iov_len, int locked) +{ + struct conn_info *conn_info = (struct conn_info *)conn; + struct sembuf sop; + int res; + int i; + char buf; + + for (i = 0; i < iov_len; i++) { + memcpy_dwrap (conn_info, iov[i].iov_base, iov[i].iov_len); + } + + buf = !list_empty (&conn_info->outq_head); + res = send (conn_info->fd, &buf, 1, MSG_NOSIGNAL); + if (res == -1 && errno == EAGAIN) { + if (locked == 0) { + pthread_mutex_lock (&conn_info->mutex); + } + conn_info->pending_semops += 1; + if (locked == 0) { + pthread_mutex_unlock (&conn_info->mutex); + } + poll_dispatch_modify (aisexec_poll_handle, conn_info->fd, + POLLIN|POLLOUT|POLLNVAL, poll_handler_connection); + } else + if (res == -1) { + ipc_disconnect (conn_info); + } + sop.sem_num = 2; + sop.sem_op = 1; + sop.sem_flg = 0; + +retry_semop: + res = semop (conn_info->semid, &sop, 1); + if ((res == -1) && (errno == EINTR || errno == EAGAIN)) { + goto retry_semop; + } else + if ((res == -1) && (errno == EINVAL || errno == EIDRM)) { + return; + } +} + +static void outq_flush (struct conn_info *conn_info) { + struct list_head *list, *list_next; + struct outq_item *outq_item; + unsigned int bytes_left; + struct iovec iov; + char buf; + int res; + + pthread_mutex_lock (&conn_info->mutex); + if (list_empty (&conn_info->outq_head)) { + buf = 3; + res = send (conn_info->fd, &buf, 1, MSG_NOSIGNAL); + pthread_mutex_unlock (&conn_info->mutex); + return; + } + for (list = conn_info->outq_head.next; + list != &conn_info->outq_head; list = list_next) { + + list_next = list->next; + outq_item = list_entry (list, struct outq_item, list); + bytes_left = shared_mem_dispatch_bytes_left (conn_info); + if (bytes_left > outq_item->mlen) { + iov.iov_base = outq_item->msg; + iov.iov_len = outq_item->mlen; + msg_send (conn_info, &iov, 1, MSG_SEND_UNLOCKED); + list_del (list); + free (iov.iov_base); + free (outq_item); + } else { + break; + } + } + pthread_mutex_unlock (&conn_info->mutex); +} + +static int priv_change (struct conn_info *conn_info) +{ + mar_req_priv_change req_priv_change; + unsigned int res; + union semun semun; + struct semid_ds ipc_set; + int i; + +retry_recv: + res = recv (conn_info->fd, &req_priv_change, + sizeof (mar_req_priv_change), + MSG_NOSIGNAL); + if (res == -1 && errno == EINTR) { + goto retry_recv; + } + if (res == -1 && errno == EAGAIN) { + goto retry_recv; + } + if (res == -1 && errno != EAGAIN) { + return (-1); + } +#if defined(CS_SOLARIS) || defined(CS_BSD) || defined(CS_DARWIN) + /* Error on socket, EOF is detected when recv return 0 + */ + if (res == 0) { + return (-1); + } +#endif + + ipc_set.sem_perm.uid = req_priv_change.euid; + ipc_set.sem_perm.gid = req_priv_change.egid; + ipc_set.sem_perm.mode = 0600; + + semun.buf = &ipc_set; + + for (i = 0; i < 3; i++) { + res = semctl (conn_info->semid, 0, IPC_SET, semun); + if (res == -1) { + return (-1); + } + } + return (0); +} + +static void msg_send_or_queue (void *conn, struct iovec *iov, int iov_len) +{ + struct conn_info *conn_info = (struct conn_info *)conn; + unsigned int bytes_left; + unsigned int bytes_msg = 0; + int i; + struct outq_item *outq_item; + char *write_buf = 0; + + /* + * Exit transmission if the connection is dead + */ + pthread_mutex_lock (&conn_info->mutex); + if (conn_info->destroyed || conn_info->disconnect_requested) { + pthread_mutex_unlock (&conn_info->mutex); + return; + } + pthread_mutex_unlock (&conn_info->mutex); + + bytes_left = shared_mem_dispatch_bytes_left (conn_info); + for (i = 0; i < iov_len; i++) { + bytes_msg += iov[i].iov_len; + } + if (bytes_left < bytes_msg || list_empty (&conn_info->outq_head) == 0) { + outq_item = malloc (sizeof (struct outq_item)); + if (outq_item == NULL) { + ipc_disconnect (conn); + return; + } + outq_item->msg = malloc (bytes_msg); + if (outq_item->msg == 0) { + free (outq_item); + ipc_disconnect (conn); + return; + } + + write_buf = outq_item->msg; + for (i = 0; i < iov_len; i++) { + memcpy (write_buf, iov[i].iov_base, iov[i].iov_len); + write_buf += iov[i].iov_len; + } + outq_item->mlen = bytes_msg; + list_init (&outq_item->list); + pthread_mutex_lock (&conn_info->mutex); + if (list_empty (&conn_info->outq_head)) { + conn_info->notify_flow_control_enabled = 1; + poll_dispatch_modify (aisexec_poll_handle, + conn_info->fd, POLLOUT|POLLIN|POLLNVAL, + poll_handler_connection); + } + list_add_tail (&outq_item->list, &conn_info->outq_head); + pthread_mutex_unlock (&conn_info->mutex); + return; + } + msg_send (conn, iov, iov_len, MSG_SEND_LOCKED); +} + +void cs_conn_refcount_inc (void *conn) { struct conn_info *conn_info = (struct conn_info *)conn; - cs_flow_control_create ( - conn_info->flow_control_handle, - service, - id, - id_len, - flow_control_state_set_fn, - context); - conn_info->conn_info_partner->flow_control_handle = conn_info->flow_control_handle; + pthread_mutex_lock (&conn_info->mutex); + conn_info->refcount++; + pthread_mutex_unlock (&conn_info->mutex); } -void cs_ipc_flow_control_destroy ( - void *conn, - unsigned int service, - unsigned char *id, - int id_len) +void cs_conn_refcount_dec (void *conn) { struct conn_info *conn_info = (struct conn_info *)conn; - cs_flow_control_destroy ( - conn_info->flow_control_handle, - service, - id, - id_len); + pthread_mutex_lock (&conn_info->mutex); + conn_info->refcount--; + pthread_mutex_unlock (&conn_info->mutex); } -void cs_ipc_flow_control_local_increment ( - void *conn) +int cs_dispatch_send (void *conn, void *msg, int mlen) { - struct conn_info *conn_info = (struct conn_info *)conn; + struct iovec iov; - pthread_mutex_lock (&conn_info->flow_control_mutex); + iov.iov_base = msg; + iov.iov_len = mlen; - conn_info->flow_control_local_count++; - - pthread_mutex_unlock (&conn_info->flow_control_mutex); + msg_send_or_queue (conn, &iov, 1); + return (0); } -void cs_ipc_flow_control_local_decrement ( - void *conn) +int cs_dispatch_iov_send (void *conn, struct iovec *iov, int iov_len) { - struct conn_info *conn_info = (struct conn_info *)conn; - - pthread_mutex_lock (&conn_info->flow_control_mutex); - - conn_info->flow_control_local_count--; - - pthread_mutex_unlock (&conn_info->flow_control_mutex); + msg_send_or_queue (conn, iov, iov_len); + return (0); } diff --git a/exec/ipc.h b/exec/ipc.h index 85edc204..0a231ec2 100644 --- a/exec/ipc.h +++ b/exec/ipc.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2006-2007 Red Hat, Inc. + * Copyright (c) 2006-2009 Red Hat, Inc. * * All rights reserved. * @@ -35,63 +35,29 @@ #ifndef IPC_H_DEFINED #define IPC_H_DEFINED -#include "flow.h" - -#ifndef TIMER_HANDLE -typedef void * timer_handle; -#define TIMER_HANDLE -#endif - extern void message_source_set (mar_message_source_t *source, void *conn); extern int message_source_is_local (mar_message_source_t *source); -extern void *cs_conn_partner_get (void *conn); +extern void cs_ipc_init ( + void (*serialize_lock_fn) (void), + void (*serialize_unlock_fn) (void), + unsigned int gid_valid); extern void *cs_conn_private_data_get (void *conn); -extern int cs_conn_send_response (void *conn, void *msg, int mlen); +extern int cs_response_send (void *conn, void *msg, int mlen); -extern int cs_conn_send_response_no_fcc (void *conn, void *msg,int mlen); +extern int cs_response_iov_send (void *conn, struct iovec *iov, int iov_len); -extern void cs_ipc_init ( - void (*serialize_lock_fn) (void), - void (*serialize_unlock_fn) (void), - unsigned int gid_valid); +extern int cs_dispatch_send (void *conn, void *msg, int mlen); -extern int cs_ipc_timer_add ( - void *conn, - void (*timer_fn) (void *data), - void *data, - unsigned int msec_in_future, - timer_handle *handle); +extern int cs_dispatch_iov_send (void *conn, struct iovec *iov, int iov_len); -extern void cs_ipc_timer_del ( - void *conn, - timer_handle timer_handle); +extern void cs_conn_refcount_inc (void *conn); -extern void cs_ipc_timer_del_data ( - void *conn, - timer_handle timer_handle); +extern void cs_conn_refcount_dec (void *conn); -extern void cs_ipc_flow_control_create ( - void *conn, - unsigned int service, - char *id, - int id_len, - void (*flow_control_state_set_fn) (void *context, enum cs_flow_control_state flow_control_state_set), - void *context); - -extern void cs_ipc_flow_control_destroy ( - void *conn, - unsigned int service, - unsigned char *id, - int id_len); - -extern void cs_ipc_flow_control_local_increment ( - void *conn); - -extern void cs_ipc_flow_control_local_decrement ( - void *conn); +extern void cs_ipc_exit (void); #endif /* IPC_H_DEFINED */ diff --git a/exec/main.c b/exec/main.c index 00ced1cd..8fe095e5 100644 --- a/exec/main.c +++ b/exec/main.c @@ -72,11 +72,9 @@ #include "main.h" #include "sync.h" #include "tlist.h" -#include "flow.h" #include "ipc.h" #include "timer.h" #include "util.h" -#include "flow.h" #include "apidef.h" #include "service.h" #include "version.h" @@ -141,8 +139,9 @@ static void *aisexec_exit (void *arg) } #endif + poll_stop (0); totempg_finalize (); - + cs_ipc_exit (); corosync_exit_error (AIS_DONE_EXIT); /* never reached */ @@ -303,10 +302,11 @@ static void aisexec_tty_detach (void) /* * child which is disconnected, run this process */ -/* setset(); */ +/* setset(); close (0); close (1); close (2); +*/ break; default: exit (0); @@ -672,8 +672,6 @@ int main (int argc, char **argv) sync_register (corosync_sync_callbacks_retrieve, corosync_sync_completed); - res = cs_flow_control_initialize (); - /* * Drop root privleges to user 'ais' * TODO: Don't really need full root capabilities; @@ -686,10 +684,7 @@ int main (int argc, char **argv) aisexec_mempool_init (); - cs_ipc_init ( - serialize_mutex_lock, - serialize_mutex_unlock, - main_config.gid); + cs_ipc_init (serialize_mutex_lock, serialize_mutex_unlock, main_config.gid); /* * Start main processing loop diff --git a/exec/vsf_quorum.c b/exec/vsf_quorum.c index ab0972b5..3ba6c66e 100644 --- a/exec/vsf_quorum.c +++ b/exec/vsf_quorum.c @@ -375,7 +375,7 @@ static void send_library_notification(void *conn) /* Send it to all interested parties */ if (conn) { - corosync_api->ipc_conn_send_response(conn, res_lib_quorum_notification, size); + corosync_api->ipc_response_send(conn, res_lib_quorum_notification, size); } else { struct quorum_pd *qpd; @@ -384,8 +384,8 @@ static void send_library_notification(void *conn) qpd = list_entry(tmp, struct quorum_pd, list); - corosync_api->ipc_conn_send_response(corosync_api->ipc_conn_partner_get(qpd->conn), - res_lib_quorum_notification, size); + corosync_api->ipc_dispatch_send(qpd->conn, + res_lib_quorum_notification, size); } } return; @@ -402,7 +402,7 @@ static void message_handler_req_lib_quorum_getquorate (void *conn, void *msg) res_lib_quorum_getquorate.header.size = sizeof(res_lib_quorum_getquorate); res_lib_quorum_getquorate.header.id = MESSAGE_RES_QUORUM_GETQUORATE; res_lib_quorum_getquorate.header.error = CS_OK; - corosync_api->ipc_conn_send_response(conn, &res_lib_quorum_getquorate, sizeof(res_lib_quorum_getquorate)); + corosync_api->ipc_response_send(conn, &res_lib_quorum_getquorate, sizeof(res_lib_quorum_getquorate)); } @@ -421,7 +421,7 @@ static void message_handler_req_lib_quorum_trackstart (void *conn, void *msg) if (req_lib_quorum_trackstart->track_flags & CS_TRACK_CURRENT || req_lib_quorum_trackstart->track_flags & CS_TRACK_CHANGES) { log_printf(LOG_LEVEL_DEBUG, "sending initial status to %p\n", conn); - send_library_notification(corosync_api->ipc_conn_partner_get (conn)); + send_library_notification(conn); } /* @@ -440,7 +440,7 @@ static void message_handler_req_lib_quorum_trackstart (void *conn, void *msg) res.size = sizeof(res); res.id = MESSAGE_RES_QUORUM_TRACKSTART; res.error = CS_OK; - corosync_api->ipc_conn_send_response(conn, &res, sizeof(mar_res_header_t)); + corosync_api->ipc_response_send(conn, &res, sizeof(mar_res_header_t)); } static void message_handler_req_lib_quorum_trackstop (void *conn, void *msg) @@ -463,6 +463,6 @@ static void message_handler_req_lib_quorum_trackstop (void *conn, void *msg) res.size = sizeof(res); res.id = MESSAGE_RES_QUORUM_TRACKSTOP; res.error = CS_OK; - corosync_api->ipc_conn_send_response(conn, &res, sizeof(mar_res_header_t)); + corosync_api->ipc_response_send(conn, &res, sizeof(mar_res_header_t)); } diff --git a/include/corosync/ais_util.h b/include/corosync/coroipc.h similarity index 75% rename from include/corosync/ais_util.h rename to include/corosync/coroipc.h index 18c87058..4d4229b7 100644 --- a/include/corosync/ais_util.h +++ b/include/corosync/coroipc.h @@ -1,5 +1,6 @@ /* - * Copyright (c) 2002-2005 MontaVista Software, Inc. + * Copyright (c) 2002-2003 MontaVista Software, Inc. + * Copyright (c) 2006-2007 Red Hat, Inc. * * All rights reserved. * @@ -31,13 +32,14 @@ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef AIS_UTIL_H_DEFINED -#define AIS_UTIL_H_DEFINED + +#ifndef COROIPC_H_DEFINED +#define COROIPC_H_DEFINED #include #include #include - +#include #include /* Debug macro @@ -65,36 +67,44 @@ struct saHandleDatabase { }; -struct saVersionDatabase { - int versionCount; - cs_version_t *versionsSupported; -}; - -cs_error_t saSendMsgRetry ( - int s, - struct iovec *iov, - int iov_len); - -cs_error_t saSendMsgReceiveReply ( - int s, - struct iovec *iov, - int iov_len, - void *responseMessage, - int responseLen); - -cs_error_t saSendReceiveReply ( - int s, - void *requestMessage, - int requestLen, - void *responseMessage, - int responseLen); +cs_error_t +cslib_service_connect ( + enum service_types service, + void **ipc_context); cs_error_t -saPollRetry ( - struct pollfd *ufds, - unsigned int nfds, +cslib_service_disconnect ( + void *ipc_context); + +int +cslib_fd_get ( + void *ipc_context); + +int +cslib_dispatch_recv ( + void *ipc_context, + void *buf, int timeout); +int +cslib_dispatch_flow_control_get ( + void *ipc_context); + +cs_error_t +cslib_msg_send_reply_receive ( + void *ipc_context, + struct iovec *iov, + int iov_len, + void *res_msg, + int res_len); + +cs_error_t +cslib_msg_send_reply_receive_in_buf ( + void *ipc_context, + struct iovec *iov, + int iov_len, + void **res_msg); + cs_error_t saHandleCreate ( struct saHandleDatabase *handleDatabase, @@ -117,21 +127,7 @@ saHandleInstancePut ( struct saHandleDatabase *handleDatabase, uint64_t handle); -cs_error_t -saVersionVerify ( - struct saVersionDatabase *versionDatabase, - cs_version_t *version); - #define offset_of(type,member) (int)(&(((type *)0)->member)) -cs_time_t -clustTimeNow(void); +#endif /* COROIPC_H_DEFINED */ -extern cs_error_t saServiceConnect ( - int *responseOut, int *callbackOut, enum service_types service); - -extern cs_error_t saRecvRetry (int s, void *msg, size_t len); - -extern cs_error_t saSendRetry (int s, const void *msg, size_t len); - -#endif /* AIS_UTIL_H_DEFINED */ diff --git a/include/corosync/engine/coroapi.h b/include/corosync/engine/coroapi.h index 5f0b7848..d4f3bf4d 100644 --- a/include/corosync/engine/coroapi.h +++ b/include/corosync/engine/coroapi.h @@ -390,35 +390,11 @@ struct corosync_api_v1 { int (*ipc_response_send) (void *conn, void *msg, int mlen); - int (*ipc_response_no_fcc) (void *conn, void *msg, int mlen); + int (*ipc_response_iov_send) (void *conn, struct iovec *iov, int iov_len); int (*ipc_dispatch_send) (void *conn, void *msg, int mlen); - /* - * DEPRECATED - */ - int (*ipc_conn_send_response) (void *conn, void *msg, int mlen); - - /* - * DEPRECATED - */ - void *(*ipc_conn_partner_get) (void *conn); - - void (*ipc_fc_create) ( - void *conn, - unsigned int service, - char *id, - int id_len, - void (*flow_control_state_set_fn) - (void *context, - enum cs_flow_control_state flow_control_state_set), - void *context); - - void (*ipc_fc_destroy) ( - void *conn, - unsigned int service, - unsigned char *id, - int id_len); + int (*ipc_dispatch_iov_send) (void *conn, struct iovec *iov, int iov_len); void (*ipc_refcnt_inc) (void *conn); diff --git a/include/corosync/ipc_cfg.h b/include/corosync/ipc_cfg.h index 06125e2b..0dfbb9f9 100644 --- a/include/corosync/ipc_cfg.h +++ b/include/corosync/ipc_cfg.h @@ -171,10 +171,14 @@ struct res_lib_cfg_tryshutdown { }; struct req_lib_cfg_replytoshutdown { - mar_res_header_t header __attribute__((aligned(8))); + mar_req_header_t header __attribute__((aligned(8))); unsigned int response; }; +struct res_lib_cfg_replytoshutdown { + mar_res_header_t header __attribute__((aligned(8))); +}; + struct res_lib_cfg_testshutdown { mar_res_header_t header __attribute__((aligned(8))); unsigned int flags; diff --git a/include/corosync/ipc_cpg.h b/include/corosync/ipc_cpg.h index 6e898c46..4f175e92 100644 --- a/include/corosync/ipc_cpg.h +++ b/include/corosync/ipc_cpg.h @@ -122,7 +122,6 @@ struct req_lib_cpg_mcast { struct res_lib_cpg_mcast { mar_res_header_t header __attribute__((aligned(8))); - mar_uint32_t flow_control_state __attribute__((aligned(8))); }; /* Message from another node */ @@ -132,7 +131,6 @@ struct res_lib_cpg_deliver_callback { mar_uint32_t msglen __attribute__((aligned(8))); mar_uint32_t nodeid __attribute__((aligned(8))); mar_uint32_t pid __attribute__((aligned(8))); - mar_uint32_t flow_control_state __attribute__((aligned(8))); mar_uint8_t message[] __attribute__((aligned(8))); }; diff --git a/include/corosync/ipc_gen.h b/include/corosync/ipc_gen.h index 718a66f0..3c41c1cc 100644 --- a/include/corosync/ipc_gen.h +++ b/include/corosync/ipc_gen.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2002-2005 MontaVista Software, Inc. + * Copyright (c) 2006-2009 Red Hat, Inc. * * All rights reserved. * @@ -57,6 +58,21 @@ enum req_init_types { MESSAGE_REQ_DISPATCH_INIT = 1 }; +#define MESSAGE_REQ_CHANGE_EUID 1 +#define MESSAGE_REQ_OUTQ_FLUSH 2 + +#define REQ_SIZE 1000000 +#define RES_SIZE 1000000 +#define DISPATCH_SIZE 1000000 + +struct shared_memory { + unsigned char req_buffer[REQ_SIZE]; + unsigned char res_buffer[RES_SIZE]; + unsigned char dispatch_buffer[DISPATCH_SIZE]; + unsigned int read; + unsigned int write; +}; + enum res_init_types { MESSAGE_RES_INIT }; @@ -66,6 +82,16 @@ typedef struct { int id __attribute__((aligned(8))); } mar_req_header_t __attribute__((aligned(8))); +typedef struct { + int service __attribute__((aligned(8))); + unsigned long long shmkey __attribute__((aligned(8))); + unsigned long long semkey __attribute__((aligned(8))); +} mar_req_setup_t __attribute__((aligned(8))); + +typedef struct { + int error __attribute__((aligned(8))); +} mar_res_setup_t __attribute__((aligned(8))); + static inline void swab_mar_req_header_t (mar_req_header_t *to_swab) { swab_mar_int32_t (&to_swab->size); @@ -79,19 +105,9 @@ typedef struct { } mar_res_header_t __attribute__((aligned(8))); typedef struct { - int size __attribute__((aligned(8))); - int id __attribute__((aligned(8))); - int service __attribute__((aligned(8))); -} mar_req_lib_resdis_init_t __attribute__((aligned(8))); - -typedef struct { - mar_req_lib_resdis_init_t resdis_header __attribute__((aligned(8))); -} mar_req_lib_response_init_t __attribute__((aligned(8))); - -typedef struct { - mar_req_lib_resdis_init_t resdis_header __attribute__((aligned(8))); - mar_uint64_t conn_info __attribute__((aligned(8))); -} mar_req_lib_dispatch_init_t __attribute__((aligned(8))); + uid_t euid __attribute__((aligned(8))); + gid_t egid __attribute__((aligned(8))); +} mar_req_priv_change __attribute__((aligned(8))); typedef struct { mar_res_header_t header __attribute__((aligned(8))); diff --git a/lib/Makefile b/lib/Makefile index ae748273..194f1655 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -43,87 +43,87 @@ all: libcpg.a libcpg.so.2.0.0 \ libcfg.a libcfg.so.2.0.0 \ libquorum.a libquorum.so.2.0.0 \ libpload.a libpload.so.2.0.0 \ - libcoroutil.a libcoroutil.so.2.0.0 \ + libcoroipc.a libcoroipc.so.2.0.0 \ libvotequorum.a libvotequorum.so.2.0.0 -libcoroutil.a: util.o - $(AR) -rc libcoroutil.a util.o +libcoroipc.a: coroipc.o + $(AR) -rc libcoroipc.a coroipc.o ifeq (${COROSYNC_COMPAT}, DARWIN) DARWIN_OPTS=-dynamiclib -bind_at_load -current_version 2.0.0 -compatibility_version 2.0.0 -libcoroutil.so.2.0.0: util.o - $(CC) $(LDFLAGS) $(DARWIN_OPTS) util.o -o $@ +libcoroipc.so.2.0.0: coroipc.o + $(CC) $(LDFLAGS) $(DARWIN_OPTS) coroipc.o -o $@ -libevs.so.2.0.0: util.o evs.o - $(CC) $(LDFLAGS) $(DARWIN_OPTS) util.o evs.o -o $@ +libevs.so.2.0.0: coroipc.o evs.o + $(CC) $(LDFLAGS) $(DARWIN_OPTS) coroipc.o evs.o -o $@ -libcpg.so.2.0.0: util.o cpg.o - $(CC) $(DARWIN_OPTS) util.o cpg.o -o $@ +libcpg.so.2.0.0: coroipc.o cpg.o + $(CC) $(DARWIN_OPTS) coroipc.o cpg.o -o $@ -libquorum.so.2.0.0: util.o quorum.o - $(CC) $(DARWIN_OPTS) util.o quorum.o -o $@ +libquorum.so.2.0.0: coroipc.o quorum.o + $(CC) $(DARWIN_OPTS) coroipc.o quorum.o -o $@ -libvotequorum.so.2.0.0: util.o votequorum.o - $(CC) $(DARWIN_OPTS) util.o votequorum.o -o $@ +libvotequorum.so.2.0.0: coroipc.o votequorum.o + $(CC) $(DARWIN_OPTS) coroipc.o votequorum.o -o $@ -libconfdb.so.2.0.0: util.o confdb.o sa-confdb.o - $(CC) $(LDFLAGS) $(DARWIN_OPTS) util.o confdb.o sa-confdb.o ../lcr/lcr_ifact.o -o $@ +libconfdb.so.2.0.0: coroipc.o confdb.o sa-confdb.o + $(CC) $(LDFLAGS) $(DARWIN_OPTS) coroipc.o confdb.o sa-confdb.o ../lcr/lcr_ifact.o -o $@ -libcfg.so.2.0.0: util.o cfg.o - $(CC) $(DARWIN_OPTS) util.o cfg.o -o $@ +libcfg.so.2.0.0: coroipc.o cfg.o + $(CC) $(DARWIN_OPTS) coroipc.o cfg.o -o $@ -libpload.so.2.0.0: util.o pload.o - $(CC) $(DARWIN_OPTS) util.o pload.o -o $@ +libpload.so.2.0.0: coroipc.o pload.o + $(CC) $(DARWIN_OPTS) coroipc.o pload.o -o $@ else -libcoroutil.so.2.0.0: util.o - $(CC) $(LDFLAGS) -shared -Wl,-soname,libcoroutil.so.2,-version-script=$(srcdir)$(subdir)libcoroutil.versions util.o -o $@ +libcoroipc.so.2.0.0: coroipc.o + $(CC) $(LDFLAGS) -shared -Wl,-soname,libcoroipc.so.2,-version-script=$(srcdir)$(subdir)libcoroipc.versions coroipc.o -o $@ -libevs.so.2.0.0: util.o evs.o - $(CC) $(LDFLAGS) -shared -Wl,-soname,libevs.so.2,-version-script=$(srcdir)$(subdir)libevs.versions util.o evs.o -o $@ +libevs.so.2.0.0: coroipc.o evs.o + $(CC) $(LDFLAGS) -shared -Wl,-soname,libevs.so.2,-version-script=$(srcdir)$(subdir)libevs.versions coroipc.o evs.o -o $@ -libcpg.so.2.0.0: util.o cpg.o - $(CC) -shared -Wl,-soname,libcpg.so.2,-version-script=$(srcdir)$(subdir)libcpg.versions util.o cpg.o -o $@ +libcpg.so.2.0.0: coroipc.o cpg.o + $(CC) -shared -Wl,-soname,libcpg.so.2,-version-script=$(srcdir)$(subdir)libcpg.versions coroipc.o cpg.o -o $@ -libquorum.so.2.0.0: util.o quorum.o - $(CC) -shared -Wl,-soname,libquorum.so.2,-version-script=$(srcdir)$(subdir)libquorum.versions util.o quorum.o -o $@ +libquorum.so.2.0.0: coroipc.o quorum.o + $(CC) -shared -Wl,-soname,libquorum.so.2,-version-script=$(srcdir)$(subdir)libquorum.versions coroipc.o quorum.o -o $@ -libvotequorum.so.2.0.0: util.o votequorum.o - $(CC) -shared -Wl,-soname,libvotequorum.so.2,-version-script=$(srcdir)$(subdir)libvotequorum.versions util.o votequorum.o -o $@ +libvotequorum.so.2.0.0: coroipc.o votequorum.o + $(CC) -shared -Wl,-soname,libvotequorum.so.2,-version-script=$(srcdir)$(subdir)libvotequorum.versions coroipc.o votequorum.o -o $@ -libconfdb.so.2.0.0: util.o confdb.o sa-confdb.o - $(CC) $(LDFLAGS) -shared -Wl,-soname,libconfdb.so.2,-version-script=$(srcdir)$(subdir)libconfdb.versions util.o confdb.o sa-confdb.o ../lcr/lcr_ifact.o -o $@ +libconfdb.so.2.0.0: coroipc.o confdb.o sa-confdb.o + $(CC) $(LDFLAGS) -shared -Wl,-soname,libconfdb.so.2,-version-script=$(srcdir)$(subdir)libconfdb.versions coroipc.o confdb.o sa-confdb.o ../lcr/lcr_ifact.o -o $@ -libcfg.so.2.0.0: util.o cfg.o - $(CC) -shared -Wl,-soname,libcfg.so.2,-version-script=$(srcdir)$(subdir)libcfg.versions util.o cfg.o -o $@ +libcfg.so.2.0.0: coroipc.o cfg.o + $(CC) -shared -Wl,-soname,libcfg.so.2,-version-script=$(srcdir)$(subdir)libcfg.versions coroipc.o cfg.o -o $@ -libpload.so.2.0.0: util.o pload.o - $(CC) -shared -Wl,-soname,libpload.so.2,-version-script=$(srcdir)$(subdir)libpload.versions util.o cfg.o -o $@ +libpload.so.2.0.0: coroipc.o pload.o + $(CC) -shared -Wl,-soname,libpload.so.2,-version-script=$(srcdir)$(subdir)libpload.versions coroipc.o cfg.o -o $@ endif -libevs.a: util.o evs.o - $(AR) -rc libevs.a util.o evs.o +libevs.a: coroipc.o evs.o + $(AR) -rc libevs.a coroipc.o evs.o -libcpg.a: util.o cpg.o - $(AR) -rc libcpg.a util.o cpg.o +libcpg.a: coroipc.o cpg.o + $(AR) -rc libcpg.a coroipc.o cpg.o -libquorum.a: util.o quorum.o - $(AR) -rc libquorum.a util.o quorum.o +libquorum.a: coroipc.o quorum.o + $(AR) -rc libquorum.a coroipc.o quorum.o -libvotequorum.a: util.o votequorum.o - $(AR) -rc libvotequorum.a util.o votequorum.o +libvotequorum.a: coroipc.o votequorum.o + $(AR) -rc libvotequorum.a coroipc.o votequorum.o -libconfdb.a: util.o confdb.o sa-confdb.o - $(AR) -rc libconfdb.a util.o confdb.o sa-confdb.o ../lcr/lcr_ifact.o +libconfdb.a: coroipc.o confdb.o sa-confdb.o + $(AR) -rc libconfdb.a coroipc.o confdb.o sa-confdb.o ../lcr/lcr_ifact.o -libcfg.a: util.o cfg.o - $(AR) -rc libcfg.a util.o cfg.o +libcfg.a: coroipc.o cfg.o + $(AR) -rc libcfg.a coroipc.o cfg.o -libpload.a: util.o pload.o - $(AR) -rc libpload.a util.o pload.o +libpload.a: coroipc.o pload.o + $(AR) -rc libpload.a coroipc.o pload.o clean: rm -f *.o *.a *.so* *.da *.bb *.bbg diff --git a/lib/cfg.c b/lib/cfg.c index a6c30aa6..5cbb4065 100644 --- a/lib/cfg.c +++ b/lib/cfg.c @@ -52,7 +52,7 @@ #include #include #include -#include +#include struct cfg_res_overlay { mar_res_header_t header; @@ -63,8 +63,7 @@ struct cfg_res_overlay { * Data structure for instance data */ struct cfg_instance { - int response_fd; - int dispatch_fd; + void *ipc_ctx; corosync_cfg_callbacks_t callbacks; cs_name_t comp_name; int comp_registered; @@ -114,12 +113,7 @@ corosync_cfg_initialize ( goto error_destroy; } - cfg_instance->response_fd = -1; - - cfg_instance->dispatch_fd = -1; - - error = saServiceConnect (&cfg_instance->response_fd, - &cfg_instance->dispatch_fd, CFG_SERVICE); + error = cslib_service_connect (CFG_SERVICE, &cfg_instance->ipc_ctx); if (error != CS_OK) { goto error_put_destroy; } @@ -157,7 +151,7 @@ corosync_cfg_fd_get ( return (error); } - *selection_fd = cfg_instance->dispatch_fd; + *selection_fd = cslib_fd_get (cfg_instance->ipc_ctx); (void)saHandleInstancePut (&cfg_hdb, cfg_handle); return (CS_OK); @@ -168,7 +162,6 @@ corosync_cfg_dispatch ( corosync_cfg_handle_t cfg_handle, cs_dispatch_flags_t dispatch_flags) { - struct pollfd ufds; int timeout = -1; cs_error_t error; int cont = 1; /* always continue do loop except when set to 0 */ @@ -199,24 +192,8 @@ corosync_cfg_dispatch ( } do { - /* - * Read data directly from socket - */ - ufds.fd = cfg_instance->dispatch_fd; - ufds.events = POLLIN; - ufds.revents = 0; - - error = saPollRetry (&ufds, 1, timeout); - if (error != CS_OK) { - goto error_nounlock; - } - - pthread_mutex_lock (&cfg_instance->dispatch_mutex); - - error = saPollRetry (&ufds, 1, 0); - if (error != CS_OK) { - goto error_nounlock; - } + dispatch_avail = cslib_dispatch_recv (cfg_instance->ipc_ctx, + (void *)&dispatch_data, timeout); /* * Handle has been finalized in another thread @@ -227,7 +204,6 @@ corosync_cfg_dispatch ( goto error_unlock; } - dispatch_avail = ufds.revents & POLLIN; if (dispatch_avail == 0 && dispatch_flags == CS_DISPATCH_ALL) { pthread_mutex_unlock (&cfg_instance->dispatch_mutex); break; /* exit do while cont is 1 loop */ @@ -237,27 +213,6 @@ corosync_cfg_dispatch ( continue; /* next poll */ } - if (ufds.revents & POLLIN) { - /* - * Queue empty, read response from socket - */ - error = saRecvRetry (cfg_instance->dispatch_fd, &dispatch_data.header, - sizeof (mar_res_header_t)); - if (error != CS_OK) { - goto error_unlock; - } - if (dispatch_data.header.size > sizeof (mar_res_header_t)) { - error = saRecvRetry (cfg_instance->dispatch_fd, &dispatch_data.data, - dispatch_data.header.size - sizeof (mar_res_header_t)); - if (error != CS_OK) { - goto error_unlock; - } - } - } else { - pthread_mutex_unlock (&cfg_instance->dispatch_mutex); - continue; - } - /* * Make copy of callbacks, message data, unlock instance, and call callback * A risk of this dispatch method is that the callback routines may @@ -330,6 +285,8 @@ corosync_cfg_finalize ( cfg_instance->finalize = 1; + cslib_service_disconnect (cfg_instance->ipc_ctx); + pthread_mutex_unlock (&cfg_instance->response_mutex); pthread_mutex_unlock (&cfg_instance->dispatch_mutex); @@ -338,16 +295,9 @@ corosync_cfg_finalize ( pthread_mutex_destroy (&cfg_instance->dispatch_mutex); - (void)saHandleDestroy (&cfg_hdb, cfg_handle); + cslib_service_disconnect (&cfg_instance->ipc_ctx); - if (cfg_instance->response_fd != -1) { - shutdown (cfg_instance->response_fd, 0); - close (cfg_instance->response_fd); - } - if (cfg_instance->dispatch_fd != -1) { - shutdown (cfg_instance->dispatch_fd, 0); - close (cfg_instance->dispatch_fd); - } + (void)saHandleDestroy (&cfg_hdb, cfg_handle); (void)saHandleInstancePut (&cfg_hdb, cfg_handle); @@ -366,6 +316,7 @@ corosync_cfg_ring_status_get ( struct res_lib_cfg_ringstatusget res_lib_cfg_ringstatusget; unsigned int i; cs_error_t error; + struct iovec iov; error = saHandleInstanceGet (&cfg_hdb, cfg_handle, (void *)&cfg_instance); if (error != CS_OK) { @@ -375,11 +326,14 @@ corosync_cfg_ring_status_get ( req_lib_cfg_ringstatusget.header.size = sizeof (struct req_lib_cfg_ringstatusget); req_lib_cfg_ringstatusget.header.id = MESSAGE_REQ_CFG_RINGSTATUSGET; + iov.iov_base = &req_lib_cfg_ringstatusget, + iov.iov_len = sizeof (struct req_lib_cfg_ringstatusget), + pthread_mutex_lock (&cfg_instance->response_mutex); - error = saSendReceiveReply (cfg_instance->response_fd, - &req_lib_cfg_ringstatusget, - sizeof (struct req_lib_cfg_ringstatusget), + error = cslib_msg_send_reply_receive(cfg_instance->ipc_ctx, + &iov, + 1, &res_lib_cfg_ringstatusget, sizeof (struct res_lib_cfg_ringstatusget)); @@ -442,6 +396,7 @@ corosync_cfg_ring_reenable ( struct req_lib_cfg_ringreenable req_lib_cfg_ringreenable; struct res_lib_cfg_ringreenable res_lib_cfg_ringreenable; cs_error_t error; + struct iovec iov; error = saHandleInstanceGet (&cfg_hdb, cfg_handle, (void *)&cfg_instance); if (error != CS_OK) { @@ -451,11 +406,14 @@ corosync_cfg_ring_reenable ( req_lib_cfg_ringreenable.header.size = sizeof (struct req_lib_cfg_ringreenable); req_lib_cfg_ringreenable.header.id = MESSAGE_REQ_CFG_RINGREENABLE; + iov.iov_base = &req_lib_cfg_ringreenable, + iov.iov_len = sizeof (struct req_lib_cfg_ringreenable); + pthread_mutex_lock (&cfg_instance->response_mutex); - error = saSendReceiveReply (cfg_instance->response_fd, - &req_lib_cfg_ringreenable, - sizeof (struct req_lib_cfg_ringreenable), + error = cslib_msg_send_reply_receive (cfg_instance->ipc_ctx, + &iov, + 1, &res_lib_cfg_ringreenable, sizeof (struct res_lib_cfg_ringreenable)); @@ -475,6 +433,7 @@ corosync_cfg_service_load ( struct req_lib_cfg_serviceload req_lib_cfg_serviceload; struct res_lib_cfg_serviceload res_lib_cfg_serviceload; cs_error_t error; + struct iovec iov; error = saHandleInstanceGet (&cfg_hdb, cfg_handle, (void *)&cfg_instance); if (error != CS_OK) { @@ -489,11 +448,14 @@ corosync_cfg_service_load ( sizeof (req_lib_cfg_serviceload.service_name) - 1); req_lib_cfg_serviceload.service_ver = service_ver; + iov.iov_base = &req_lib_cfg_serviceload; + iov.iov_len = sizeof (req_lib_cfg_serviceload); + pthread_mutex_lock (&cfg_instance->response_mutex); - error = saSendReceiveReply (cfg_instance->response_fd, - &req_lib_cfg_serviceload, - sizeof (struct req_lib_cfg_serviceload), + error = cslib_msg_send_reply_receive (cfg_instance->ipc_ctx, + &iov, + 1, &res_lib_cfg_serviceload, sizeof (struct res_lib_cfg_serviceload)); @@ -513,6 +475,7 @@ corosync_cfg_service_unload ( struct req_lib_cfg_serviceunload req_lib_cfg_serviceunload; struct res_lib_cfg_serviceunload res_lib_cfg_serviceunload; cs_error_t error; + struct iovec iov; error = saHandleInstanceGet (&cfg_hdb, cfg_handle, (void *)&cfg_instance); if (error != CS_OK) { @@ -527,11 +490,14 @@ corosync_cfg_service_unload ( sizeof (req_lib_cfg_serviceunload.service_name) - 1); req_lib_cfg_serviceunload.service_ver = service_ver; + iov.iov_base = &req_lib_cfg_serviceunload; + iov.iov_len = sizeof (req_lib_cfg_serviceunload); + pthread_mutex_lock (&cfg_instance->response_mutex); - error = saSendReceiveReply (cfg_instance->response_fd, - &req_lib_cfg_serviceunload, - sizeof (struct req_lib_cfg_serviceunload), + error = cslib_msg_send_reply_receive (cfg_instance->ipc_ctx, + &iov, + 1, &res_lib_cfg_serviceunload, sizeof (struct res_lib_cfg_serviceunload)); @@ -550,6 +516,7 @@ corosync_cfg_state_track ( struct req_lib_cfg_statetrack req_lib_cfg_statetrack; struct res_lib_cfg_statetrack res_lib_cfg_statetrack; cs_error_t error; + struct iovec iov; req_lib_cfg_statetrack.header.size = sizeof (struct req_lib_cfg_statetrack); req_lib_cfg_statetrack.header.id = MESSAGE_REQ_CFG_STATETRACKSTART; @@ -562,11 +529,14 @@ corosync_cfg_state_track ( return (error); } + iov.iov_base = &req_lib_cfg_statetrack, + iov.iov_len = sizeof (struct req_lib_cfg_statetrack), + pthread_mutex_lock (&cfg_instance->response_mutex); - error = saSendReceiveReply (cfg_instance->response_fd, - &req_lib_cfg_statetrack, - sizeof (struct req_lib_cfg_statetrack), + error = cslib_msg_send_reply_receive (cfg_instance->ipc_ctx, + &iov, + 1, &res_lib_cfg_statetrack, sizeof (struct res_lib_cfg_statetrack)); @@ -585,6 +555,7 @@ corosync_cfg_state_track_stop ( struct req_lib_cfg_statetrackstop req_lib_cfg_statetrackstop; struct res_lib_cfg_statetrackstop res_lib_cfg_statetrackstop; cs_error_t error; + struct iovec iov; error = saHandleInstanceGet (&cfg_hdb, cfg_handle, (void *)&cfg_instance); @@ -595,11 +566,13 @@ corosync_cfg_state_track_stop ( req_lib_cfg_statetrackstop.header.size = sizeof (struct req_lib_cfg_statetrackstop); req_lib_cfg_statetrackstop.header.id = MESSAGE_REQ_CFG_STATETRACKSTOP; + iov.iov_base = &req_lib_cfg_statetrackstop, + iov.iov_len = sizeof (struct req_lib_cfg_statetrackstop), pthread_mutex_lock (&cfg_instance->response_mutex); - error = saSendReceiveReply (cfg_instance->response_fd, - &req_lib_cfg_statetrackstop, - sizeof (struct req_lib_cfg_statetrackstop), + error = cslib_msg_send_reply_receive (cfg_instance->ipc_ctx, + &iov, + 1, &res_lib_cfg_statetrackstop, sizeof (struct res_lib_cfg_statetrackstop)); @@ -620,6 +593,7 @@ corosync_cfg_admin_state_get ( struct req_lib_cfg_administrativestateget req_lib_cfg_administrativestateget; struct res_lib_cfg_administrativestateget res_lib_cfg_administrativestateget; cs_error_t error; + struct iovec iov; error = saHandleInstanceGet (&cfg_hdb, cfg_handle, (void *)&cfg_instance); @@ -633,9 +607,14 @@ corosync_cfg_admin_state_get ( pthread_mutex_lock (&cfg_instance->response_mutex); - error = saSendReceiveReply (cfg_instance->response_fd, - &req_lib_cfg_administrativestateget, - sizeof (struct req_lib_cfg_administrativestateget), + iov.iov_base = &req_lib_cfg_administrativestateget, + iov.iov_len = sizeof (struct req_lib_cfg_administrativestateget), + + pthread_mutex_lock (&cfg_instance->response_mutex); + + error = cslib_msg_send_reply_receive (cfg_instance->ipc_ctx, + &iov, + 1, &res_lib_cfg_administrativestateget, sizeof (struct res_lib_cfg_administrativestateget)); @@ -658,6 +637,7 @@ corosync_cfg_admin_state_set ( struct req_lib_cfg_administrativestateset req_lib_cfg_administrativestateset; struct res_lib_cfg_administrativestateset res_lib_cfg_administrativestateset; cs_error_t error; + struct iovec iov; error = saHandleInstanceGet (&cfg_hdb, cfg_handle, (void *)&cfg_instance); @@ -672,9 +652,14 @@ corosync_cfg_admin_state_set ( pthread_mutex_lock (&cfg_instance->response_mutex); - error = saSendReceiveReply (cfg_instance->response_fd, - &req_lib_cfg_administrativestateset, - sizeof (struct req_lib_cfg_administrativestateset), + iov.iov_base = &req_lib_cfg_administrativestateset, + iov.iov_len = sizeof (struct req_lib_cfg_administrativestateset), + + pthread_mutex_lock (&cfg_instance->response_mutex); + + error = cslib_msg_send_reply_receive (cfg_instance->ipc_ctx, + &iov, + 1, &res_lib_cfg_administrativestateset, sizeof (struct res_lib_cfg_administrativestateset)); @@ -697,6 +682,7 @@ corosync_cfg_kill_node ( struct req_lib_cfg_killnode req_lib_cfg_killnode; struct res_lib_cfg_killnode res_lib_cfg_killnode; cs_error_t error; + struct iovec iov; if (strlen(reason) >= CS_MAX_NAME_LENGTH) return CS_ERR_NAME_TOO_LONG; @@ -713,11 +699,14 @@ corosync_cfg_kill_node ( strcpy((char *)req_lib_cfg_killnode.reason.value, reason); req_lib_cfg_killnode.reason.length = strlen(reason)+1; + iov.iov_base = &req_lib_cfg_killnode; + iov.iov_len = sizeof (struct req_lib_cfg_killnode); + pthread_mutex_lock (&cfg_instance->response_mutex); - error = saSendReceiveReply (cfg_instance->response_fd, - &req_lib_cfg_killnode, - sizeof (struct req_lib_cfg_killnode), + error = cslib_msg_send_reply_receive (cfg_instance->ipc_ctx, + &iov, + 1, &res_lib_cfg_killnode, sizeof (struct res_lib_cfg_killnode)); @@ -739,6 +728,7 @@ corosync_cfg_try_shutdown ( struct req_lib_cfg_tryshutdown req_lib_cfg_tryshutdown; struct res_lib_cfg_tryshutdown res_lib_cfg_tryshutdown; cs_error_t error; + struct iovec iov; error = saHandleInstanceGet (&cfg_hdb, cfg_handle, (void *)&cfg_instance); @@ -750,11 +740,14 @@ corosync_cfg_try_shutdown ( req_lib_cfg_tryshutdown.header.size = sizeof (struct req_lib_cfg_tryshutdown); req_lib_cfg_tryshutdown.flags = flags; + iov.iov_base = &req_lib_cfg_tryshutdown; + iov.iov_len = sizeof (req_lib_cfg_tryshutdown); + pthread_mutex_lock (&cfg_instance->response_mutex); - error = saSendReceiveReply (cfg_instance->response_fd, - &req_lib_cfg_tryshutdown, - sizeof (struct req_lib_cfg_tryshutdown), + error = cslib_msg_send_reply_receive (cfg_instance->ipc_ctx, + &iov, + 1, &res_lib_cfg_tryshutdown, sizeof (struct res_lib_cfg_tryshutdown)); @@ -772,6 +765,7 @@ corosync_cfg_replyto_shutdown ( { struct cfg_instance *cfg_instance; struct req_lib_cfg_replytoshutdown req_lib_cfg_replytoshutdown; + struct res_lib_cfg_replytoshutdown res_lib_cfg_replytoshutdown; struct iovec iov; cs_error_t error; @@ -789,8 +783,12 @@ corosync_cfg_replyto_shutdown ( iov.iov_len = sizeof (struct req_lib_cfg_replytoshutdown); pthread_mutex_lock (&cfg_instance->response_mutex); - error = saSendMsgRetry (cfg_instance->response_fd, - &iov, 1); + + error = cslib_msg_send_reply_receive (cfg_instance->ipc_ctx, + &iov, + 1, + &res_lib_cfg_replytoshutdown, + sizeof (struct res_lib_cfg_replytoshutdown)); pthread_mutex_unlock (&cfg_instance->response_mutex); @@ -805,13 +803,13 @@ cs_error_t corosync_cfg_get_node_addrs ( corosync_cfg_node_address_t *addrs) { cs_error_t error; - char buf[PIPE_BUF]; struct req_lib_cfg_get_node_addrs req_lib_cfg_get_node_addrs; - struct res_lib_cfg_get_node_addrs * res_lib_cfg_get_node_addrs = (struct res_lib_cfg_get_node_addrs *)buf; + struct res_lib_cfg_get_node_addrs *res_lib_cfg_get_node_addrs; struct cfg_instance *cfg_instance; int addrlen; int i; - struct iovec iov[2]; + struct iovec iov; + void *return_address; error = saHandleInstanceGet (&cfg_hdb, cfg_handle, (void *)&cfg_instance); @@ -819,22 +817,21 @@ cs_error_t corosync_cfg_get_node_addrs ( return (error); } - pthread_mutex_lock (&cfg_instance->response_mutex); - req_lib_cfg_get_node_addrs.header.size = sizeof (req_lib_cfg_get_node_addrs); req_lib_cfg_get_node_addrs.header.id = MESSAGE_REQ_CFG_GET_NODE_ADDRS; req_lib_cfg_get_node_addrs.nodeid = nodeid; - iov[0].iov_base = (char *)&req_lib_cfg_get_node_addrs; - iov[0].iov_len = sizeof (req_lib_cfg_get_node_addrs); + iov.iov_base = (char *)&req_lib_cfg_get_node_addrs; + iov.iov_len = sizeof (req_lib_cfg_get_node_addrs); - error = saSendMsgReceiveReply (cfg_instance->response_fd, iov, 1, - res_lib_cfg_get_node_addrs, sizeof (mar_res_header_t)); + pthread_mutex_lock (&cfg_instance->response_mutex); + + error = cslib_msg_send_reply_receive_in_buf (cfg_instance->ipc_ctx, + &iov, + 1, + &return_address); + res_lib_cfg_get_node_addrs = return_address; - if (error == CS_OK && res_lib_cfg_get_node_addrs->header.size > sizeof(mar_res_header_t)) { - error = saRecvRetry (cfg_instance->response_fd, (char *)res_lib_cfg_get_node_addrs + sizeof (mar_res_header_t), - res_lib_cfg_get_node_addrs->header.size - sizeof (mar_res_header_t)); - } pthread_mutex_unlock (&cfg_instance->response_mutex); if (error != CS_OK) { @@ -867,7 +864,6 @@ cs_error_t corosync_cfg_get_node_addrs ( error_exit: - pthread_mutex_unlock (&cfg_instance->response_mutex); return (error); } @@ -894,8 +890,12 @@ cs_error_t corosync_cfg_local_get ( pthread_mutex_lock (&cfg_inst->response_mutex); - error = saSendMsgReceiveReply (cfg_inst->response_fd, &iov, 1, - &res_lib_cfg_local_get, sizeof (res_lib_cfg_local_get)); + error = cslib_msg_send_reply_receive ( + cfg_inst->ipc_ctx, + &iov, + 1, + &res_lib_cfg_local_get, + sizeof (struct res_lib_cfg_local_get)); pthread_mutex_unlock (&cfg_inst->response_mutex); diff --git a/lib/confdb.c b/lib/confdb.c index ab21e45e..8135640f 100644 --- a/lib/confdb.c +++ b/lib/confdb.c @@ -46,7 +46,7 @@ #include #include #include -#include +#include #include #include "sa-confdb.h" @@ -62,8 +62,7 @@ struct iter_context { }; struct confdb_inst { - int response_fd; - int dispatch_fd; + void *ipc_ctx; int finalize; int standalone; confdb_callbacks_t callbacks; @@ -159,9 +158,7 @@ cs_error_t confdb_initialize ( confdb_inst->standalone = 1; } else { - error = saServiceConnect (&confdb_inst->dispatch_fd, - &confdb_inst->response_fd, - CONFDB_SERVICE); + error = cslib_service_connect (CONFDB_SERVICE, &confdb_inst->ipc_ctx); } if (error != CS_OK) goto error_put_destroy; @@ -213,6 +210,10 @@ cs_error_t confdb_finalize ( pthread_mutex_unlock (&confdb_inst->response_mutex); + if (!confdb_inst->standalone) { + cslib_service_disconnect (&confdb_inst->ipc_ctx); + } + (void)saHandleDestroy (&confdb_handle_t_db, handle); /* Free saved context handles */ @@ -220,19 +221,6 @@ cs_error_t confdb_finalize ( free_context_list(confdb_inst, &confdb_inst->object_iter_head); free_context_list(confdb_inst, &confdb_inst->key_iter_head); - if (!confdb_inst->standalone) { - /* - * Disconnect from the server - */ - if (confdb_inst->response_fd != -1) { - shutdown(confdb_inst->response_fd, 0); - close(confdb_inst->response_fd); - } - if (confdb_inst->dispatch_fd != -1) { - shutdown(confdb_inst->dispatch_fd, 0); - close(confdb_inst->dispatch_fd); - } - } (void)saHandleInstancePut (&confdb_handle_t_db, handle); return (CS_OK); @@ -250,7 +238,7 @@ cs_error_t confdb_fd_get ( return (error); } - *fd = confdb_inst->dispatch_fd; + *fd = cslib_fd_get (confdb_inst->ipc_ctx); (void)saHandleInstancePut (&confdb_handle_t_db, handle); @@ -323,7 +311,7 @@ cs_error_t confdb_dispatch ( if (confdb_inst->standalone) { error = CS_ERR_NOT_SUPPORTED; - goto error_unlock; + goto error_put; } /* @@ -335,24 +323,11 @@ cs_error_t confdb_dispatch ( } do { - ufds.fd = confdb_inst->dispatch_fd; - ufds.events = POLLIN; - ufds.revents = 0; - - error = saPollRetry (&ufds, 1, timeout); - if (error != CS_OK) { - goto error_nounlock; - } - pthread_mutex_lock (&confdb_inst->dispatch_mutex); - /* - * Regather poll data in case ufds has changed since taking lock - */ - error = saPollRetry (&ufds, 1, timeout); - if (error != CS_OK) { - goto error_nounlock; - } + dispatch_avail = cslib_dispatch_recv (confdb_inst->ipc_ctx, + (void *)&dispatch_data, timeout); + /* * Handle has been finalized in another thread @@ -360,41 +335,17 @@ cs_error_t confdb_dispatch ( if (confdb_inst->finalize == 1) { error = CS_OK; pthread_mutex_unlock (&confdb_inst->dispatch_mutex); - goto error_unlock; + goto error_put; } dispatch_avail = ufds.revents & POLLIN; if (dispatch_avail == 0 && dispatch_types == CONFDB_DISPATCH_ALL) { - pthread_mutex_unlock (&confdb_inst->dispatch_mutex); break; /* exit do while cont is 1 loop */ } else if (dispatch_avail == 0) { - pthread_mutex_unlock (&confdb_inst->dispatch_mutex); continue; /* next poll */ } - if (ufds.revents & POLLIN) { - /* - * Queue empty, read response from socket - */ - error = saRecvRetry (confdb_inst->dispatch_fd, &dispatch_data.header, - sizeof (mar_res_header_t)); - if (error != CS_OK) { - goto error_unlock; - } - if (dispatch_data.header.size > sizeof (mar_res_header_t)) { - error = saRecvRetry (confdb_inst->dispatch_fd, &dispatch_data.data, - dispatch_data.header.size - sizeof (mar_res_header_t)); - - if (error != CS_OK) { - goto error_unlock; - } - } - } else { - pthread_mutex_unlock (&confdb_inst->dispatch_mutex); - continue; - } - /* * Make copy of callbacks, message data, unlock instance, and call callback * A risk of this dispatch method is that the callback routines may @@ -403,6 +354,7 @@ cs_error_t confdb_dispatch ( memcpy (&callbacks, &confdb_inst->callbacks, sizeof (confdb_callbacks_t)); pthread_mutex_unlock (&confdb_inst->dispatch_mutex); + /* * Dispatch incoming message */ @@ -443,7 +395,7 @@ cs_error_t confdb_dispatch ( default: error = CS_ERR_LIBRARY; - goto error_nounlock; + goto error_noput; break; } @@ -461,9 +413,9 @@ cs_error_t confdb_dispatch ( } } while (cont); -error_unlock: +error_put: (void)saHandleInstancePut (&confdb_handle_t_db, handle); -error_nounlock: +error_noput: return (error); } @@ -476,7 +428,7 @@ cs_error_t confdb_object_create ( { cs_error_t error; struct confdb_inst *confdb_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_confdb_object_create req_lib_confdb_object_create; struct res_lib_confdb_object_create res_lib_confdb_object_create; @@ -501,13 +453,17 @@ cs_error_t confdb_object_create ( memcpy(req_lib_confdb_object_create.object_name.value, object_name, object_name_len); req_lib_confdb_object_create.object_name.length = object_name_len; - iov[0].iov_base = (char *)&req_lib_confdb_object_create; - iov[0].iov_len = sizeof (struct req_lib_confdb_object_create); + iov.iov_base = (char *)&req_lib_confdb_object_create; + iov.iov_len = sizeof (struct req_lib_confdb_object_create); pthread_mutex_lock (&confdb_inst->response_mutex); - error = saSendMsgReceiveReply (confdb_inst->response_fd, iov, 1, - &res_lib_confdb_object_create, sizeof (struct res_lib_confdb_object_create)); + error = cslib_msg_send_reply_receive ( + confdb_inst->ipc_ctx, + &iov, + 1, + &res_lib_confdb_object_create, + sizeof (struct res_lib_confdb_object_create)); pthread_mutex_unlock (&confdb_inst->response_mutex); if (error != CS_OK) { @@ -529,7 +485,7 @@ cs_error_t confdb_object_destroy ( { cs_error_t error; struct confdb_inst *confdb_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_confdb_object_destroy req_lib_confdb_object_destroy; mar_res_header_t res; @@ -550,13 +506,17 @@ cs_error_t confdb_object_destroy ( req_lib_confdb_object_destroy.header.id = MESSAGE_REQ_CONFDB_OBJECT_DESTROY; req_lib_confdb_object_destroy.object_handle = object_handle; - iov[0].iov_base = (char *)&req_lib_confdb_object_destroy; - iov[0].iov_len = sizeof (struct req_lib_confdb_object_destroy); + iov.iov_base = (char *)&req_lib_confdb_object_destroy; + iov.iov_len = sizeof (struct req_lib_confdb_object_destroy); pthread_mutex_lock (&confdb_inst->response_mutex); - error = saSendMsgReceiveReply (confdb_inst->response_fd, iov, 1, - &res, sizeof ( mar_res_header_t)); + error = cslib_msg_send_reply_receive ( + confdb_inst->ipc_ctx, + &iov, + 1, + &res, + sizeof (mar_res_header_t)); pthread_mutex_unlock (&confdb_inst->response_mutex); if (error != CS_OK) { @@ -578,7 +538,7 @@ cs_error_t confdb_object_parent_get ( { cs_error_t error; struct confdb_inst *confdb_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_confdb_object_parent_get req_lib_confdb_object_parent_get; struct res_lib_confdb_object_parent_get res_lib_confdb_object_parent_get; @@ -599,13 +559,17 @@ cs_error_t confdb_object_parent_get ( req_lib_confdb_object_parent_get.header.id = MESSAGE_REQ_CONFDB_OBJECT_PARENT_GET; req_lib_confdb_object_parent_get.object_handle = object_handle; - iov[0].iov_base = (char *)&req_lib_confdb_object_parent_get; - iov[0].iov_len = sizeof (struct req_lib_confdb_object_parent_get); + iov.iov_base = (char *)&req_lib_confdb_object_parent_get; + iov.iov_len = sizeof (struct req_lib_confdb_object_parent_get); pthread_mutex_lock (&confdb_inst->response_mutex); - error = saSendMsgReceiveReply (confdb_inst->response_fd, iov, 1, - &res_lib_confdb_object_parent_get, sizeof (struct res_lib_confdb_object_parent_get)); + error = cslib_msg_send_reply_receive ( + confdb_inst->ipc_ctx, + &iov, + 1, + &res_lib_confdb_object_parent_get, + sizeof (struct res_lib_confdb_object_parent_get)); pthread_mutex_unlock (&confdb_inst->response_mutex); if (error != CS_OK) { @@ -626,7 +590,7 @@ static cs_error_t do_find_destroy( unsigned int find_handle) { cs_error_t error; - struct iovec iov[2]; + struct iovec iov; struct req_lib_confdb_object_find_destroy req_lib_confdb_object_find_destroy; mar_res_header_t res; @@ -645,13 +609,17 @@ static cs_error_t do_find_destroy( req_lib_confdb_object_find_destroy.header.id = MESSAGE_REQ_CONFDB_OBJECT_FIND_DESTROY; req_lib_confdb_object_find_destroy.find_handle = find_handle; - iov[0].iov_base = (char *)&req_lib_confdb_object_find_destroy; - iov[0].iov_len = sizeof (struct req_lib_confdb_object_find_destroy); + iov.iov_base = (char *)&req_lib_confdb_object_find_destroy; + iov.iov_len = sizeof (struct req_lib_confdb_object_find_destroy); pthread_mutex_lock (&confdb_inst->response_mutex); - error = saSendMsgReceiveReply (confdb_inst->response_fd, iov, 1, - &res, sizeof (mar_res_header_t)); + error = cslib_msg_send_reply_receive ( + confdb_inst->ipc_ctx, + &iov, + 1, + &res, + sizeof (mar_res_header_t)); pthread_mutex_unlock (&confdb_inst->response_mutex); if (error != CS_OK) { @@ -724,7 +692,7 @@ cs_error_t confdb_key_create ( { cs_error_t error; struct confdb_inst *confdb_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_confdb_key_create req_lib_confdb_key_create; mar_res_header_t res; @@ -751,13 +719,17 @@ cs_error_t confdb_key_create ( memcpy(req_lib_confdb_key_create.value.value, value, value_len); req_lib_confdb_key_create.value.length = value_len; - iov[0].iov_base = (char *)&req_lib_confdb_key_create; - iov[0].iov_len = sizeof (struct req_lib_confdb_key_create); + iov.iov_base = (char *)&req_lib_confdb_key_create; + iov.iov_len = sizeof (struct req_lib_confdb_key_create); pthread_mutex_lock (&confdb_inst->response_mutex); - error = saSendMsgReceiveReply (confdb_inst->response_fd, iov, 1, - &res, sizeof (res)); + error = cslib_msg_send_reply_receive ( + confdb_inst->ipc_ctx, + &iov, + 1, + &res, + sizeof (res)); pthread_mutex_unlock (&confdb_inst->response_mutex); if (error != CS_OK) { @@ -782,7 +754,7 @@ cs_error_t confdb_key_delete ( { cs_error_t error; struct confdb_inst *confdb_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_confdb_key_delete req_lib_confdb_key_delete; mar_res_header_t res; @@ -809,13 +781,17 @@ cs_error_t confdb_key_delete ( memcpy(req_lib_confdb_key_delete.value.value, value, value_len); req_lib_confdb_key_delete.value.length = value_len; - iov[0].iov_base = (char *)&req_lib_confdb_key_delete; - iov[0].iov_len = sizeof (struct req_lib_confdb_key_delete); + iov.iov_base = (char *)&req_lib_confdb_key_delete; + iov.iov_len = sizeof (struct req_lib_confdb_key_delete); pthread_mutex_lock (&confdb_inst->response_mutex); - error = saSendMsgReceiveReply (confdb_inst->response_fd, iov, 1, - &res, sizeof (res)); + error = cslib_msg_send_reply_receive ( + confdb_inst->ipc_ctx, + &iov, + 1, + &res, + sizeof (res)); pthread_mutex_unlock (&confdb_inst->response_mutex); if (error != CS_OK) { @@ -840,7 +816,7 @@ cs_error_t confdb_key_get ( { cs_error_t error; struct confdb_inst *confdb_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_confdb_key_get req_lib_confdb_key_get; struct res_lib_confdb_key_get res_lib_confdb_key_get; @@ -865,13 +841,17 @@ cs_error_t confdb_key_get ( memcpy(req_lib_confdb_key_get.key_name.value, key_name, key_name_len); req_lib_confdb_key_get.key_name.length = key_name_len; - iov[0].iov_base = (char *)&req_lib_confdb_key_get; - iov[0].iov_len = sizeof (struct req_lib_confdb_key_get); + iov.iov_base = (char *)&req_lib_confdb_key_get; + iov.iov_len = sizeof (struct req_lib_confdb_key_get); pthread_mutex_lock (&confdb_inst->response_mutex); - error = saSendMsgReceiveReply (confdb_inst->response_fd, iov, 1, - &res_lib_confdb_key_get, sizeof (struct res_lib_confdb_key_get)); + error = cslib_msg_send_reply_receive ( + confdb_inst->ipc_ctx, + &iov, + 1, + &res_lib_confdb_key_get, + sizeof (struct res_lib_confdb_key_get)); pthread_mutex_unlock (&confdb_inst->response_mutex); if (error != CS_OK) { @@ -899,7 +879,7 @@ cs_error_t confdb_key_increment ( { cs_error_t error; struct confdb_inst *confdb_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_confdb_key_get req_lib_confdb_key_get; struct res_lib_confdb_key_incdec res_lib_confdb_key_incdec; @@ -924,13 +904,17 @@ cs_error_t confdb_key_increment ( memcpy(req_lib_confdb_key_get.key_name.value, key_name, key_name_len); req_lib_confdb_key_get.key_name.length = key_name_len; - iov[0].iov_base = (char *)&req_lib_confdb_key_get; - iov[0].iov_len = sizeof (struct req_lib_confdb_key_get); + iov.iov_base = (char *)&req_lib_confdb_key_get; + iov.iov_len = sizeof (struct req_lib_confdb_key_get); pthread_mutex_lock (&confdb_inst->response_mutex); - error = saSendMsgReceiveReply (confdb_inst->response_fd, iov, 1, - &res_lib_confdb_key_incdec, sizeof (struct res_lib_confdb_key_incdec)); + error = cslib_msg_send_reply_receive ( + confdb_inst->ipc_ctx, + &iov, + 1, + &res_lib_confdb_key_incdec, + sizeof (struct res_lib_confdb_key_incdec)); pthread_mutex_unlock (&confdb_inst->response_mutex); if (error != CS_OK) { @@ -957,7 +941,7 @@ cs_error_t confdb_key_decrement ( { cs_error_t error; struct confdb_inst *confdb_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_confdb_key_get req_lib_confdb_key_get; struct res_lib_confdb_key_incdec res_lib_confdb_key_incdec; @@ -982,13 +966,17 @@ cs_error_t confdb_key_decrement ( memcpy(req_lib_confdb_key_get.key_name.value, key_name, key_name_len); req_lib_confdb_key_get.key_name.length = key_name_len; - iov[0].iov_base = (char *)&req_lib_confdb_key_get; - iov[0].iov_len = sizeof (struct req_lib_confdb_key_get); + iov.iov_base = (char *)&req_lib_confdb_key_get; + iov.iov_len = sizeof (struct req_lib_confdb_key_get); pthread_mutex_lock (&confdb_inst->response_mutex); - error = saSendMsgReceiveReply (confdb_inst->response_fd, iov, 1, - &res_lib_confdb_key_incdec, sizeof (struct res_lib_confdb_key_incdec)); + error = cslib_msg_send_reply_receive ( + confdb_inst->ipc_ctx, + &iov, + 1, + &res_lib_confdb_key_incdec, + sizeof (struct res_lib_confdb_key_incdec)); pthread_mutex_unlock (&confdb_inst->response_mutex); if (error != CS_OK) { @@ -1018,7 +1006,7 @@ cs_error_t confdb_key_replace ( { cs_error_t error; struct confdb_inst *confdb_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_confdb_key_replace req_lib_confdb_key_replace; mar_res_header_t res; @@ -1047,13 +1035,17 @@ cs_error_t confdb_key_replace ( memcpy(req_lib_confdb_key_replace.new_value.value, new_value, new_value_len); req_lib_confdb_key_replace.new_value.length = new_value_len; - iov[0].iov_base = (char *)&req_lib_confdb_key_replace; - iov[0].iov_len = sizeof (struct req_lib_confdb_key_replace); + iov.iov_base = (char *)&req_lib_confdb_key_replace; + iov.iov_len = sizeof (struct req_lib_confdb_key_replace); pthread_mutex_lock (&confdb_inst->response_mutex); - error = saSendMsgReceiveReply (confdb_inst->response_fd, iov, 1, - &res, sizeof (res)); + error = cslib_msg_send_reply_receive ( + confdb_inst->ipc_ctx, + &iov, + 1, + &res, + sizeof (res)); pthread_mutex_unlock (&confdb_inst->response_mutex); if (error != CS_OK) { @@ -1183,7 +1175,7 @@ cs_error_t confdb_object_find ( { cs_error_t error; struct confdb_inst *confdb_inst; - struct iovec iov[2]; + struct iovec iov; struct iter_context *context; struct req_lib_confdb_object_find req_lib_confdb_object_find; struct res_lib_confdb_object_find res_lib_confdb_object_find; @@ -1219,13 +1211,17 @@ cs_error_t confdb_object_find ( memcpy(req_lib_confdb_object_find.object_name.value, object_name, object_name_len); req_lib_confdb_object_find.object_name.length = object_name_len; - iov[0].iov_base = (char *)&req_lib_confdb_object_find; - iov[0].iov_len = sizeof (struct req_lib_confdb_object_find); + iov.iov_base = (char *)&req_lib_confdb_object_find; + iov.iov_len = sizeof (struct req_lib_confdb_object_find); pthread_mutex_lock (&confdb_inst->response_mutex); - error = saSendMsgReceiveReply (confdb_inst->response_fd, iov, 1, - &res_lib_confdb_object_find, sizeof (struct res_lib_confdb_object_find)); + error = cslib_msg_send_reply_receive ( + confdb_inst->ipc_ctx, + &iov, + 1, + &res_lib_confdb_object_find, + sizeof (struct res_lib_confdb_object_find)); pthread_mutex_unlock (&confdb_inst->response_mutex); if (error != CS_OK) { @@ -1252,7 +1248,7 @@ cs_error_t confdb_object_iter ( { cs_error_t error; struct confdb_inst *confdb_inst; - struct iovec iov[2]; + struct iovec iov; struct iter_context *context; struct req_lib_confdb_object_iter req_lib_confdb_object_iter; struct res_lib_confdb_object_iter res_lib_confdb_object_iter; @@ -1287,13 +1283,17 @@ cs_error_t confdb_object_iter ( req_lib_confdb_object_iter.parent_object_handle = parent_object_handle; req_lib_confdb_object_iter.find_handle = context->find_handle; - iov[0].iov_base = (char *)&req_lib_confdb_object_iter; - iov[0].iov_len = sizeof (struct req_lib_confdb_object_iter); + iov.iov_base = (char *)&req_lib_confdb_object_iter; + iov.iov_len = sizeof (struct req_lib_confdb_object_iter); pthread_mutex_lock (&confdb_inst->response_mutex); - error = saSendMsgReceiveReply (confdb_inst->response_fd, iov, 1, - &res_lib_confdb_object_iter, sizeof (struct res_lib_confdb_object_iter)); + error = cslib_msg_send_reply_receive ( + confdb_inst->ipc_ctx, + &iov, + 1, + &res_lib_confdb_object_iter, + sizeof (struct res_lib_confdb_object_iter)); pthread_mutex_unlock (&confdb_inst->response_mutex); if (error != CS_OK) { @@ -1325,7 +1325,7 @@ cs_error_t confdb_key_iter ( { cs_error_t error; struct confdb_inst *confdb_inst; - struct iovec iov[2]; + struct iovec iov; struct iter_context *context; struct req_lib_confdb_key_iter req_lib_confdb_key_iter; struct res_lib_confdb_key_iter res_lib_confdb_key_iter; @@ -1358,13 +1358,17 @@ cs_error_t confdb_key_iter ( req_lib_confdb_key_iter.parent_object_handle = parent_object_handle; req_lib_confdb_key_iter.next_entry= context->next_entry; - iov[0].iov_base = (char *)&req_lib_confdb_key_iter; - iov[0].iov_len = sizeof (struct req_lib_confdb_key_iter); + iov.iov_base = (char *)&req_lib_confdb_key_iter; + iov.iov_len = sizeof (struct req_lib_confdb_key_iter); pthread_mutex_lock (&confdb_inst->response_mutex); - error = saSendMsgReceiveReply (confdb_inst->response_fd, iov, 1, - &res_lib_confdb_key_iter, sizeof (struct res_lib_confdb_key_iter)); + error = cslib_msg_send_reply_receive ( + confdb_inst->ipc_ctx, + &iov, + 1, + &res_lib_confdb_key_iter, + sizeof (struct res_lib_confdb_key_iter)); pthread_mutex_unlock (&confdb_inst->response_mutex); if (error != CS_OK) { @@ -1394,7 +1398,7 @@ cs_error_t confdb_write ( { cs_error_t error; struct confdb_inst *confdb_inst; - struct iovec iov[2]; + struct iovec iov; mar_req_header_t req; struct res_lib_confdb_write res_lib_confdb_write; @@ -1414,13 +1418,17 @@ cs_error_t confdb_write ( req.size = sizeof (mar_req_header_t); req.id = MESSAGE_REQ_CONFDB_WRITE; - iov[0].iov_base = (char *)&req; - iov[0].iov_len = sizeof (mar_req_header_t); + iov.iov_base = (char *)&req; + iov.iov_len = sizeof (mar_req_header_t); pthread_mutex_lock (&confdb_inst->response_mutex); - error = saSendMsgReceiveReply (confdb_inst->response_fd, iov, 1, - &res_lib_confdb_write, sizeof ( struct res_lib_confdb_write)); + error = cslib_msg_send_reply_receive ( + confdb_inst->ipc_ctx, + &iov, + 1, + &res_lib_confdb_write, + sizeof (struct res_lib_confdb_write)); pthread_mutex_unlock (&confdb_inst->response_mutex); if (error != CS_OK) { @@ -1444,7 +1452,7 @@ cs_error_t confdb_reload ( { cs_error_t error; struct confdb_inst *confdb_inst; - struct iovec iov[2]; + struct iovec iov; struct res_lib_confdb_reload res_lib_confdb_reload; struct req_lib_confdb_reload req_lib_confdb_reload; @@ -1465,13 +1473,17 @@ cs_error_t confdb_reload ( req_lib_confdb_reload.header.id = MESSAGE_REQ_CONFDB_RELOAD; req_lib_confdb_reload.flush = flush; - iov[0].iov_base = (char *)&req_lib_confdb_reload; - iov[0].iov_len = sizeof (req_lib_confdb_reload); + iov.iov_base = (char *)&req_lib_confdb_reload; + iov.iov_len = sizeof (req_lib_confdb_reload); pthread_mutex_lock (&confdb_inst->response_mutex); - error = saSendMsgReceiveReply (confdb_inst->response_fd, iov, 1, - &res_lib_confdb_reload, sizeof (struct res_lib_confdb_reload)); + error = cslib_msg_send_reply_receive ( + confdb_inst->ipc_ctx, + &iov, + 1, + &res_lib_confdb_reload, + sizeof (struct res_lib_confdb_reload)); pthread_mutex_unlock (&confdb_inst->response_mutex); @@ -1496,7 +1508,7 @@ cs_error_t confdb_track_changes ( { cs_error_t error; struct confdb_inst *confdb_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_confdb_object_track_start req; mar_res_header_t res; @@ -1515,13 +1527,17 @@ cs_error_t confdb_track_changes ( req.object_handle = object_handle; req.flags = flags; - iov[0].iov_base = (char *)&req; - iov[0].iov_len = sizeof (struct req_lib_confdb_object_track_start); + iov.iov_base = (char *)&req; + iov.iov_len = sizeof (struct req_lib_confdb_object_track_start); pthread_mutex_lock (&confdb_inst->response_mutex); - error = saSendMsgReceiveReply (confdb_inst->response_fd, iov, 1, - &res, sizeof ( mar_res_header_t)); + error = cslib_msg_send_reply_receive ( + confdb_inst->ipc_ctx, + &iov, + 1, + &res, + sizeof (mar_res_header_t)); pthread_mutex_unlock (&confdb_inst->response_mutex); if (error != CS_OK) { @@ -1540,7 +1556,7 @@ cs_error_t confdb_stop_track_changes (confdb_handle_t handle) { cs_error_t error; struct confdb_inst *confdb_inst; - struct iovec iov[2]; + struct iovec iov; mar_req_header_t req; mar_res_header_t res; @@ -1557,13 +1573,17 @@ cs_error_t confdb_stop_track_changes (confdb_handle_t handle) req.size = sizeof (mar_req_header_t); req.id = MESSAGE_REQ_CONFDB_TRACK_STOP; - iov[0].iov_base = (char *)&req; - iov[0].iov_len = sizeof (mar_req_header_t); + iov.iov_base = (char *)&req; + iov.iov_len = sizeof (mar_req_header_t); pthread_mutex_lock (&confdb_inst->response_mutex); - error = saSendMsgReceiveReply (confdb_inst->response_fd, iov, 1, - &res, sizeof ( mar_res_header_t)); + error = cslib_msg_send_reply_receive ( + confdb_inst->ipc_ctx, + &iov, + 1, + &res, + sizeof (mar_res_header_t)); pthread_mutex_unlock (&confdb_inst->response_mutex); if (error != CS_OK) { diff --git a/lib/coroipc.c b/lib/coroipc.c new file mode 100644 index 00000000..1df32326 --- /dev/null +++ b/lib/coroipc.c @@ -0,0 +1,867 @@ +/* + * vi: set autoindent tabstop=4 shiftwidth=4 : + * + * Copyright (c) 2002-2006 MontaVista Software, Inc. + * Copyright (c) 2006-2009 Red Hat, Inc. + * + * All rights reserved. + * + * Author: Steven Dake (sdake@redhat.com) + * + * This software licensed under BSD license, the text of which follows: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * - Neither the name of the MontaVista Software, Inc. nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +enum SA_HANDLE_STATE { + SA_HANDLE_STATE_EMPTY, + SA_HANDLE_STATE_PENDINGREMOVAL, + SA_HANDLE_STATE_ACTIVE +}; + +struct saHandle { + int state; + void *instance; + int refCount; + uint32_t check; +}; + +struct ipc_segment { + int fd; + int shmid; + int semid; + int flow_control_state; + struct shared_memory *shared_memory; + uid_t euid; +}; + + +#if defined(COROSYNC_LINUX) +/* SUN_LEN is broken for abstract namespace + */ +#define AIS_SUN_LEN(a) sizeof(*(a)) + +static char *socketname = "libais.socket"; +#else +#define AIS_SUN_LEN(a) SUN_LEN(a) + +static char *socketname = "/var/run/libais.socket"; +#endif + +#ifdef SO_NOSIGPIPE +void socket_nosigpipe(int s) +{ + int on = 1; + setsockopt(s, SOL_SOCKET, SO_NOSIGPIPE, (void *)&on, sizeof(on)); +} +#endif + +static int +cslib_send ( + int s, + const void *msg, + size_t len) +{ + int result; + struct msghdr msg_send; + struct iovec iov_send; + char *rbuf = (char *)msg; + int processed = 0; + + msg_send.msg_iov = &iov_send; + msg_send.msg_iovlen = 1; + msg_send.msg_name = 0; + msg_send.msg_namelen = 0; + msg_send.msg_control = 0; + msg_send.msg_controllen = 0; + msg_send.msg_flags = 0; + +retry_send: + iov_send.iov_base = (void *)&rbuf[processed]; + iov_send.iov_len = len - processed; + + result = sendmsg (s, &msg_send, MSG_NOSIGNAL); + + /* + * return immediately on any kind of syscall error that maps to + * CS_ERR if no part of message has been sent + */ + if (result == -1 && processed == 0) { + if (errno == EINTR) { + goto error_exit; + } + if (errno == EAGAIN) { + goto error_exit; + } + if (errno == EFAULT) { + goto error_exit; + } + } + + /* + * retry read operations that are already started except + * for fault in that case, return ERR_LIBRARY + */ + if (result == -1 && processed > 0) { + if (errno == EINTR) { + goto retry_send; + } + if (errno == EAGAIN) { + goto retry_send; + } + if (errno == EFAULT) { + goto error_exit; + } + } + + /* + * return ERR_LIBRARY on any other syscall error + */ + if (result == -1) { + goto error_exit; + } + + processed += result; + if (processed != len) { + goto retry_send; + } + + return (0); + +error_exit: + return (-1); +} + +static int +cslib_recv ( + int s, + void *msg, + size_t len) +{ + int error = 0; + int result; + struct msghdr msg_recv; + struct iovec iov_recv; + char *rbuf = (char *)msg; + int processed = 0; + + msg_recv.msg_iov = &iov_recv; + msg_recv.msg_iovlen = 1; + msg_recv.msg_name = 0; + msg_recv.msg_namelen = 0; + msg_recv.msg_control = 0; + msg_recv.msg_controllen = 0; + msg_recv.msg_flags = 0; + +retry_recv: + iov_recv.iov_base = (void *)&rbuf[processed]; + iov_recv.iov_len = len - processed; + + result = recvmsg (s, &msg_recv, MSG_NOSIGNAL|MSG_WAITALL); + if (result == -1 && errno == EINTR) { + goto retry_recv; + } + if (result == -1 && errno == EAGAIN) { + goto retry_recv; + } +#if defined(OPENAIS_SOLARIS) || defined(OPENAIS_BSD) || defined(OPENAIS_DARWIN) + /* On many OS poll never return POLLHUP or POLLERR. + * EOF is detected when recvmsg return 0. + */ + if (result == 0) { + error = -1; + goto error_exit; + } +#endif + if (result == -1 || result == 0) { + error = -1; + goto error_exit; + } + processed += result; + if (processed != len) { + goto retry_recv; + } + assert (processed == len); +error_exit: + return (0); +} + +static int +priv_change_send (struct ipc_segment *ipc_segment) +{ + char buf_req; + mar_req_priv_change req_priv_change; + unsigned int res; + + req_priv_change.euid = geteuid(); + /* + * Don't resend request unless euid has changed + */ + if (ipc_segment->euid == req_priv_change.euid) { + return (0); + } + req_priv_change.egid = getegid(); + + buf_req = MESSAGE_REQ_CHANGE_EUID; + res = cslib_send (ipc_segment->fd, &buf_req, 1); + if (res == -1) { + return (-1); + } + + res = cslib_send (ipc_segment->fd, &req_priv_change, + sizeof (req_priv_change)); + if (res == -1) { + return (-1); + } + + ipc_segment->euid = req_priv_change.euid; + return (0); +} + +cs_error_t +cslib_service_connect ( + enum service_types service, + void **shmseg) +{ + int request_fd; + struct sockaddr_un address; + cs_error_t error; + struct ipc_segment *ipc_segment; + key_t shmkey = 0; + key_t semkey = 0; + int res; + mar_req_setup_t req_setup; + mar_res_setup_t res_setup; + + res_setup.error = CS_ERR_LIBRARY; + + request_fd = socket (PF_UNIX, SOCK_STREAM, 0); + if (request_fd == -1) { + return (-1); + } + + memset (&address, 0, sizeof (struct sockaddr_un)); +#if defined(OPENAIS_BSD) || defined(OPENAIS_DARWIN) + address.sun_len = sizeof(struct sockaddr_un); +#endif + address.sun_family = PF_UNIX; +#if defined(COROSYNC_LINUX) + strcpy (address.sun_path + 1, socketname); +#else + strcpy (address.sun_path, socketname); +#endif + res = connect (request_fd, (struct sockaddr *)&address, + AIS_SUN_LEN(&address)); + if (res == -1) { + close (request_fd); + return (CS_ERR_TRY_AGAIN); + } + + ipc_segment = malloc (sizeof (struct ipc_segment)); + if (ipc_segment == NULL) { + close (request_fd); + return (-1); + } + bzero (ipc_segment, sizeof (struct ipc_segment)); + + /* + * Allocate a shared memory segment + */ + do { + shmkey = random(); + ipc_segment->shmid = shmget (shmkey, sizeof (struct shared_memory), + IPC_CREAT|IPC_EXCL|0600); + } while (ipc_segment->shmid == -1); + + /* + * Allocate a semaphore segment + */ + do { + semkey = random(); + ipc_segment->semid = semget (semkey, 3, IPC_CREAT|IPC_EXCL|0600); + ipc_segment->euid = geteuid (); + } while (ipc_segment->semid == -1); + + /* + * Attach to shared memory segment + */ + ipc_segment->shared_memory = shmat (ipc_segment->shmid, NULL, 0); + if (ipc_segment->shared_memory == (void *)-1) { + goto error_exit; + } + + res = semctl (ipc_segment->semid, 0, SETVAL, 0); + if (res != 0) { + goto error_exit; + } + + res = semctl (ipc_segment->semid, 1, SETVAL, 0); + if (res != 0) { + goto error_exit; + } + + req_setup.shmkey = shmkey; + req_setup.semkey = semkey; + req_setup.service = service; + + error = cslib_send (request_fd, &req_setup, sizeof (mar_req_setup_t)); + if (error != 0) { + goto error_exit; + } + error = cslib_recv (request_fd, &res_setup, sizeof (mar_res_setup_t)); + if (error != 0) { + goto error_exit; + } + + ipc_segment->fd = request_fd; + ipc_segment->flow_control_state = 0; + *shmseg = ipc_segment; + + /* + * Something go wrong with server + * Cleanup all + */ + if (res_setup.error == CS_ERR_TRY_AGAIN) { + goto error_exit; + } + + return (res_setup.error); + +error_exit: + close (request_fd); + if (ipc_segment->shmid > 0) + shmctl (ipc_segment->shmid, IPC_RMID, NULL); + if (ipc_segment->semid > 0) + semctl (ipc_segment->semid, 0, IPC_RMID); + return (res_setup.error); +} + +cs_error_t +cslib_service_disconnect ( + void *ipc_context) +{ + struct ipc_segment *ipc_segment = (struct ipc_segment *)ipc_context; + + shutdown (ipc_segment->fd, SHUT_RDWR); + close (ipc_segment->fd); + shmdt (ipc_segment->shared_memory); + free (ipc_segment); + return (CS_OK); +} + +int +cslib_dispatch_flow_control_get ( + void *ipc_context) +{ + struct ipc_segment *ipc_segment = (struct ipc_segment *)ipc_context; + + return (ipc_segment->flow_control_state); +} + + +int +cslib_fd_get (void *ipc_ctx) +{ + struct ipc_segment *ipc_segment = (struct ipc_segment *)ipc_ctx; + + return (ipc_segment->fd); +} + +static void memcpy_swrap ( + void *dest, void *src, int len, unsigned int *read) +{ + char *dest_chr = (char *)dest; + char *src_chr = (char *)src; + + unsigned int first_read; + unsigned int second_read; + + first_read = len; + second_read = 0; + + if (len + *read >= DISPATCH_SIZE) { + first_read = DISPATCH_SIZE - *read; + second_read = (len + *read) % DISPATCH_SIZE; + } + memcpy (dest_chr, &src_chr[*read], first_read); + if (second_read) { + memcpy (&dest_chr[first_read], src_chr, + second_read); + } + *read = (*read + len) % (DISPATCH_SIZE); +} +int original_flow = -1; + +int +cslib_dispatch_recv (void *ipc_ctx, void *data, int timeout) +{ + struct pollfd ufds; + struct sembuf sop; + int poll_events; + mar_res_header_t *header; + char buf; + struct ipc_segment *ipc_segment = (struct ipc_segment *)ipc_ctx; + int res; + unsigned int my_read; + char buf_two = 1; + + ufds.fd = ipc_segment->fd; + ufds.events = POLLIN; + ufds.revents = 0; + +retry_poll: + poll_events = poll (&ufds, 1, timeout); + if (poll_events == -1 && errno == EINTR) { + goto retry_poll; + } else + if (poll_events == -1) { + return (-1); + } else + if (poll_events == 0) { + return (0); + } + if (poll_events == 1 && (ufds.revents & (POLLERR|POLLHUP))) { + return (-1); + } +retry_recv: + res = recv (ipc_segment->fd, &buf, 1, 0); + if (res == -1 && errno == EINTR) { + goto retry_recv; + } else + if (res == -1) { + return (-1); + } + if (res == 0) { + return (-1); + } + ipc_segment->flow_control_state = 0; + if (buf == 1 || buf == 2) { + ipc_segment->flow_control_state = 1; + } + /* + * Notify executive to flush any pending dispatch messages + */ + if (ipc_segment->flow_control_state) { + buf_two = MESSAGE_REQ_OUTQ_FLUSH; + res = cslib_send (ipc_segment->fd, &buf_two, 1); + assert (res == 0); //TODO + } + /* + * This is just a notification of flow control starting at the addition + * of a new pending message, not a message to dispatch + */ + if (buf == 2) { + return (0); + } + if (buf == 3) { + return (0); + } + + sop.sem_num = 2; + sop.sem_op = -1; + sop.sem_flg = 0; + +retry_semop: + res = semop (ipc_segment->semid, &sop, 1); + if (res == -1 && errno == EINTR) { + goto retry_semop; + } else + if (res == -1 && errno == EACCES) { + priv_change_send (ipc_segment); + goto retry_semop; + } else + if (res == -1) { + return (-1); + } + + if (ipc_segment->shared_memory->read + sizeof (mar_res_header_t) >= DISPATCH_SIZE) { + my_read = ipc_segment->shared_memory->read; + memcpy_swrap (data, + ipc_segment->shared_memory->dispatch_buffer, + sizeof (mar_res_header_t), + &ipc_segment->shared_memory->read); + header = (mar_res_header_t *)data; + memcpy_swrap ( + data + sizeof (mar_res_header_t), + ipc_segment->shared_memory->dispatch_buffer, + header->size - sizeof (mar_res_header_t), + &ipc_segment->shared_memory->read); + } else { + header = (mar_res_header_t *)&ipc_segment->shared_memory->dispatch_buffer[ipc_segment->shared_memory->read]; + memcpy_swrap ( + data, + ipc_segment->shared_memory->dispatch_buffer, + header->size, + &ipc_segment->shared_memory->read); + } + + return (1); +} + +static cs_error_t +cslib_msg_send ( + void *ipc_context, + struct iovec *iov, + int iov_len) +{ + struct ipc_segment *ipc_segment = (struct ipc_segment *)ipc_context; + struct sembuf sop; + int i; + int res; + int req_buffer_idx = 0; + + for (i = 0; i < iov_len; i++) { + memcpy (&ipc_segment->shared_memory->req_buffer[req_buffer_idx], + iov[i].iov_base, + iov[i].iov_len); + req_buffer_idx += iov[i].iov_len; + } + /* + * Signal semaphore #0 indicting a new message from client + * to server request queue + */ + sop.sem_num = 0; + sop.sem_op = 1; + sop.sem_flg = 0; + +retry_semop: + res = semop (ipc_segment->semid, &sop, 1); + if (res == -1 && errno == EINTR) { + goto retry_semop; + } else + if (res == -1 && errno == EACCES) { + priv_change_send (ipc_segment); + goto retry_semop; + } else + if (res == -1) { + return (CS_ERR_LIBRARY); + } + return (CS_OK); +} + +static cs_error_t +cslib_reply_receive ( + void *ipc_context, + void *res_msg, int res_len) +{ + struct sembuf sop; + struct ipc_segment *ipc_segment = (struct ipc_segment *)ipc_context; + int res; + + /* + * Wait for semaphore #1 indicating a new message from server + * to client in the response queue + */ + sop.sem_num = 1; + sop.sem_op = -1; + sop.sem_flg = 0; + +retry_semop: + res = semop (ipc_segment->semid, &sop, 1); + if (res == -1 && errno == EINTR) { + goto retry_semop; + } else + if (res == -1 && errno == EACCES) { + priv_change_send (ipc_segment); + goto retry_semop; + } else + if (res == -1) { + return (CS_ERR_LIBRARY); + } + + memcpy (res_msg, ipc_segment->shared_memory->res_buffer, res_len); + return (CS_OK); +} + +static cs_error_t +cslib_reply_receive_in_buf ( + void *ipc_context, + void **res_msg) +{ + struct sembuf sop; + struct ipc_segment *ipc_segment = (struct ipc_segment *)ipc_context; + int res; + + /* + * Wait for semaphore #1 indicating a new message from server + * to client in the response queue + */ + sop.sem_num = 1; + sop.sem_op = -1; + sop.sem_flg = 0; + +retry_semop: + res = semop (ipc_segment->semid, &sop, 1); + if (res == -1 && errno == EINTR) { + goto retry_semop; + } else + if (res == -1 && errno == EACCES) { + priv_change_send (ipc_segment); + goto retry_semop; + } else + if (res == -1) { + return (CS_ERR_LIBRARY); + } + + *res_msg = (char *)ipc_segment->shared_memory->res_buffer; + return (CS_OK); +} + +cs_error_t +cslib_msg_send_reply_receive ( + void *ipc_context, + struct iovec *iov, + int iov_len, + void *res_msg, + int res_len) +{ + cs_error_t res; + + res = cslib_msg_send (ipc_context, iov, iov_len); + if (res != CS_OK) { + return (res); + } + + res = cslib_reply_receive (ipc_context, res_msg, res_len); + if (res != CS_OK) { + return (res); + } + + return (CS_OK); +} + +cs_error_t +cslib_msg_send_reply_receive_in_buf ( + void *ipc_context, + struct iovec *iov, + int iov_len, + void **res_msg) +{ + unsigned int res; + + res = cslib_msg_send (ipc_context, iov, iov_len); + if (res != CS_OK) { + return (res); + } + + res = cslib_reply_receive_in_buf (ipc_context, res_msg); + if (res != CS_OK) { + return (res); + } + + return (CS_OK); +} + +cs_error_t +saHandleCreate ( + struct saHandleDatabase *handleDatabase, + int instanceSize, + uint64_t *handleOut) +{ + uint32_t handle; + uint32_t check; + void *newHandles = NULL; + int found = 0; + void *instance; + int i; + + pthread_mutex_lock (&handleDatabase->mutex); + + for (handle = 0; handle < handleDatabase->handleCount; handle++) { + if (handleDatabase->handles[handle].state == SA_HANDLE_STATE_EMPTY) { + found = 1; + break; + } + } + + if (found == 0) { + handleDatabase->handleCount += 1; + newHandles = (struct saHandle *)realloc (handleDatabase->handles, + sizeof (struct saHandle) * handleDatabase->handleCount); + if (newHandles == NULL) { + pthread_mutex_unlock (&handleDatabase->mutex); + return (CS_ERR_NO_MEMORY); + } + handleDatabase->handles = newHandles; + } + + instance = malloc (instanceSize); + if (instance == 0) { + free (newHandles); + pthread_mutex_unlock (&handleDatabase->mutex); + return (CS_ERR_NO_MEMORY); + } + + + /* + * This code makes sure the random number isn't zero + * We use 0 to specify an invalid handle out of the 1^64 address space + * If we get 0 200 times in a row, the RNG may be broken + */ + for (i = 0; i < 200; i++) { + check = random(); + if (check != 0) { + break; + } + } + + memset (instance, 0, instanceSize); + + handleDatabase->handles[handle].state = SA_HANDLE_STATE_ACTIVE; + + handleDatabase->handles[handle].instance = instance; + + handleDatabase->handles[handle].refCount = 1; + + handleDatabase->handles[handle].check = check; + + *handleOut = (uint64_t)((uint64_t)check << 32 | handle); + + pthread_mutex_unlock (&handleDatabase->mutex); + + return (CS_OK); +} + + +cs_error_t +saHandleDestroy ( + struct saHandleDatabase *handleDatabase, + uint64_t inHandle) +{ + cs_error_t error = CS_OK; + uint32_t check = inHandle >> 32; + uint32_t handle = inHandle & 0xffffffff; + + pthread_mutex_lock (&handleDatabase->mutex); + + if (check != handleDatabase->handles[handle].check) { + pthread_mutex_unlock (&handleDatabase->mutex); + error = CS_ERR_BAD_HANDLE; + return (error); + } + + handleDatabase->handles[handle].state = SA_HANDLE_STATE_PENDINGREMOVAL; + + pthread_mutex_unlock (&handleDatabase->mutex); + + saHandleInstancePut (handleDatabase, inHandle); + + return (error); +} + + +cs_error_t +saHandleInstanceGet ( + struct saHandleDatabase *handleDatabase, + uint64_t inHandle, + void **instance) +{ + uint32_t check = inHandle >> 32; + uint32_t handle = inHandle & 0xffffffff; + + cs_error_t error = CS_OK; + pthread_mutex_lock (&handleDatabase->mutex); + + if (handle >= (uint64_t)handleDatabase->handleCount) { + error = CS_ERR_BAD_HANDLE; + goto error_exit; + } + if (handleDatabase->handles[handle].state != SA_HANDLE_STATE_ACTIVE) { + error = CS_ERR_BAD_HANDLE; + goto error_exit; + } + if (check != handleDatabase->handles[handle].check) { + error = CS_ERR_BAD_HANDLE; + goto error_exit; + } + + + *instance = handleDatabase->handles[handle].instance; + + handleDatabase->handles[handle].refCount += 1; + +error_exit: + pthread_mutex_unlock (&handleDatabase->mutex); + + return (error); +} + + +cs_error_t +saHandleInstancePut ( + struct saHandleDatabase *handleDatabase, + uint64_t inHandle) +{ + void *instance; + cs_error_t error = CS_OK; + uint32_t check = inHandle >> 32; + uint32_t handle = inHandle & 0xffffffff; + + pthread_mutex_lock (&handleDatabase->mutex); + + if (check != handleDatabase->handles[handle].check) { + error = CS_ERR_BAD_HANDLE; + goto error_exit; + } + + handleDatabase->handles[handle].refCount -= 1; + assert (handleDatabase->handles[handle].refCount >= 0); + + if (handleDatabase->handles[handle].refCount == 0) { + instance = (handleDatabase->handles[handle].instance); + handleDatabase->handleInstanceDestructor (instance); + free (instance); + memset (&handleDatabase->handles[handle], 0, sizeof (struct saHandle)); + } + +error_exit: + pthread_mutex_unlock (&handleDatabase->mutex); + + return (error); +} diff --git a/lib/cpg.c b/lib/cpg.c index edc15fde..3e988073 100644 --- a/lib/cpg.c +++ b/lib/cpg.c @@ -5,7 +5,7 @@ * * All rights reserved. * - * Author: Christine Caulfield (ccaulfie@redhat.com) + * Author: Patrick Caulfield (pcaulfie@redhat.com) * * This software licensed under BSD license, the text of which follows: * @@ -34,7 +34,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. */ /* - * Provides a closed process group API using the corosync executive + * Provides a closed process group API using the cslib executive */ #include @@ -49,13 +49,11 @@ #include #include #include -#include +#include struct cpg_inst { - int response_fd; - int dispatch_fd; + void *ipc_ctx; int finalize; - cpg_flow_control_state_t flow_control_state; cpg_callbacks_t callbacks; void *context; pthread_mutex_t response_mutex; @@ -72,7 +70,7 @@ static struct saHandleDatabase cpg_handle_t_db = { }; /* - * Clean up function for a cpg instance (cpg_initialize) handle + * Clean up function for a cpg instance (cpg_nitialize) handle */ static void cpg_instance_destructor (void *instance) { @@ -84,8 +82,8 @@ static void cpg_instance_destructor (void *instance) /** - * @defgroup cpg_corosync The closed process group API - * @ingroup corosync + * @defgroup cpg_cslib The closed process group API + * @ingroup cslib * * @{ */ @@ -107,9 +105,7 @@ cs_error_t cpg_initialize ( goto error_destroy; } - error = saServiceConnect (&cpg_inst->dispatch_fd, - &cpg_inst->response_fd, - CPG_SERVICE); + error = cslib_service_connect (CPG_SERVICE, &cpg_inst->ipc_ctx); if (error != CS_OK) { goto error_put_destroy; } @@ -120,14 +116,14 @@ cs_error_t cpg_initialize ( pthread_mutex_init (&cpg_inst->dispatch_mutex, NULL); - (void)saHandleInstancePut (&cpg_handle_t_db, *handle); + saHandleInstancePut (&cpg_handle_t_db, *handle); return (CS_OK); error_put_destroy: - (void)saHandleInstancePut (&cpg_handle_t_db, *handle); + saHandleInstancePut (&cpg_handle_t_db, *handle); error_destroy: - (void)saHandleDestroy (&cpg_handle_t_db, *handle); + saHandleDestroy (&cpg_handle_t_db, *handle); error_no_destroy: return (error); } @@ -150,30 +146,21 @@ cs_error_t cpg_finalize ( */ if (cpg_inst->finalize) { pthread_mutex_unlock (&cpg_inst->response_mutex); - (void)saHandleInstancePut (&cpg_handle_t_db, handle); - return (CS_ERR_BAD_HANDLE); + saHandleInstancePut (&cpg_handle_t_db, handle); + return (CPG_ERR_BAD_HANDLE); } cpg_inst->finalize = 1; + cslib_service_disconnect (cpg_inst->ipc_ctx); + pthread_mutex_unlock (&cpg_inst->response_mutex); - (void)saHandleDestroy (&cpg_handle_t_db, handle); + saHandleDestroy (&cpg_handle_t_db, handle); - /* - * Disconnect from the server - */ - if (cpg_inst->response_fd != -1) { - shutdown(cpg_inst->response_fd, 0); - close(cpg_inst->response_fd); - } - if (cpg_inst->dispatch_fd != -1) { - shutdown(cpg_inst->dispatch_fd, 0); - close(cpg_inst->dispatch_fd); - } - (void)saHandleInstancePut (&cpg_handle_t_db, handle); + saHandleInstancePut (&cpg_handle_t_db, handle); - return (CS_OK); + return (CPG_OK); } cs_error_t cpg_fd_get ( @@ -188,9 +175,9 @@ cs_error_t cpg_fd_get ( return (error); } - *fd = cpg_inst->dispatch_fd; + *fd = cslib_fd_get (cpg_inst->ipc_ctx); - (void)saHandleInstancePut (&cpg_handle_t_db, handle); + saHandleInstancePut (&cpg_handle_t_db, handle); return (CS_OK); } @@ -209,7 +196,7 @@ cs_error_t cpg_context_get ( *context = cpg_inst->context; - (void)saHandleInstancePut (&cpg_handle_t_db, handle); + saHandleInstancePut (&cpg_handle_t_db, handle); return (CS_OK); } @@ -228,12 +215,12 @@ cs_error_t cpg_context_set ( cpg_inst->context = context; - (void)saHandleInstancePut (&cpg_handle_t_db, handle); + saHandleInstancePut (&cpg_handle_t_db, handle); return (CS_OK); } -struct cpg_res_overlay { +struct res_overlay { mar_res_header_t header __attribute__((aligned(8))); char data[512000]; }; @@ -242,18 +229,16 @@ cs_error_t cpg_dispatch ( cpg_handle_t handle, cs_dispatch_flags_t dispatch_types) { - struct pollfd ufds; int timeout = -1; cs_error_t error; int cont = 1; /* always continue do loop except when set to 0 */ int dispatch_avail; struct cpg_inst *cpg_inst; - struct res_lib_cpg_flowcontrol_callback *res_cpg_flowcontrol_callback; struct res_lib_cpg_confchg_callback *res_cpg_confchg_callback; struct res_lib_cpg_deliver_callback *res_cpg_deliver_callback; - struct res_lib_cpg_groups_get_callback *res_lib_cpg_groups_get_callback; cpg_callbacks_t callbacks; - struct cpg_res_overlay dispatch_data; + struct res_overlay dispatch_data; + int ignore_dispatch = 0; struct cpg_address member_list[CPG_MEMBERS_MAX]; struct cpg_address left_list[CPG_MEMBERS_MAX]; struct cpg_address joined_list[CPG_MEMBERS_MAX]; @@ -271,41 +256,23 @@ cs_error_t cpg_dispatch ( * Timeout instantly for SA_DISPATCH_ONE or SA_DISPATCH_ALL and * wait indefinately for SA_DISPATCH_BLOCKING */ - if (dispatch_types == CS_DISPATCH_ALL) { + if (dispatch_types == CPG_DISPATCH_ALL) { timeout = 0; } do { - ufds.fd = cpg_inst->dispatch_fd; - ufds.events = POLLIN; - ufds.revents = 0; - - error = saPollRetry (&ufds, 1, timeout); - if (error != CS_OK) { - goto error_nounlock; - } - pthread_mutex_lock (&cpg_inst->dispatch_mutex); - /* - * Regather poll data in case ufds has changed since taking lock - */ - error = saPollRetry (&ufds, 1, timeout); + dispatch_avail = cslib_dispatch_recv (cpg_inst->ipc_ctx, + (void *)&dispatch_data, timeout); + + pthread_mutex_unlock (&cpg_inst->dispatch_mutex); + if (error != CS_OK) { - goto error_nounlock; + goto error_put; } - /* - * Handle has been finalized in another thread - */ - if (cpg_inst->finalize == 1) { - error = CS_OK; - pthread_mutex_unlock (&cpg_inst->dispatch_mutex); - goto error_unlock; - } - - dispatch_avail = ufds.revents & POLLIN; - if (dispatch_avail == 0 && dispatch_types == CS_DISPATCH_ALL) { + if (dispatch_avail == 0 && dispatch_types == CPG_DISPATCH_ALL) { pthread_mutex_unlock (&cpg_inst->dispatch_mutex); break; /* exit do while cont is 1 loop */ } else @@ -313,37 +280,21 @@ cs_error_t cpg_dispatch ( pthread_mutex_unlock (&cpg_inst->dispatch_mutex); continue; /* next poll */ } - - if (ufds.revents & POLLIN) { - /* - * Queue empty, read response from socket - */ - error = saRecvRetry (cpg_inst->dispatch_fd, &dispatch_data.header, - sizeof (mar_res_header_t)); - if (error != CS_OK) { - goto error_unlock; + if (dispatch_avail == -1) { + if (cpg_inst->finalize == 1) { + error = CS_OK; + } else { + error = CS_ERR_LIBRARY; } - if (dispatch_data.header.size > sizeof (mar_res_header_t)) { - error = saRecvRetry (cpg_inst->dispatch_fd, &dispatch_data.data, - dispatch_data.header.size - sizeof (mar_res_header_t)); - - if (error != CS_OK) { - goto error_unlock; - } - } - } else { - pthread_mutex_unlock (&cpg_inst->dispatch_mutex); - continue; + goto error_put; } /* * Make copy of callbacks, message data, unlock instance, and call callback * A risk of this dispatch method is that the callback routines may * operate at the same time that cpgFinalize has been called. - */ + */ memcpy (&callbacks, &cpg_inst->callbacks, sizeof (cpg_callbacks_t)); - - pthread_mutex_unlock (&cpg_inst->dispatch_mutex); /* * Dispatch incoming message */ @@ -351,7 +302,6 @@ cs_error_t cpg_dispatch ( case MESSAGE_RES_CPG_DELIVER_CALLBACK: res_cpg_deliver_callback = (struct res_lib_cpg_deliver_callback *)&dispatch_data; - cpg_inst->flow_control_state = res_cpg_deliver_callback->flow_control_state; marshall_from_mar_cpg_name_t ( &group_name, &res_cpg_deliver_callback->group_name); @@ -398,33 +348,9 @@ cs_error_t cpg_dispatch ( res_cpg_confchg_callback->joined_list_entries); break; - case MESSAGE_RES_CPG_GROUPS_CALLBACK: - res_lib_cpg_groups_get_callback = (struct res_lib_cpg_groups_get_callback *)&dispatch_data; - marshall_from_mar_cpg_name_t ( - &group_name, - &res_lib_cpg_groups_get_callback->group_name); - for (i = 0; i < res_lib_cpg_groups_get_callback->num_members; i++) { - marshall_from_mar_cpg_address_t (&member_list[i], - &res_lib_cpg_groups_get_callback->member_list[i]); - } - - callbacks.cpg_groups_get_fn(handle, - res_lib_cpg_groups_get_callback->group_num, - res_lib_cpg_groups_get_callback->total_groups, - &group_name, - member_list, - res_lib_cpg_groups_get_callback->num_members); - - break; - - case MESSAGE_RES_CPG_FLOWCONTROL_CALLBACK: - res_cpg_flowcontrol_callback = (struct res_lib_cpg_flowcontrol_callback *)&dispatch_data; - cpg_inst->flow_control_state = res_cpg_flowcontrol_callback->flow_control_state; - break; - default: error = CS_ERR_LIBRARY; - goto error_nounlock; + goto error_put; break; } @@ -432,19 +358,25 @@ cs_error_t cpg_dispatch ( * Determine if more messages should be processed * */ switch (dispatch_types) { - case CS_DISPATCH_ONE: - cont = 0; + case CPG_DISPATCH_ONE: + if (ignore_dispatch) { + ignore_dispatch = 0; + } else { + cont = 0; + } break; - case CS_DISPATCH_ALL: + case CPG_DISPATCH_ALL: + if (ignore_dispatch) { + ignore_dispatch = 0; + } break; - case CS_DISPATCH_BLOCKING: + case CPG_DISPATCH_BLOCKING: break; } } while (cont); -error_unlock: - (void)saHandleInstancePut (&cpg_handle_t_db, handle); -error_nounlock: +error_put: + saHandleInstancePut (&cpg_handle_t_db, handle); return (error); } @@ -473,10 +405,10 @@ cs_error_t cpg_join ( marshall_to_mar_cpg_name_t (&req_lib_cpg_trackstart.group_name, group); - iov[0].iov_base = (char *)&req_lib_cpg_trackstart; + iov[0].iov_base = &req_lib_cpg_trackstart; iov[0].iov_len = sizeof (struct req_lib_cpg_trackstart); - error = saSendMsgReceiveReply (cpg_inst->dispatch_fd, iov, 1, + error = cslib_msg_send_reply_receive (cpg_inst->ipc_ctx, iov, 1, &res_lib_cpg_trackstart, sizeof (struct res_lib_cpg_trackstart)); if (error != CS_OK) { @@ -491,10 +423,10 @@ cs_error_t cpg_join ( marshall_to_mar_cpg_name_t (&req_lib_cpg_join.group_name, group); - iov[0].iov_base = (char *)&req_lib_cpg_join; + iov[0].iov_base = &req_lib_cpg_join; iov[0].iov_len = sizeof (struct req_lib_cpg_join); - error = saSendMsgReceiveReply (cpg_inst->response_fd, iov, 1, + error = cslib_msg_send_reply_receive (cpg_inst->ipc_ctx, iov, 1, &res_lib_cpg_join, sizeof (struct res_lib_cpg_join)); pthread_mutex_unlock (&cpg_inst->response_mutex); @@ -506,7 +438,7 @@ cs_error_t cpg_join ( error = res_lib_cpg_join.header.error; error_exit: - (void)saHandleInstancePut (&cpg_handle_t_db, handle); + saHandleInstancePut (&cpg_handle_t_db, handle); return (error); } @@ -532,12 +464,12 @@ cs_error_t cpg_leave ( marshall_to_mar_cpg_name_t (&req_lib_cpg_leave.group_name, group); - iov[0].iov_base = (char *)&req_lib_cpg_leave; + iov[0].iov_base = &req_lib_cpg_leave; iov[0].iov_len = sizeof (struct req_lib_cpg_leave); pthread_mutex_lock (&cpg_inst->response_mutex); - error = saSendMsgReceiveReply (cpg_inst->response_fd, iov, 1, + error = cslib_msg_send_reply_receive (cpg_inst->ipc_ctx, iov, 1, &res_lib_cpg_leave, sizeof (struct res_lib_cpg_leave)); pthread_mutex_unlock (&cpg_inst->response_mutex); @@ -548,7 +480,7 @@ cs_error_t cpg_leave ( error = res_lib_cpg_leave.header.error; error_exit: - (void)saHandleInstancePut (&cpg_handle_t_db, handle); + saHandleInstancePut (&cpg_handle_t_db, handle); return (error); } @@ -583,14 +515,14 @@ cs_error_t cpg_mcast_joined ( req_lib_cpg_mcast.guarantee = guarantee; req_lib_cpg_mcast.msglen = msg_len; - iov[0].iov_base = (char *)&req_lib_cpg_mcast; + iov[0].iov_base = &req_lib_cpg_mcast; iov[0].iov_len = sizeof (struct req_lib_cpg_mcast); memcpy (&iov[1], iovec, iov_len * sizeof (struct iovec)); pthread_mutex_lock (&cpg_inst->response_mutex); - error = saSendMsgReceiveReply (cpg_inst->response_fd, iov, iov_len + 1, - &res_lib_cpg_mcast, sizeof (res_lib_cpg_mcast)); + error = cslib_msg_send_reply_receive (cpg_inst->ipc_ctx, iov, + iov_len + 1, &res_lib_cpg_mcast, sizeof (res_lib_cpg_mcast)); pthread_mutex_unlock (&cpg_inst->response_mutex); @@ -598,19 +530,10 @@ cs_error_t cpg_mcast_joined ( goto error_exit; } -/* Only update the flow control state when the return value is OK. - * Otherwise the flow control state is not guaranteed to be valid in the - * return message. - * Also, don't set to ENABLED if the return value is TRY_AGAIN as this can lead - * to Flow Control State sync issues between AIS LIB and EXEC. - */ - if (res_lib_cpg_mcast.header.error == CS_OK) { - cpg_inst->flow_control_state = res_lib_cpg_mcast.flow_control_state; - } error = res_lib_cpg_mcast.header.error; error_exit: - (void)saHandleInstancePut (&cpg_handle_t_db, handle); + saHandleInstancePut (&cpg_handle_t_db, handle); return (error); } @@ -625,71 +548,46 @@ cs_error_t cpg_membership_get ( struct cpg_inst *cpg_inst; struct iovec iov; struct req_lib_cpg_membership req_lib_cpg_membership_get; - struct res_lib_cpg_confchg_callback *res_lib_cpg_membership_get; - mar_res_header_t header; - unsigned int i, bytesleft; - char *buffer = NULL; + struct res_lib_cpg_confchg_callback res_lib_cpg_membership_get; + unsigned int i; error = saHandleInstanceGet (&cpg_handle_t_db, handle, (void *)&cpg_inst); if (error != CS_OK) { return (error); } - req_lib_cpg_membership_get.header.size = sizeof (req_lib_cpg_membership_get); + req_lib_cpg_membership_get.header.size = sizeof (mar_req_header_t); req_lib_cpg_membership_get.header.id = MESSAGE_REQ_CPG_MEMBERSHIP; - iov.iov_base = (char *)&req_lib_cpg_membership_get; - iov.iov_len = sizeof (req_lib_cpg_membership_get); + iov.iov_base = &req_lib_cpg_membership_get; + iov.iov_len = sizeof (mar_req_header_t); pthread_mutex_lock (&cpg_inst->response_mutex); - error = saSendMsgReceiveReply (cpg_inst->response_fd, &iov, 1, - &header, sizeof (header)); + error = cslib_msg_send_reply_receive (cpg_inst->ipc_ctx, &iov, 1, + &res_lib_cpg_membership_get, sizeof (mar_res_header_t)); + + pthread_mutex_unlock (&cpg_inst->response_mutex); + if (error != CS_OK) { goto error_exit; } - buffer = malloc(header.size); - if (buffer == NULL) { - error = CS_ERR_NO_MEMORY; - goto error_exit; - } - - memcpy (buffer, &header, sizeof (header)); - bytesleft = header.size - sizeof (header); - - error = saRecvRetry (cpg_inst->response_fd, - buffer + sizeof (header), bytesleft); - if (error != CS_OK) { - goto error_exit; - } - - error = header.error; - if (error != CS_OK) { - goto error_exit; - } - - res_lib_cpg_membership_get = (struct res_lib_cpg_confchg_callback *) buffer; + error = res_lib_cpg_membership_get.header.error; /* * Copy results to caller */ - *member_list_entries = res_lib_cpg_membership_get->member_list_entries; + *member_list_entries = res_lib_cpg_membership_get.member_list_entries; if (member_list) { - for (i = 0; i < res_lib_cpg_membership_get->member_list_entries; i++) { + for (i = 0; i < res_lib_cpg_membership_get.member_list_entries; i++) { marshall_from_mar_cpg_address_t (&member_list[i], - &res_lib_cpg_membership_get->member_list[i]); + &res_lib_cpg_membership_get.member_list[i]); } } error_exit: - - if (buffer != NULL) - free(buffer); - - pthread_mutex_unlock (&cpg_inst->response_mutex); - - (void)saHandleInstancePut (&cpg_handle_t_db, handle); + saHandleInstancePut (&cpg_handle_t_db, handle); return (error); } @@ -717,7 +615,7 @@ cs_error_t cpg_local_get ( pthread_mutex_lock (&cpg_inst->response_mutex); - error = saSendMsgReceiveReply (cpg_inst->response_fd, &iov, 1, + error = cslib_msg_send_reply_receive (cpg_inst->ipc_ctx, &iov, 1, &res_lib_cpg_local_get, sizeof (res_lib_cpg_local_get)); pthread_mutex_unlock (&cpg_inst->response_mutex); @@ -731,49 +629,7 @@ cs_error_t cpg_local_get ( *local_nodeid = res_lib_cpg_local_get.local_nodeid; error_exit: - (void)saHandleInstancePut (&cpg_handle_t_db, handle); - - return (error); -} - -cs_error_t cpg_groups_get ( - cpg_handle_t handle, - unsigned int *num_groups) -{ - cs_error_t error; - struct cpg_inst *cpg_inst; - struct iovec iov; - struct req_lib_cpg_groups_get req_lib_cpg_groups_get; - struct res_lib_cpg_groups_get res_lib_cpg_groups_get; - - error = saHandleInstanceGet (&cpg_handle_t_db, handle, (void *)&cpg_inst); - if (error != CS_OK) { - return (error); - } - - req_lib_cpg_groups_get.header.size = sizeof (mar_req_header_t); - req_lib_cpg_groups_get.header.id = MESSAGE_REQ_CPG_GROUPS_GET; - - iov.iov_base = &req_lib_cpg_groups_get; - iov.iov_len = sizeof (struct req_lib_cpg_groups_get); - - pthread_mutex_lock (&cpg_inst->response_mutex); - - error = saSendMsgReceiveReply (cpg_inst->response_fd, &iov, 1, - &res_lib_cpg_groups_get, sizeof (res_lib_cpg_groups_get)); - - pthread_mutex_unlock (&cpg_inst->response_mutex); - - if (error != CS_OK) { - goto error_exit; - } - - *num_groups = res_lib_cpg_groups_get.num_groups; - error = res_lib_cpg_groups_get.header.error; - - /* Real output is delivered via a callback */ -error_exit: - (void)saHandleInstancePut (&cpg_handle_t_db, handle); + saHandleInstancePut (&cpg_handle_t_db, handle); return (error); } @@ -789,10 +645,10 @@ cs_error_t cpg_flow_control_state_get ( if (error != CS_OK) { return (error); } + + *flow_control_state = cslib_dispatch_flow_control_get (cpg_inst->ipc_ctx); - *flow_control_state = cpg_inst->flow_control_state; - - (void)saHandleInstancePut (&cpg_handle_t_db, handle); + saHandleInstancePut (&cpg_handle_t_db, handle); return (error); } diff --git a/lib/evs.c b/lib/evs.c index 1be9ef97..06e63b9b 100644 --- a/lib/evs.c +++ b/lib/evs.c @@ -2,7 +2,7 @@ * vi: set autoindent tabstop=4 shiftwidth=4 : * Copyright (c) 2004-2005 MontaVista Software, Inc. - * Copyright (c) 2006-2008 Red Hat, Inc. + * Copyright (c) 2006-2007 Red Hat, Inc. * * All rights reserved. * @@ -35,7 +35,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. */ /* - * Provides an extended virtual synchrony API using the corosync executive + * Provides an extended virtual synchrony API using the cslib executive */ #include @@ -46,23 +46,21 @@ #include #include -#include #include #include #include #include -#include +#include struct evs_inst { - int response_fd; - int dispatch_fd; + void *ipc_ctx; int finalize; evs_callbacks_t callbacks; pthread_mutex_t response_mutex; pthread_mutex_t dispatch_mutex; }; -struct evs_res_overlay { +struct res_overlay { mar_res_header_t header __attribute__((aligned(8))); char data[512000]; }; @@ -89,8 +87,8 @@ static void evs_instance_destructor (void *instance) /** - * @defgroup evs_corosync The extended virtual synchrony passthrough API - * @ingroup corosync + * @defgroup evs_cslib The extended virtual synchrony passthrough API + * @ingroup cslib * * @{ */ @@ -98,9 +96,9 @@ static void evs_instance_destructor (void *instance) * test * @param handle The handle of evs initialize * @param callbacks The callbacks for evs_initialize - * @returns CS_OK + * @returns EVS_OK */ -cs_error_t evs_initialize ( +evs_error_t evs_initialize ( evs_handle_t *handle, evs_callbacks_t *callbacks) { @@ -117,10 +115,8 @@ cs_error_t evs_initialize ( goto error_destroy; } - error = saServiceConnect (&evs_inst->response_fd, - &evs_inst->dispatch_fd, - EVS_SERVICE); - if (error != CS_OK) { + error = cslib_service_connect (EVS_SERVICE, &evs_inst->ipc_ctx); + if (error != EVS_OK) { goto error_put_destroy; } @@ -130,19 +126,19 @@ cs_error_t evs_initialize ( pthread_mutex_init (&evs_inst->dispatch_mutex, NULL); - (void)saHandleInstancePut (&evs_handle_t_db, *handle); + saHandleInstancePut (&evs_handle_t_db, *handle); return (CS_OK); error_put_destroy: - (void)saHandleInstancePut (&evs_handle_t_db, *handle); + saHandleInstancePut (&evs_handle_t_db, *handle); error_destroy: - (void)saHandleDestroy (&evs_handle_t_db, *handle); + saHandleDestroy (&evs_handle_t_db, *handle); error_no_destroy: return (error); } -cs_error_t evs_finalize ( +evs_error_t evs_finalize ( evs_handle_t handle) { struct evs_inst *evs_inst; @@ -152,7 +148,6 @@ cs_error_t evs_finalize ( if (error != CS_OK) { return (error); } -// TODO is the locking right here pthread_mutex_lock (&evs_inst->response_mutex); /* @@ -160,33 +155,24 @@ cs_error_t evs_finalize ( */ if (evs_inst->finalize) { pthread_mutex_unlock (&evs_inst->response_mutex); - (void)saHandleInstancePut (&evs_handle_t_db, handle); - return (CS_ERR_BAD_HANDLE); + saHandleInstancePut (&evs_handle_t_db, handle); + return (EVS_ERR_BAD_HANDLE); } evs_inst->finalize = 1; + cslib_service_disconnect (evs_inst->ipc_ctx); + pthread_mutex_unlock (&evs_inst->response_mutex); - (void)saHandleDestroy (&evs_handle_t_db, handle); - /* - * Disconnect from the server - */ - if (evs_inst->response_fd != -1) { - shutdown(evs_inst->response_fd, 0); - close(evs_inst->response_fd); - } - if (evs_inst->dispatch_fd != -1) { - shutdown(evs_inst->dispatch_fd, 0); - close(evs_inst->dispatch_fd); - } - (void)saHandleInstancePut (&evs_handle_t_db, handle); + saHandleDestroy (&evs_handle_t_db, handle); + saHandleInstancePut (&evs_handle_t_db, handle); - return (CS_OK); + return (EVS_OK); } -cs_error_t evs_fd_get ( +evs_error_t evs_fd_get ( evs_handle_t handle, int *fd) { @@ -198,18 +184,17 @@ cs_error_t evs_fd_get ( return (error); } - *fd = evs_inst->dispatch_fd; + *fd = cslib_fd_get (evs_inst->ipc_ctx); - (void)saHandleInstancePut (&evs_handle_t_db, handle); + saHandleInstancePut (&evs_handle_t_db, handle); return (CS_OK); } -cs_error_t evs_dispatch ( +evs_error_t evs_dispatch ( evs_handle_t handle, cs_dispatch_flags_t dispatch_types) { - struct pollfd ufds; int timeout = -1; cs_error_t error; int cont = 1; /* always continue do loop except when set to 0 */ @@ -218,7 +203,8 @@ cs_error_t evs_dispatch ( struct res_evs_confchg_callback *res_evs_confchg_callback; struct res_evs_deliver_callback *res_evs_deliver_callback; evs_callbacks_t callbacks; - struct evs_res_overlay dispatch_data; + struct res_overlay dispatch_data; + int ignore_dispatch = 0; error = saHandleInstanceGet (&evs_handle_t_db, handle, (void *)&evs_inst); if (error != CS_OK) { @@ -229,69 +215,36 @@ cs_error_t evs_dispatch ( * Timeout instantly for SA_DISPATCH_ONE or SA_DISPATCH_ALL and * wait indefinately for SA_DISPATCH_BLOCKING */ - if (dispatch_types == CS_DISPATCH_ALL) { + if (dispatch_types == EVS_DISPATCH_ALL) { timeout = 0; } do { - ufds.fd = evs_inst->dispatch_fd; - ufds.events = POLLIN; - ufds.revents = 0; - - error = saPollRetry (&ufds, 1, timeout); - if (error != CS_OK) { + dispatch_avail = cslib_dispatch_recv (evs_inst->ipc_ctx, (void *)&dispatch_data, timeout); + if (dispatch_avail == -1) { + error = CS_ERR_LIBRARY; goto error_nounlock; } + pthread_mutex_lock (&evs_inst->dispatch_mutex); - /* - * Regather poll data in case ufds has changed since taking lock - */ - error = saPollRetry (&ufds, 1, 0); - if (error != CS_OK) { - goto error_nounlock; - } - /* * Handle has been finalized in another thread */ if (evs_inst->finalize == 1) { - error = CS_OK; + error = EVS_OK; pthread_mutex_unlock (&evs_inst->dispatch_mutex); goto error_unlock; } - dispatch_avail = ufds.revents & POLLIN; - if (dispatch_avail == 0 && dispatch_types == CS_DISPATCH_ALL) { + if (dispatch_avail == 0 && dispatch_types == EVS_DISPATCH_ALL) { pthread_mutex_unlock (&evs_inst->dispatch_mutex); break; /* exit do while cont is 1 loop */ } else if (dispatch_avail == 0) { pthread_mutex_unlock (&evs_inst->dispatch_mutex); - continue; /* next poll */ - } - - if (ufds.revents & POLLIN) { - /* - * Queue empty, read response from socket - */ - error = saRecvRetry (evs_inst->dispatch_fd, &dispatch_data.header, - sizeof (mar_res_header_t)); - if (error != CS_OK) { - goto error_unlock; - } - if (dispatch_data.header.size > sizeof (mar_res_header_t)) { - error = saRecvRetry (evs_inst->dispatch_fd, &dispatch_data.data, - dispatch_data.header.size - sizeof (mar_res_header_t)); - - if (error != CS_OK) { - goto error_unlock; - } - } - } else { - pthread_mutex_unlock (&evs_inst->dispatch_mutex); - continue; + continue; /* next dispatch event */ } /* @@ -335,35 +288,42 @@ cs_error_t evs_dispatch ( * Determine if more messages should be processed * */ switch (dispatch_types) { - case CS_DISPATCH_ONE: - cont = 0; + case EVS_DISPATCH_ONE: + if (ignore_dispatch) { + ignore_dispatch = 0; + } else { + cont = 0; + } break; - case CS_DISPATCH_ALL: + case EVS_DISPATCH_ALL: + if (ignore_dispatch) { + ignore_dispatch = 0; + } break; - case CS_DISPATCH_BLOCKING: + case EVS_DISPATCH_BLOCKING: break; } } while (cont); error_unlock: - (void)saHandleInstancePut (&evs_handle_t_db, handle); + saHandleInstancePut (&evs_handle_t_db, handle); error_nounlock: return (error); } -cs_error_t evs_join ( +evs_error_t evs_join ( evs_handle_t handle, struct evs_group *groups, int group_entries) { - cs_error_t error; + evs_error_t error; struct evs_inst *evs_inst; struct iovec iov[2]; struct req_lib_evs_join req_lib_evs_join; struct res_lib_evs_join res_lib_evs_join; error = saHandleInstanceGet (&evs_handle_t_db, handle, (void *)&evs_inst); - if (error != CS_OK) { + if (error != EVS_OK) { return (error); } @@ -372,14 +332,14 @@ cs_error_t evs_join ( req_lib_evs_join.header.id = MESSAGE_REQ_EVS_JOIN; req_lib_evs_join.group_entries = group_entries; - iov[0].iov_base = (char *)&req_lib_evs_join; + iov[0].iov_base = &req_lib_evs_join; iov[0].iov_len = sizeof (struct req_lib_evs_join); - iov[1].iov_base = (char *)groups; + iov[1].iov_base = groups; iov[1].iov_len = (group_entries * sizeof (struct evs_group)); pthread_mutex_lock (&evs_inst->response_mutex); - error = saSendMsgReceiveReply (evs_inst->response_fd, iov, 2, + error = cslib_msg_send_reply_receive (evs_inst->ipc_ctx, iov, 2, &res_lib_evs_join, sizeof (struct res_lib_evs_join)); pthread_mutex_unlock (&evs_inst->response_mutex); @@ -391,17 +351,17 @@ cs_error_t evs_join ( error = res_lib_evs_join.header.error; error_exit: - (void)saHandleInstancePut (&evs_handle_t_db, handle); + saHandleInstancePut (&evs_handle_t_db, handle); return (error); } -cs_error_t evs_leave ( +evs_error_t evs_leave ( evs_handle_t handle, struct evs_group *groups, int group_entries) { - cs_error_t error; + evs_error_t error; struct evs_inst *evs_inst; struct iovec iov[2]; struct req_lib_evs_leave req_lib_evs_leave; @@ -417,14 +377,14 @@ cs_error_t evs_leave ( req_lib_evs_leave.header.id = MESSAGE_REQ_EVS_LEAVE; req_lib_evs_leave.group_entries = group_entries; - iov[0].iov_base = (char *)&req_lib_evs_leave; + iov[0].iov_base = &req_lib_evs_leave; iov[0].iov_len = sizeof (struct req_lib_evs_leave); - iov[1].iov_base = (char *)groups; + iov[1].iov_base = groups; iov[1].iov_len = (group_entries * sizeof (struct evs_group)); pthread_mutex_lock (&evs_inst->response_mutex); - error = saSendMsgReceiveReply (evs_inst->response_fd, iov, 2, + error = cslib_msg_send_reply_receive (evs_inst->ipc_ctx, iov, 2, &res_lib_evs_leave, sizeof (struct res_lib_evs_leave)); pthread_mutex_unlock (&evs_inst->response_mutex); @@ -436,19 +396,19 @@ cs_error_t evs_leave ( error = res_lib_evs_leave.header.error; error_exit: - (void)saHandleInstancePut (&evs_handle_t_db, handle); + saHandleInstancePut (&evs_handle_t_db, handle); return (error); } -cs_error_t evs_mcast_joined ( +evs_error_t evs_mcast_joined ( evs_handle_t handle, evs_guarantee_t guarantee, struct iovec *iovec, int iov_len) { int i; - cs_error_t error; + evs_error_t error; struct evs_inst *evs_inst; struct iovec iov[64]; struct req_lib_evs_mcast_joined req_lib_evs_mcast_joined; @@ -471,14 +431,16 @@ cs_error_t evs_mcast_joined ( req_lib_evs_mcast_joined.guarantee = guarantee; req_lib_evs_mcast_joined.msg_len = msg_len; - iov[0].iov_base = (char *)&req_lib_evs_mcast_joined; + iov[0].iov_base = &req_lib_evs_mcast_joined; iov[0].iov_len = sizeof (struct req_lib_evs_mcast_joined); memcpy (&iov[1], iovec, iov_len * sizeof (struct iovec)); pthread_mutex_lock (&evs_inst->response_mutex); - error = saSendMsgReceiveReply (evs_inst->response_fd, iov, iov_len + 1, - &res_lib_evs_mcast_joined, sizeof (struct res_lib_evs_mcast_joined)); + error = cslib_msg_send_reply_receive (evs_inst->ipc_ctx, iov, + iov_len + 1, + &res_lib_evs_mcast_joined, + sizeof (struct res_lib_evs_mcast_joined)); pthread_mutex_unlock (&evs_inst->response_mutex); @@ -489,12 +451,12 @@ cs_error_t evs_mcast_joined ( error = res_lib_evs_mcast_joined.header.error; error_exit: - (void)saHandleInstancePut (&evs_handle_t_db, handle); + saHandleInstancePut (&evs_handle_t_db, handle); return (error); } -cs_error_t evs_mcast_groups ( +evs_error_t evs_mcast_groups ( evs_handle_t handle, evs_guarantee_t guarantee, struct evs_group *groups, @@ -503,7 +465,7 @@ cs_error_t evs_mcast_groups ( int iov_len) { int i; - cs_error_t error; + evs_error_t error; struct evs_inst *evs_inst; struct iovec iov[64]; struct req_lib_evs_mcast_groups req_lib_evs_mcast_groups; @@ -524,16 +486,18 @@ cs_error_t evs_mcast_groups ( req_lib_evs_mcast_groups.msg_len = msg_len; req_lib_evs_mcast_groups.group_entries = group_entries; - iov[0].iov_base = (char *)&req_lib_evs_mcast_groups; + iov[0].iov_base = &req_lib_evs_mcast_groups; iov[0].iov_len = sizeof (struct req_lib_evs_mcast_groups); - iov[1].iov_base = (char *)groups; + iov[1].iov_base = groups; iov[1].iov_len = (group_entries * sizeof (struct evs_group)); memcpy (&iov[2], iovec, iov_len * sizeof (struct iovec)); pthread_mutex_lock (&evs_inst->response_mutex); - error = saSendMsgReceiveReply (evs_inst->response_fd, iov, iov_len + 2, - &res_lib_evs_mcast_groups, sizeof (struct res_lib_evs_mcast_groups)); + error = cslib_msg_send_reply_receive (evs_inst->ipc_ctx, iov, + iov_len + 2, + &res_lib_evs_mcast_groups, + sizeof (struct res_lib_evs_mcast_groups)); pthread_mutex_unlock (&evs_inst->response_mutex); if (error != CS_OK) { @@ -543,18 +507,18 @@ cs_error_t evs_mcast_groups ( error = res_lib_evs_mcast_groups.header.error; error_exit: - (void)saHandleInstancePut (&evs_handle_t_db, handle); + saHandleInstancePut (&evs_handle_t_db, handle); return (error); } -cs_error_t evs_membership_get ( +evs_error_t evs_membership_get ( evs_handle_t handle, unsigned int *local_nodeid, unsigned int *member_list, unsigned int *member_list_entries) { - cs_error_t error; + evs_error_t error; struct evs_inst *evs_inst; struct iovec iov; struct req_lib_evs_membership_get req_lib_evs_membership_get; @@ -568,13 +532,16 @@ cs_error_t evs_membership_get ( req_lib_evs_membership_get.header.size = sizeof (struct req_lib_evs_membership_get); req_lib_evs_membership_get.header.id = MESSAGE_REQ_EVS_MEMBERSHIP_GET; - iov.iov_base = (char *)&req_lib_evs_membership_get; + iov.iov_base = &req_lib_evs_membership_get; iov.iov_len = sizeof (struct req_lib_evs_membership_get); pthread_mutex_lock (&evs_inst->response_mutex); - error = saSendMsgReceiveReply (evs_inst->response_fd, &iov, 1, - &res_lib_evs_membership_get, sizeof (struct res_lib_evs_membership_get)); + error = cslib_msg_send_reply_receive (evs_inst->ipc_ctx, + &iov, + 1, + &res_lib_evs_membership_get, + sizeof (struct res_lib_evs_membership_get)); pthread_mutex_unlock (&evs_inst->response_mutex); @@ -598,7 +565,7 @@ cs_error_t evs_membership_get ( } error_exit: - (void)saHandleInstancePut (&evs_handle_t_db, handle); + saHandleInstancePut (&evs_handle_t_db, handle); return (error); } diff --git a/lib/libcfg.versions b/lib/libcfg.versions index b3d7a697..54e8c82c 100644 --- a/lib/libcfg.versions +++ b/lib/libcfg.versions @@ -17,17 +17,15 @@ COROSYNC_CFG_0.82 { local: + cslib_service_connect; + cslib_service_disconnect; + cslib_dispatch_flow_control_get; + cslib_fd_get; + cslib_dispatch_recv; + cslib_msg_send_reply_receive; + cslib_msg_send_reply_receive_in_buf; saHandleCreate; saHandleDestroy; saHandleInstanceGet; saHandleInstancePut; - saPollRetry; - saRecvRetry; - saSendMsgReceiveReply; - saSendMsgRetry; - saSendReceiveReply; - saSendRetry; - saServiceConnect; - saVersionVerify; - clustTimeNow; }; diff --git a/lib/libconfdb.versions b/lib/libconfdb.versions index 8b7c91b0..6ebece81 100644 --- a/lib/libconfdb.versions +++ b/lib/libconfdb.versions @@ -23,16 +23,15 @@ COROSYNC_CONFDB_1.0 { confdb_key_iter; local: + cslib_service_connect; + cslib_service_disconnect; + cslib_dispatch_flow_control_get; + cslib_fd_get; + cslib_dispatch_recv; + cslib_msg_send_reply_receive; + cslib_msg_send_reply_receive_in_buf; saHandleCreate; saHandleDestroy; saHandleInstanceGet; saHandleInstancePut; - saRecvRetry; - saSelectRetry; - saSendMsgReceiveReply; - saSendMsgRetry; - saSendReceiveReply; - saSendRetry; - saServiceConnect; - saVersionVerify; }; diff --git a/lib/libcoroipc.versions b/lib/libcoroipc.versions new file mode 100644 index 00000000..866f2155 --- /dev/null +++ b/lib/libcoroipc.versions @@ -0,0 +1,16 @@ +# Version and symbol export for libipcutil.so + +COROSYNC_IPCUTIL_2.0 { + global: + cslib_service_connect; + cslib_service_disconnect; + cslib_dispatch_flow_control_get; + cslib_fd_get; + cslib_dispatch_recv; + cslib_msg_send_reply_receive; + cslib_msg_send_reply_receive_in_buf; + saHandleCreate; + saHandleDestroy; + saHandleInstanceGet; + saHandleInstancePut; +}; diff --git a/lib/libcoroutil.versions b/lib/libcoroutil.versions deleted file mode 100644 index 6ecb089f..00000000 --- a/lib/libcoroutil.versions +++ /dev/null @@ -1,18 +0,0 @@ -# Version and symbol export for libaisutil.so - -COROSYNC_UTIL_2.0 { - global: - saHandleCreate; - saHandleDestroy; - saHandleInstanceGet; - saHandleInstancePut; - saRecvRetry; - saSelectRetry; - saSendMsgReceiveReply; - saSendMsgRetry; - saSendReceiveReply; - saSendRetry; - saServiceConnect; - saVersionVerify; - clustTimeNow; -}; diff --git a/lib/libcpg.versions b/lib/libcpg.versions index 9c6aca2f..ef6434db 100644 --- a/lib/libcpg.versions +++ b/lib/libcpg.versions @@ -14,17 +14,15 @@ COROSYNC_CPG_1.0 { cpg_context_set; local: + cslib_service_connect; + cslib_service_disconnect; + cslib_dispatch_flow_control_get; + cslib_fd_get; + cslib_dispatch_recv; + cslib_msg_send_reply_receive; + cslib_msg_send_reply_receive_in_buf; saHandleCreate; saHandleDestroy; saHandleInstanceGet; saHandleInstancePut; - saRecvRetry; - saSelectRetry; - saSendMsgReceiveReply; - saSendMsgRetry; - saSendReceiveReply; - saSendRetry; - saServiceConnect; - saVersionVerify; - clustTimeNow; }; diff --git a/lib/libevs.versions b/lib/libevs.versions index 64c35487..dd6e67b9 100644 --- a/lib/libevs.versions +++ b/lib/libevs.versions @@ -13,17 +13,15 @@ COROSYNC_EVS_2.0 { evs_membership_get; local: + cslib_service_connect; + cslib_service_disconnect; + cslib_dispatch_flow_control_get; + cslib_fd_get; + cslib_dispatch_recv; + cslib_msg_send_reply_receive; + cslib_msg_send_reply_receive_in_buf; saHandleCreate; saHandleDestroy; saHandleInstanceGet; saHandleInstancePut; - saRecvRetry; - saSelectRetry; - saSendMsgReceiveReply; - saSendMsgRetry; - saSendReceiveReply; - saSendRetry; - saServiceConnect; - saVersionVerify; - clustTimeNow; }; diff --git a/lib/libpload.versions b/lib/libpload.versions index 73a79e79..2b36dec0 100644 --- a/lib/libpload.versions +++ b/lib/libpload.versions @@ -5,17 +5,15 @@ COROSYNC_PLOAD_1.0 { pload_start; local: + cslib_service_connect; + cslib_service_disconnect; + cslib_dispatch_flow_control_get; + cslib_fd_get; + cslib_dispatch_recv; + cslib_msg_send_reply_receive; + cslib_msg_send_reply_receive_in_buf; saHandleCreate; saHandleDestroy; saHandleInstanceGet; saHandleInstancePut; - saRecvRetry; - saSelectRetry; - saSendMsgReceiveReply; - saSendMsgRetry; - saSendReceiveReply; - saSendRetry; - saServiceConnect; - saVersionVerify; - clustTimeNow; }; diff --git a/lib/libquorum.versions b/lib/libquorum.versions index 5bdf23dd..ecc8f6e7 100644 --- a/lib/libquorum.versions +++ b/lib/libquorum.versions @@ -10,41 +10,15 @@ OPENAIS_QUORUM_1.0 { quorum_dispatch; local: + cslib_service_connect; + cslib_service_disconnect; + cslib_dispatch_flow_control_get; + cslib_fd_get; + cslib_dispatch_recv; + cslib_msg_send_reply_receive; + cslib_msg_send_reply_receive_in_buf; saHandleCreate; saHandleDestroy; saHandleInstanceGet; saHandleInstancePut; - saRecvRetry; - saSelectRetry; - saSendMsgReceiveReply; - saSendMsgRetry; - saSendReceiveReply; - saSendRetry; - saServiceConnect; - saVersionVerify; - clustTimeNow; -}; -# Version and symbol export for libquorum.so - -COROSYNC_QUORUM_1.0 { - global: - quorum_initialize; - quorum_finalize; - quorum_getquorate; - quorum_dispatch; - - local: - saHandleCreate; - saHandleDestroy; - saHandleInstanceGet; - saHandleInstancePut; - saRecvRetry; - saSelectRetry; - saSendMsgReceiveReply; - saSendMsgRetry; - saSendReceiveReply; - saSendRetry; - saServiceConnect; - saVersionVerify; - clustTimeNow; }; diff --git a/lib/libvotequorum.versions b/lib/libvotequorum.versions index a8269d9d..c74aa915 100644 --- a/lib/libvotequorum.versions +++ b/lib/libvotequorum.versions @@ -19,53 +19,15 @@ OPENAIS_VOTEQUORUM_1.0 { votequorum_context_set; local: + cslib_service_connect; + cslib_service_disconnect; + cslib_dispatch_flow_control_get; + cslib_fd_get; + cslib_dispatch_recv; + cslib_msg_send_reply_receive; + cslib_msg_send_reply_receive_in_buf; saHandleCreate; saHandleDestroy; saHandleInstanceGet; saHandleInstancePut; - saRecvRetry; - saSelectRetry; - saSendMsgReceiveReply; - saSendMsgRetry; - saSendReceiveReply; - saSendRetry; - saServiceConnect; - saVersionVerify; - clustTimeNow; -}; -# Version and symbol export for libvotequorum.so - -COROSYNC_VOTEQUORUM_1.0 { - global: - votequorum_initialize; - votequorum_finalize; - votequorum_getinfo; - votequorum_setexpected; - votequorum_setvotes; - votequorum_qdisk_register; - votequorum_qdisk_unregister; - votequorum_qdisk_poll; - votequorum_qdisk_getinfo; - votequorum_setdirty; - votequorum_killnode; - votequorum_leaving; - votequorum_trackstart; - votequorum_trackstop; - votequorum_context_get; - votequorum_context_set; - - local: - saHandleCreate; - saHandleDestroy; - saHandleInstanceGet; - saHandleInstancePut; - saRecvRetry; - saSelectRetry; - saSendMsgReceiveReply; - saSendMsgRetry; - saSendReceiveReply; - saSendRetry; - saServiceConnect; - saVersionVerify; - clustTimeNow; }; diff --git a/lib/pload.c b/lib/pload.c index 15e06249..db3ccbc5 100644 --- a/lib/pload.c +++ b/lib/pload.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008 Red Hat, Inc. + * Copyright (c) 2008-2009 Red Hat, Inc. * * All rights reserved. * @@ -44,13 +44,12 @@ #include #include #include -#include +#include static void pload_instance_destructor (void *instance); struct pload_inst { - int dispatch_fd; - int response_fd; + void *ipc_ctx; pthread_mutex_t response_mutex; pthread_mutex_t dispatch_mutex; unsigned int finalize; @@ -104,9 +103,7 @@ unsigned int pload_initialize ( goto error_destroy; } - error = saServiceConnect (&pload_inst->response_fd, - &pload_inst->dispatch_fd, - PLOAD_SERVICE); + error = cslib_service_connect (PLOAD_SERVICE, pload_inst->ipc_ctx); if (error != CS_OK) { goto error_put_destroy; } @@ -137,7 +134,7 @@ unsigned int pload_finalize ( if (error != CS_OK) { return (error); } -// TODO is the locking right here + pthread_mutex_lock (&pload_inst->response_mutex); /* @@ -151,22 +148,13 @@ unsigned int pload_finalize ( pload_inst->finalize = 1; + cslib_service_disconnect(pload_inst->ipc_ctx); + pthread_mutex_unlock (&pload_inst->response_mutex); (void)saHandleDestroy (&pload_handle_t_db, handle); - /* - * Disconnect from the server - */ - if (pload_inst->response_fd != -1) { - shutdown(pload_inst->response_fd, 0); - close(pload_inst->response_fd); - } - if (pload_inst->dispatch_fd != -1) { - shutdown(pload_inst->dispatch_fd, 0); - close(pload_inst->dispatch_fd); - } - (void)saHandleInstancePut (&pload_handle_t_db, handle); + (void)saHandleInstancePut (&pload_handle_t_db, handle); return (PLOAD_OK); } @@ -183,7 +171,7 @@ unsigned int pload_fd_get ( return (error); } - *fd = pload_inst->dispatch_fd; + *fd = cslib_fd_get (pload_inst->ipc_ctx); (void)saHandleInstancePut (&pload_handle_t_db, handle); @@ -218,8 +206,11 @@ unsigned int pload_start ( pthread_mutex_lock (&pload_inst->response_mutex); - error = saSendMsgReceiveReply (pload_inst->response_fd, &iov, 1, - &res_lib_pload_start, sizeof (struct res_lib_pload_start)); + error = cslib_msg_send_reply_receive(pload_inst->ipc_ctx, + &iov, + 1, + &res_lib_pload_start, + sizeof (struct res_lib_pload_start)); pthread_mutex_unlock (&pload_inst->response_mutex); diff --git a/lib/quorum.c b/lib/quorum.c index b5b3f316..00c701f3 100644 --- a/lib/quorum.c +++ b/lib/quorum.c @@ -46,13 +46,12 @@ #include #include #include -#include +#include #include "corosync/quorum.h" #include "corosync/ipc_quorum.h" struct quorum_inst { - int response_fd; - int dispatch_fd; + void *ipc_ctx; int finalize; void *context; quorum_callbacks_t callbacks; @@ -96,9 +95,7 @@ cs_error_t quorum_initialize ( goto error_destroy; } - error = saServiceConnect (&quorum_inst->dispatch_fd, - &quorum_inst->response_fd, - QUORUM_SERVICE); + error = cslib_service_connect (QUORUM_SERVICE, quorum_inst->ipc_ctx); if (error != CS_OK) { goto error_put_destroy; } @@ -146,17 +143,12 @@ cs_error_t quorum_finalize ( quorum_inst->finalize = 1; + cslib_service_disconnect (quorum_inst->ipc_ctx); + pthread_mutex_unlock (&quorum_inst->response_mutex); (void)saHandleDestroy (&quorum_handle_t_db, handle); - /* - * Disconnect from the server - */ - if (quorum_inst->response_fd != -1) { - shutdown(quorum_inst->response_fd, 0); - close(quorum_inst->response_fd); - } (void)saHandleInstancePut (&quorum_handle_t_db, handle); return (CS_OK); @@ -168,7 +160,7 @@ cs_error_t quorum_getquorate ( { cs_error_t error; struct quorum_inst *quorum_inst; - struct iovec iov[2]; + struct iovec iov; mar_req_header_t req; struct res_lib_quorum_getquorate res_lib_quorum_getquorate; @@ -182,11 +174,15 @@ cs_error_t quorum_getquorate ( req.size = sizeof (req); req.id = MESSAGE_REQ_QUORUM_GETQUORATE; - iov[0].iov_base = (char *)&req; - iov[0].iov_len = sizeof (req); + iov.iov_base = (char *)&req; + iov.iov_len = sizeof (req); - error = saSendMsgReceiveReply (quorum_inst->response_fd, iov, 1, - &res_lib_quorum_getquorate, sizeof (struct res_lib_quorum_getquorate)); + error = cslib_msg_send_reply_receive ( + quorum_inst->ipc_ctx, + &iov, + 1, + &res_lib_quorum_getquorate, + sizeof (struct res_lib_quorum_getquorate)); pthread_mutex_unlock (&quorum_inst->response_mutex); @@ -216,7 +212,7 @@ cs_error_t quorum_fd_get ( return (error); } - *fd = quorum_inst->dispatch_fd; + *fd = cslib_fd_get (quorum_inst->ipc_ctx); (void)saHandleInstancePut (&quorum_handle_t_db, handle); @@ -269,7 +265,7 @@ cs_error_t quorum_trackstart ( { cs_error_t error; struct quorum_inst *quorum_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_quorum_trackstart req_lib_quorum_trackstart; mar_res_header_t res; @@ -284,11 +280,15 @@ cs_error_t quorum_trackstart ( req_lib_quorum_trackstart.header.id = MESSAGE_REQ_QUORUM_TRACKSTART; req_lib_quorum_trackstart.track_flags = flags; - iov[0].iov_base = (char *)&req_lib_quorum_trackstart; - iov[0].iov_len = sizeof (struct req_lib_quorum_trackstart); + iov.iov_base = (char *)&req_lib_quorum_trackstart; + iov.iov_len = sizeof (struct req_lib_quorum_trackstart); - error = saSendMsgReceiveReply (quorum_inst->response_fd, iov, 1, - &res, sizeof (res)); + error = cslib_msg_send_reply_receive ( + quorum_inst->ipc_ctx, + &iov, + 1, + &res, + sizeof (res)); pthread_mutex_unlock (&quorum_inst->response_mutex); @@ -309,7 +309,7 @@ cs_error_t quorum_trackstop ( { cs_error_t error; struct quorum_inst *quorum_inst; - struct iovec iov[2]; + struct iovec iov; mar_req_header_t req; mar_res_header_t res; @@ -323,11 +323,15 @@ cs_error_t quorum_trackstop ( req.size = sizeof (req); req.id = MESSAGE_REQ_QUORUM_TRACKSTOP; - iov[0].iov_base = (char *)&req; - iov[0].iov_len = sizeof (req); + iov.iov_base = (char *)&req; + iov.iov_len = sizeof (req); - error = saSendMsgReceiveReply (quorum_inst->response_fd, iov, 1, - &res, sizeof (res)); + error = cslib_msg_send_reply_receive ( + quorum_inst->ipc_ctx, + &iov, + 1, + &res, + sizeof (res)); pthread_mutex_unlock (&quorum_inst->response_mutex); @@ -352,7 +356,6 @@ cs_error_t quorum_dispatch ( quorum_handle_t handle, cs_dispatch_flags_t dispatch_types) { - struct pollfd ufds; int timeout = -1; cs_error_t error; int cont = 1; /* always continue do loop except when set to 0 */ @@ -384,16 +387,10 @@ cs_error_t quorum_dispatch ( } do { - ufds.fd = quorum_inst->dispatch_fd; - ufds.events = POLLIN; - ufds.revents = 0; - pthread_mutex_lock (&quorum_inst->dispatch_mutex); - error = saPollRetry (&ufds, 1, timeout); - if (error != CS_OK) { - goto error_unlock; - } + dispatch_avail = cslib_dispatch_recv (quorum_inst->ipc_ctx, + (void *)&dispatch_data, timeout); /* * Handle has been finalized in another thread @@ -403,12 +400,6 @@ cs_error_t quorum_dispatch ( goto error_unlock; } - if ((ufds.revents & (POLLERR|POLLHUP|POLLNVAL)) != 0) { - error = CS_ERR_BAD_HANDLE; - goto error_unlock; - } - - dispatch_avail = ufds.revents & POLLIN; if (dispatch_avail == 0 && dispatch_types == CS_DISPATCH_ALL) { pthread_mutex_unlock (&quorum_inst->dispatch_mutex); break; /* exit do while cont is 1 loop */ @@ -418,24 +409,6 @@ cs_error_t quorum_dispatch ( continue; /* next poll */ } - if (ufds.revents & POLLIN) { - error = saRecvRetry (quorum_inst->dispatch_fd, &dispatch_data.header, - sizeof (mar_res_header_t)); - if (error != CS_OK) { - goto error_unlock; - } - if (dispatch_data.header.size > sizeof (mar_res_header_t)) { - error = saRecvRetry (quorum_inst->dispatch_fd, &dispatch_data.data, - dispatch_data.header.size - sizeof (mar_res_header_t)); - if (error != CS_OK) { - goto error_unlock; - } - } - } else { - pthread_mutex_unlock (&quorum_inst->dispatch_mutex); - continue; - } - /* * Make copy of callbacks, message data, unlock instance, and call callback * A risk of this dispatch method is that the callback routines may @@ -456,10 +429,10 @@ cs_error_t quorum_dispatch ( res_lib_quorum_notification = (struct res_lib_quorum_notification *)&dispatch_data; callbacks.quorum_notify_fn ( handle, - res_lib_quorum_notification->quorate, - res_lib_quorum_notification->ring_seq, - res_lib_quorum_notification->view_list_entries, - res_lib_quorum_notification->view_list); + res_lib_quorum_notification->quorate, + res_lib_quorum_notification->ring_seq, + res_lib_quorum_notification->view_list_entries, + res_lib_quorum_notification->view_list); break; default: diff --git a/lib/sa-confdb.c b/lib/sa-confdb.c index 28d15764..d350d705 100644 --- a/lib/sa-confdb.c +++ b/lib/sa-confdb.c @@ -44,7 +44,7 @@ #include #include -#include +#include #include #include #include diff --git a/lib/util.c b/lib/util.c deleted file mode 100644 index 71935467..00000000 --- a/lib/util.c +++ /dev/null @@ -1,788 +0,0 @@ -/* - * vi: set autoindent tabstop=4 shiftwidth=4 : - * - * Copyright (c) 2002-2006 MontaVista Software, Inc. - * Copyright (c) 2006-2008 Red Hat, Inc. - * - * All rights reserved. - * - * Author: Steven Dake (sdake@redhat.com) - * - * This software licensed under BSD license, the text of which follows: - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - Neither the name of the MontaVista Software, Inc. nor the names of its - * contributors may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -enum SA_HANDLE_STATE { - SA_HANDLE_STATE_EMPTY, - SA_HANDLE_STATE_PENDINGREMOVAL, - SA_HANDLE_STATE_ACTIVE -}; - -struct saHandle { - int state; - void *instance; - int refCount; - uint32_t check; -}; - -#ifdef COROSYNC_SOLARIS -#define MSG_NOSIGNAL 0 -#endif - -#if defined(COROSYNC_LINUX) || defined(COROSYNC_SOLARIS) -/* SUN_LEN is broken for abstract namespace - */ -#define AIS_SUN_LEN(a) sizeof(*(a)) -#else -#define AIS_SUN_LEN(a) SUN_LEN(a) -#endif - -#ifdef COROSYNC_LINUX -static char *socketname = "libcorosync.socket"; -#else -static char *socketname = "/var/run/libcorosync.socket"; -#endif - -#ifdef SO_NOSIGPIPE -void socket_nosigpipe(int s) -{ - int on = 1; - setsockopt(s, SOL_SOCKET, SO_NOSIGPIPE, (void *)&on, sizeof(on)); -} -#endif - -cs_error_t -saServiceConnect ( - int *responseOut, - int *callbackOut, - enum service_types service) -{ - int responseFD; - int callbackFD; - int result; - struct sockaddr_un address; - mar_req_lib_response_init_t req_lib_response_init; - mar_res_lib_response_init_t res_lib_response_init; - mar_req_lib_dispatch_init_t req_lib_dispatch_init; - mar_res_lib_dispatch_init_t res_lib_dispatch_init; - cs_error_t error; - gid_t egid; - - /* - * Allow set group id binaries to be authenticated - */ - egid = getegid(); - setregid (egid, -1); - - memset (&address, 0, sizeof (struct sockaddr_un)); -#if defined(COROSYNC_BSD) || defined(COROSYNC_DARWIN) - address.sun_len = sizeof(struct sockaddr_un); -#endif - address.sun_family = PF_UNIX; -#if defined(COROSYNC_LINUX) - strcpy (address.sun_path + 1, socketname); -#else - strcpy (address.sun_path, socketname); -#endif - responseFD = socket (PF_UNIX, SOCK_STREAM, 0); - if (responseFD == -1) { - return (CS_ERR_NO_RESOURCES); - } - - socket_nosigpipe (responseFD); - - result = connect (responseFD, (struct sockaddr *)&address, AIS_SUN_LEN(&address)); - if (result == -1) { - close (responseFD); - return (CS_ERR_TRY_AGAIN); - } - - req_lib_response_init.resdis_header.size = sizeof (req_lib_response_init); - req_lib_response_init.resdis_header.id = MESSAGE_REQ_RESPONSE_INIT; - req_lib_response_init.resdis_header.service = service; - - error = saSendRetry (responseFD, &req_lib_response_init, - sizeof (mar_req_lib_response_init_t)); - if (error != CS_OK) { - goto error_exit; - } - error = saRecvRetry (responseFD, &res_lib_response_init, - sizeof (mar_res_lib_response_init_t)); - if (error != CS_OK) { - goto error_exit; - } - - /* - * Check for security errors - */ - if (res_lib_response_init.header.error != CS_OK) { - error = res_lib_response_init.header.error; - goto error_exit; - } - - *responseOut = responseFD; - -/* if I comment out the 4 lines below the executive crashes */ - callbackFD = socket (PF_UNIX, SOCK_STREAM, 0); - if (callbackFD == -1) { - close (responseFD); - return (CS_ERR_NO_RESOURCES); - } - - socket_nosigpipe (callbackFD); - - result = connect (callbackFD, (struct sockaddr *)&address, AIS_SUN_LEN(&address)); - if (result == -1) { - close (callbackFD); - close (responseFD); - return (CS_ERR_TRY_AGAIN); - } - - req_lib_dispatch_init.resdis_header.size = sizeof (req_lib_dispatch_init); - req_lib_dispatch_init.resdis_header.id = MESSAGE_REQ_DISPATCH_INIT; - req_lib_dispatch_init.resdis_header.service = service; - - req_lib_dispatch_init.conn_info = res_lib_response_init.conn_info; - - error = saSendRetry (callbackFD, &req_lib_dispatch_init, - sizeof (mar_req_lib_dispatch_init_t)); - if (error != CS_OK) { - goto error_exit_two; - } - error = saRecvRetry (callbackFD, &res_lib_dispatch_init, - sizeof (mar_res_lib_dispatch_init_t)); - if (error != CS_OK) { - goto error_exit_two; - } - - /* - * Check for security errors - */ - if (res_lib_dispatch_init.header.error != CS_OK) { - error = res_lib_dispatch_init.header.error; - goto error_exit; - } - - *callbackOut = callbackFD; - return (CS_OK); - -error_exit_two: - close (callbackFD); -error_exit: - close (responseFD); - return (error); -} - -cs_error_t -saRecvRetry ( - int s, - void *msg, - size_t len) -{ - cs_error_t error = CS_OK; - ssize_t result; - struct msghdr msg_recv; - struct iovec iov_recv; - char *rbuf = (char *)msg; - int processed = 0; - - msg_recv.msg_iov = &iov_recv; - msg_recv.msg_iovlen = 1; - msg_recv.msg_name = 0; - msg_recv.msg_namelen = 0; -#ifndef COROSYNC_SOLARIS - msg_recv.msg_control = 0; - msg_recv.msg_controllen = 0; - msg_recv.msg_flags = 0; -#else - msg_recv.msg_accrights = NULL; - msg_recv.msg_accrightslen = 0; -#endif - -retry_recv: - iov_recv.iov_base = (void *)&rbuf[processed]; - iov_recv.iov_len = len - processed; - - result = recvmsg (s, &msg_recv, MSG_NOSIGNAL); - if (result == -1 && errno == EINTR) { - goto retry_recv; - } - if (result == -1 && errno == EAGAIN) { - goto retry_recv; - } -#if defined(COROSYNC_SOLARIS) || defined(COROSYNC_BSD) || defined(COROSYNC_DARWIN) - /* On many OS poll never return POLLHUP or POLLERR. - * EOF is detected when recvmsg return 0. - */ - if (result == 0) { - error = CS_ERR_LIBRARY; - goto error_exit; - } -#endif - if (result == -1 || result == 0) { - error = CS_ERR_LIBRARY; - goto error_exit; - } - processed += result; - if (processed != len) { - goto retry_recv; - } - assert (processed == len); -error_exit: - return (error); -} - -cs_error_t -saSendRetry ( - int s, - const void *msg, - size_t len) -{ - cs_error_t error = CS_OK; - ssize_t result; - struct msghdr msg_send; - struct iovec iov_send; - char *rbuf = (char *)msg; - int processed = 0; - - msg_send.msg_iov = &iov_send; - msg_send.msg_iovlen = 1; - msg_send.msg_name = 0; - msg_send.msg_namelen = 0; -#ifndef COROSYNC_SOLARIS - msg_send.msg_control = 0; - msg_send.msg_controllen = 0; - msg_send.msg_flags = 0; -#else - msg_send.msg_accrights = NULL; - msg_send.msg_accrightslen = 0; -#endif - -retry_send: - iov_send.iov_base = (void *)&rbuf[processed]; - iov_send.iov_len = len - processed; - - result = sendmsg (s, &msg_send, MSG_NOSIGNAL); - - /* - * return immediately on any kind of syscall error that maps to - * SA_AIS_ERR if no part of message has been sent - */ - if (result == -1 && processed == 0) { - if (errno == EINTR) { - error = CS_ERR_TRY_AGAIN; - goto error_exit; - } - if (errno == EAGAIN) { - error = CS_ERR_TRY_AGAIN; - goto error_exit; - } - if (errno == EFAULT) { - error = CS_ERR_INVALID_PARAM; - goto error_exit; - } - } - - /* - * retry read operations that are already started except - * for fault in that case, return ERR_LIBRARY - */ - if (result == -1 && processed > 0) { - if (errno == EINTR) { - goto retry_send; - } - if (errno == EAGAIN) { - goto retry_send; - } - if (errno == EFAULT) { - error = CS_ERR_LIBRARY; - goto error_exit; - } - } - - /* - * return ERR_LIBRARY on any other syscall error - */ - if (result == -1) { - error = CS_ERR_LIBRARY; - goto error_exit; - } - - processed += result; - if (processed != len) { - goto retry_send; - } - -error_exit: - return (error); -} - -cs_error_t saSendMsgRetry ( - int s, - struct iovec *iov, - int iov_len) -{ - cs_error_t error = CS_OK; - ssize_t result; - int total_size = 0; - int i; - int csize; - int csize_cntr; - int total_sent = 0; - int iov_len_sendmsg = iov_len; - struct iovec *iov_sendmsg = iov; - struct iovec iovec_save; - int iovec_saved_position = -1; - - struct msghdr msg_send; - - for (i = 0; i < iov_len; i++) { - total_size += iov[i].iov_len; - } - msg_send.msg_iov = iov_sendmsg; - msg_send.msg_iovlen = iov_len_sendmsg; - msg_send.msg_name = 0; - msg_send.msg_namelen = 0; -#ifndef COROSYNC_SOLARIS - msg_send.msg_control = 0; - msg_send.msg_controllen = 0; - msg_send.msg_flags = 0; -#else - msg_send.msg_accrights = NULL; - msg_send.msg_accrightslen = 0; -#endif - -retry_sendmsg: - result = sendmsg (s, &msg_send, MSG_NOSIGNAL); - /* - * Can't send now, and message not committed, so don't retry send - */ - if (result == -1 && iovec_saved_position == -1) { - if (errno == EINTR) { - error = CS_ERR_TRY_AGAIN; - goto error_exit; - } - if (errno == EAGAIN) { - error = CS_ERR_TRY_AGAIN; - goto error_exit; - } - if (errno == EFAULT) { - error = CS_ERR_INVALID_PARAM; - goto error_exit; - } - } - - /* - * Retry (and block) if portion of message has already been written - */ - if (result == -1 && iovec_saved_position != -1) { - if (errno == EINTR) { - goto retry_sendmsg; - } - if (errno == EAGAIN) { - goto retry_sendmsg; - } - if (errno == EFAULT) { - error = CS_ERR_LIBRARY; - goto error_exit; - } - } - - /* - * ERR_LIBRARY for any other syscall error - */ - if (result == -1) { - error = CS_ERR_LIBRARY; - goto error_exit; - } - - if (iovec_saved_position != -1) { - memcpy (&iov[iovec_saved_position], &iovec_save, sizeof (struct iovec)); - } - - total_sent += result; - if (total_sent != total_size) { - for (i = 0, csize = 0, csize_cntr = 0; i < iov_len; i++) { - csize += iov[i].iov_len; - if (csize > total_sent) { - break; - } - - csize_cntr += iov[i].iov_len; - } - memcpy (&iovec_save, &iov[i], sizeof (struct iovec)); - iovec_saved_position = i; - iov[i].iov_base = ((char *)(iov[i].iov_base)) + - (total_sent - csize_cntr); - iov[i].iov_len = total_size - total_sent; - msg_send.msg_iov = &iov[i]; - msg_send.msg_iovlen = iov_len - i; - - goto retry_sendmsg; - } - -error_exit: - return (error); -} - -cs_error_t saSendMsgReceiveReply ( - int s, - struct iovec *iov, - int iov_len, - void *responseMessage, - int responseLen) -{ - cs_error_t error = CS_OK; - - error = saSendMsgRetry (s, iov, iov_len); - if (error != CS_OK) { - goto error_exit; - } - - error = saRecvRetry (s, responseMessage, responseLen); - if (error != CS_OK) { - goto error_exit; - } - -error_exit: - return (error); -} - -cs_error_t saSendReceiveReply ( - int s, - void *requestMessage, - int requestLen, - void *responseMessage, - int responseLen) -{ - cs_error_t error = CS_OK; - - error = saSendRetry (s, requestMessage, requestLen); - if (error != CS_OK) { - goto error_exit; - } - - error = saRecvRetry (s, responseMessage, responseLen); - if (error != CS_OK) { - goto error_exit; - } - -error_exit: - return (error); -} - -cs_error_t -saPollRetry ( - struct pollfd *ufds, - unsigned int nfds, - int timeout) -{ - cs_error_t error = CS_OK; - int result; - -retry_poll: - result = poll (ufds, nfds, timeout); - if (result == -1 && errno == EINTR) { - goto retry_poll; - } - if (result == -1) { - error = CS_ERR_LIBRARY; - } - - return (error); -} - - -cs_error_t -saHandleCreate ( - struct saHandleDatabase *handleDatabase, - int instanceSize, - uint64_t *handleOut) -{ - uint32_t handle; - uint32_t check; - void *newHandles = NULL; - int found = 0; - void *instance; - int i; - - pthread_mutex_lock (&handleDatabase->mutex); - - for (handle = 0; handle < handleDatabase->handleCount; handle++) { - if (handleDatabase->handles[handle].state == SA_HANDLE_STATE_EMPTY) { - found = 1; - break; - } - } - - if (found == 0) { - handleDatabase->handleCount += 1; - newHandles = (struct saHandle *)realloc (handleDatabase->handles, - sizeof (struct saHandle) * handleDatabase->handleCount); - if (newHandles == NULL) { - pthread_mutex_unlock (&handleDatabase->mutex); - return (CS_ERR_NO_MEMORY); - } - handleDatabase->handles = newHandles; - } - - instance = malloc (instanceSize); - if (instance == 0) { - free (newHandles); - pthread_mutex_unlock (&handleDatabase->mutex); - return (CS_ERR_NO_MEMORY); - } - - - /* - * This code makes sure the random number isn't zero - * We use 0 to specify an invalid handle out of the 1^64 address space - * If we get 0 200 times in a row, the RNG may be broken - */ - for (i = 0; i < 200; i++) { - check = random(); - if (check != 0) { - break; - } - } - - memset (instance, 0, instanceSize); - - handleDatabase->handles[handle].state = SA_HANDLE_STATE_ACTIVE; - - handleDatabase->handles[handle].instance = instance; - - handleDatabase->handles[handle].refCount = 1; - - handleDatabase->handles[handle].check = check; - - *handleOut = (uint64_t)((uint64_t)check << 32 | handle); - - pthread_mutex_unlock (&handleDatabase->mutex); - - return (CS_OK); -} - - -cs_error_t -saHandleDestroy ( - struct saHandleDatabase *handleDatabase, - uint64_t inHandle) -{ - cs_error_t error = CS_OK; - uint32_t check = inHandle >> 32; - uint32_t handle = inHandle & 0xffffffff; - - pthread_mutex_lock (&handleDatabase->mutex); - - if (check != handleDatabase->handles[handle].check) { - pthread_mutex_unlock (&handleDatabase->mutex); - error = CS_ERR_BAD_HANDLE; - return (error); - } - - handleDatabase->handles[handle].state = SA_HANDLE_STATE_PENDINGREMOVAL; - - pthread_mutex_unlock (&handleDatabase->mutex); - - (void)saHandleInstancePut (handleDatabase, inHandle); - - return (error); -} - - -cs_error_t -saHandleInstanceGet ( - struct saHandleDatabase *handleDatabase, - uint64_t inHandle, - void **instance) -{ - uint32_t check = inHandle >> 32; - uint32_t handle = inHandle & 0xffffffff; - - cs_error_t error = CS_OK; - pthread_mutex_lock (&handleDatabase->mutex); - - if (handle >= (uint64_t)handleDatabase->handleCount) { - error = CS_ERR_BAD_HANDLE; - goto error_exit; - } - if (handleDatabase->handles[handle].state != SA_HANDLE_STATE_ACTIVE) { - error = CS_ERR_BAD_HANDLE; - goto error_exit; - } - if (check != handleDatabase->handles[handle].check) { - error = CS_ERR_BAD_HANDLE; - goto error_exit; - } - - - *instance = handleDatabase->handles[handle].instance; - - handleDatabase->handles[handle].refCount += 1; - -error_exit: - pthread_mutex_unlock (&handleDatabase->mutex); - - return (error); -} - - -cs_error_t -saHandleInstancePut ( - struct saHandleDatabase *handleDatabase, - uint64_t inHandle) -{ - void *instance; - cs_error_t error = CS_OK; - uint32_t check = inHandle >> 32; - uint32_t handle = inHandle & 0xffffffff; - - pthread_mutex_lock (&handleDatabase->mutex); - - if (check != handleDatabase->handles[handle].check) { - error = CS_ERR_BAD_HANDLE; - goto error_exit; - } - - handleDatabase->handles[handle].refCount -= 1; - assert (handleDatabase->handles[handle].refCount >= 0); - - if (handleDatabase->handles[handle].refCount == 0) { - instance = (handleDatabase->handles[handle].instance); - handleDatabase->handleInstanceDestructor (instance); - free (instance); - memset (&handleDatabase->handles[handle], 0, sizeof (struct saHandle)); - } - -error_exit: - pthread_mutex_unlock (&handleDatabase->mutex); - - return (error); -} - - -cs_error_t -saVersionVerify ( - struct saVersionDatabase *versionDatabase, - cs_version_t *version) -{ - int i; - cs_error_t error = CS_ERR_VERSION; - - if (version == 0) { - return (CS_ERR_INVALID_PARAM); - } - - /* - * Look for a release code that we support. If we find it then - * make sure that the supported major version is >= to the required one. - * In any case we return what we support in the version structure. - */ - for (i = 0; i < versionDatabase->versionCount; i++) { - - /* - * Check if the caller requires and old release code that we don't support. - */ - if (version->releaseCode < versionDatabase->versionsSupported[i].releaseCode) { - break; - } - - /* - * Check if we can support this release code. - */ - if (version->releaseCode == versionDatabase->versionsSupported[i].releaseCode) { - - /* - * Check if we can support the major version requested. - */ - if (versionDatabase->versionsSupported[i].majorVersion >= version->majorVersion) { - error = CS_OK; - break; - } - - /* - * We support the release code, but not the major version. - */ - break; - } - } - - /* - * If we fall out of the if loop, the caller requires a release code - * beyond what we support. - */ - if (i == versionDatabase->versionCount) { - i = versionDatabase->versionCount - 1; - } - - /* - * Tell the caller what we support - */ - memcpy(version, &versionDatabase->versionsSupported[i], sizeof(*version)); - return (error); -} - -/* - * Get the time of day and convert to nanoseconds - */ -cs_time_t clustTimeNow(void) -{ - struct timeval tv; - cs_time_t time_now; - - if (gettimeofday(&tv, 0)) { - return 0ULL; - } - - time_now = (cs_time_t)(tv.tv_sec) * 1000000000ULL; - time_now += (cs_time_t)(tv.tv_usec) * 1000ULL; - - return time_now; -} - diff --git a/lib/votequorum.c b/lib/votequorum.c index a4ad461e..002f179a 100644 --- a/lib/votequorum.c +++ b/lib/votequorum.c @@ -46,13 +46,12 @@ #include #include -#include +#include #include "corosync/votequorum.h" #include "corosync/ipc_votequorum.h" struct votequorum_inst { - int response_fd; - int dispatch_fd; + void *ipc_ctx; int finalize; void *context; votequorum_callbacks_t callbacks; @@ -96,9 +95,7 @@ cs_error_t votequorum_initialize ( goto error_destroy; } - error = saServiceConnect (&votequorum_inst->dispatch_fd, - &votequorum_inst->response_fd, - VOTEQUORUM_SERVICE); + error = cslib_service_connect (VOTEQUORUM_SERVICE, votequorum_inst->ipc_ctx); if (error != CS_OK) { goto error_put_destroy; } @@ -146,17 +143,12 @@ cs_error_t votequorum_finalize ( votequorum_inst->finalize = 1; + cslib_service_disconnect (votequorum_inst->ipc_ctx); + pthread_mutex_unlock (&votequorum_inst->response_mutex); saHandleDestroy (&votequorum_handle_t_db, handle); - /* - * Disconnect from the server - */ - if (votequorum_inst->response_fd != -1) { - shutdown(votequorum_inst->response_fd, 0); - close(votequorum_inst->response_fd); - } saHandleInstancePut (&votequorum_handle_t_db, handle); return (CS_OK); @@ -170,7 +162,7 @@ cs_error_t votequorum_getinfo ( { cs_error_t error; struct votequorum_inst *votequorum_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_votequorum_getinfo req_lib_votequorum_getinfo; struct res_lib_votequorum_getinfo res_lib_votequorum_getinfo; @@ -179,17 +171,21 @@ cs_error_t votequorum_getinfo ( return (error); } - pthread_mutex_lock (&votequorum_inst->response_mutex); - req_lib_votequorum_getinfo.header.size = sizeof (struct req_lib_votequorum_getinfo); req_lib_votequorum_getinfo.header.id = MESSAGE_REQ_VOTEQUORUM_GETINFO; req_lib_votequorum_getinfo.nodeid = nodeid; - iov[0].iov_base = (char *)&req_lib_votequorum_getinfo; - iov[0].iov_len = sizeof (struct req_lib_votequorum_getinfo); + iov.iov_base = (char *)&req_lib_votequorum_getinfo; + iov.iov_len = sizeof (struct req_lib_votequorum_getinfo); - error = saSendMsgReceiveReply (votequorum_inst->response_fd, iov, 1, - &res_lib_votequorum_getinfo, sizeof (struct res_lib_votequorum_getinfo)); + pthread_mutex_lock (&votequorum_inst->response_mutex); + + error = cslib_msg_send_reply_receive ( + votequorum_inst->ipc_ctx, + &iov, + 1, + &res_lib_votequorum_getinfo, + sizeof (struct res_lib_votequorum_getinfo)); pthread_mutex_unlock (&votequorum_inst->response_mutex); @@ -219,7 +215,7 @@ cs_error_t votequorum_setexpected ( { cs_error_t error; struct votequorum_inst *votequorum_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_votequorum_setexpected req_lib_votequorum_setexpected; struct res_lib_votequorum_status res_lib_votequorum_status; @@ -228,17 +224,22 @@ cs_error_t votequorum_setexpected ( return (error); } - pthread_mutex_lock (&votequorum_inst->response_mutex); req_lib_votequorum_setexpected.header.size = sizeof (struct req_lib_votequorum_setexpected); req_lib_votequorum_setexpected.header.id = MESSAGE_REQ_VOTEQUORUM_SETEXPECTED; req_lib_votequorum_setexpected.expected_votes = expected_votes; - iov[0].iov_base = (char *)&req_lib_votequorum_setexpected; - iov[0].iov_len = sizeof (struct req_lib_votequorum_setexpected); + iov.iov_base = (char *)&req_lib_votequorum_setexpected; + iov.iov_len = sizeof (struct req_lib_votequorum_setexpected); - error = saSendMsgReceiveReply (votequorum_inst->response_fd, iov, 1, - &res_lib_votequorum_status, sizeof (struct res_lib_votequorum_status)); + pthread_mutex_lock (&votequorum_inst->response_mutex); + + error = cslib_msg_send_reply_receive ( + votequorum_inst->ipc_ctx, + &iov, + 1, + &res_lib_votequorum_status, + sizeof (struct res_lib_votequorum_status)); pthread_mutex_unlock (&votequorum_inst->response_mutex); @@ -261,7 +262,7 @@ cs_error_t votequorum_setvotes ( { cs_error_t error; struct votequorum_inst *votequorum_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_votequorum_setvotes req_lib_votequorum_setvotes; struct res_lib_votequorum_status res_lib_votequorum_status; @@ -270,18 +271,22 @@ cs_error_t votequorum_setvotes ( return (error); } - pthread_mutex_lock (&votequorum_inst->response_mutex); - req_lib_votequorum_setvotes.header.size = sizeof (struct req_lib_votequorum_setvotes); req_lib_votequorum_setvotes.header.id = MESSAGE_REQ_VOTEQUORUM_SETVOTES; req_lib_votequorum_setvotes.nodeid = nodeid; req_lib_votequorum_setvotes.votes = votes; - iov[0].iov_base = (char *)&req_lib_votequorum_setvotes; - iov[0].iov_len = sizeof (struct req_lib_votequorum_setvotes); + iov.iov_base = (char *)&req_lib_votequorum_setvotes; + iov.iov_len = sizeof (struct req_lib_votequorum_setvotes); - error = saSendMsgReceiveReply (votequorum_inst->response_fd, iov, 1, - &res_lib_votequorum_status, sizeof (struct res_lib_votequorum_status)); + pthread_mutex_lock (&votequorum_inst->response_mutex); + + error = cslib_msg_send_reply_receive ( + votequorum_inst->ipc_ctx, + &iov, + 1, + &res_lib_votequorum_status, + sizeof (struct res_lib_votequorum_status)); pthread_mutex_unlock (&votequorum_inst->response_mutex); @@ -304,7 +309,7 @@ cs_error_t votequorum_qdisk_register ( { cs_error_t error; struct votequorum_inst *votequorum_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_votequorum_qdisk_register req_lib_votequorum_qdisk_register; struct res_lib_votequorum_status res_lib_votequorum_status; @@ -316,18 +321,23 @@ cs_error_t votequorum_qdisk_register ( return (error); } - pthread_mutex_lock (&votequorum_inst->response_mutex); req_lib_votequorum_qdisk_register.header.size = sizeof (struct req_lib_votequorum_qdisk_register); req_lib_votequorum_qdisk_register.header.id = MESSAGE_REQ_VOTEQUORUM_QDISK_REGISTER; strcpy(req_lib_votequorum_qdisk_register.name, name); req_lib_votequorum_qdisk_register.votes = votes; - iov[0].iov_base = (char *)&req_lib_votequorum_qdisk_register; - iov[0].iov_len = sizeof (struct req_lib_votequorum_qdisk_register); + iov.iov_base = (char *)&req_lib_votequorum_qdisk_register; + iov.iov_len = sizeof (struct req_lib_votequorum_qdisk_register); - error = saSendMsgReceiveReply (votequorum_inst->response_fd, iov, 1, - &res_lib_votequorum_status, sizeof (struct res_lib_votequorum_status)); + pthread_mutex_lock (&votequorum_inst->response_mutex); + + error = cslib_msg_send_reply_receive ( + votequorum_inst->ipc_ctx, + &iov, + 1, + &res_lib_votequorum_status, + sizeof (struct res_lib_votequorum_status)); pthread_mutex_unlock (&votequorum_inst->response_mutex); @@ -349,7 +359,7 @@ cs_error_t votequorum_qdisk_poll ( { cs_error_t error; struct votequorum_inst *votequorum_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_votequorum_qdisk_poll req_lib_votequorum_qdisk_poll; struct res_lib_votequorum_status res_lib_votequorum_status; @@ -358,17 +368,22 @@ cs_error_t votequorum_qdisk_poll ( return (error); } - pthread_mutex_lock (&votequorum_inst->response_mutex); req_lib_votequorum_qdisk_poll.header.size = sizeof (struct req_lib_votequorum_qdisk_poll); req_lib_votequorum_qdisk_poll.header.id = MESSAGE_REQ_VOTEQUORUM_QDISK_POLL; req_lib_votequorum_qdisk_poll.state = state; - iov[0].iov_base = (char *)&req_lib_votequorum_qdisk_poll; - iov[0].iov_len = sizeof (struct req_lib_votequorum_qdisk_poll); + iov.iov_base = (char *)&req_lib_votequorum_qdisk_poll; + iov.iov_len = sizeof (struct req_lib_votequorum_qdisk_poll); - error = saSendMsgReceiveReply (votequorum_inst->response_fd, iov, 1, - &res_lib_votequorum_status, sizeof (struct res_lib_votequorum_status)); + pthread_mutex_lock (&votequorum_inst->response_mutex); + + error = cslib_msg_send_reply_receive ( + votequorum_inst->ipc_ctx, + &iov, + 1, + &res_lib_votequorum_status, + sizeof (struct res_lib_votequorum_status)); pthread_mutex_unlock (&votequorum_inst->response_mutex); @@ -389,7 +404,7 @@ cs_error_t votequorum_qdisk_unregister ( { cs_error_t error; struct votequorum_inst *votequorum_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_votequorum_general req_lib_votequorum_general; struct res_lib_votequorum_status res_lib_votequorum_status; @@ -403,11 +418,15 @@ cs_error_t votequorum_qdisk_unregister ( req_lib_votequorum_general.header.size = sizeof (struct req_lib_votequorum_general); req_lib_votequorum_general.header.id = MESSAGE_REQ_VOTEQUORUM_QDISK_UNREGISTER; - iov[0].iov_base = (char *)&req_lib_votequorum_general; - iov[0].iov_len = sizeof (struct req_lib_votequorum_general); + iov.iov_base = (char *)&req_lib_votequorum_general; + iov.iov_len = sizeof (struct req_lib_votequorum_general); - error = saSendMsgReceiveReply (votequorum_inst->response_fd, iov, 1, - &res_lib_votequorum_status, sizeof (struct res_lib_votequorum_status)); + error = cslib_msg_send_reply_receive ( + votequorum_inst->ipc_ctx, + &iov, + 1, + &res_lib_votequorum_status, + sizeof (struct res_lib_votequorum_status)); pthread_mutex_unlock (&votequorum_inst->response_mutex); @@ -431,7 +450,7 @@ cs_error_t votequorum_qdisk_getinfo ( { cs_error_t error; struct votequorum_inst *votequorum_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_votequorum_general req_lib_votequorum_general; struct res_lib_votequorum_qdisk_getinfo res_lib_votequorum_qdisk_getinfo; @@ -440,16 +459,21 @@ cs_error_t votequorum_qdisk_getinfo ( return (error); } - pthread_mutex_lock (&votequorum_inst->response_mutex); req_lib_votequorum_general.header.size = sizeof (struct req_lib_votequorum_general); req_lib_votequorum_general.header.id = MESSAGE_REQ_VOTEQUORUM_QDISK_GETINFO; - iov[0].iov_base = (char *)&req_lib_votequorum_general; - iov[0].iov_len = sizeof (struct req_lib_votequorum_general); + iov.iov_base = (char *)&req_lib_votequorum_general; + iov.iov_len = sizeof (struct req_lib_votequorum_general); - error = saSendMsgReceiveReply (votequorum_inst->response_fd, iov, 1, - &res_lib_votequorum_qdisk_getinfo, sizeof (struct res_lib_votequorum_qdisk_getinfo)); + pthread_mutex_lock (&votequorum_inst->response_mutex); + + error = cslib_msg_send_reply_receive ( + votequorum_inst->ipc_ctx, + &iov, + 1, + &res_lib_votequorum_qdisk_getinfo, + sizeof (struct res_lib_votequorum_qdisk_getinfo)); pthread_mutex_unlock (&votequorum_inst->response_mutex); @@ -475,7 +499,7 @@ cs_error_t votequorum_setstate ( { cs_error_t error; struct votequorum_inst *votequorum_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_votequorum_general req_lib_votequorum_general; struct res_lib_votequorum_status res_lib_votequorum_status; @@ -484,16 +508,20 @@ cs_error_t votequorum_setstate ( return (error); } - pthread_mutex_lock (&votequorum_inst->response_mutex); - req_lib_votequorum_general.header.size = sizeof (struct req_lib_votequorum_general); req_lib_votequorum_general.header.id = MESSAGE_REQ_VOTEQUORUM_SETSTATE; - iov[0].iov_base = (char *)&req_lib_votequorum_general; - iov[0].iov_len = sizeof (struct req_lib_votequorum_general); + iov.iov_base = (char *)&req_lib_votequorum_general; + iov.iov_len = sizeof (struct req_lib_votequorum_general); - error = saSendMsgReceiveReply (votequorum_inst->response_fd, iov, 1, - &res_lib_votequorum_status, sizeof (struct res_lib_votequorum_status)); + pthread_mutex_lock (&votequorum_inst->response_mutex); + + error = cslib_msg_send_reply_receive ( + votequorum_inst->ipc_ctx, + &iov, + 1, + &res_lib_votequorum_status, + sizeof (struct res_lib_votequorum_status)); pthread_mutex_unlock (&votequorum_inst->response_mutex); @@ -514,7 +542,7 @@ cs_error_t votequorum_leaving ( { cs_error_t error; struct votequorum_inst *votequorum_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_votequorum_general req_lib_votequorum_general; struct res_lib_votequorum_status res_lib_votequorum_status; @@ -523,16 +551,21 @@ cs_error_t votequorum_leaving ( return (error); } - pthread_mutex_lock (&votequorum_inst->response_mutex); req_lib_votequorum_general.header.size = sizeof (struct req_lib_votequorum_general); req_lib_votequorum_general.header.id = MESSAGE_REQ_VOTEQUORUM_LEAVING; - iov[0].iov_base = (char *)&req_lib_votequorum_general; - iov[0].iov_len = sizeof (struct req_lib_votequorum_general); + iov.iov_base = (char *)&req_lib_votequorum_general; + iov.iov_len = sizeof (struct req_lib_votequorum_general); - error = saSendMsgReceiveReply (votequorum_inst->response_fd, iov, 1, - &res_lib_votequorum_status, sizeof (struct res_lib_votequorum_status)); + pthread_mutex_lock (&votequorum_inst->response_mutex); + + error = cslib_msg_send_reply_receive ( + votequorum_inst->ipc_ctx, + &iov, + 1, + &res_lib_votequorum_status, + sizeof (struct res_lib_votequorum_status)); pthread_mutex_unlock (&votequorum_inst->response_mutex); @@ -551,11 +584,11 @@ error_exit: cs_error_t votequorum_trackstart ( votequorum_handle_t handle, uint64_t context, - unsigned int flags ) + unsigned int flags) { cs_error_t error; struct votequorum_inst *votequorum_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_votequorum_trackstart req_lib_votequorum_trackstart; struct res_lib_votequorum_status res_lib_votequorum_status; @@ -564,18 +597,22 @@ cs_error_t votequorum_trackstart ( return (error); } - pthread_mutex_lock (&votequorum_inst->response_mutex); - req_lib_votequorum_trackstart.header.size = sizeof (struct req_lib_votequorum_trackstart); req_lib_votequorum_trackstart.header.id = MESSAGE_REQ_VOTEQUORUM_TRACKSTART; req_lib_votequorum_trackstart.track_flags = flags; req_lib_votequorum_trackstart.context = context; - iov[0].iov_base = (char *)&req_lib_votequorum_trackstart; - iov[0].iov_len = sizeof (struct req_lib_votequorum_trackstart); + iov.iov_base = (char *)&req_lib_votequorum_trackstart; + iov.iov_len = sizeof (struct req_lib_votequorum_trackstart); - error = saSendMsgReceiveReply (votequorum_inst->response_fd, iov, 1, - &res_lib_votequorum_status, sizeof (struct res_lib_votequorum_status)); + pthread_mutex_lock (&votequorum_inst->response_mutex); + + error = cslib_msg_send_reply_receive ( + votequorum_inst->ipc_ctx, + &iov, + 1, + &res_lib_votequorum_status, + sizeof (struct res_lib_votequorum_status)); pthread_mutex_unlock (&votequorum_inst->response_mutex); @@ -596,7 +633,7 @@ cs_error_t votequorum_trackstop ( { cs_error_t error; struct votequorum_inst *votequorum_inst; - struct iovec iov[2]; + struct iovec iov; struct req_lib_votequorum_general req_lib_votequorum_general; struct res_lib_votequorum_status res_lib_votequorum_status; @@ -605,16 +642,20 @@ cs_error_t votequorum_trackstop ( return (error); } - pthread_mutex_lock (&votequorum_inst->response_mutex); - req_lib_votequorum_general.header.size = sizeof (struct req_lib_votequorum_general); req_lib_votequorum_general.header.id = MESSAGE_REQ_VOTEQUORUM_TRACKSTOP; - iov[0].iov_base = (char *)&req_lib_votequorum_general; - iov[0].iov_len = sizeof (struct req_lib_votequorum_general); + iov.iov_base = (char *)&req_lib_votequorum_general; + iov.iov_len = sizeof (struct req_lib_votequorum_general); - error = saSendMsgReceiveReply (votequorum_inst->response_fd, iov, 1, - &res_lib_votequorum_status, sizeof (struct res_lib_votequorum_status)); + pthread_mutex_lock (&votequorum_inst->response_mutex); + + error = cslib_msg_send_reply_receive ( + votequorum_inst->ipc_ctx, + &iov, + 1, + &res_lib_votequorum_status, + sizeof (struct res_lib_votequorum_status)); pthread_mutex_unlock (&votequorum_inst->response_mutex); @@ -682,7 +723,7 @@ cs_error_t votequorum_fd_get ( return (error); } - *fd = votequorum_inst->dispatch_fd; + *fd = cslib_fd_get (votequorum_inst->ipc_ctx); (void)saHandleInstancePut (&votequorum_handle_t_db, handle); @@ -699,7 +740,6 @@ cs_error_t votequorum_dispatch ( votequorum_handle_t handle, cs_dispatch_flags_t dispatch_types) { - struct pollfd ufds; int timeout = -1; cs_error_t error; int cont = 1; /* always continue do loop except when set to 0 */ @@ -731,16 +771,12 @@ cs_error_t votequorum_dispatch ( } do { - ufds.fd = votequorum_inst->dispatch_fd; - ufds.events = POLLIN; - ufds.revents = 0; - pthread_mutex_lock (&votequorum_inst->dispatch_mutex); - error = saPollRetry (&ufds, 1, timeout); - if (error != CS_OK) { - goto error_unlock; - } + dispatch_avail = cslib_dispatch_recv ( + votequorum_inst->ipc_ctx, + (void *)&dispatch_data, timeout); + /* * Handle has been finalized in another thread @@ -750,12 +786,6 @@ cs_error_t votequorum_dispatch ( goto error_unlock; } - if ((ufds.revents & (POLLERR|POLLHUP|POLLNVAL)) != 0) { - error = CS_ERR_BAD_HANDLE; - goto error_unlock; - } - - dispatch_avail = ufds.revents & POLLIN; if (dispatch_avail == 0 && dispatch_types == CS_DISPATCH_ALL) { pthread_mutex_unlock (&votequorum_inst->dispatch_mutex); break; /* exit do while cont is 1 loop */ @@ -765,24 +795,6 @@ cs_error_t votequorum_dispatch ( continue; /* next poll */ } - if (ufds.revents & POLLIN) { - error = saRecvRetry (votequorum_inst->dispatch_fd, &dispatch_data.header, - sizeof (mar_res_header_t)); - if (error != CS_OK) { - goto error_unlock; - } - if (dispatch_data.header.size > sizeof (mar_res_header_t)) { - error = saRecvRetry (votequorum_inst->dispatch_fd, &dispatch_data.data, - dispatch_data.header.size - sizeof (mar_res_header_t)); - if (error != CS_OK) { - goto error_unlock; - } - } - } else { - pthread_mutex_unlock (&votequorum_inst->dispatch_mutex); - continue; - } - /* * Make copy of callbacks, message data, unlock instance, and call callback * A risk of this dispatch method is that the callback routines may diff --git a/services/cfg.c b/services/cfg.c index 79a92dcd..9540aaa4 100644 --- a/services/cfg.c +++ b/services/cfg.c @@ -398,14 +398,14 @@ static void send_test_shutdown(void * conn, int status) if (conn) { TRACE1("sending testshutdown to %p", conn); - api->ipc_conn_send_response(conn, &res_lib_cfg_testshutdown, + api->ipc_response_send(conn, &res_lib_cfg_testshutdown, sizeof(res_lib_cfg_testshutdown)); } else { for (iter = trackers_list.next; iter != &trackers_list; iter = iter->next) { struct cfg_info *ci = list_entry(iter, struct cfg_info, list); TRACE1("sending testshutdown to %p", ci->tracker_conn); - api->ipc_conn_send_response(ci->tracker_conn, &res_lib_cfg_testshutdown, + api->ipc_dispatch_send(ci->tracker_conn, &res_lib_cfg_testshutdown, sizeof(res_lib_cfg_testshutdown)); } } @@ -448,7 +448,7 @@ static void check_shutdown_status() /* * Tell originator that shutdown was confirmed */ - api->ipc_conn_send_response(shutdown_con->conn, &res_lib_cfg_tryshutdown, + api->ipc_response_send(shutdown_con->conn, &res_lib_cfg_tryshutdown, sizeof(res_lib_cfg_tryshutdown)); shutdown_con = NULL; } @@ -462,7 +462,7 @@ static void check_shutdown_status() /* * Tell originator that shutdown was cancelled */ - api->ipc_conn_send_response(shutdown_con->conn, &res_lib_cfg_tryshutdown, + api->ipc_response_send(shutdown_con->conn, &res_lib_cfg_tryshutdown, sizeof(res_lib_cfg_tryshutdown)); shutdown_con = NULL; } @@ -570,7 +570,7 @@ static void message_handler_req_exec_cfg_ringreenable ( res_lib_cfg_ringreenable.header.id = MESSAGE_RES_CFG_RINGREENABLE; res_lib_cfg_ringreenable.header.size = sizeof (struct res_lib_cfg_ringreenable); res_lib_cfg_ringreenable.header.error = CS_OK; - api->ipc_conn_send_response ( + api->ipc_response_send ( req_exec_cfg_ringreenable->source.conn, &res_lib_cfg_ringreenable, sizeof (struct res_lib_cfg_ringreenable)); @@ -660,7 +660,7 @@ static void message_handler_req_lib_cfg_ringstatusget ( strcpy ((char *)&res_lib_cfg_ringstatusget.interface_name[i], totem_ip_string); } - api->ipc_conn_send_response ( + api->ipc_response_send ( conn, &res_lib_cfg_ringstatusget, sizeof (struct res_lib_cfg_ringstatusget)); @@ -705,7 +705,7 @@ static void message_handler_req_lib_cfg_statetrack ( */ if (list_empty(&ci->list)) { list_add(&ci->list, &trackers_list); - ci->tracker_conn = api->ipc_conn_partner_get (conn); + ci->tracker_conn = conn; if (shutdown_con) { /* @@ -713,7 +713,7 @@ static void message_handler_req_lib_cfg_statetrack ( */ ci->shutdown_reply = SHUTDOWN_REPLY_UNKNOWN; shutdown_expected++; - send_test_shutdown(ci->tracker_conn, CS_OK); + send_test_shutdown(conn, CS_OK); } } @@ -721,7 +721,7 @@ static void message_handler_req_lib_cfg_statetrack ( res_lib_cfg_statetrack.header.id = MESSAGE_RES_CFG_STATETRACKSTART; res_lib_cfg_statetrack.header.error = CS_OK; - api->ipc_conn_send_response(conn, &res_lib_cfg_statetrack, + api->ipc_response_send(conn, &res_lib_cfg_statetrack, sizeof(res_lib_cfg_statetrack)); LEAVE(); @@ -774,7 +774,7 @@ static void message_handler_req_lib_cfg_serviceload ( res_lib_cfg_serviceload.header.id = MESSAGE_RES_CFG_SERVICEUNLOAD; res_lib_cfg_serviceload.header.size = sizeof (struct res_lib_cfg_serviceload); res_lib_cfg_serviceload.header.error = CS_OK; - api->ipc_conn_send_response ( + api->ipc_response_send ( conn, &res_lib_cfg_serviceload, sizeof (struct res_lib_cfg_serviceload)); @@ -797,7 +797,7 @@ static void message_handler_req_lib_cfg_serviceunload ( res_lib_cfg_serviceunload.header.id = MESSAGE_RES_CFG_SERVICEUNLOAD; res_lib_cfg_serviceunload.header.size = sizeof (struct res_lib_cfg_serviceunload); res_lib_cfg_serviceunload.header.error = CS_OK; - api->ipc_conn_send_response ( + api->ipc_response_send ( conn, &res_lib_cfg_serviceunload, sizeof (struct res_lib_cfg_serviceunload)); @@ -832,7 +832,7 @@ static void message_handler_req_lib_cfg_killnode ( res_lib_cfg_killnode.header.id = MESSAGE_RES_CFG_KILLNODE; res_lib_cfg_killnode.header.error = CS_OK; - api->ipc_conn_send_response(conn, &res_lib_cfg_killnode, + api->ipc_response_send(conn, &res_lib_cfg_killnode, sizeof(res_lib_cfg_killnode)); LEAVE(); @@ -860,7 +860,7 @@ static void message_handler_req_lib_cfg_tryshutdown ( res_lib_cfg_tryshutdown.header.size = sizeof(struct res_lib_cfg_tryshutdown); res_lib_cfg_tryshutdown.header.id = MESSAGE_RES_CFG_TRYSHUTDOWN; res_lib_cfg_tryshutdown.header.error = CS_OK; - api->ipc_conn_send_response(conn, &res_lib_cfg_tryshutdown, + api->ipc_response_send(conn, &res_lib_cfg_tryshutdown, sizeof(res_lib_cfg_tryshutdown)); LEAVE(); @@ -877,7 +877,7 @@ static void message_handler_req_lib_cfg_tryshutdown ( res_lib_cfg_tryshutdown.header.id = MESSAGE_RES_CFG_TRYSHUTDOWN; res_lib_cfg_tryshutdown.header.error = CS_ERR_EXIST; - api->ipc_conn_send_response(conn, &res_lib_cfg_tryshutdown, + api->ipc_response_send(conn, &res_lib_cfg_tryshutdown, sizeof(res_lib_cfg_tryshutdown)); @@ -1009,7 +1009,7 @@ static void message_handler_req_lib_cfg_get_node_addrs (void *conn, void *msg) else { res_lib_cfg_get_node_addrs->header.error = CS_ERR_NOT_EXIST; } - api->ipc_conn_send_response(conn, res_lib_cfg_get_node_addrs, res_lib_cfg_get_node_addrs->header.size); + api->ipc_response_send(conn, res_lib_cfg_get_node_addrs, res_lib_cfg_get_node_addrs->header.size); } static void message_handler_req_lib_cfg_local_get (void *conn, void *message) @@ -1021,6 +1021,6 @@ static void message_handler_req_lib_cfg_local_get (void *conn, void *message) res_lib_cfg_local_get.header.error = CS_OK; res_lib_cfg_local_get.local_nodeid = api->totem_nodeid_get (); - api->ipc_conn_send_response(conn, &res_lib_cfg_local_get, + api->ipc_response_send(conn, &res_lib_cfg_local_get, sizeof(res_lib_cfg_local_get)); } diff --git a/services/confdb.c b/services/confdb.c index 8fa34551..0c779dec 100644 --- a/services/confdb.c +++ b/services/confdb.c @@ -278,10 +278,10 @@ static int confdb_lib_exit_fn (void *conn) log_printf(LOG_LEVEL_DEBUG, "exit_fn for conn=%p\n", conn); /* cleanup the object trackers for this client. */ api->object_track_stop(confdb_notify_lib_of_key_change, - confdb_notify_lib_of_new_object, - confdb_notify_lib_of_destroyed_object, - NULL, - api->ipc_conn_partner_get (conn)); + confdb_notify_lib_of_new_object, + confdb_notify_lib_of_destroyed_object, + NULL, + conn); return (0); } @@ -302,7 +302,7 @@ static void message_handler_req_lib_confdb_object_create (void *conn, void *mess res_lib_confdb_object_create.header.size = sizeof(res_lib_confdb_object_create); res_lib_confdb_object_create.header.id = MESSAGE_RES_CONFDB_OBJECT_CREATE; res_lib_confdb_object_create.header.error = ret; - api->ipc_conn_send_response(conn, &res_lib_confdb_object_create, sizeof(res_lib_confdb_object_create)); + api->ipc_response_send(conn, &res_lib_confdb_object_create, sizeof(res_lib_confdb_object_create)); } static void message_handler_req_lib_confdb_object_destroy (void *conn, void *message) @@ -317,7 +317,7 @@ static void message_handler_req_lib_confdb_object_destroy (void *conn, void *mes res.size = sizeof(res); res.id = MESSAGE_RES_CONFDB_OBJECT_DESTROY; res.error = ret; - api->ipc_conn_send_response(conn, &res, sizeof(res)); + api->ipc_response_send(conn, &res, sizeof(res)); } static void message_handler_req_lib_confdb_object_find_destroy (void *conn, void *message) @@ -332,7 +332,7 @@ static void message_handler_req_lib_confdb_object_find_destroy (void *conn, void res.size = sizeof(res); res.id = MESSAGE_RES_CONFDB_OBJECT_FIND_DESTROY; res.error = ret; - api->ipc_conn_send_response(conn, &res, sizeof(res)); + api->ipc_response_send(conn, &res, sizeof(res)); } @@ -352,7 +352,7 @@ static void message_handler_req_lib_confdb_key_create (void *conn, void *message res.size = sizeof(res); res.id = MESSAGE_RES_CONFDB_KEY_CREATE; res.error = ret; - api->ipc_conn_send_response(conn, &res, sizeof(res)); + api->ipc_response_send(conn, &res, sizeof(res)); } static void message_handler_req_lib_confdb_key_get (void *conn, void *message) @@ -377,7 +377,7 @@ static void message_handler_req_lib_confdb_key_get (void *conn, void *message) res_lib_confdb_key_get.header.size = sizeof(res_lib_confdb_key_get); res_lib_confdb_key_get.header.id = MESSAGE_RES_CONFDB_KEY_GET; res_lib_confdb_key_get.header.error = ret; - api->ipc_conn_send_response(conn, &res_lib_confdb_key_get, sizeof(res_lib_confdb_key_get)); + api->ipc_response_send(conn, &res_lib_confdb_key_get, sizeof(res_lib_confdb_key_get)); } static void message_handler_req_lib_confdb_key_increment (void *conn, void *message) @@ -395,7 +395,7 @@ static void message_handler_req_lib_confdb_key_increment (void *conn, void *mess res_lib_confdb_key_incdec.header.size = sizeof(res_lib_confdb_key_incdec); res_lib_confdb_key_incdec.header.id = MESSAGE_RES_CONFDB_KEY_INCREMENT; res_lib_confdb_key_incdec.header.error = ret; - api->ipc_conn_send_response(conn, &res_lib_confdb_key_incdec, sizeof(res_lib_confdb_key_incdec)); + api->ipc_response_send(conn, &res_lib_confdb_key_incdec, sizeof(res_lib_confdb_key_incdec)); } static void message_handler_req_lib_confdb_key_decrement (void *conn, void *message) @@ -413,7 +413,7 @@ static void message_handler_req_lib_confdb_key_decrement (void *conn, void *mess res_lib_confdb_key_incdec.header.size = sizeof(res_lib_confdb_key_incdec); res_lib_confdb_key_incdec.header.id = MESSAGE_RES_CONFDB_KEY_DECREMENT; res_lib_confdb_key_incdec.header.error = ret; - api->ipc_conn_send_response(conn, &res_lib_confdb_key_incdec, sizeof(res_lib_confdb_key_incdec)); + api->ipc_response_send(conn, &res_lib_confdb_key_incdec, sizeof(res_lib_confdb_key_incdec)); } static void message_handler_req_lib_confdb_key_replace (void *conn, void *message) @@ -434,7 +434,7 @@ static void message_handler_req_lib_confdb_key_replace (void *conn, void *messag res.size = sizeof(res); res.id = MESSAGE_RES_CONFDB_KEY_REPLACE; res.error = ret; - api->ipc_conn_send_response(conn, &res, sizeof(res)); + api->ipc_response_send(conn, &res, sizeof(res)); } static void message_handler_req_lib_confdb_key_delete (void *conn, void *message) @@ -453,7 +453,7 @@ static void message_handler_req_lib_confdb_key_delete (void *conn, void *message res.size = sizeof(res); res.id = MESSAGE_RES_CONFDB_KEY_DELETE; res.error = ret; - api->ipc_conn_send_response(conn, &res, sizeof(res)); + api->ipc_response_send(conn, &res, sizeof(res)); } static void message_handler_req_lib_confdb_object_parent_get (void *conn, void *message) @@ -471,7 +471,7 @@ static void message_handler_req_lib_confdb_object_parent_get (void *conn, void * res_lib_confdb_object_parent_get.header.size = sizeof(res_lib_confdb_object_parent_get); res_lib_confdb_object_parent_get.header.id = MESSAGE_RES_CONFDB_OBJECT_CREATE; res_lib_confdb_object_parent_get.header.error = ret; - api->ipc_conn_send_response(conn, &res_lib_confdb_object_parent_get, sizeof(res_lib_confdb_object_parent_get)); + api->ipc_response_send(conn, &res_lib_confdb_object_parent_get, sizeof(res_lib_confdb_object_parent_get)); } @@ -502,7 +502,7 @@ static void message_handler_req_lib_confdb_key_iter (void *conn, void *message) res_lib_confdb_key_iter.header.id = MESSAGE_RES_CONFDB_KEY_ITER; res_lib_confdb_key_iter.header.error = ret; - api->ipc_conn_send_response(conn, &res_lib_confdb_key_iter, sizeof(res_lib_confdb_key_iter)); + api->ipc_response_send(conn, &res_lib_confdb_key_iter, sizeof(res_lib_confdb_key_iter)); } static void message_handler_req_lib_confdb_object_iter (void *conn, void *message) @@ -536,7 +536,7 @@ static void message_handler_req_lib_confdb_object_iter (void *conn, void *messag res_lib_confdb_object_iter.header.id = MESSAGE_RES_CONFDB_OBJECT_ITER; res_lib_confdb_object_iter.header.error = ret; - api->ipc_conn_send_response(conn, &res_lib_confdb_object_iter, sizeof(res_lib_confdb_object_iter)); + api->ipc_response_send(conn, &res_lib_confdb_object_iter, sizeof(res_lib_confdb_object_iter)); } static void message_handler_req_lib_confdb_object_find (void *conn, void *message) @@ -565,7 +565,7 @@ static void message_handler_req_lib_confdb_object_find (void *conn, void *messag res_lib_confdb_object_find.header.error = ret; - api->ipc_conn_send_response(conn, &res_lib_confdb_object_find, sizeof(res_lib_confdb_object_find)); + api->ipc_response_send(conn, &res_lib_confdb_object_find, sizeof(res_lib_confdb_object_find)); } static void message_handler_req_lib_confdb_write (void *conn, void *message) @@ -586,7 +586,7 @@ static void message_handler_req_lib_confdb_write (void *conn, void *message) } else res_lib_confdb_write.error.length = 0; - api->ipc_conn_send_response(conn, &res_lib_confdb_write, sizeof(res_lib_confdb_write)); + api->ipc_response_send(conn, &res_lib_confdb_write, sizeof(res_lib_confdb_write)); } static void message_handler_req_lib_confdb_reload (void *conn, void *message) @@ -609,7 +609,7 @@ static void message_handler_req_lib_confdb_reload (void *conn, void *message) } else res_lib_confdb_reload.error.length = 0; - api->ipc_conn_send_response(conn, &res_lib_confdb_reload, sizeof(res_lib_confdb_reload)); + api->ipc_response_send(conn, &res_lib_confdb_reload, sizeof(res_lib_confdb_reload)); } static void confdb_notify_lib_of_key_change(object_change_type_t change_type, @@ -639,7 +639,7 @@ static void confdb_notify_lib_of_key_change(object_change_type_t change_type, memcpy(res.key_value.value, key_value_pt, key_value_len); res.key_value.length = key_value_len; - api->ipc_conn_send_response(priv_data_pt, &res, sizeof(res)); + api->ipc_dispatch_send(priv_data_pt, &res, sizeof(res)); } static void confdb_notify_lib_of_new_object(unsigned int parent_object_handle, @@ -657,7 +657,7 @@ static void confdb_notify_lib_of_new_object(unsigned int parent_object_handle, memcpy(res.name.value, name_pt, name_len); res.name.length = name_len; - api->ipc_conn_send_response(priv_data_pt, &res, sizeof(res)); + api->ipc_dispatch_send(priv_data_pt, &res, sizeof(res)); } static void confdb_notify_lib_of_destroyed_object(unsigned int parent_object_handle, @@ -673,7 +673,7 @@ static void confdb_notify_lib_of_destroyed_object(unsigned int parent_object_han memcpy(res.name.value, name_pt, name_len); res.name.length = name_len; - api->ipc_conn_send_response(priv_data_pt, &res, sizeof(res)); + api->ipc_dispatch_send(priv_data_pt, &res, sizeof(res)); } @@ -682,16 +682,17 @@ static void message_handler_req_lib_confdb_track_start (void *conn, void *messag struct req_lib_confdb_object_track_start *req = (struct req_lib_confdb_object_track_start *)message; mar_res_header_t res; - api->object_track_start(req->object_handle, req->flags, - confdb_notify_lib_of_key_change, - confdb_notify_lib_of_new_object, - confdb_notify_lib_of_destroyed_object, - NULL, - api->ipc_conn_partner_get (conn)); + api->object_track_start(req->object_handle, + req->flags, + confdb_notify_lib_of_key_change, + confdb_notify_lib_of_new_object, + confdb_notify_lib_of_destroyed_object, + NULL, + conn); res.size = sizeof(res); res.id = MESSAGE_RES_CONFDB_TRACK_START; res.error = CS_OK; - api->ipc_conn_send_response(conn, &res, sizeof(res)); + api->ipc_response_send(conn, &res, sizeof(res)); } static void message_handler_req_lib_confdb_track_stop (void *conn, void *message) @@ -699,15 +700,15 @@ static void message_handler_req_lib_confdb_track_stop (void *conn, void *message mar_res_header_t res; api->object_track_stop(confdb_notify_lib_of_key_change, - confdb_notify_lib_of_new_object, - confdb_notify_lib_of_destroyed_object, - NULL, - api->ipc_conn_partner_get (conn)); + confdb_notify_lib_of_new_object, + confdb_notify_lib_of_destroyed_object, + NULL, + conn); res.size = sizeof(res); res.id = MESSAGE_RES_CONFDB_TRACK_STOP; res.error = CS_OK; - api->ipc_conn_send_response(conn, &res, sizeof(res)); + api->ipc_response_send(conn, &res, sizeof(res)); } diff --git a/services/cpg.c b/services/cpg.c index 5b3d100e..c0dc5765 100644 --- a/services/cpg.c +++ b/services/cpg.c @@ -99,7 +99,6 @@ struct process_info { void *conn; void *trackerconn; struct group_info *group; - enum cs_flow_control_state flow_control_state; struct list_head list; /* on the group_info members list */ }; @@ -440,14 +439,14 @@ static int notify_lib_joinlist( } if (conn) { - api->ipc_conn_send_response(conn, buf, size); + api->ipc_response_send(conn, buf, size); } else { /* Send it to all listeners */ for (iter = gi->members.next, tmp=iter->next; iter != &gi->members; iter = tmp, tmp=iter->next) { struct process_info *pi = list_entry(iter, struct process_info, list); if (pi->trackerconn && (pi->flags & PI_FLAG_MEMBER)) { - if (api->ipc_conn_send_response(pi->trackerconn, buf, size) == -1) { + if (api->ipc_response_send(pi->trackerconn, buf, size) == -1) { // Error ?? } } @@ -537,72 +536,6 @@ static struct group_info *get_group(mar_cpg_name_t *name) return gi; } -static void send_group_list_callbacks(int num_groups, void *conn) -{ - struct list_head *iter, *piter; - struct group_info *gi; - uint32_t hash; - int max_proc_count=0; - int size = 0; - int group_counter = 0; - char *buf = NULL; - struct res_lib_cpg_groups_get_callback *res; - mar_cpg_address_t *retgi; - - for (hash=0; hash < GROUP_HASH_SIZE; hash++) { - for (iter = group_lists[hash].next; iter != &group_lists[hash]; iter = iter->next) { - gi = list_entry(iter, struct group_info, list); - int proc_count = 0; - - /* First, we need to know how many processes are in the list */ - for (piter = gi->members.next; piter != &gi->members; piter = piter->next) { - struct process_info *pi = list_entry(piter, struct process_info, list); - if (pi->pid) - proc_count++; - } - - /* Make sure we have adequate buffer space */ - if (proc_count > max_proc_count) { - max_proc_count = proc_count+10; - size = max_proc_count*sizeof(mar_cpg_address_t) + - sizeof(struct res_lib_cpg_groups_get_callback); - buf = realloc(buf, size); - if (!buf) { - log_printf(LOG_LEVEL_WARNING, "Unable to allocate group_list struct"); - return; - } - } - if (!buf) - continue; - - res = (struct res_lib_cpg_groups_get_callback *)buf; - retgi = res->member_list; - - res->header.size = size; - res->header.id = MESSAGE_RES_CPG_GROUPS_CALLBACK; - - - memcpy(&res->group_name, &gi->group_name, sizeof(mar_cpg_name_t)); - res->num_members = proc_count; - res->group_num = ++group_counter; - res->total_groups = num_groups; - - for (piter = gi->members.next; piter != &gi->members; piter = piter->next) { - struct process_info *pi = list_entry(piter, struct process_info, list); - if (pi->pid) { - retgi->nodeid = pi->nodeid; - retgi->pid = pi->pid; - retgi->reason = 0; - retgi++; - } - } - api->ipc_conn_send_response(conn, buf, size); - } - } - if (buf) - free(buf); -} - static int cpg_node_joinleave_send (struct group_info *gi, struct process_info *pi, int fn, int reason) { struct req_exec_cpg_procjoin req_exec_cpg_procjoin; @@ -737,34 +670,6 @@ static void cpg_confchg_fn ( } } -static void cpg_flow_control_state_set_fn ( - void *context, - enum cs_flow_control_state flow_control_state) -{ - struct res_lib_cpg_flowcontrol_callback res_lib_cpg_flowcontrol_callback; - struct process_info *process_info = (struct process_info *)context; - - process_info->flow_control_state = flow_control_state; - /* - * Send disabled flow control if a disabled occurs. This prevents - * the condition where a disabled occurs after all messages have been - * delivered and then there is no valid way to retrieve the flow - * control state - */ - if (flow_control_state == CPG_FLOW_CONTROL_DISABLED) { - res_lib_cpg_flowcontrol_callback.header.id = MESSAGE_RES_CPG_FLOWCONTROL_CALLBACK; - res_lib_cpg_flowcontrol_callback.header.size = sizeof (struct res_lib_cpg_flowcontrol_callback); - res_lib_cpg_flowcontrol_callback.flow_control_state = flow_control_state; - - if (process_info->trackerconn) { - api->ipc_response_no_fcc ( - process_info->trackerconn, - &res_lib_cpg_flowcontrol_callback, - sizeof (struct res_lib_cpg_flowcontrol_callback)); - } - } -} - /* Can byteswap join & leave messages */ static void exec_cpg_procjoin_endian_convert (void *msg) { @@ -1004,7 +909,6 @@ static void message_handler_req_exec_cpg_mcast ( res_lib_cpg_mcast->msglen = msglen; res_lib_cpg_mcast->pid = req_exec_cpg_mcast->pid; res_lib_cpg_mcast->nodeid = nodeid; - res_lib_cpg_mcast->flow_control_state = CPG_FLOW_CONTROL_DISABLED; if (api->ipc_source_is_local (&req_exec_cpg_mcast->source)) { api->ipc_refcnt_dec (req_exec_cpg_mcast->source.conn); } @@ -1017,8 +921,7 @@ static void message_handler_req_exec_cpg_mcast ( for (iter = gi->members.next; iter != &gi->members; iter = iter->next) { struct process_info *pi = list_entry(iter, struct process_info, list); if (pi->trackerconn && (pi->flags & PI_FLAG_MEMBER)) { - res_lib_cpg_mcast->flow_control_state = pi->flow_control_state; - api->ipc_conn_send_response( + api->ipc_dispatch_send( pi->trackerconn, buf, res_lib_cpg_mcast->header.size); @@ -1124,14 +1027,6 @@ static void message_handler_req_lib_cpg_join (void *conn, void *message) goto join_err; } - api->ipc_fc_create ( - conn, - CPG_SERVICE, - req_lib_cpg_join->group_name.value, - req_lib_cpg_join->group_name.length, - cpg_flow_control_state_set_fn, - pi); - /* Add a node entry for us */ pi->nodeid = api->totem_nodeid_get(); pi->pid = req_lib_cpg_join->pid; @@ -1145,7 +1040,7 @@ join_err: res_lib_cpg_join.header.size = sizeof(res_lib_cpg_join); res_lib_cpg_join.header.id = MESSAGE_RES_CPG_JOIN; res_lib_cpg_join.header.error = error; - api->ipc_conn_send_response(conn, &res_lib_cpg_join, sizeof(res_lib_cpg_join)); + api->ipc_response_send(conn, &res_lib_cpg_join, sizeof(res_lib_cpg_join)); } /* Leave message from the library */ @@ -1169,18 +1064,12 @@ static void message_handler_req_lib_cpg_leave (void *conn, void *message) cpg_node_joinleave_send(gi, pi, MESSAGE_REQ_EXEC_CPG_PROCLEAVE, CONFCHG_CPG_REASON_LEAVE); pi->group = NULL; - api->ipc_fc_destroy ( - conn, - CPG_SERVICE, - (unsigned char *)gi->group_name.value, - (unsigned int)gi->group_name.length); - leave_ret: /* send return */ res_lib_cpg_leave.header.size = sizeof(res_lib_cpg_leave); res_lib_cpg_leave.header.id = MESSAGE_RES_CPG_LEAVE; res_lib_cpg_leave.header.error = error; - api->ipc_conn_send_response(conn, &res_lib_cpg_leave, sizeof(res_lib_cpg_leave)); + api->ipc_response_send(conn, &res_lib_cpg_leave, sizeof(res_lib_cpg_leave)); } /* Mcast message from the library */ @@ -1202,8 +1091,7 @@ static void message_handler_req_lib_cpg_mcast (void *conn, void *message) res_lib_cpg_mcast.header.size = sizeof(res_lib_cpg_mcast); res_lib_cpg_mcast.header.id = MESSAGE_RES_CPG_MCAST; res_lib_cpg_mcast.header.error = CS_ERR_ACCESS; /* TODO Better error code ?? */ - res_lib_cpg_mcast.flow_control_state = CPG_FLOW_CONTROL_DISABLED; - api->ipc_conn_send_response(conn, &res_lib_cpg_mcast, + api->ipc_response_send(conn, &res_lib_cpg_mcast, sizeof(res_lib_cpg_mcast)); return; } @@ -1229,8 +1117,7 @@ static void message_handler_req_lib_cpg_mcast (void *conn, void *message) res_lib_cpg_mcast.header.size = sizeof(res_lib_cpg_mcast); res_lib_cpg_mcast.header.id = MESSAGE_RES_CPG_MCAST; res_lib_cpg_mcast.header.error = CS_OK; - res_lib_cpg_mcast.flow_control_state = pi->flow_control_state; - api->ipc_conn_send_response(conn, &res_lib_cpg_mcast, + api->ipc_response_send(conn, &res_lib_cpg_mcast, sizeof(res_lib_cpg_mcast)); } @@ -1244,7 +1131,7 @@ static void message_handler_req_lib_cpg_membership (void *conn, void *message) res.size = sizeof(res); res.id = MESSAGE_RES_CPG_MEMBERSHIP; res.error = CS_ERR_ACCESS; /* TODO Better error code */ - api->ipc_conn_send_response(conn, &res, sizeof(res)); + api->ipc_response_send(conn, &res, sizeof(res)); return; } @@ -1258,7 +1145,6 @@ static void message_handler_req_lib_cpg_trackstart (void *conn, void *message) struct res_lib_cpg_trackstart res_lib_cpg_trackstart; struct group_info *gi; struct process_info *otherpi; - void *otherconn; cs_error_t error = CS_OK; log_printf(LOG_LEVEL_DEBUG, "got trackstart request on %p\n", conn); @@ -1270,7 +1156,6 @@ static void message_handler_req_lib_cpg_trackstart (void *conn, void *message) } /* Find the partner connection and add us to it's process_info struct */ - otherconn = api->ipc_conn_partner_get (conn); otherpi = (struct process_info *)api->ipc_private_data_get (conn); otherpi->trackerconn = conn; @@ -1278,7 +1163,7 @@ tstart_ret: res_lib_cpg_trackstart.header.size = sizeof(res_lib_cpg_trackstart); res_lib_cpg_trackstart.header.id = MESSAGE_RES_CPG_TRACKSTART; res_lib_cpg_trackstart.header.error = CS_OK; - api->ipc_conn_send_response(conn, &res_lib_cpg_trackstart, sizeof(res_lib_cpg_trackstart)); + api->ipc_response_send(conn, &res_lib_cpg_trackstart, sizeof(res_lib_cpg_trackstart)); } static void message_handler_req_lib_cpg_trackstop (void *conn, void *message) @@ -1286,7 +1171,6 @@ static void message_handler_req_lib_cpg_trackstop (void *conn, void *message) struct req_lib_cpg_trackstop *req_lib_cpg_trackstop = (struct req_lib_cpg_trackstop *)message; struct res_lib_cpg_trackstop res_lib_cpg_trackstop; struct process_info *otherpi; - void *otherconn; struct group_info *gi; cs_error_t error = CS_OK; @@ -1299,7 +1183,6 @@ static void message_handler_req_lib_cpg_trackstop (void *conn, void *message) } /* Find the partner connection and add us to it's process_info struct */ - otherconn = api->ipc_conn_partner_get (conn); otherpi = (struct process_info *)api->ipc_private_data_get (conn); otherpi->trackerconn = NULL; @@ -1307,7 +1190,7 @@ tstop_ret: res_lib_cpg_trackstop.header.size = sizeof(res_lib_cpg_trackstop); res_lib_cpg_trackstop.header.id = MESSAGE_RES_CPG_TRACKSTOP; res_lib_cpg_trackstop.header.error = CS_OK; - api->ipc_conn_send_response(conn, &res_lib_cpg_trackstop.header, sizeof(res_lib_cpg_trackstop)); + api->ipc_response_send(conn, &res_lib_cpg_trackstop.header, sizeof(res_lib_cpg_trackstop)); } static void message_handler_req_lib_cpg_local_get (void *conn, void *message) @@ -1319,7 +1202,7 @@ static void message_handler_req_lib_cpg_local_get (void *conn, void *message) res_lib_cpg_local_get.header.error = CS_OK; res_lib_cpg_local_get.local_nodeid = api->totem_nodeid_get (); - api->ipc_conn_send_response(conn, &res_lib_cpg_local_get, + api->ipc_response_send(conn, &res_lib_cpg_local_get, sizeof(res_lib_cpg_local_get)); } @@ -1332,11 +1215,7 @@ static void message_handler_req_lib_cpg_groups_get (void *conn, void *message) res_lib_cpg_groups_get.header.error = CS_OK; res_lib_cpg_groups_get.num_groups = count_groups(); - api->ipc_conn_send_response(conn, &res_lib_cpg_groups_get, + api->ipc_response_send(conn, &res_lib_cpg_groups_get, sizeof(res_lib_cpg_groups_get)); - - /* Now do the callbacks for each group */ - send_group_list_callbacks(res_lib_cpg_groups_get.num_groups, - api->ipc_conn_partner_get (conn)); } diff --git a/services/evs.c b/services/evs.c index 6ce9f429..e3eb65bb 100644 --- a/services/evs.c +++ b/services/evs.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2006 MontaVista Software, Inc. - * Copyright (c) 2006-2008 Red Hat, Inc. + * Copyright (c) 2006-2009 Red Hat, Inc. * * All rights reserved. * @@ -242,7 +242,7 @@ static void evs_confchg_fn ( */ for (list = confchg_notify.next; list != &confchg_notify; list = list->next) { evs_pd = list_entry (list, struct evs_pd, list); - api->ipc_conn_send_response (evs_pd->conn, + api->ipc_response_send (evs_pd->conn, &res_evs_confchg_callback, sizeof (res_evs_confchg_callback)); } @@ -260,7 +260,7 @@ static int evs_lib_init_fn (void *conn) list_init (&evs_pd->list); list_add (&evs_pd->list, &confchg_notify); - api->ipc_conn_send_response (conn, &res_evs_confchg_callback, + api->ipc_response_send (conn, &res_evs_confchg_callback, sizeof (res_evs_confchg_callback)); return (0); @@ -306,7 +306,7 @@ exit_error: res_lib_evs_join.header.id = MESSAGE_RES_EVS_JOIN; res_lib_evs_join.header.error = error; - api->ipc_conn_send_response (conn, &res_lib_evs_join, + api->ipc_response_send (conn, &res_lib_evs_join, sizeof (struct res_lib_evs_join)); } @@ -352,7 +352,7 @@ static void message_handler_req_evs_leave (void *conn, void *msg) res_lib_evs_leave.header.id = MESSAGE_RES_EVS_LEAVE; res_lib_evs_leave.header.error = error; - api->ipc_conn_send_response (conn, &res_lib_evs_leave, + api->ipc_response_send (conn, &res_lib_evs_leave, sizeof (struct res_lib_evs_leave)); } @@ -395,7 +395,7 @@ static void message_handler_req_evs_mcast_joined (void *conn, void *msg) res_lib_evs_mcast_joined.header.id = MESSAGE_RES_EVS_MCAST_JOINED; res_lib_evs_mcast_joined.header.error = error; - api->ipc_conn_send_response (conn, &res_lib_evs_mcast_joined, + api->ipc_response_send (conn, &res_lib_evs_mcast_joined, sizeof (struct res_lib_evs_mcast_joined)); } @@ -441,7 +441,7 @@ static void message_handler_req_evs_mcast_groups (void *conn, void *msg) res_lib_evs_mcast_groups.header.id = MESSAGE_RES_EVS_MCAST_GROUPS; res_lib_evs_mcast_groups.header.error = error; - api->ipc_conn_send_response (conn, &res_lib_evs_mcast_groups, + api->ipc_response_send (conn, &res_lib_evs_mcast_groups, sizeof (struct res_lib_evs_mcast_groups)); } @@ -460,7 +460,7 @@ static void message_handler_req_evs_membership_get (void *conn, void *msg) res_lib_evs_membership_get.member_list_entries = res_evs_confchg_callback.member_list_entries; - api->ipc_conn_send_response (conn, &res_lib_evs_membership_get, + api->ipc_response_send (conn, &res_lib_evs_membership_get, sizeof (struct res_lib_evs_membership_get)); } @@ -484,6 +484,7 @@ static void message_handler_req_exec_mcast ( int found = 0; int i, j; struct evs_pd *evs_pd; + struct iovec iov[2]; res_evs_deliver_callback.header.size = sizeof (struct res_evs_deliver_callback) + req_exec_evs_mcast->msg_len; @@ -515,10 +516,15 @@ static void message_handler_req_exec_mcast ( if (found) { res_evs_deliver_callback.local_nodeid = nodeid; - api->ipc_conn_send_response (evs_pd->conn, &res_evs_deliver_callback, - sizeof (struct res_evs_deliver_callback)); - api->ipc_conn_send_response (evs_pd->conn, msg_addr, - req_exec_evs_mcast->msg_len); + iov[0].iov_base = &res_evs_deliver_callback; + iov[0].iov_len = sizeof (struct res_evs_deliver_callback); + iov[1].iov_base = msg_addr; + iov[1].iov_len = req_exec_evs_mcast->msg_len; + + api->ipc_dispatch_iov_send ( + evs_pd->conn, + iov, + 2); } } } diff --git a/services/votequorum.c b/services/votequorum.c index ead0bf3b..38c71306 100644 --- a/services/votequorum.c +++ b/services/votequorum.c @@ -612,7 +612,7 @@ static int send_quorum_notification(void *conn, uint64_t context) /* Send it to all interested parties */ if (conn) { - int ret = corosync_api->ipc_conn_send_response(conn, buf, size); + int ret = corosync_api->ipc_dispatch_send(conn, buf, size); LEAVE(); return ret; } @@ -622,7 +622,7 @@ static int send_quorum_notification(void *conn, uint64_t context) list_iterate(tmp, &trackers_list) { qpd = list_entry(tmp, struct quorum_pd, list); res_lib_votequorum_notification->context = qpd->tracking_context; - corosync_api->ipc_conn_send_response(corosync_api->ipc_conn_partner_get(qpd->conn), buf, size); + corosync_api->ipc_dispatch_send(qpd->conn, buf, size); } } LEAVE(); @@ -1197,7 +1197,7 @@ static void message_handler_req_lib_votequorum_getinfo (void *conn, void *messag res_lib_votequorum_getinfo.header.size = sizeof(res_lib_votequorum_getinfo); res_lib_votequorum_getinfo.header.id = MESSAGE_RES_VOTEQUORUM_GETINFO; res_lib_votequorum_getinfo.header.error = error; - corosync_api->ipc_conn_send_response(conn, &res_lib_votequorum_getinfo, sizeof(res_lib_votequorum_getinfo)); + corosync_api->ipc_response_send(conn, &res_lib_votequorum_getinfo, sizeof(res_lib_votequorum_getinfo)); log_printf(LOG_LEVEL_DEBUG, "getinfo response error: %d\n", error); } @@ -1236,7 +1236,7 @@ error_exit: res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status); res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS; res_lib_votequorum_status.header.error = error; - corosync_api->ipc_conn_send_response(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); + corosync_api->ipc_response_send(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); LEAVE(); } @@ -1281,7 +1281,7 @@ error_exit: res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status); res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS; res_lib_votequorum_status.header.error = error; - corosync_api->ipc_conn_send_response(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); + corosync_api->ipc_response_send(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); LEAVE(); } @@ -1307,7 +1307,7 @@ static void message_handler_req_lib_votequorum_leaving (void *conn, void *messag res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status); res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS; res_lib_votequorum_status.header.error = error; - corosync_api->ipc_conn_send_response(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); + corosync_api->ipc_response_send(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); LEAVE(); } @@ -1355,7 +1355,7 @@ static void message_handler_req_lib_votequorum_qdisk_register (void *conn, void res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status); res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS; res_lib_votequorum_status.header.error = error; - corosync_api->ipc_conn_send_response(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); + corosync_api->ipc_response_send(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); LEAVE(); } @@ -1382,7 +1382,7 @@ static void message_handler_req_lib_votequorum_qdisk_unregister (void *conn, voi res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status); res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS; res_lib_votequorum_status.header.error = error; - corosync_api->ipc_conn_send_response(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); + corosync_api->ipc_response_send(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); LEAVE(); } @@ -1421,7 +1421,7 @@ static void message_handler_req_lib_votequorum_qdisk_poll (void *conn, void *mes res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status); res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS; res_lib_votequorum_status.header.error = error; - corosync_api->ipc_conn_send_response(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); + corosync_api->ipc_response_send(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); LEAVE(); } @@ -1450,7 +1450,7 @@ static void message_handler_req_lib_votequorum_qdisk_getinfo (void *conn, void * res_lib_votequorum_qdisk_getinfo.header.size = sizeof(res_lib_votequorum_qdisk_getinfo); res_lib_votequorum_qdisk_getinfo.header.id = MESSAGE_RES_VOTEQUORUM_GETINFO; res_lib_votequorum_qdisk_getinfo.header.error = error; - corosync_api->ipc_conn_send_response(conn, &res_lib_votequorum_qdisk_getinfo, sizeof(res_lib_votequorum_qdisk_getinfo)); + corosync_api->ipc_response_send(conn, &res_lib_votequorum_qdisk_getinfo, sizeof(res_lib_votequorum_qdisk_getinfo)); LEAVE(); } @@ -1468,7 +1468,7 @@ static void message_handler_req_lib_votequorum_setstate (void *conn, void *messa res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status); res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS; res_lib_votequorum_status.header.error = error; - corosync_api->ipc_conn_send_response(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); + corosync_api->ipc_response_send(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); LEAVE(); } @@ -1487,7 +1487,7 @@ static void message_handler_req_lib_votequorum_trackstart (void *conn, void *msg if (req_lib_votequorum_trackstart->track_flags & CS_TRACK_CURRENT || req_lib_votequorum_trackstart->track_flags & CS_TRACK_CHANGES) { log_printf(LOG_LEVEL_DEBUG, "sending initial status to %p\n", conn); - send_quorum_notification(corosync_api->ipc_conn_partner_get (conn), req_lib_votequorum_trackstart->context); + send_quorum_notification(conn, req_lib_votequorum_trackstart->context); } /* @@ -1507,7 +1507,7 @@ static void message_handler_req_lib_votequorum_trackstart (void *conn, void *msg res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status); res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS; res_lib_votequorum_status.header.error = CS_OK; - corosync_api->ipc_conn_send_response(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); + corosync_api->ipc_response_send(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); LEAVE(); } @@ -1533,7 +1533,7 @@ static void message_handler_req_lib_votequorum_trackstop (void *conn, void *msg) res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status); res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS; res_lib_votequorum_status.header.error = error; - corosync_api->ipc_conn_send_response(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); + corosync_api->ipc_response_send(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); LEAVE(); } diff --git a/test/cpgbench.c b/test/cpgbench.c index a41c6694..585d3afd 100644 --- a/test/cpgbench.c +++ b/test/cpgbench.c @@ -1,4 +1,4 @@ -#define _BSD_SOURCE +#include /* * Copyright (c) 2006 Red Hat, Inc. * @@ -156,10 +156,11 @@ static struct cpg_name group_name = { int main (void) { cpg_handle_t handle; - unsigned int size = 1; + unsigned int size; int i; unsigned int res; + size = 1000; signal (SIGALRM, sigalrm_handler); res = cpg_initialize (&handle, &callbacks); if (res != CS_OK) { diff --git a/test/evsbench.c b/test/evsbench.c index 028960d8..379de57a 100644 --- a/test/evsbench.c +++ b/test/evsbench.c @@ -66,17 +66,12 @@ volatile static int alarm_notice = 0; -int outstanding = 0; - - - void evs_deliver_fn ( unsigned int nodeid, void *msg, int msg_len) { - outstanding--; -// printf ("Delivering message %s\n", msg); + printf ("Delivering message %s\n", msg); } void evs_confchg_fn ( @@ -136,13 +131,10 @@ void evs_benchmark (evs_handle_t handle, iov.iov_len = write_size; do { sprintf (buffer, "This is message %d\n", write_count); - if (outstanding < 10) { - result = evs_mcast_joined (handle, EVS_TYPE_AGREED, &iov, 1); + result = evs_mcast_joined (handle, EVS_TYPE_AGREED, &iov, 1); - if (result != CS_ERR_TRY_AGAIN) { - write_count += 1; - outstanding++; - } + if (result != CS_ERR_TRY_AGAIN) { + write_count += 1; } result = evs_dispatch (handle, CS_DISPATCH_ALL); } while (alarm_notice == 0); @@ -185,8 +177,6 @@ int main (void) { printf ("Init result %d\n", result); result = evs_join (handle, groups, 3); printf ("Join result %d\n", result); - result = evs_leave (handle, &groups[0], 1); - printf ("Leave result %d\n", result); size = 1; diff --git a/test/evsverify.c b/test/evsverify.c index 7044923d..01c51a07 100644 --- a/test/evsverify.c +++ b/test/evsverify.c @@ -44,7 +44,7 @@ #include "../exec/crypto.h" char *delivery_string; -struct msg { +struct my_msg { unsigned int msg_size; unsigned char sha1[20]; unsigned char buffer[0]; @@ -56,7 +56,7 @@ void evs_deliver_fn ( void *m, int msg_len) { - struct msg *msg2 = (struct msg *)m; + struct my_msg *msg2 = (struct my_msg *)m; unsigned char sha1_compare[20]; hash_state sha1_hash; unsigned int i; @@ -111,7 +111,7 @@ struct evs_group groups[3] = { { "key3" } }; -struct msg msg; +struct my_msg msg; unsigned char buffer[200000]; int main (void) @@ -123,7 +123,7 @@ int main (void) unsigned int member_list[32]; unsigned int local_nodeid; unsigned int member_list_entries = 32; - struct msg msg; + struct my_msg msg; hash_state sha1_hash; struct iovec iov[2]; @@ -150,7 +150,7 @@ int main (void) delivery_string = "evs_mcast_joined"; iov[0].iov_base = &msg; - iov[0].iov_len = sizeof (struct msg); + iov[0].iov_len = sizeof (struct my_msg); iov[1].iov_base = buffer; /* diff --git a/test/testcpg.c b/test/testcpg.c index 8d511849..9278912a 100644 --- a/test/testcpg.c +++ b/test/testcpg.c @@ -141,36 +141,9 @@ void ConfchgCallback ( } } -void GroupsGetCallback(cpg_handle_t handle, - uint32_t groupnum, - uint32_t groupmax, - struct cpg_name *group_name, - struct cpg_address *member_list, int member_list_entries) -{ - int i; - struct in_addr saddr; - - printf("Groups List Callback %d/%d: ", groupnum, groupmax); - print_cpgname(group_name); - printf("\n"); - for (i=0; i