diff --git a/include/corosync/ipc_votequorum.h b/include/corosync/ipc_votequorum.h index 5a820da6..5cf1bb20 100644 --- a/include/corosync/ipc_votequorum.h +++ b/include/corosync/ipc_votequorum.h @@ -58,7 +58,8 @@ enum res_votequorum_types { MESSAGE_RES_VOTEQUORUM_GETINFO, MESSAGE_RES_VOTEQUORUM_QDISK_GETINFO, MESSAGE_RES_VOTEQUORUM_TRACKSTART, - MESSAGE_RES_VOTEQUORUM_NOTIFICATION + MESSAGE_RES_VOTEQUORUM_NOTIFICATION, + MESSAGE_RES_VOTEQUORUM_EXPECTEDVOTES_NOTIFICATION }; struct req_lib_votequorum_setvotes { @@ -142,4 +143,10 @@ struct res_lib_votequorum_notification { struct votequorum_node node_list[] __attribute__((aligned(8))); }; +struct res_lib_votequorum_expectedvotes_notification { + mar_res_header_t header __attribute__((aligned(8))); + mar_uint64_t context __attribute__((aligned(8))); + mar_uint32_t expected_votes __attribute__((aligned(8))); +}; + #endif diff --git a/include/corosync/votequorum.h b/include/corosync/votequorum.h index b6807ae8..bcb593d5 100644 --- a/include/corosync/votequorum.h +++ b/include/corosync/votequorum.h @@ -84,8 +84,15 @@ typedef void (*votequorum_notification_fn_t) ( votequorum_node_t node_list[] ); +typedef void (*votequorum_expectedvotes_notification_fn_t) ( + votequorum_handle_t handle, + uint64_t context, + uint32_t expected_votes + ); + typedef struct { votequorum_notification_fn_t votequorum_notify_fn; + votequorum_expectedvotes_notification_fn_t votequorum_expectedvotes_notify_fn; } votequorum_callbacks_t; diff --git a/lib/votequorum.c b/lib/votequorum.c index f241421b..e67d5357 100644 --- a/lib/votequorum.c +++ b/lib/votequorum.c @@ -103,9 +103,9 @@ cs_error_t votequorum_initialize ( pthread_mutex_init (&votequorum_inst->response_mutex, NULL); pthread_mutex_init (&votequorum_inst->dispatch_mutex, NULL); if (callbacks) - memcpy(&votequorum_inst->callbacks, callbacks, sizeof (callbacks)); + memcpy(&votequorum_inst->callbacks, callbacks, sizeof (*callbacks)); else - memset(&votequorum_inst->callbacks, 0, sizeof (callbacks)); + memset(&votequorum_inst->callbacks, 0, sizeof (*callbacks)); saHandleInstancePut (&votequorum_handle_t_db, *handle); @@ -748,6 +748,7 @@ cs_error_t votequorum_dispatch ( votequorum_callbacks_t callbacks; struct res_overlay dispatch_data; struct res_lib_votequorum_notification *res_lib_votequorum_notification; + struct res_lib_votequorum_expectedvotes_notification *res_lib_votequorum_expectedvotes_notification; if (dispatch_types != CS_DISPATCH_ONE && dispatch_types != CS_DISPATCH_ALL && @@ -764,7 +765,7 @@ cs_error_t votequorum_dispatch ( /* * Timeout instantly for CS_DISPATCH_ONE or CS_DISPATCH_ALL and - * wait indefinately for CS_DISPATCH_BLOCKING + * wait indefinitely for CS_DISPATCH_BLOCKING */ if (dispatch_types == CS_DISPATCH_ALL) { timeout = 0; @@ -822,6 +823,17 @@ cs_error_t votequorum_dispatch ( ; break; + case MESSAGE_RES_VOTEQUORUM_EXPECTEDVOTES_NOTIFICATION: + if (callbacks.votequorum_expectedvotes_notify_fn == NULL) { + continue; + } + res_lib_votequorum_expectedvotes_notification = (struct res_lib_votequorum_expectedvotes_notification *)&dispatch_data; + + callbacks.votequorum_expectedvotes_notify_fn ( handle, + res_lib_votequorum_expectedvotes_notification->context, + res_lib_votequorum_expectedvotes_notification->expected_votes); + break; + default: error = CS_ERR_LIBRARY; goto error_put; diff --git a/services/votequorum.c b/services/votequorum.c index 6f274af1..05ae2e50 100644 --- a/services/votequorum.c +++ b/services/votequorum.c @@ -223,6 +223,7 @@ static int quorum_exec_send_killnode(int nodeid, unsigned int reason); static void add_votequorum_config_notification(hdb_handle_t quorum_object_handle); +static void recalculate_quorum(int allow_decrease, int by_current_nodes); /* * Library Handler Definition @@ -543,6 +544,8 @@ static int votequorum_exec_init_fn (struct corosync_api_v1 *api) if (corosync_api->object_find_next(find_handle, &object_handle) == 0) { read_quorum_config(object_handle); } + recalculate_quorum(0, 0); + /* Listen for changes */ add_votequorum_config_notification(object_handle); corosync_api->object_find_destroy(find_handle); @@ -629,6 +632,27 @@ static int send_quorum_notification(void *conn, uint64_t context) return 0; } +static void send_expectedvotes_notification() +{ + struct res_lib_votequorum_expectedvotes_notification res_lib_votequorum_expectedvotes_notification; + struct quorum_pd *qpd; + struct list_head *tmp; + + log_printf(LOG_DEBUG, "Sending expected votes callback\n"); + + res_lib_votequorum_expectedvotes_notification.header.id = MESSAGE_RES_VOTEQUORUM_EXPECTEDVOTES_NOTIFICATION; + res_lib_votequorum_expectedvotes_notification.header.size = sizeof(res_lib_votequorum_expectedvotes_notification); + res_lib_votequorum_expectedvotes_notification.header.error = CS_OK; + res_lib_votequorum_expectedvotes_notification.expected_votes = us->expected_votes; + + list_iterate(tmp, &trackers_list) { + qpd = list_entry(tmp, struct quorum_pd, list); + res_lib_votequorum_expectedvotes_notification.context = qpd->tracking_context; + corosync_api->ipc_dispatch_send(qpd->conn, &res_lib_votequorum_expectedvotes_notification, + sizeof(struct res_lib_votequorum_expectedvotes_notification)); + } +} + static void set_quorate(int total_votes) { int quorate; @@ -673,6 +697,7 @@ static int calculate_quorum(int allow_decrease, int max_expected, unsigned int * unsigned int total_nodes = 0; ENTER(); + list_iterate(nodelist, &cluster_members_list) { node = list_entry(nodelist, struct cluster_node, list); @@ -725,25 +750,33 @@ static int calculate_quorum(int allow_decrease, int max_expected, unsigned int * /* Recalculate cluster quorum, set quorate and notify changes */ static void recalculate_quorum(int allow_decrease, int by_current_nodes) { - unsigned int total_votes; + unsigned int total_votes = 0; int cluster_members = 0; struct list_head *nodelist; struct cluster_node *node; ENTER(); - if (by_current_nodes) { - list_iterate(nodelist, &cluster_members_list) { - node = list_entry(nodelist, struct cluster_node, list); + list_iterate(nodelist, &cluster_members_list) { + node = list_entry(nodelist, struct cluster_node, list); - if (node->state == NODESTATE_MEMBER) { + if (node->state == NODESTATE_MEMBER) { + if (by_current_nodes) cluster_members++; - } + total_votes += node->votes; } } + /* Keep expected_votes at the highest number of votes in the cluster */ + log_printf(LOG_DEBUG, "total_votes=%d, expected_votes=%d\n", total_votes, us->expected_votes); + if (total_votes > us->expected_votes) { + us->expected_votes = total_votes; + send_expectedvotes_notification(); + } + quorum = calculate_quorum(allow_decrease, cluster_members, &total_votes); set_quorate(total_votes); + send_quorum_notification(NULL, 0L); LEAVE(); } @@ -1080,8 +1113,6 @@ static void message_handler_req_exec_quorum_reconfigure ( switch(req_exec_quorum_reconfigure->param) { case RECONFIG_PARAM_EXPECTED_VOTES: - node->expected_votes = req_exec_quorum_reconfigure->value; - list_iterate(nodelist, &cluster_members_list) { node = list_entry(nodelist, struct cluster_node, list); if (node->state == NODESTATE_MEMBER && @@ -1089,6 +1120,7 @@ static void message_handler_req_exec_quorum_reconfigure ( node->expected_votes = req_exec_quorum_reconfigure->value; } } + send_expectedvotes_notification(); recalculate_quorum(1, 0); /* Allow decrease */ break; diff --git a/test/testvotequorum1.c b/test/testvotequorum1.c index 92a677d6..158e09e5 100644 --- a/test/testvotequorum1.c +++ b/test/testvotequorum1.c @@ -66,6 +66,18 @@ static char *node_state(int state) } } + +static void votequorum_expectedvotes_notification_fn( + votequorum_handle_t handle, + uint64_t context, + uint32_t expected_votes + ) +{ + printf("votequorum expectedvotes notification called \n"); + printf(" expected_votes = %d\n", expected_votes); +} + + static void votequorum_notification_fn( votequorum_handle_t handle, uint64_t context, @@ -99,6 +111,8 @@ int main(int argc, char *argv[]) } callbacks.votequorum_notify_fn = votequorum_notification_fn; + callbacks.votequorum_expectedvotes_notify_fn = votequorum_expectedvotes_notification_fn; + if ( (err=votequorum_initialize(&handle, &callbacks)) != CS_OK) fprintf(stderr, "votequorum_initialize FAILED: %d\n", err);