From 8217e33e86bca79dc307e112f0c0c4a278f9a568 Mon Sep 17 00:00:00 2001 From: Jan Friesse Date: Mon, 31 Aug 2020 16:59:58 +0200 Subject: [PATCH] qdevice: Port qdevice to use pr-poll-loop Signed-off-by: Jan Friesse --- qdevices/Makefile.am | 2 - qdevices/corosync-qdevice.c | 63 ++- qdevices/qdevice-instance.h | 3 +- qdevices/qdevice-ipc.c | 153 ++++++ qdevices/qdevice-ipc.h | 1 - qdevices/qdevice-model-net.c | 354 ++++++++----- qdevices/qdevice-model-net.h | 7 +- qdevices/qdevice-model.c | 24 +- qdevices/qdevice-model.h | 30 +- qdevices/qdevice-net-cast-vote-timer.c | 7 +- qdevices/qdevice-net-disconnect-reason.h | 6 +- qdevices/qdevice-net-echo-request-timer.c | 9 +- qdevices/qdevice-net-heuristics.c | 6 +- qdevices/qdevice-net-instance.c | 91 +--- qdevices/qdevice-net-instance.h | 11 +- qdevices/qdevice-net-msg-received.c | 4 +- qdevices/qdevice-net-poll-array-user-data.h | 64 --- qdevices/qdevice-net-poll.c | 546 -------------------- qdevices/qdevice-net-poll.h | 47 -- qdevices/qdevice-net-socket.c | 204 ++++++++ qdevices/qdevice-net-socket.h | 6 +- qdevices/qdevice-pr-poll-loop-cb.c | 52 ++ 22 files changed, 772 insertions(+), 918 deletions(-) delete mode 100644 qdevices/qdevice-net-poll-array-user-data.h delete mode 100644 qdevices/qdevice-net-poll.c delete mode 100644 qdevices/qdevice-net-poll.h diff --git a/qdevices/Makefile.am b/qdevices/Makefile.am index e75b90e..11d3840 100644 --- a/qdevices/Makefile.am +++ b/qdevices/Makefile.am @@ -110,7 +110,6 @@ corosync_qdevice_SOURCES = corosync-qdevice.c \ pr-poll-array.c pr-poll-array.h dynar-simple-lex.c dynar-simple-lex.h \ dynar-str.c dynar-str.h qdevice-ipc-cmd.c qdevice-ipc-cmd.h \ qdevice-net-ipc-cmd.c qdevice-net-ipc-cmd.h \ - qdevice-net-poll.c qdevice-net-poll.h \ qdevice-net-send.c qdevice-net-send.h \ qdevice-net-votequorum.c qdevice-net-votequorum.h \ qdevice-net-socket.c qdevice-net-socket.h \ @@ -123,7 +122,6 @@ corosync_qdevice_SOURCES = corosync-qdevice.c \ qdevice-net-algo-ffsplit.c qdevice-net-algo-ffsplit.h \ qdevice-net-algo-2nodelms.c qdevice-net-algo-2nodelms.h \ qdevice-net-algo-lms.c qdevice-net-algo-lms.h \ - qdevice-net-poll-array-user-data.h \ qdevice-config.h qnet-config.h qdevice-net-disconnect-reason.h \ qdevice-model-type.h qdevice-advanced-settings.c \ qdevice-advanced-settings.h dynar-getopt-lex.c dynar-getopt-lex.h \ diff --git a/qdevices/corosync-qdevice.c b/qdevices/corosync-qdevice.c index 3cad898..dd9f758 100644 --- a/qdevices/corosync-qdevice.c +++ b/qdevices/corosync-qdevice.c @@ -102,6 +102,54 @@ signal_handlers_register(void) sigaction(SIGPIPE, &act, NULL); } +static int +qdevice_run_main_loop(struct qdevice_instance *instance) +{ + int res; + enum qdevice_model_post_poll_loop_exit_reason exit_reason; + int restart_loop; + + log(LOG_DEBUG, "Running QDevice main loop"); + + restart_loop = 1; + + while (restart_loop) { + res = qdevice_model_pre_poll_loop(instance); + if (res == -1) { + return (-1); + } + + while ((res = pr_poll_loop_exec(&instance->main_poll_loop)) == 0) { + } + + if (res == -2) { + log(LOG_CRIT, "pr_poll_loop_exec returned -2 - internal error"); + return (-1); + } else if (res == -3) { + log_nss(LOG_CRIT, "pr_poll_loop_exec returned -3 - PR_Poll error"); + return (-1); + } + + exit_reason = QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_MODEL; + if (instance->votequorum_closed) { + exit_reason = QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_VOTEQUORUM_CLOSED; + } else if (instance->cmap_closed) { + exit_reason = QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_CMAP_CLOSED; + } else if (instance->heuristics_closed) { + exit_reason = QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_HEURISTICS_CLOSED; + } else if (qdevice_ipc_is_closed(instance)) { + exit_reason = QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_IPC_SOCKET_CLOSED; + } + + res = qdevice_model_post_poll_loop(instance, exit_reason); + if (res == 0 || res == -1) { + restart_loop = 0; + } + } + + return (res); +} + static void usage(void) { @@ -187,7 +235,7 @@ main(int argc, char * const argv[]) int bump_log_priority; int lock_file; int another_instance_running; - int model_run_res; + int main_loop_res; if (qdevice_advanced_settings_init(&advanced_settings) != 0) { errx(EXIT_FAILURE, "Can't alloc memory for advanced settings"); @@ -269,19 +317,20 @@ main(int argc, char * const argv[]) return (EXIT_FAILURE); } + global_instance = &instance; + signal_handlers_register(); + log(LOG_DEBUG, "Waiting for initial heuristics exec result"); if (qdevice_pr_poll_loop_wait_for_initial_heuristics_exec_result(&instance) != 0) { return (EXIT_FAILURE); } - global_instance = &instance; - signal_handlers_register(); - - log(LOG_DEBUG, "Running qdevice model"); + log(LOG_DEBUG, "Qdevice ready to provide service"); #ifdef HAVE_LIBSYSTEMD sd_notify (0, "READY=1"); #endif - model_run_res = qdevice_model_run(&instance); + + main_loop_res = qdevice_run_main_loop(&instance); log(LOG_DEBUG, "Removing cmap tracking"); /* @@ -309,5 +358,5 @@ main(int argc, char * const argv[]) qdevice_advanced_settings_destroy(&advanced_settings); - return (model_run_res == 0 ? EXIT_SUCCESS : EXIT_FAILURE); + return (main_loop_res == 0 ? EXIT_SUCCESS : EXIT_FAILURE); } diff --git a/qdevices/qdevice-instance.h b/qdevices/qdevice-instance.h index 32c2e05..4a4cc17 100644 --- a/qdevices/qdevice-instance.h +++ b/qdevices/qdevice-instance.h @@ -123,9 +123,10 @@ struct qdevice_instance { struct pr_poll_loop main_poll_loop; /* - * Set by poll handler when votequorum connection is closed + * Set by poll handler when votequorum/cmap connection is closed */ int votequorum_closed; + int cmap_closed; /* * Set by poll handler when one of the heuristics pipes becomes closed */ diff --git a/qdevices/qdevice-ipc.c b/qdevices/qdevice-ipc.c index a7451b6..03cf525 100644 --- a/qdevices/qdevice-ipc.c +++ b/qdevices/qdevice-ipc.c @@ -40,9 +40,139 @@ #include "dynar-str.h" #include "qdevice-ipc-cmd.h" +/* + * Callbacks + */ + +/* + * IPC server socket + */ +static int +ipc_socket_poll_loop_set_events_cb(int fd, short *events, void *user_data1, void *user_data2) +{ + struct qdevice_instance *instance = (struct qdevice_instance *)user_data1; + + if (qdevice_ipc_is_closed(instance)) { + log(LOG_DEBUG, "Listening socket is closed"); + + return (-2); + } + + return (0); +} + +static int +ipc_socket_poll_loop_read_cb(int fd, void *user_data1, void *user_data2) +{ + struct qdevice_instance *instance = (struct qdevice_instance *)user_data1; + struct unix_socket_client *ipc_client; + + qdevice_ipc_accept(instance, &ipc_client); + + return (0); +} + +static int +ipc_socket_poll_loop_err_cb(int fd, short revents, void *user_data1, void *user_data2) +{ + + if (revents != POLLNVAL) { + /* + * Poll ERR on listening socket is fatal error. + * POLL_NVAL is used as a signal to quit poll loop. + */ + log(LOG_CRIT, "POLL_ERR (%u) on listening socket", revents); + } else { + log(LOG_DEBUG, "Listening socket is closed"); + } + + return (-1); +} + +/* + * IPC client sockets + */ +static int +ipc_client_socket_poll_loop_set_events_cb(int fd, short *events, void *user_data1, void *user_data2) +{ + struct qdevice_instance *instance = (struct qdevice_instance *)user_data1; + struct unix_socket_client *ipc_client = (struct unix_socket_client *)user_data2; + + if (ipc_client->schedule_disconnect) { + qdevice_ipc_client_disconnect(instance, ipc_client); + + if (pr_poll_loop_del_fd(&instance->main_poll_loop, fd) == -1) { + log(LOG_ERR, "pr_poll_loop_del_fd for ipc client socket failed"); + + return (-2); + } + + return (-1); + } + + if (!ipc_client->reading_line && !ipc_client->writing_buffer) { + return (-1); + } + + if (ipc_client->reading_line) { + *events |= POLLIN; + } + + if (ipc_client->writing_buffer) { + *events |= POLLOUT; + } + + return (0); +} + +static int +ipc_client_socket_poll_loop_read_cb(int fd, void *user_data1, void *user_data2) +{ + struct qdevice_instance *instance = (struct qdevice_instance *)user_data1; + struct unix_socket_client *ipc_client = (struct unix_socket_client *)user_data2; + + if (!ipc_client->schedule_disconnect) { + qdevice_ipc_io_read(instance, ipc_client); + } + + return (0); +} + +static int +ipc_client_socket_poll_loop_write_cb(int fd, void *user_data1, void *user_data2) +{ + struct qdevice_instance *instance = (struct qdevice_instance *)user_data1; + struct unix_socket_client *ipc_client = (struct unix_socket_client *)user_data2; + + if (!ipc_client->schedule_disconnect) { + qdevice_ipc_io_write(instance, ipc_client); + } + + return (0); +} + +static int +ipc_client_socket_poll_loop_err_cb(int fd, short revents, void *user_data1, void *user_data2) +{ + struct unix_socket_client *ipc_client = (struct unix_socket_client *)user_data2; + + if (!ipc_client->schedule_disconnect) { + log(LOG_DEBUG, "POLL_ERR (%u) on ipc client socket." + " Disconnecting.", revents); + + ipc_client->schedule_disconnect = 1; + } + + return (0); +} + +/* + * Exported functions + */ int qdevice_ipc_init(struct qdevice_instance *instance) { + if (unix_socket_ipc_init(&instance->local_ipc, instance->advanced_settings->local_socket_file, instance->advanced_settings->local_socket_backlog, @@ -54,6 +184,16 @@ qdevice_ipc_init(struct qdevice_instance *instance) return (-1); } + if (pr_poll_loop_add_fd(&instance->main_poll_loop, instance->local_ipc.socket, POLLIN, + ipc_socket_poll_loop_set_events_cb, + ipc_socket_poll_loop_read_cb, + NULL, + ipc_socket_poll_loop_err_cb, instance, NULL) == -1) { + log(LOG_ERR, "Can't add IPC socket to main poll loop"); + + return (-1); + } + return (0); } @@ -137,6 +277,19 @@ qdevice_ipc_accept(struct qdevice_instance *instance, struct unix_socket_client memset((*res_client)->user_data, 0, sizeof(struct qdevice_ipc_user_data)); } + if (pr_poll_loop_add_fd(&instance->main_poll_loop, (*res_client)->socket, 0, + ipc_client_socket_poll_loop_set_events_cb, + ipc_client_socket_poll_loop_read_cb, + ipc_client_socket_poll_loop_write_cb, + ipc_client_socket_poll_loop_err_cb, instance, *res_client) == -1) { + log(LOG_ERR, "Can't add IPC client socket to main poll loop"); + res = -1; + + qdevice_ipc_client_disconnect(instance, *res_client); + + goto return_res; + } + return_res: return (res); } diff --git a/qdevices/qdevice-ipc.h b/qdevices/qdevice-ipc.h index 8185799..b407612 100644 --- a/qdevices/qdevice-ipc.h +++ b/qdevices/qdevice-ipc.h @@ -42,7 +42,6 @@ extern "C" { #endif struct qdevice_ipc_user_data { - void *model_data; int shutdown_requested; }; diff --git a/qdevices/qdevice-model-net.c b/qdevices/qdevice-model-net.c index 98347c8..e8b2aa9 100644 --- a/qdevices/qdevice-model-net.c +++ b/qdevices/qdevice-model-net.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2019 Red Hat, Inc. + * Copyright (c) 2015-2020 Red Hat, Inc. * * All rights reserved. * @@ -42,12 +42,47 @@ #include "qdevice-net-ipc-cmd.h" #include "qdevice-net-algorithm.h" #include "qdevice-net-heuristics.h" -#include "qdevice-net-poll.h" #include "qdevice-net-send.h" +#include "qdevice-net-socket.h" #include "qdevice-net-votequorum.h" #include "qnet-config.h" #include "nss-sock.h" +/* + * Callbacks + */ +static int +check_schedule_disconnect_cb(void *user_data1, void *user_data2) +{ + struct qdevice_net_instance *instance = (struct qdevice_net_instance *)user_data1; + + if (instance->schedule_disconnect) { + return (-1); + } + + return (0); +} + +static int +qdevice_model_net_timer_connect_timeout(void *data1, void *data2) +{ + struct qdevice_net_instance *instance; + + instance = (struct qdevice_net_instance *)data1; + + log(LOG_ERR, "Connect timeout"); + + instance->schedule_disconnect = 1; + + instance->connect_timer = NULL; + instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_CONNECT_TO_THE_SERVER; + + return (0); +} + +/* + * Exported functions + */ int qdevice_model_net_init(struct qdevice_instance *instance) { @@ -120,22 +155,6 @@ qdevice_model_net_destroy(struct qdevice_instance *instance) return (0); } -static int -qdevice_model_net_timer_connect_timeout(void *data1, void *data2) -{ - struct qdevice_net_instance *instance; - - instance = (struct qdevice_net_instance *)data1; - - log(LOG_ERR, "Connect timeout"); - - instance->schedule_disconnect = 1; - - instance->connect_timer = NULL; - instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_CONNECT_TO_THE_SERVER; - - return (0); -} static PRIntn qdevice_model_net_get_af(const struct qdevice_net_instance *instance) @@ -154,134 +173,213 @@ qdevice_model_net_get_af(const struct qdevice_net_instance *instance) return (af); } +/* + * 0 - Continue + * -1 - End loop + */ int -qdevice_model_net_run(struct qdevice_instance *instance) +qdevice_model_net_pre_poll_loop(struct qdevice_instance *instance) { struct qdevice_net_instance *net_instance; - int try_connect; int res; - enum tlv_vote vote; - int delay_before_reconnect; - int ret_val; net_instance = instance->model_data; - log(LOG_DEBUG, "Executing qdevice-net"); + net_instance->state = QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT; + net_instance->socket = NULL; + net_instance->connect_timer = timer_list_add(pr_poll_loop_get_timer_list(&instance->main_poll_loop), + net_instance->connect_timeout, qdevice_model_net_timer_connect_timeout, + (void *)net_instance, NULL); + + if (net_instance->connect_timer == NULL) { + log(LOG_CRIT, "Can't schedule connect timer"); + + return (-1); + } + + log(LOG_DEBUG, "Trying connect to qnetd server %s:%u (timeout = %ums)", + net_instance->host_addr, net_instance->host_port, net_instance->connect_timeout); + + res = nss_sock_non_blocking_client_init(net_instance->host_addr, + net_instance->host_port, qdevice_model_net_get_af(net_instance), + &net_instance->non_blocking_client); + if (res == -1) { + log_nss(LOG_ERR, "Can't initialize non blocking client connection"); + } + + res = nss_sock_non_blocking_client_try_next(&net_instance->non_blocking_client); + if (res == -1) { + log_nss(LOG_ERR, "Can't connect to qnetd host"); + } + + res = qdevice_net_socket_add_to_main_poll_loop(net_instance); + if (res == -1) { + goto error_free_non_blocking_client; + } + + res = pr_poll_loop_add_pre_poll_cb(&instance->main_poll_loop, check_schedule_disconnect_cb, + net_instance, NULL); + if (res == -1) { + log(LOG_CRIT, "Can't add pre poll callback to main loop"); + goto error_del_from_main_poll_loop; + } + + return (0); + +error_del_from_main_poll_loop: + (void)qdevice_net_socket_del_from_main_poll_loop(net_instance); + +error_free_non_blocking_client: + nss_sock_non_blocking_client_destroy(&net_instance->non_blocking_client); + return (-1); +} + +/* + * 1 - Restart loop + * 0 - End loop with no error + * -1 - End loop with error -1 + */ +int +qdevice_model_net_post_poll_loop(struct qdevice_instance *instance, + enum qdevice_model_post_poll_loop_exit_reason exit_reason) +{ + struct qdevice_net_instance *net_instance; + int restart_loop; + int ret_val; + enum tlv_vote vote; + int delay_before_reconnect; + + net_instance = instance->model_data; + + restart_loop = 1; ret_val = -1; - try_connect = 1; - while (try_connect) { - net_instance->state = QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT; + if (net_instance->connect_timer != NULL) { + timer_list_delete(pr_poll_loop_get_timer_list(&instance->main_poll_loop), + net_instance->connect_timer); + net_instance->connect_timer = NULL; + } + + if (net_instance->echo_request_timer != NULL) { + timer_list_delete(pr_poll_loop_get_timer_list(&instance->main_poll_loop), + net_instance->echo_request_timer); + net_instance->echo_request_timer = NULL; + } + + /* + * Map qdevice exit_reason to qdevice-net disconnect reason + */ + switch (exit_reason) { + case QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_MODEL: + /* + * Disconnect reason should be already set + */ + break; + case QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_VOTEQUORUM_CLOSED: + net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED; + break; + case QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_CMAP_CLOSED: + net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED; + break; + case QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_HEURISTICS_CLOSED: + net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_WORKER_CLOSED; + break; + case QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_IPC_SOCKET_CLOSED: + net_instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_LOCAL_SOCKET_CLOSED; + break; + /* + * Default not handled intentionally. Compiler shows warning when new exit reason is added + */ + } + + restart_loop = qdevice_net_disconnect_reason_try_reconnect(net_instance->disconnect_reason); + + /* + * Unpause cast vote timer, because if it is paused we cannot remove tracking + */ + qdevice_net_cast_vote_timer_set_paused(net_instance, 0); + + vote = TLV_VOTE_NO_CHANGE; + + if (qdevice_net_algorithm_disconnected(net_instance, + net_instance->disconnect_reason, &restart_loop, &vote) != 0) { + log(LOG_ERR, "Algorithm returned error, force exit"); + return (-1); + } else { + log(LOG_DEBUG, "Algorithm result vote is %s", + tlv_vote_to_str(vote)); + } + + if (qdevice_net_cast_vote_timer_update(net_instance, vote) != 0) { + log(LOG_ERR, "qdevice_model_net_run fatal error. " + " Can't update cast vote timer vote"); + } + + if (qdevice_net_disconnect_reason_force_disconnect(net_instance->disconnect_reason)) { + restart_loop = 0; + } + + /* + * Return 0 only when local socket was closed -> regular exit + */ + if (net_instance->disconnect_reason == QDEVICE_NET_DISCONNECT_REASON_LOCAL_SOCKET_CLOSED) { + ret_val = 0; + } + + /* + * Remove pre poll cb + */ + if (pr_poll_loop_del_pre_poll_cb(&instance->main_poll_loop, check_schedule_disconnect_cb) == -1) { + log(LOG_ERR, "Can't delete pre poll callback from main loop"); + restart_loop = 0; + ret_val = -1; + } + + /* + * Remove socket from loop + */ + if (qdevice_net_socket_del_from_main_poll_loop(net_instance) == -1) { + restart_loop = 0; + ret_val = -1; + } + + if (net_instance->socket != NULL) { + if (PR_Close(net_instance->socket) != PR_SUCCESS) { + log_nss(LOG_WARNING, "Unable to close connection"); + } net_instance->socket = NULL; + } - net_instance->connect_timer = timer_list_add(&net_instance->main_timer_list, - net_instance->connect_timeout, qdevice_model_net_timer_connect_timeout, - (void *)net_instance, NULL); + if (!net_instance->non_blocking_client.destroyed) { + nss_sock_non_blocking_client_destroy(&net_instance->non_blocking_client); + } - if (net_instance->connect_timer == NULL) { - log(LOG_CRIT, "Can't schedule connect timer"); - - try_connect = 0; - break; + if (net_instance->non_blocking_client.socket != NULL) { + if (PR_Close(net_instance->non_blocking_client.socket) != PR_SUCCESS) { + log_nss(LOG_WARNING, "Unable to close non-blocking client connection"); } + net_instance->non_blocking_client.socket = NULL; + } - log(LOG_DEBUG, "Trying connect to qnetd server %s:%u (timeout = %ums)", - net_instance->host_addr, net_instance->host_port, net_instance->connect_timeout); - - res = nss_sock_non_blocking_client_init(net_instance->host_addr, - net_instance->host_port, qdevice_model_net_get_af(net_instance), - &net_instance->non_blocking_client); - if (res == -1) { - log_nss(LOG_ERR, "Can't initialize non blocking client connection"); - } - - res = nss_sock_non_blocking_client_try_next(&net_instance->non_blocking_client); - if (res == -1) { - log_nss(LOG_ERR, "Can't connect to qnetd host"); - nss_sock_non_blocking_client_destroy(&net_instance->non_blocking_client); - } - - while (qdevice_net_poll(net_instance) == 0) { - }; - - if (net_instance->connect_timer != NULL) { - timer_list_delete(&net_instance->main_timer_list, net_instance->connect_timer); - net_instance->connect_timer = NULL; - } - - if (net_instance->echo_request_timer != NULL) { - timer_list_delete(&net_instance->main_timer_list, net_instance->echo_request_timer); - net_instance->echo_request_timer = NULL; - } - - try_connect = qdevice_net_disconnect_reason_try_reconnect(net_instance->disconnect_reason); - + if (restart_loop && + net_instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT) { /* - * Unpause cast vote timer, because if it is paused we cannot remove tracking + * Give qnetd server a little time before reconnect */ - qdevice_net_cast_vote_timer_set_paused(net_instance, 0); + delay_before_reconnect = random() % + (int)(net_instance->cast_vote_timer_interval * 0.9); - vote = TLV_VOTE_NO_CHANGE; - - if (qdevice_net_algorithm_disconnected(net_instance, - net_instance->disconnect_reason, &try_connect, &vote) != 0) { - log(LOG_ERR, "Algorithm returned error, force exit"); - return (-1); - } else { - log(LOG_DEBUG, "Algorithm result vote is %s", - tlv_vote_to_str(vote)); - } - - if (qdevice_net_cast_vote_timer_update(net_instance, vote) != 0) { - log(LOG_ERR, "qdevice_model_net_run fatal error. " - " Can't update cast vote timer vote"); - } - - if (qdevice_net_disconnect_reason_force_disconnect(net_instance->disconnect_reason)) { - try_connect = 0; - } - - /* - * Return 0 only when local socket was closed -> regular exit - */ - if (net_instance->disconnect_reason == QDEVICE_NET_DISCONNECT_REASON_LOCAL_SOCKET_CLOSED) { - ret_val = 0; - } - - if (net_instance->socket != NULL) { - if (PR_Close(net_instance->socket) != PR_SUCCESS) { - log_nss(LOG_WARNING, "Unable to close connection"); - } - net_instance->socket = NULL; - } - - if (!net_instance->non_blocking_client.destroyed) { - nss_sock_non_blocking_client_destroy(&net_instance->non_blocking_client); - } - - if (net_instance->non_blocking_client.socket != NULL) { - if (PR_Close(net_instance->non_blocking_client.socket) != PR_SUCCESS) { - log_nss(LOG_WARNING, "Unable to close non-blocking client connection"); - } - net_instance->non_blocking_client.socket = NULL; - } - - if (try_connect && - net_instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT) { - /* - * Give qnetd server a little time before reconnect - */ - delay_before_reconnect = random() % - (int)(net_instance->cast_vote_timer_interval * 0.9); - - log(LOG_DEBUG, "Sleeping for %u ms before reconnect", - delay_before_reconnect); - (void)poll(NULL, 0, delay_before_reconnect); - } + log(LOG_DEBUG, "Sleeping for %u ms before reconnect", + delay_before_reconnect); + (void)poll(NULL, 0, delay_before_reconnect); + } - qdevice_net_instance_clean(net_instance); + qdevice_net_instance_clean(net_instance); + + if (restart_loop) { + return (1); } return (ret_val); @@ -679,7 +777,8 @@ static struct qdevice_model qdevice_model_net = { .name = "net", .init = qdevice_model_net_init, .destroy = qdevice_model_net_destroy, - .run = qdevice_model_net_run, + .pre_poll_loop = qdevice_model_net_pre_poll_loop, + .post_poll_loop = qdevice_model_net_post_poll_loop, .get_config_node_list_failed = qdevice_model_net_get_config_node_list_failed, .config_node_list_changed = qdevice_model_net_config_node_list_changed, .votequorum_quorum_notify = qdevice_model_net_votequorum_quorum_notify, @@ -693,5 +792,6 @@ static struct qdevice_model qdevice_model_net = { int qdevice_model_net_register(void) { + return (qdevice_model_register(QDEVICE_MODEL_TYPE_NET, &qdevice_model_net)); } diff --git a/qdevices/qdevice-model-net.h b/qdevices/qdevice-model-net.h index 199205c..6169aab 100644 --- a/qdevices/qdevice-model-net.h +++ b/qdevices/qdevice-model-net.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017 Red Hat, Inc. + * Copyright (c) 2015-2020 Red Hat, Inc. * * All rights reserved. * @@ -46,7 +46,10 @@ extern int qdevice_model_net_init(struct qdevice_instance *instance); extern int qdevice_model_net_destroy(struct qdevice_instance *instance); -extern int qdevice_model_net_run(struct qdevice_instance *instance); +extern int qdevice_model_net_pre_poll_loop(struct qdevice_instance *instance); + +extern int qdevice_model_net_post_poll_loop(struct qdevice_instance *instance, + enum qdevice_model_post_poll_loop_exit_reason exit_reason); extern int qdevice_model_net_get_config_node_list_failed(struct qdevice_instance *instance); diff --git a/qdevices/qdevice-model.c b/qdevices/qdevice-model.c index a4167f0..dc1e5ee 100644 --- a/qdevices/qdevice-model.c +++ b/qdevices/qdevice-model.c @@ -65,16 +65,30 @@ qdevice_model_destroy(struct qdevice_instance *instance) } int -qdevice_model_run(struct qdevice_instance *instance) +qdevice_model_pre_poll_loop(struct qdevice_instance *instance) { if (instance->model_type >= QDEVICE_MODEL_TYPE_ARRAY_SIZE || qdevice_model_array[instance->model_type] == NULL) { - log(LOG_CRIT, "qdevice_model_run unhandled model"); + log(LOG_CRIT, "qdevice_model_pre_poll_loop unhandled model"); exit(EXIT_FAILURE); } - return (qdevice_model_array[instance->model_type]->run(instance)); + return (qdevice_model_array[instance->model_type]->pre_poll_loop(instance)); +} + +int +qdevice_model_post_poll_loop(struct qdevice_instance *instance, + enum qdevice_model_post_poll_loop_exit_reason exit_reason) +{ + + if (instance->model_type >= QDEVICE_MODEL_TYPE_ARRAY_SIZE || + qdevice_model_array[instance->model_type] == NULL) { + log(LOG_CRIT, "qdevice_model_post_poll_loop unhandled model"); + exit(EXIT_FAILURE); + } + + return (qdevice_model_array[instance->model_type]->post_poll_loop(instance, exit_reason)); } int @@ -83,7 +97,7 @@ qdevice_model_get_config_node_list_failed(struct qdevice_instance *instance) if (instance->model_type >= QDEVICE_MODEL_TYPE_ARRAY_SIZE || qdevice_model_array[instance->model_type] == NULL) { - log(LOG_CRIT, "qdevice_model_run unhandled model"); + log(LOG_CRIT, "qdevice_model_get_config_node_list_failed unhandled model"); exit(EXIT_FAILURE); } @@ -97,7 +111,7 @@ qdevice_model_config_node_list_changed(struct qdevice_instance *instance, if (instance->model_type >= QDEVICE_MODEL_TYPE_ARRAY_SIZE || qdevice_model_array[instance->model_type] == NULL) { - log(LOG_CRIT, "qdevice_model_run unhandled model"); + log(LOG_CRIT, "qdevice_model_config_node_list_changed unhandled model"); exit(EXIT_FAILURE); } diff --git a/qdevices/qdevice-model.h b/qdevices/qdevice-model.h index 6afa5f9..4323f0a 100644 --- a/qdevices/qdevice-model.h +++ b/qdevices/qdevice-model.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017 Red Hat, Inc. + * Copyright (c) 2015-2020 Red Hat, Inc. * * All rights reserved. * @@ -44,11 +44,33 @@ extern "C" { #endif +enum qdevice_model_post_poll_loop_exit_reason { + /* Some of model callbacks made poll loop exit */ + QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_MODEL, + /* Votequorum connection closed */ + QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_VOTEQUORUM_CLOSED, + /* Cmap connection closed */ + QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_CMAP_CLOSED, + /* One of heuristics pipes closed */ + QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_HEURISTICS_CLOSED, + /* IPC socket closed (user initiated shutdown) */ + QDEVICE_MODEL_POST_POLL_LOOP_EXIT_REASON_IPC_SOCKET_CLOSED +}; + extern int qdevice_model_init(struct qdevice_instance *instance); extern int qdevice_model_destroy(struct qdevice_instance *instance); -extern int qdevice_model_run(struct qdevice_instance *instance); +/* + * Return codes: 0 - Continue, -1 - End loop + */ +extern int qdevice_model_pre_poll_loop(struct qdevice_instance *instance); + +/* + * Return codes: 1 - Restart loop, 0 - End loop with no error, -1 - End loop with error -1 + */ +extern int qdevice_model_post_poll_loop(struct qdevice_instance *instance, + enum qdevice_model_post_poll_loop_exit_reason exit_reason); extern int qdevice_model_get_config_node_list_failed(struct qdevice_instance *instance); @@ -78,7 +100,9 @@ struct qdevice_model { const char *name; int (*init)(struct qdevice_instance *instance); int (*destroy)(struct qdevice_instance *instance); - int (*run)(struct qdevice_instance *instance); + int (*pre_poll_loop)(struct qdevice_instance *instance); + int (*post_poll_loop)(struct qdevice_instance *instance, + enum qdevice_model_post_poll_loop_exit_reason exit_reason); int (*get_config_node_list_failed)(struct qdevice_instance *instance); int (*config_node_list_changed)(struct qdevice_instance *instance, const struct node_list *nlist, int config_version_set, uint64_t config_version); diff --git a/qdevices/qdevice-net-cast-vote-timer.c b/qdevices/qdevice-net-cast-vote-timer.c index 33004db..7b1c998 100644 --- a/qdevices/qdevice-net-cast-vote-timer.c +++ b/qdevices/qdevice-net-cast-vote-timer.c @@ -136,7 +136,8 @@ qdevice_net_cast_vote_timer_update(struct qdevice_net_instance *instance, enum t if (timer_needs_running) { if (instance->cast_vote_timer == NULL) { - instance->cast_vote_timer = timer_list_add(&instance->main_timer_list, + instance->cast_vote_timer = timer_list_add( + pr_poll_loop_get_timer_list(&instance->qdevice_instance_ptr->main_poll_loop), instance->cast_vote_timer_interval, qdevice_net_cast_vote_timer_callback, (void *)instance, NULL); @@ -161,7 +162,9 @@ qdevice_net_cast_vote_timer_update(struct qdevice_net_instance *instance, enum t } } else { if (instance->cast_vote_timer != NULL) { - timer_list_delete(&instance->main_timer_list, instance->cast_vote_timer); + timer_list_delete( + pr_poll_loop_get_timer_list(&instance->qdevice_instance_ptr->main_poll_loop), + instance->cast_vote_timer); instance->cast_vote_timer = NULL; log(LOG_DEBUG, "Cast vote timer is now stopped."); } else { diff --git a/qdevices/qdevice-net-disconnect-reason.h b/qdevices/qdevice-net-disconnect-reason.h index 8c86b64..321c55f 100644 --- a/qdevices/qdevice-net-disconnect-reason.h +++ b/qdevices/qdevice-net-disconnect-reason.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017 Red Hat, Inc. + * Copyright (c) 2015-2020 Red Hat, Inc. * * All rights reserved. * @@ -121,7 +121,6 @@ enum qdevice_net_disconnect_reason { QDEVICE_NET_DISCONNECT_REASON_ALGO_HEURISTICS_CHANGE_REPLY_ERR, QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_WORKER_CLOSED, - QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_CANT_SEND_RECEIVE_MSG, }; #define qdevice_net_disconnect_reason_try_reconnect(reason) ( \ @@ -142,8 +141,7 @@ enum qdevice_net_disconnect_reason { reason == QDEVICE_NET_DISCONNECT_REASON_CANT_SCHEDULE_VOTING_TIMER || \ reason == QDEVICE_NET_DISCONNECT_REASON_CANT_REGISTER_VOTEQUORUM_CALLBACK || \ reason == QDEVICE_NET_DISCONNECT_REASON_CANT_REGISTER_CMAP_CALLBACK || \ - reason == QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_WORKER_CLOSED || \ - reason == QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_CANT_SEND_RECEIVE_MSG) + reason == QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_WORKER_CLOSED) #ifdef __cplusplus } diff --git a/qdevices/qdevice-net-echo-request-timer.c b/qdevices/qdevice-net-echo-request-timer.c index 7b3f95c..0c0c547 100644 --- a/qdevices/qdevice-net-echo-request-timer.c +++ b/qdevices/qdevice-net-echo-request-timer.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2019 Red Hat, Inc. + * Copyright (c) 2015-2020 Red Hat, Inc. * * All rights reserved. * @@ -85,12 +85,15 @@ qdevice_net_echo_request_timer_schedule(struct qdevice_net_instance *instance) instance->echo_reply_received_msg_seq_num = 0; if (instance->echo_request_timer != NULL) { - timer_list_delete(&instance->main_timer_list, instance->echo_request_timer); + timer_list_delete( + pr_poll_loop_get_timer_list(&instance->qdevice_instance_ptr->main_poll_loop), + instance->echo_request_timer); instance->echo_request_timer = NULL; } log(LOG_DEBUG, "Scheduling send of heartbeat every %"PRIu32"ms", instance->heartbeat_interval); - instance->echo_request_timer = timer_list_add(&instance->main_timer_list, + instance->echo_request_timer = timer_list_add( + pr_poll_loop_get_timer_list(&instance->qdevice_instance_ptr->main_poll_loop), instance->heartbeat_interval, qdevice_net_echo_request_timer_callback, (void *)instance, NULL); diff --git a/qdevices/qdevice-net-heuristics.c b/qdevices/qdevice-net-heuristics.c index 050821e..91f9ea5 100644 --- a/qdevices/qdevice-net-heuristics.c +++ b/qdevices/qdevice-net-heuristics.c @@ -350,7 +350,8 @@ qdevice_net_heuristics_stop_timer(struct qdevice_net_instance *net_instance) if (net_instance->regular_heuristics_timer != NULL) { log(LOG_DEBUG, "Regular heuristics timer stopped"); - timer_list_delete(&net_instance->main_timer_list, net_instance->regular_heuristics_timer); + timer_list_delete(pr_poll_loop_get_timer_list(&instance->main_poll_loop), + net_instance->regular_heuristics_timer); net_instance->regular_heuristics_timer = NULL; if (qdevice_heuristics_result_notifier_list_set_active(&heuristics_instance->exec_result_notifier_list, @@ -397,7 +398,8 @@ qdevice_net_heuristics_schedule_timer(struct qdevice_net_instance *net_instance) log(LOG_DEBUG, "Scheduling next regular heuristics in %"PRIu32"ms", interval); - net_instance->regular_heuristics_timer = timer_list_add(&net_instance->main_timer_list, + net_instance->regular_heuristics_timer = timer_list_add( + pr_poll_loop_get_timer_list(&instance->main_poll_loop), interval, qdevice_net_heuristics_timer_callback, (void *)net_instance, NULL); diff --git a/qdevices/qdevice-net-instance.c b/qdevices/qdevice-net-instance.c index cd5818e..53338db 100644 --- a/qdevices/qdevice-net-instance.c +++ b/qdevices/qdevice-net-instance.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2019 Red Hat, Inc. + * Copyright (c) 2015-2020 Red Hat, Inc. * * All rights reserved. * @@ -39,14 +39,8 @@ #include "qdevice-net-instance.h" #include "qnet-config.h" #include "utils.h" -#include "qdevice-net-poll-array-user-data.h" #include "qdevice-ipc.h" -/* - * Needed for creating nspr handle from unix fd - */ -#include - int qdevice_net_instance_init(struct qdevice_net_instance *instance, enum tlv_tls_supported tls_supported, @@ -86,45 +80,8 @@ qdevice_net_instance_init(struct qdevice_net_instance *instance, send_buffer_list_init(&instance->send_buffer_list, advanced_settings->net_max_send_buffers, advanced_settings->net_initial_msg_send_size); - timer_list_init(&instance->main_timer_list); - - pr_poll_array_init(&instance->poll_array, sizeof(struct qdevice_net_poll_array_user_data)); - instance->tls_supported = tls_supported; - if ((instance->cmap_poll_fd = PR_CreateSocketPollFd(cmap_fd)) == NULL) { - log_nss(LOG_CRIT, "Can't create NSPR cmap poll fd"); - return (-1); - } - - if ((instance->votequorum_poll_fd = PR_CreateSocketPollFd(votequorum_fd)) == NULL) { - log_nss(LOG_CRIT, "Can't create NSPR votequorum poll fd"); - return (-1); - } - - if ((instance->ipc_socket_poll_fd = PR_CreateSocketPollFd(local_socket_fd)) == NULL) { - log_nss(LOG_CRIT, "Can't create NSPR IPC socket poll fd"); - return (-1); - } - - if ((instance->heuristics_pipe_cmd_send_poll_fd = - PR_CreateSocketPollFd(heuristics_pipe_cmd_send_fd)) == NULL) { - log_nss(LOG_CRIT, "Can't create NSPR heuristics pipe command send poll fd"); - return (-1); - } - - if ((instance->heuristics_pipe_cmd_recv_poll_fd = - PR_CreateSocketPollFd(heuristics_pipe_cmd_recv_fd)) == NULL) { - log_nss(LOG_CRIT, "Can't create NSPR heuristics pipe command recv poll fd"); - return (-1); - } - - if ((instance->heuristics_pipe_log_recv_poll_fd = - PR_CreateSocketPollFd(heuristics_pipe_log_recv_fd)) == NULL) { - log_nss(LOG_CRIT, "Can't create NSPR heuristics pipe log recv poll fd"); - return (-1); - } - return (0); } @@ -152,60 +109,14 @@ qdevice_net_instance_clean(struct qdevice_net_instance *instance) int qdevice_net_instance_destroy(struct qdevice_net_instance *instance) { - struct unix_socket_client *ipc_client; - const struct unix_socket_client_list *ipc_client_list; - struct qdevice_ipc_user_data *qdevice_ipc_user_data; - PRFileDesc *prfd; - - ipc_client_list = &instance->qdevice_instance_ptr->local_ipc.clients; - - TAILQ_FOREACH(ipc_client, ipc_client_list, entries) { - qdevice_ipc_user_data = (struct qdevice_ipc_user_data *)ipc_client->user_data; - prfd = (PRFileDesc *)qdevice_ipc_user_data->model_data; - - if (PR_DestroySocketPollFd(prfd) != PR_SUCCESS) { - log_nss(LOG_WARNING, "Unable to destroy client IPC poll socket fd"); - } - } dynar_destroy(&instance->receive_buffer); send_buffer_list_free(&instance->send_buffer_list); - pr_poll_array_destroy(&instance->poll_array); - - timer_list_free(&instance->main_timer_list); - free((void *)instance->cluster_name); free((void *)instance->host_addr); - if (PR_DestroySocketPollFd(instance->votequorum_poll_fd) != PR_SUCCESS) { - log_nss(LOG_WARNING, "Unable to close votequorum connection fd"); - } - - if (PR_DestroySocketPollFd(instance->cmap_poll_fd) != PR_SUCCESS) { - log_nss(LOG_WARNING, "Unable to close votequorum connection fd"); - } - - if (PR_DestroySocketPollFd(instance->ipc_socket_poll_fd) != PR_SUCCESS) { - log_nss(LOG_WARNING, "Unable to close local socket poll fd"); - } - - if (PR_DestroySocketPollFd(instance->heuristics_pipe_cmd_send_poll_fd) != PR_SUCCESS) { - log_nss(LOG_WARNING, "Unable to close heuristics pipe command send poll fd"); - return (-1); - } - - if (PR_DestroySocketPollFd(instance->heuristics_pipe_cmd_recv_poll_fd) != PR_SUCCESS) { - log_nss(LOG_WARNING, "Unable to close heuristics pipe command recv poll fd"); - return (-1); - } - - if (PR_DestroySocketPollFd(instance->heuristics_pipe_log_recv_poll_fd) != PR_SUCCESS) { - log_nss(LOG_WARNING, "Unable to close heuristics pipe log recv poll fd"); - return (-1); - } - return (0); } diff --git a/qdevices/qdevice-net-instance.h b/qdevices/qdevice-net-instance.h index 0b2c40c..303570d 100644 --- a/qdevices/qdevice-net-instance.h +++ b/qdevices/qdevice-net-instance.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017 Red Hat, Inc. + * Copyright (c) 2015-2020 Red Hat, Inc. * * All rights reserved. * @@ -46,7 +46,6 @@ #include "dynar.h" #include "node-list.h" -#include "pr-poll-array.h" #include "qdevice-net-disconnect-reason.h" #include "send-buffer-list.h" #include "tlv.h" @@ -89,12 +88,8 @@ struct qdevice_net_instance { uint16_t host_port; const char *cluster_name; enum tlv_decision_algorithm_type decision_algorithm; - struct timer_list main_timer_list; struct timer_list_entry *echo_request_timer; int schedule_disconnect; - PRFileDesc *votequorum_poll_fd; - PRFileDesc *cmap_poll_fd; - PRFileDesc *ipc_socket_poll_fd; struct tlv_ring_id last_sent_ring_id; struct tlv_tie_breaker tie_breaker; void *algorithm_data; @@ -103,13 +98,9 @@ struct qdevice_net_instance { struct nss_sock_non_blocking_client non_blocking_client; struct timer_list_entry *connect_timer; int force_ip_version; - struct pr_poll_array poll_array; time_t last_echo_reply_received_time; time_t connected_since_time; const struct qdevice_advanced_settings *advanced_settings; - PRFileDesc *heuristics_pipe_cmd_send_poll_fd; - PRFileDesc *heuristics_pipe_cmd_recv_poll_fd; - PRFileDesc *heuristics_pipe_log_recv_poll_fd; struct timer_list_entry *regular_heuristics_timer; int server_supports_heuristics; enum tlv_heuristics latest_regular_heuristics_result; diff --git a/qdevices/qdevice-net-msg-received.c b/qdevices/qdevice-net-msg-received.c index 4d923e1..ce0ceaf 100644 --- a/qdevices/qdevice-net-msg-received.c +++ b/qdevices/qdevice-net-msg-received.c @@ -377,7 +377,9 @@ qdevice_net_msg_received_init_reply(struct qdevice_net_instance *instance, * Finally fully connected so it's possible to remove connection timer */ if (instance->connect_timer != NULL) { - timer_list_delete(&instance->main_timer_list, instance->connect_timer); + timer_list_delete( + pr_poll_loop_get_timer_list(&instance->qdevice_instance_ptr->main_poll_loop), + instance->connect_timer); instance->connect_timer = NULL; } diff --git a/qdevices/qdevice-net-poll-array-user-data.h b/qdevices/qdevice-net-poll-array-user-data.h deleted file mode 100644 index b696690..0000000 --- a/qdevices/qdevice-net-poll-array-user-data.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2015-2017 Red Hat, Inc. - * - * All rights reserved. - * - * Author: Jan Friesse (jfriesse@redhat.com) - * - * This software licensed under BSD license, the text of which follows: - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - Neither the name of the Red Hat, Inc. nor the names of its - * contributors may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _QDEVICE_NET_POLL_ARRAY_USER_DATA_H_ -#define _QDEVICE_NET_POLL_ARRAY_USER_DATA_H_ - -#include "unix-socket-client.h" - -#ifdef __cplusplus -extern "C" { -#endif - -enum qdevice_net_poll_array_user_data_type { - QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_VOTEQUORUM, - QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_CMAP, - QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET, - QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_SOCKET, - QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT, - QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_SEND, - QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_RECV, - QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_LOG_RECV, -}; - -struct qdevice_net_poll_array_user_data { - enum qdevice_net_poll_array_user_data_type type; - struct unix_socket_client *ipc_client; -}; - -#ifdef __cplusplus -} -#endif - -#endif /* _QDEVICE_NET_POLL_ARRAY_USER_DATA_H_ */ diff --git a/qdevices/qdevice-net-poll.c b/qdevices/qdevice-net-poll.c deleted file mode 100644 index 893c4bf..0000000 --- a/qdevices/qdevice-net-poll.c +++ /dev/null @@ -1,546 +0,0 @@ -/* - * Copyright (c) 2015-2020 Red Hat, Inc. - * - * All rights reserved. - * - * Author: Jan Friesse (jfriesse@redhat.com) - * - * This software licensed under BSD license, the text of which follows: - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - Neither the name of the Red Hat, Inc. nor the names of its - * contributors may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "log.h" -#include "qdevice-cmap.h" -#include "qdevice-net-poll.h" -#include "qdevice-net-send.h" -#include "qdevice-net-socket.h" -#include "qdevice-votequorum.h" -#include "qdevice-ipc.h" -#include "qdevice-net-poll-array-user-data.h" -#include "qdevice-heuristics.h" -#include "qdevice-heuristics-cmd.h" - -/* - * Needed for creating nspr handle from unix fd - */ -#include - -static void -qdevice_net_poll_read_socket(struct qdevice_net_instance *instance) -{ - - if (qdevice_net_socket_read(instance) == -1) { - instance->schedule_disconnect = 1; - } -} - -static void -qdevice_net_poll_read_votequorum(struct qdevice_net_instance *instance) -{ - - if (qdevice_votequorum_dispatch(instance->qdevice_instance_ptr) == -1) { - instance->schedule_disconnect = 1; - instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED; - } -} - -static void -qdevice_net_poll_read_cmap(struct qdevice_net_instance *instance) -{ - - if (qdevice_cmap_dispatch(instance->qdevice_instance_ptr) == -1) { - instance->schedule_disconnect = 1; - instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED; - } -} - -static void -qdevice_net_poll_write_socket(struct qdevice_net_instance *instance, const PRPollDesc *pfd) -{ - int res; - - if (instance->state == QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT) { - res = nss_sock_non_blocking_client_succeeded(pfd); - if (res == -1) { - /* - * Connect failed -> try next - */ - res = nss_sock_non_blocking_client_try_next(&instance->non_blocking_client); - if (res == -1) { - log_nss(LOG_ERR, "Can't connect to qnetd host."); - nss_sock_non_blocking_client_destroy(&instance->non_blocking_client); - } - } else if (res == 0) { - /* - * Poll again - */ - } else if (res == 1) { - /* - * Connect success - */ - instance->socket = instance->non_blocking_client.socket; - nss_sock_non_blocking_client_destroy(&instance->non_blocking_client); - instance->non_blocking_client.socket = NULL; - - instance->state = QDEVICE_NET_INSTANCE_STATE_SENDING_PREINIT_REPLY; - - log(LOG_DEBUG, "Sending preinit msg to qnetd"); - if (qdevice_net_send_preinit(instance) != 0) { - instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER; - instance->schedule_disconnect = 1; - } - } else { - log(LOG_CRIT, "Unhandled nss_sock_non_blocking_client_succeeded"); - exit(EXIT_FAILURE); - } - } else { - if (qdevice_net_socket_write(instance) == -1) { - instance->schedule_disconnect = 1; - } - } -} - -static void -qdevice_net_poll_err_socket(struct qdevice_net_instance *instance, const PRPollDesc *pfd) -{ - - if (instance->state == QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT) { - /* - * Workaround for RHEL<7. Pollout is never set for nonblocking connect (doesn't work - * only with poll, select works as expected!???). - * So test if client is still valid and if pollout was not already called (ensured - * by default because of order in PR_Poll). - * If both applies it's possible to emulate pollout set by calling poll_write. - */ - if (!instance->non_blocking_client.destroyed) { - qdevice_net_poll_write_socket(instance, pfd); - } - } else { - log(LOG_ERR, "POLL_ERR (%u) on main socket", pfd->out_flags); - - instance->schedule_disconnect = 1; - instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_SERVER_CLOSED_CONNECTION; - } -} - -static void -qdevice_net_poll_read_heuristics_log(struct qdevice_net_instance *instance) -{ - int res; - - res = qdevice_heuristics_log_read_from_pipe(&instance->qdevice_instance_ptr->heuristics_instance); - if (res == -1) { - instance->schedule_disconnect = 1; - instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_WORKER_CLOSED; - } -} - -static void -qdevice_net_poll_read_heuristics_cmd(struct qdevice_net_instance *instance) -{ - int res; - - res = qdevice_heuristics_cmd_read_from_pipe(&instance->qdevice_instance_ptr->heuristics_instance); - if (res == -1) { - instance->schedule_disconnect = 1; - instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_WORKER_CLOSED; - } -} - -static void -qdevice_net_poll_write_heuristics_cmd(struct qdevice_net_instance *instance) -{ - int res; - - res = qdevice_heuristics_cmd_write(&instance->qdevice_instance_ptr->heuristics_instance); - if (res == -1) { - instance->schedule_disconnect = 1; - instance->disconnect_reason = - QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_CANT_SEND_RECEIVE_MSG; - } -} - -static void -qdevice_net_poll_read_ipc_socket(struct qdevice_net_instance *instance) -{ - struct unix_socket_client *client; - PRFileDesc *prfd; - struct qdevice_ipc_user_data *user_data; - - if (qdevice_ipc_accept(instance->qdevice_instance_ptr, &client) != 0) { - return ; - } - - prfd = PR_CreateSocketPollFd(client->socket); - if (prfd == NULL) { - log_nss(LOG_CRIT, "Can't create NSPR poll fd for IPC client. " - "Disconnecting client"); - qdevice_ipc_client_disconnect(instance->qdevice_instance_ptr, client); - - return ; - } - - user_data = (struct qdevice_ipc_user_data *)client->user_data; - user_data->model_data = (void *)prfd; -} - -static PRPollDesc * -qdevice_net_pr_poll_array_create(struct qdevice_net_instance *instance) -{ - struct pr_poll_array *poll_array; - PRPollDesc *poll_desc; - struct qdevice_net_poll_array_user_data *user_data; - struct unix_socket_client *ipc_client; - const struct unix_socket_client_list *ipc_client_list; - struct qdevice_ipc_user_data *qdevice_ipc_user_data; - - poll_array = &instance->poll_array; - ipc_client_list = &instance->qdevice_instance_ptr->local_ipc.clients; - - if (qdevice_ipc_is_closed(instance->qdevice_instance_ptr)) { - log(LOG_DEBUG, "Local socket is closed"); - instance->schedule_disconnect = 1; - instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_LOCAL_SOCKET_CLOSED; - - return (NULL); - } - - pr_poll_array_clean(poll_array); - - if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) { - return (NULL); - } - poll_desc->fd = instance->votequorum_poll_fd; - poll_desc->in_flags = PR_POLL_READ; - user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_VOTEQUORUM; - - if (!instance->qdevice_instance_ptr->sync_in_progress) { - if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) { - return (NULL); - } - poll_desc->fd = instance->cmap_poll_fd; - poll_desc->in_flags = PR_POLL_READ; - user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_CMAP; - } - - if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) { - return (NULL); - } - poll_desc->fd = instance->ipc_socket_poll_fd; - poll_desc->in_flags = PR_POLL_READ; - user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET; - - if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) { - return (NULL); - } - - poll_desc->fd = instance->heuristics_pipe_log_recv_poll_fd; - poll_desc->in_flags = PR_POLL_READ; - user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_LOG_RECV; - - if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) { - return (NULL); - } - - poll_desc->fd = instance->heuristics_pipe_cmd_recv_poll_fd; - poll_desc->in_flags = PR_POLL_READ; - user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_RECV; - - if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) { - return (NULL); - } - - if (!send_buffer_list_empty( - &instance->qdevice_instance_ptr->heuristics_instance.cmd_out_buffer_list)) { - poll_desc->fd = instance->heuristics_pipe_cmd_send_poll_fd; - poll_desc->in_flags = PR_POLL_WRITE; - user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_SEND; - - if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) { - return (NULL); - } - } - - if (instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT || - !instance->non_blocking_client.destroyed) { - if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) { - return (NULL); - } - - user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_SOCKET; - - if (instance->state == QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT) { - poll_desc->fd = instance->non_blocking_client.socket; - poll_desc->in_flags = PR_POLL_WRITE | PR_POLL_EXCEPT; - } else { - poll_desc->fd = instance->socket; - poll_desc->in_flags = PR_POLL_READ; - - if (!send_buffer_list_empty(&instance->send_buffer_list)) { - poll_desc->in_flags |= PR_POLL_WRITE; - } - } - } - - TAILQ_FOREACH(ipc_client, ipc_client_list, entries) { - if (!ipc_client->reading_line && !ipc_client->writing_buffer) { - continue; - } - - if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) { - return (NULL); - } - - qdevice_ipc_user_data = (struct qdevice_ipc_user_data *)ipc_client->user_data; - poll_desc->fd = (PRFileDesc *)qdevice_ipc_user_data->model_data; - if (ipc_client->reading_line) { - poll_desc->in_flags |= PR_POLL_READ; - } - - if (ipc_client->writing_buffer) { - poll_desc->in_flags |= PR_POLL_WRITE; - } - - user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT; - user_data->ipc_client = ipc_client; - } - - pr_poll_array_gc(poll_array); - - return (poll_array->array); -} - -int -qdevice_net_poll(struct qdevice_net_instance *instance) -{ - PRPollDesc *pfds; - PRFileDesc *prfd; - PRInt32 poll_res; - ssize_t i; - struct qdevice_net_poll_array_user_data *user_data; - struct unix_socket_client *ipc_client; - struct qdevice_ipc_user_data *qdevice_ipc_user_data; - int case_processed; - - pfds = qdevice_net_pr_poll_array_create(instance); - if (pfds == NULL) { - return (-1); - } - - instance->schedule_disconnect = 0; - - if ((poll_res = PR_Poll(pfds, pr_poll_array_size(&instance->poll_array), - timer_list_time_to_expire(&instance->main_timer_list))) > 0) { - for (i = 0; i < pr_poll_array_size(&instance->poll_array); i++) { - user_data = pr_poll_array_get_user_data(&instance->poll_array, i); - - ipc_client = user_data->ipc_client; - - if (pfds[i].out_flags & PR_POLL_READ) { - case_processed = 0; - - switch (user_data->type) { - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_SOCKET: - case_processed = 1; - qdevice_net_poll_read_socket(instance); - break; - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_VOTEQUORUM: - case_processed = 1; - qdevice_net_poll_read_votequorum(instance); - break; - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_CMAP: - case_processed = 1; - qdevice_net_poll_read_cmap(instance); - break; - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET: - case_processed = 1; - qdevice_net_poll_read_ipc_socket(instance); - break; - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT: - case_processed = 1; - qdevice_ipc_io_read(instance->qdevice_instance_ptr, ipc_client); - break; - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_SEND: - /* - * Read on heuristics cmd send fd shouldn't happen - */ - break; - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_RECV: - case_processed = 1; - qdevice_net_poll_read_heuristics_cmd(instance); - break; - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_LOG_RECV: - case_processed = 1; - qdevice_net_poll_read_heuristics_log(instance); - break; - /* - * Default is not defined intentionally. Compiler shows warning when - * new poll_array_user_data_type is added - */ - } - - if (!case_processed) { - log(LOG_CRIT, "Unhandled read on poll descriptor %zu", i); - exit(EXIT_FAILURE); - } - } - - if (!instance->schedule_disconnect && pfds[i].out_flags & PR_POLL_WRITE) { - case_processed = 0; - - switch (user_data->type) { - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_SOCKET: - case_processed = 1; - qdevice_net_poll_write_socket(instance, &pfds[i]); - break; - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT: - case_processed = 1; - qdevice_ipc_io_write(instance->qdevice_instance_ptr, ipc_client); - break; - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_SEND: - case_processed = 1; - qdevice_net_poll_write_heuristics_cmd(instance); - break; - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_VOTEQUORUM: - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_CMAP: - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET: - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_RECV: - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_LOG_RECV: - /* - * Write on votequorum, cmap, ipc socket and - * heuristics log shouldn't happen. - */ - break; - /* - * Default is not defined intentionally. Compiler shows warning when - * new poll_array_user_data_type is added - */ - } - - if (!case_processed) { - log(LOG_CRIT, "Unhandled write on poll descriptor %zu", i); - exit(EXIT_FAILURE); - } - } - - if (!instance->schedule_disconnect && - (pfds[i].out_flags & (PR_POLL_ERR|PR_POLL_NVAL|PR_POLL_HUP|PR_POLL_EXCEPT)) && - !(pfds[i].out_flags & (PR_POLL_READ|PR_POLL_WRITE))) { - case_processed = 0; - - switch (user_data->type) { - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_SOCKET: - case_processed = 1; - qdevice_net_poll_err_socket(instance, &pfds[i]); - break; - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET: - case_processed = 1; - if (pfds[i].out_flags != PR_POLL_NVAL) { - log(LOG_CRIT, "POLLERR (%u) on local socket", - pfds[i].out_flags); - exit(EXIT_FAILURE); - } else { - log(LOG_DEBUG, "Local socket is closed"); - instance->schedule_disconnect = 1; - instance->disconnect_reason = - QDEVICE_NET_DISCONNECT_REASON_LOCAL_SOCKET_CLOSED; - } - break; - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT: - case_processed = 1; - log(LOG_DEBUG, "POLL_ERR (%u) on ipc client socket. " - "Disconnecting.", pfds[i].out_flags); - ipc_client->schedule_disconnect = 1; - break; - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_VOTEQUORUM: - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_CMAP: - case_processed = 1; - log(LOG_DEBUG, "POLL_ERR (%u) on corosync socket. " - "Disconnecting.", pfds[i].out_flags); - - instance->schedule_disconnect = 1; - instance->disconnect_reason = - QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED; - break; - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_LOG_RECV: - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_RECV: - case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_HEURISTICS_CMD_SEND: - case_processed = 1; - - /* - * Closed pipe doesn't mean return of PR_POLL_READ. To display - * better log message, we call read log as if POLL_READ would - * be set. - */ - qdevice_net_poll_read_heuristics_log(instance); - - log(LOG_DEBUG, "POLL_ERR (%u) on heuristics pipe. " - "Disconnecting.", pfds[i].out_flags); - - instance->schedule_disconnect = 1; - instance->disconnect_reason = - QDEVICE_NET_DISCONNECT_REASON_HEURISTICS_WORKER_CLOSED; - break; - /* - * Default is not defined intentionally. Compiler shows warning when - * new poll_array_user_data_type is added - */ - } - - if (!case_processed) { - log(LOG_CRIT, "Unhandled error on poll descriptor %zu", i); - exit(EXIT_FAILURE); - } - } - - if (user_data->type == QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT && - ipc_client->schedule_disconnect) { - qdevice_ipc_user_data = (struct qdevice_ipc_user_data *)ipc_client->user_data; - prfd = (PRFileDesc *)qdevice_ipc_user_data->model_data; - - if (PR_DestroySocketPollFd(prfd) != PR_SUCCESS) { - log_nss(LOG_WARNING, "Unable to destroy client IPC poll socket fd"); - } - - qdevice_ipc_client_disconnect(instance->qdevice_instance_ptr, ipc_client); - } - } - } - - if (!instance->schedule_disconnect) { - timer_list_expire(&instance->main_timer_list); - } - - if (instance->schedule_disconnect) { - /* - * Schedule disconnect can be set by this function, by some timer_list callback - * or cmap/votequorum callbacks - */ - return (-1); - } - - return (0); -} diff --git a/qdevices/qdevice-net-poll.h b/qdevices/qdevice-net-poll.h deleted file mode 100644 index fe7732f..0000000 --- a/qdevices/qdevice-net-poll.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2015-2016 Red Hat, Inc. - * - * All rights reserved. - * - * Author: Jan Friesse (jfriesse@redhat.com) - * - * This software licensed under BSD license, the text of which follows: - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - Neither the name of the Red Hat, Inc. nor the names of its - * contributors may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _QDEVICE_NET_POLL_H_ -#define _QDEVICE_NET_POLL_H_ - - -#include "qdevice-net-instance.h" - -extern int qdevice_net_poll(struct qdevice_net_instance *instance); - -#ifdef __cplusplus -} -#endif - -#endif /* _QDEVICE_NET_POLL_H_ */ diff --git a/qdevices/qdevice-net-socket.c b/qdevices/qdevice-net-socket.c index afd851c..1b3d31e 100644 --- a/qdevices/qdevice-net-socket.c +++ b/qdevices/qdevice-net-socket.c @@ -41,6 +41,150 @@ #include "qdevice-net-send.h" #include "qdevice-net-socket.h" +/* + * Socket callbacks + */ +static int +socket_set_events_cb(PRFileDesc *prfd, short *events, void *user_data1, void *user_data2) +{ + struct qdevice_net_instance *instance = (struct qdevice_net_instance *)user_data1; + + if (!send_buffer_list_empty(&instance->send_buffer_list)) { + *events |= POLLOUT; + } + + return (0); +} + +static int +socket_read_cb(PRFileDesc *prfd, const PRPollDesc *pd, void *user_data1, void *user_data2) +{ + struct qdevice_net_instance *instance = (struct qdevice_net_instance *)user_data1; + + if (qdevice_net_socket_read(instance) == -1) { + instance->schedule_disconnect = 1; + + return (-1); + } + + return (0); +} + +static int +socket_write_cb(PRFileDesc *prfd, const PRPollDesc *pd, void *user_data1, void *user_data2) +{ + struct qdevice_net_instance *instance = (struct qdevice_net_instance *)user_data1; + + if (qdevice_net_socket_write(instance) == -1) { + instance->schedule_disconnect = 1; + + return (-1); + } + + return (0); +} + +static int +non_blocking_client_socket_write_cb(PRFileDesc *prfd, const PRPollDesc *pd, void *user_data1, + void *user_data2) +{ + int res; + + struct qdevice_net_instance *instance = (struct qdevice_net_instance *)user_data1; + + res = nss_sock_non_blocking_client_succeeded(pd); + if (res == -1) { + /* + * Connect failed -> remove this fd from main loop and try next + */ + res = qdevice_net_socket_del_from_main_poll_loop(instance); + if (res == -1) { + return (-1); + } + + res = nss_sock_non_blocking_client_try_next(&instance->non_blocking_client); + if (res == -1) { + log_nss(LOG_ERR, "Can't connect to qnetd host."); + nss_sock_non_blocking_client_destroy(&instance->non_blocking_client); + } + + res = qdevice_net_socket_add_to_main_poll_loop(instance); + if (res == -1) { + return (-1); + } + } else if (res == 0) { + /* + * Poll again + */ + } else if (res == 1) { + /* + * Connect success -> delete socket from main loop and add final one + */ + res = qdevice_net_socket_del_from_main_poll_loop(instance); + if (res == -1) { + return (-1); + } + + instance->socket = instance->non_blocking_client.socket; + nss_sock_non_blocking_client_destroy(&instance->non_blocking_client); + instance->non_blocking_client.socket = NULL; + + instance->state = QDEVICE_NET_INSTANCE_STATE_SENDING_PREINIT_REPLY; + + res = qdevice_net_socket_add_to_main_poll_loop(instance); + if (res == -1) { + return (-1); + } + + log(LOG_DEBUG, "Sending preinit msg to qnetd"); + if (qdevice_net_send_preinit(instance) != 0) { + instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER; + return (-1); + } + } else { + log(LOG_CRIT, "Unhandled nss_sock_non_blocking_client_succeeded"); + exit(EXIT_FAILURE); + } + + return (0); +} + +static int +socket_err_cb(PRFileDesc *prfd, short revents, const PRPollDesc *pd, void *user_data1, void *user_data2) +{ + struct qdevice_net_instance *instance = (struct qdevice_net_instance *)user_data1; + + log(LOG_ERR, "POLL_ERR (%u) on main socket", revents); + + instance->schedule_disconnect = 1; + instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_SERVER_CLOSED_CONNECTION; + + return (-1); +} + +static int +non_blocking_client_socket_err_cb(PRFileDesc *prfd, short revents, const PRPollDesc *pd, + void *user_data1, void *user_data2) +{ + struct qdevice_net_instance *instance = (struct qdevice_net_instance *)user_data1; + + /* + * Workaround for RHEL<7. Pollout is never set for nonblocking connect (doesn't work + * only with poll, select works as expected!???). + * So test if client is still valid and if pollout was not already called (ensured + * by default because of order in PR_Poll). + * If both applies it's possible to emulate pollout set by calling poll_write. + */ + if (!instance->non_blocking_client.destroyed) { + return (non_blocking_client_socket_write_cb(prfd, pd, user_data1, user_data2)); + } + + return (0); +} +/* + * Exported functions + */ + /* * -1 means end of connection (EOF) or some other unhandled error. 0 = success */ @@ -211,3 +355,63 @@ qdevice_net_socket_write(struct qdevice_net_instance *instance) return (0); } + +int +qdevice_net_socket_add_to_main_poll_loop(struct qdevice_net_instance *instance) +{ + + if (instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT || + !instance->non_blocking_client.destroyed) { + if (instance->state == QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT) { + if (pr_poll_loop_add_prfd(&instance->qdevice_instance_ptr->main_poll_loop, + instance->non_blocking_client.socket, + POLLOUT|POLLPRI, + NULL, NULL, non_blocking_client_socket_write_cb, + non_blocking_client_socket_err_cb, + instance, NULL) != 0) { + log(LOG_ERR, "Can't add net socket (non_blocking_client) " + "fd to main poll loop"); + + return (-1); + } + } else { + if (pr_poll_loop_add_prfd(&instance->qdevice_instance_ptr->main_poll_loop, + instance->socket, + POLLIN, + socket_set_events_cb, socket_read_cb, socket_write_cb, socket_err_cb, + instance, NULL) != 0) { + log(LOG_ERR, "Can't add net socket fd to main poll loop"); + + return (-1); + } + } + } + + return (0); +} + +int +qdevice_net_socket_del_from_main_poll_loop(struct qdevice_net_instance *instance) +{ + + if (!instance->non_blocking_client.destroyed) { + if (pr_poll_loop_del_prfd(&instance->qdevice_instance_ptr->main_poll_loop, + instance->non_blocking_client.socket) != 0) { + log(LOG_ERR, "Can't remove net socket (non_blocking_client) " + "fd from main poll loop"); + + return (-1); + } + } + + if (instance->socket != NULL) { + if (pr_poll_loop_del_prfd(&instance->qdevice_instance_ptr->main_poll_loop, + instance->socket) != 0) { + log(LOG_ERR, "Can't remove net socket fd from main poll loop"); + + return (-1); + } + } + + return (0); +} diff --git a/qdevices/qdevice-net-socket.h b/qdevices/qdevice-net-socket.h index 045e4a9..58ffaf7 100644 --- a/qdevices/qdevice-net-socket.h +++ b/qdevices/qdevice-net-socket.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2016 Red Hat, Inc. + * Copyright (c) 2015-2020 Red Hat, Inc. * * All rights reserved. * @@ -45,6 +45,10 @@ extern int qdevice_net_socket_read(struct qdevice_net_instance *instance); extern int qdevice_net_socket_write(struct qdevice_net_instance *instance); +extern int qdevice_net_socket_add_to_main_poll_loop(struct qdevice_net_instance *instance); + +extern int qdevice_net_socket_del_from_main_poll_loop(struct qdevice_net_instance *instance); + #ifdef __cplusplus } #endif diff --git a/qdevices/qdevice-pr-poll-loop-cb.c b/qdevices/qdevice-pr-poll-loop-cb.c index fe90518..09d6818 100644 --- a/qdevices/qdevice-pr-poll-loop-cb.c +++ b/qdevices/qdevice-pr-poll-loop-cb.c @@ -33,6 +33,7 @@ */ #include "log.h" +#include "qdevice-cmap.h" #include "qdevice-heuristics-cmd.h" #include "qdevice-heuristics-log.h" #include "qdevice-pr-poll-loop-cb.h" @@ -149,6 +150,49 @@ votequorum_err_cb(int fd, short revents, void *user_data1, void *user_data2) return (-1); } +static int +cmap_set_events_cb(int fd, short *events, void *user_data1, void *user_data2) +{ + struct qdevice_instance *instance = (struct qdevice_instance *)user_data1; + + if (instance->sync_in_progress) { + /* + * During sync cmap is blocked -> don't add fd + */ + return (-1); + } + + return (0); +} + +static int +cmap_read_cb(int fd, void *user_data1, void *user_data2) +{ + struct qdevice_instance *instance = (struct qdevice_instance *)user_data1; + int res; + + res = qdevice_cmap_dispatch(instance); + if (res == -1) { + instance->cmap_closed = 1; + return (-1); + } + + return (0); +} + +static int +cmap_err_cb(int fd, short revents, void *user_data1, void *user_data2) +{ + struct qdevice_instance *instance = (struct qdevice_instance *)user_data1; + + instance->cmap_closed = 1; + + log(LOG_DEBUG, "POLL_ERR (%u) on corosync socket. " + "Disconnecting.", revents); + + return (-1); +} + int qdevice_pr_poll_loop_cb_register(struct qdevice_instance *instance) { @@ -186,5 +230,13 @@ qdevice_pr_poll_loop_cb_register(struct qdevice_instance *instance) return (-1); } + if (pr_poll_loop_add_fd(&instance->main_poll_loop, instance->cmap_poll_fd, + POLLIN, cmap_set_events_cb, cmap_read_cb, NULL, cmap_err_cb, + instance, NULL) != 0) { + log(LOG_ERR, "Can't add votequorum fd to main poll loop"); + + return (-1); + } + return (0); }