/* * Copyright (c) 2015-2020 Red Hat, Inc. * * All rights reserved. * * Author: Jan Friesse (jfriesse@redhat.com) * * This software licensed under BSD license, the text of which follows: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * - Neither the name of the Red Hat, Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include "log.h" #include "qdevice-config.h" #include "qdevice-cmap.h" #include "qdevice-log.h" #include "log-common.h" #include "qdevice-model.h" #include "utils.h" static uint32_t qdevice_cmap_autogenerate_node_id(const char *addr, int clear_node_high_bit) { struct addrinfo *ainfo; struct addrinfo ahints; int ret, i; memset(&ahints, 0, sizeof(ahints)); ahints.ai_socktype = SOCK_DGRAM; ahints.ai_protocol = IPPROTO_UDP; /* * Hardcoded AF_INET because autogenerated nodeid is valid only for ipv4 */ ahints.ai_family = AF_INET; ret = getaddrinfo(addr, NULL, &ahints, &ainfo); if (ret != 0) return (0); if (ainfo->ai_family != AF_INET) { freeaddrinfo(ainfo); return (0); } memcpy(&i, &(((struct sockaddr_in *)((void *)ainfo->ai_addr))->sin_addr), sizeof(struct in_addr)); freeaddrinfo(ainfo); ret = htonl(i); if (clear_node_high_bit) { ret &= 0x7FFFFFFF; } return (ret); } int qdevice_cmap_get_nodelist(cmap_handle_t cmap_handle, struct node_list *list) { cs_error_t cs_err; cmap_iter_handle_t iter_handle; char key_name[CMAP_KEYNAME_MAXLEN + 1]; char ring0_addr_key[CMAP_KEYNAME_MAXLEN + 1]; char tmp_key[CMAP_KEYNAME_MAXLEN + 1]; int res; int ret_value; unsigned int node_pos; uint32_t node_id; uint32_t data_center_id; char *tmp_str; char *addr0_str; int clear_node_high_bit; ret_value = 0; node_list_init(list); /* * Fill clear high node bit */ clear_node_high_bit = 0; if (cmap_get_string(cmap_handle, "totem.clear_node_high_bit", &tmp_str) == CS_OK) { if (strcmp (tmp_str, "yes") == 0) { clear_node_high_bit = 1; } free(tmp_str); } /* * Iterate nodelist */ cs_err = cmap_iter_init(cmap_handle, "nodelist.node.", &iter_handle); if (cs_err != CS_OK) { return (-1); } while ((cs_err = cmap_iter_next(cmap_handle, iter_handle, key_name, NULL, NULL)) == CS_OK) { res = sscanf(key_name, "nodelist.node.%u.%s", &node_pos, tmp_key); if (res != 2) { continue; } if (strcmp(tmp_key, "ring0_addr") != 0 && strcmp(tmp_key, "name") != 0) { continue; } snprintf(tmp_key, CMAP_KEYNAME_MAXLEN, "nodelist.node.%u.nodeid", node_pos); cs_err = cmap_get_uint32(cmap_handle, tmp_key, &node_id); if (cs_err == CS_ERR_NOT_EXIST) { /* * Nodeid doesn't exists -> autogenerate node id */ /* * New corosync supports configuration without ring0_addr but then * nodeid has to be specified (not needed for corosync but qdevice * needs information about all nodes) so fail when it's not possible to * read ring0_addr */ snprintf(ring0_addr_key, CMAP_KEYNAME_MAXLEN, "nodelist.node.%u.ring0_addr", node_pos); if (cmap_get_string(cmap_handle, ring0_addr_key, &addr0_str) != CS_OK) { log(LOG_ERR, "Cant find nodeid or ring0_addr for a node. " "One of them needs to be defined for every node."); return (-1); } node_id = qdevice_cmap_autogenerate_node_id(addr0_str, clear_node_high_bit); free(addr0_str); } else if (cs_err != CS_OK) { ret_value = -1; goto iter_finalize; } snprintf(tmp_key, CMAP_KEYNAME_MAXLEN, "nodelist.node.%u.datacenterid", node_pos); if (cmap_get_uint32(cmap_handle, tmp_key, &data_center_id) != CS_OK) { data_center_id = 0; } if (node_list_find_node_id(list, node_id) != NULL) { /* * Node is already in the list (both ring0_addr and name exists) */ continue ; } if (node_list_add(list, node_id, data_center_id, TLV_NODE_STATE_NOT_SET) == NULL) { ret_value = -1; goto iter_finalize; } } if (node_list_is_empty(list)) { log(LOG_ERR, "No configured nodes found - configuration without node list is not supported"); ret_value = -1; goto iter_finalize; } else { log(LOG_DEBUG, "Configuration node list:"); log_common_debug_dump_node_list(list); } iter_finalize: cmap_iter_finalize(cmap_handle, iter_handle); if (ret_value != 0) { node_list_free(list); } return (ret_value); } int qdevice_cmap_get_config_version(cmap_handle_t cmap_handle, uint64_t *config_version) { int res; if (cmap_get_uint64(cmap_handle, "totem.config_version", config_version) == CS_OK) { res = 0; } else { *config_version = 0; res = -1; } return (res); } int qdevice_cmap_store_config_node_list(struct qdevice_instance *instance) { int res; node_list_free(&instance->config_node_list); if (qdevice_cmap_get_nodelist(instance->cmap_handle, &instance->config_node_list) != 0) { log(LOG_ERR, "Can't get configuration node list."); return (-1); } res = qdevice_cmap_get_config_version(instance->cmap_handle, &instance->config_node_list_version); instance->config_node_list_version_set = (res == 0); return (0); } void qdevice_cmap_init(struct qdevice_instance *instance) { cs_error_t res; int no_retries; no_retries = 0; while ((res = cmap_initialize(&instance->cmap_handle)) == CS_ERR_TRY_AGAIN && no_retries++ < instance->advanced_settings->max_cs_try_again) { (void)poll(NULL, 0, 1000); } if (res != CS_OK) { errx(EXIT_FAILURE, "Failed to initialize the cmap API. Error %s", cs_strerror(res)); } if ((res = cmap_context_set(instance->cmap_handle, (void *)instance)) != CS_OK) { errx(EXIT_FAILURE, "Can't set cmap context. Error %s", cs_strerror(res)); } cmap_fd_get(instance->cmap_handle, &instance->cmap_poll_fd); } static void qdevice_cmap_node_list_event(struct qdevice_instance *instance) { struct node_list nlist; int config_version_set; uint64_t config_version; log(LOG_DEBUG, "Node list configuration possibly changed"); if (qdevice_cmap_get_nodelist(instance->cmap_handle, &nlist) != 0) { log(LOG_ERR, "Can't get configuration node list."); if (qdevice_model_get_config_node_list_failed(instance) != 0) { log(LOG_DEBUG, "qdevice_model_get_config_node_list_failed returned error -> exit"); exit(EXIT_FAILURE); } return ; } config_version_set = (qdevice_cmap_get_config_version(instance->cmap_handle, &config_version) == 0); if (node_list_eq(&instance->config_node_list, &nlist)) { return ; } log(LOG_DEBUG, "Node list changed"); if (config_version_set) { log(LOG_DEBUG, " config_version = "UTILS_PRI_CONFIG_VERSION, config_version); } log_common_debug_dump_node_list(&nlist); if (qdevice_model_config_node_list_changed(instance, &nlist, config_version_set, config_version) != 0) { log(LOG_DEBUG, "qdevice_model_config_node_list_changed returned error -> exit"); exit(EXIT_FAILURE); } node_list_free(&instance->config_node_list); if (node_list_clone(&instance->config_node_list, &nlist) != 0) { log(LOG_ERR, "Can't allocate instance->config_node_list clone"); node_list_free(&nlist); if (qdevice_model_get_config_node_list_failed(instance) != 0) { log(LOG_DEBUG, "qdevice_model_get_config_node_list_failed returned error -> exit"); exit(EXIT_FAILURE); } return ; } instance->config_node_list_version_set = config_version_set; if (config_version_set) { instance->config_node_list_version = config_version; } } static void qdevice_cmap_logging_event(struct qdevice_instance *instance) { log(LOG_DEBUG, "Logging configuration possibly changed"); qdevice_log_configure(instance); } static void qdevice_cmap_heuristics_event(struct qdevice_instance *instance) { log(LOG_DEBUG, "Heuristics configuration possibly changed"); if (qdevice_instance_configure_from_cmap_heuristics(instance) != 0) { log(LOG_DEBUG, "qdevice_instance_configure_from_cmap_heuristics returned error -> exit"); exit(EXIT_FAILURE); } } static void qdevice_cmap_reload_cb(cmap_handle_t cmap_handle, cmap_track_handle_t cmap_track_handle, int32_t event, const char *key_name, struct cmap_notify_value new_value, struct cmap_notify_value old_value, void *user_data) { cs_error_t cs_res; uint8_t reload; struct qdevice_instance *instance; const char *node_list_prefix_str; const char *logging_prefix_str; const char *heuristics_prefix_str; struct qdevice_cmap_change_events events; memset(&events, 0, sizeof(events)); node_list_prefix_str = "nodelist."; logging_prefix_str = "logging."; heuristics_prefix_str = "quorum.device.heuristics."; if (cmap_context_get(cmap_handle, (const void **)&instance) != CS_OK) { log(LOG_ERR, "Fatal error. Can't get cmap context"); exit(EXIT_FAILURE); } /* * Wait for full reload */ if (strcmp(key_name, "config.totemconfig_reload_in_progress") == 0 && new_value.type == CMAP_VALUETYPE_UINT8 && new_value.len == sizeof(reload)) { reload = 1; if (memcmp(new_value.data, &reload, sizeof(reload)) == 0) { /* * Ignore nodelist changes */ instance->cmap_reload_in_progress = 1; return ; } else { instance->cmap_reload_in_progress = 0; events.node_list = 1; events.logging = 1; events.heuristics = 1; } } if (instance->cmap_reload_in_progress) { return ; } if (((cs_res = cmap_get_uint8(cmap_handle, "config.totemconfig_reload_in_progress", &reload)) == CS_OK) && reload == 1) { return ; } if (strncmp(key_name, node_list_prefix_str, strlen(node_list_prefix_str)) == 0) { events.node_list = 1; } if (strncmp(key_name, logging_prefix_str, strlen(logging_prefix_str)) == 0) { events.logging = 1; } if (strncmp(key_name, heuristics_prefix_str, strlen(heuristics_prefix_str)) == 0) { events.heuristics = 1; } if (events.logging) { qdevice_cmap_logging_event(instance); } if (events.node_list) { qdevice_cmap_node_list_event(instance); } if (events.heuristics) { qdevice_cmap_heuristics_event(instance); } /* * Inform model about change */ if (qdevice_model_cmap_changed(instance, &events) != 0) { log(LOG_DEBUG, "qdevice_model_cmap_changed returned error -> exit"); exit(EXIT_FAILURE); } } int qdevice_cmap_add_track(struct qdevice_instance *instance) { cs_error_t res; res = cmap_track_add(instance->cmap_handle, "config.totemconfig_reload_in_progress", CMAP_TRACK_ADD | CMAP_TRACK_MODIFY, qdevice_cmap_reload_cb, NULL, &instance->cmap_reload_track_handle); if (res != CS_OK) { log(LOG_ERR, "Can't initialize cmap totemconfig_reload_in_progress tracking"); return (-1); } res = cmap_track_add(instance->cmap_handle, "nodelist.", CMAP_TRACK_ADD | CMAP_TRACK_DELETE | CMAP_TRACK_MODIFY | CMAP_TRACK_PREFIX, qdevice_cmap_reload_cb, NULL, &instance->cmap_nodelist_track_handle); if (res != CS_OK) { log(LOG_ERR, "Can't initialize cmap nodelist tracking"); return (-1); } res = cmap_track_add(instance->cmap_handle, "logging.", CMAP_TRACK_ADD | CMAP_TRACK_DELETE | CMAP_TRACK_MODIFY | CMAP_TRACK_PREFIX, qdevice_cmap_reload_cb, NULL, &instance->cmap_logging_track_handle); if (res != CS_OK) { log(LOG_ERR, "Can't initialize logging tracking"); return (-1); } res = cmap_track_add(instance->cmap_handle, "quorum.device.heuristics.", CMAP_TRACK_ADD | CMAP_TRACK_DELETE | CMAP_TRACK_MODIFY | CMAP_TRACK_PREFIX, qdevice_cmap_reload_cb, NULL, &instance->cmap_heuristics_track_handle); if (res != CS_OK) { log(LOG_ERR, "Can't initialize logging tracking"); return (-1); } return (0); } int qdevice_cmap_del_track(struct qdevice_instance *instance) { cs_error_t res; res = cmap_track_delete(instance->cmap_handle, instance->cmap_reload_track_handle); if (res != CS_OK) { log(LOG_WARNING, "Can't delete cmap totemconfig_reload_in_progress tracking"); } res = cmap_track_delete(instance->cmap_handle, instance->cmap_nodelist_track_handle); if (res != CS_OK) { log(LOG_WARNING, "Can't delete cmap nodelist tracking"); } res = cmap_track_delete(instance->cmap_handle, instance->cmap_logging_track_handle); if (res != CS_OK) { log(LOG_WARNING, "Can't delete cmap logging tracking"); } res = cmap_track_delete(instance->cmap_handle, instance->cmap_heuristics_track_handle); if (res != CS_OK) { log(LOG_WARNING, "Can't delete cmap heuristics tracking"); } return (0); } void qdevice_cmap_destroy(struct qdevice_instance *instance) { cs_error_t res; res = cmap_finalize(instance->cmap_handle); if (res != CS_OK) { log(LOG_WARNING, "Can't finalize cmap. Error %s", cs_strerror(res)); } } int qdevice_cmap_dispatch(struct qdevice_instance *instance) { cs_error_t res; /* * dispatch can block if corosync is during sync phase */ if (instance->sync_in_progress) { return (0); } res = cmap_dispatch(instance->cmap_handle, CS_DISPATCH_ALL); if (res != CS_OK && res != CS_ERR_TRY_AGAIN) { log(LOG_ERR, "Can't dispatch cmap messages"); return (-1); } return (0); }