diff --git a/exec/coroparse.c b/exec/coroparse.c index 35bc2bf8..54b76def 100644 --- a/exec/coroparse.c +++ b/exec/coroparse.c @@ -128,7 +128,6 @@ struct main_cp_cb_data { struct qb_list_head member_items_head; int node_number; - int ring0_addr_added; }; static int read_config_file_into_icmap( @@ -917,10 +916,6 @@ static int main_config_parser_cb(const char *path, add_as_string = 0; } - if (strcmp(key, "ring0_addr") == 0) { - data->ring0_addr_added = 1; - } - if (add_as_string) { icmap_set_string_r(config_map, key_name, value); add_as_string = 0; @@ -1011,7 +1006,6 @@ static int main_config_parser_cb(const char *path, } if (strcmp(path, "nodelist.node") == 0) { *state = MAIN_CP_CB_DATA_STATE_NODELIST_NODE; - data->ring0_addr_added = 0; } if (strcmp(path, "resources") == 0) { *state = MAIN_CP_CB_DATA_STATE_RESOURCES; @@ -1214,11 +1208,6 @@ static int main_config_parser_cb(const char *path, break; case MAIN_CP_CB_DATA_STATE_NODELIST_NODE: - if (!data->ring0_addr_added) { - *error_string = "No ring0_addr specified for node"; - - return (0); - } data->node_number++; break; case MAIN_CP_CB_DATA_STATE_NORMAL: diff --git a/exec/cpg.c b/exec/cpg.c index 4c97437a..957a2576 100644 --- a/exec/cpg.c +++ b/exec/cpg.c @@ -564,7 +564,7 @@ static void cpg_sync_init ( sizeof (unsigned int)); my_member_list_entries = member_list_entries; - last_sync_ring_id.nodeid = ring_id->rep.nodeid; + last_sync_ring_id.nodeid = ring_id->nodeid; last_sync_ring_id.seq = ring_id->seq; downlist_state = CPG_DOWNLIST_WAITING_FOR_MESSAGES; diff --git a/exec/main.c b/exec/main.c index bbe9e7f5..ba5b6af5 100644 --- a/exec/main.c +++ b/exec/main.c @@ -624,14 +624,14 @@ int main_mcast ( static void corosync_ring_id_create_or_load ( struct memb_ring_id *memb_ring_id, - const struct totem_ip_address *addr) + unsigned int nodeid) { int fd; int res = 0; char filename[PATH_MAX]; - snprintf (filename, sizeof(filename), "%s/ringid_%s", - get_run_dir(), totemip_print (addr)); + snprintf (filename, sizeof(filename), "%s/ringid_%u", + get_run_dir(), nodeid); fd = open (filename, O_RDONLY, 0700); /* * If file can be opened and read, read the ring id @@ -664,20 +664,19 @@ static void corosync_ring_id_create_or_load ( } } - totemip_copy(&memb_ring_id->rep, addr); - assert (!totemip_zero_check(&memb_ring_id->rep)); + memb_ring_id->rep = nodeid; } static void corosync_ring_id_store ( const struct memb_ring_id *memb_ring_id, - const struct totem_ip_address *addr) + unsigned int nodeid) { char filename[PATH_MAX]; int fd; int res; - snprintf (filename, sizeof(filename), "%s/ringid_%s", - get_run_dir(), totemip_print (addr)); + snprintf (filename, sizeof(filename), "%s/ringid_%u", + get_run_dir(), nodeid); fd = open (filename, O_WRONLY, 0700); if (fd == -1) { diff --git a/exec/totemconfig.c b/exec/totemconfig.c index 133750bb..d6e7ab9e 100644 --- a/exec/totemconfig.c +++ b/exec/totemconfig.c @@ -45,9 +45,12 @@ #include #include #include +#include +#include #include #include #include +#include #include #include @@ -462,6 +465,267 @@ static int totem_get_crypto(struct totem_config *totem_config, const char **erro return 0; } +static int nodelist_byname(const char *find_name, int strip_domain) +{ + icmap_iter_t iter; + const char *iter_key; + char name_str[ICMAP_KEYNAME_MAXLEN]; + int res = 0; + unsigned int node_pos; + char *name; + unsigned int namelen; + + iter = icmap_iter_init("nodelist.node."); + while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) { + res = sscanf(iter_key, "nodelist.node.%u.%s", &node_pos, name_str); + if (res != 2) { + continue; + } + if (strcmp(name_str, "name")) { + continue; + } + if (icmap_get_string(iter_key, &name) != CS_OK) { + continue; + } + namelen = strlen(name); + + if (strip_domain) { + char *dot; + dot = strchr(name, '.'); + if (dot) { + namelen = name - dot - 1; + } + } + if (strncmp(find_name, name, namelen) == 0 && + strlen(find_name) == strlen(name)) { + icmap_iter_finalize(iter); + return node_pos; + } + } + icmap_iter_finalize(iter); + return -1; +} + +/* Compare two addresses */ +static int ipaddr_equal(struct sockaddr_storage *addr1, struct sockaddr_storage *addr2) +{ + int addrlen = 0; + + if (addr1->ss_family != addr2->ss_family) + return 0; + + if (addr1->ss_family == AF_INET) { + addrlen = sizeof(struct sockaddr_in); + } + if (addr1->ss_family == AF_INET6) { + addrlen = sizeof(struct sockaddr_in6); + } + assert(addrlen); + + if (memcmp(addr1, addr2, addrlen) == 0) + return 1; + else + return 0; + +} + + +/* Finds the local node and returns its position in the nodelist. + * Uses nodelist.local_node_pos as a cache to save effort + */ +static int find_local_node(int use_cache) +{ + char nodename2[PATH_MAX]; + char name_str[ICMAP_KEYNAME_MAXLEN]; + icmap_iter_t iter; + const char *iter_key; + unsigned int cached_pos; + char *dot = NULL; + const char *node; + struct ifaddrs *ifa, *ifa_list; + struct sockaddr *sa; + int found = 0; + int node_pos = -1; + int res; + struct utsname utsname; + + /* Check for cached value first */ + if (use_cache) { + if (icmap_get_uint32("nodelist.local_node_pos", &cached_pos) == CS_OK) { + return cached_pos; + } + } + + res = uname(&utsname); + if (res) { + return -1; + } + node = utsname.nodename; + + /* 1. Exact match */ + node_pos = nodelist_byname(node, 0); + if (node_pos > -1) { + found = 1; + goto ret_found; + } + + /* 2. Try to match with increasingly more + * specific versions of it + */ + strcpy(nodename2, node); + dot = strrchr(nodename2, '.'); + while (dot) { + *dot = '\0'; + + node_pos = nodelist_byname(nodename2, 0); + if (node_pos > -1) { + found = 1; + goto ret_found; + } + dot = strrchr(nodename2, '.'); + } + + node_pos = nodelist_byname(nodename2, 1); + if (node_pos > -1) { + found = 1; + goto ret_found; + } + + /* + * The corosync.conf name may not be related to uname at all, + * they may match a hostname on some network interface. + */ + if (getifaddrs(&ifa_list)) + return -1; + + for (ifa = ifa_list; ifa; ifa = ifa->ifa_next) { + socklen_t salen = 0; + + /* Restore this */ + strcpy(nodename2, node); + sa = ifa->ifa_addr; + if (!sa) { + continue; + } + if (sa->sa_family != AF_INET && sa->sa_family != AF_INET6) { + continue; + } + + if (sa->sa_family == AF_INET) { + salen = sizeof(struct sockaddr_in); + } + if (sa->sa_family == AF_INET6) { + salen = sizeof(struct sockaddr_in6); + } + + if (getnameinfo(sa, salen, + nodename2, sizeof(nodename2), + NULL, 0, 0) == 0) { + + node_pos = nodelist_byname(nodename2, 0); + if (node_pos > -1) { + found = 1; + goto out; + } + + /* Truncate this name and try again */ + dot = strchr(nodename2, '.'); + if (dot) { + *dot = '\0'; + + node_pos = nodelist_byname(nodename2, 0); + if (node_pos > -1) { + found = 1; + goto out; + } + } + } + + /* See if it's the IP address that's in corosync.conf */ + if (getnameinfo(sa, sizeof(*sa), + nodename2, sizeof(nodename2), + NULL, 0, NI_NUMERICHOST)) + continue; + + node_pos = nodelist_byname(nodename2, 0); + if (node_pos > -1) { + found = 1; + goto out; + } + } + + out: + if (found) { + freeifaddrs(ifa_list); + goto ret_found; + } + + /* + * This section covers the usecase where the nodename specified in cluster.conf + * is an alias specified in /etc/hosts. For example: + * hostname alias1 alias2 + * and + * the above calls use uname and getnameinfo does not return aliases. + * here we take the name specified in cluster.conf, resolve it to an address + * and then compare against all known local ip addresses. + * if we have a match, we found our nodename. In theory this chunk of code + * could replace all the checks above, but let's avoid any possible regressions + * and use it as last. + */ + + iter = icmap_iter_init("nodelist.node."); + while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) { + char *dbnodename = NULL; + struct addrinfo hints; + struct addrinfo *result = NULL, *rp = NULL; + + res = sscanf(iter_key, "nodelist.node.%u.%s", &node_pos, name_str); + if (res != 2) { + continue; + } + if (strcmp(name_str, "name")) { + continue; + } + if (icmap_get_string(iter_key, &dbnodename) != CS_OK) { + continue; + } + + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_DGRAM; + hints.ai_flags = 0; + hints.ai_protocol = IPPROTO_UDP; + + if (getaddrinfo(dbnodename, NULL, &hints, &result)) { + continue; + } + + for (rp = result; rp != NULL; rp = rp->ai_next) { + for (ifa = ifa_list; ifa; ifa = ifa->ifa_next) { + if (ifa->ifa_addr && + ipaddr_equal((struct sockaddr_storage *)rp->ai_addr, + (struct sockaddr_storage *)ifa->ifa_addr)) { + freeaddrinfo(result); + found = 1; + goto out2; + } + } + } + + freeaddrinfo(result); + } +out2: + icmap_iter_finalize(iter); + freeifaddrs(ifa_list); + +ret_found: + if (found) { + res = icmap_set_uint32("nodelist.local_node_pos", node_pos); + } + + return node_pos; +} + static int totem_config_get_ip_version(struct totem_config *totem_config) { int res; @@ -583,7 +847,7 @@ static int check_for_duplicate_nodeids( continue; } - if (strcmp(tmp_key, "ring0_addr") != 0) { + if (strcmp(tmp_key, "nodeid") != 0) { continue; } @@ -612,7 +876,7 @@ static int check_for_duplicate_nodeids( continue; } - if (strcmp(tmp_key, "ring0_addr") != 0) { + if (strcmp(tmp_key, "nodeid") != 0) { continue; } @@ -648,57 +912,6 @@ static int check_for_duplicate_nodeids( } -static int find_local_node_in_nodelist(struct totem_config *totem_config) -{ - icmap_iter_t iter; - const char *iter_key; - int res = 0; - unsigned int node_pos; - int local_node_pos = -1; - struct totem_ip_address bind_addr; - int interface_up, interface_num; - char tmp_key[ICMAP_KEYNAME_MAXLEN]; - char *node_addr_str; - struct totem_ip_address node_addr; - - res = totemip_iface_check(&totem_config->interfaces[0].bindnet, - &bind_addr, &interface_up, &interface_num, - totem_config->clear_node_high_bit); - if (res == -1) { - return (-1); - } - - iter = icmap_iter_init("nodelist.node."); - while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) { - res = sscanf(iter_key, "nodelist.node.%u.%s", &node_pos, tmp_key); - if (res != 2) { - continue; - } - - if (strcmp(tmp_key, "ring0_addr") != 0) { - continue; - } - - snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.ring0_addr", node_pos); - if (icmap_get_string(tmp_key, &node_addr_str) != CS_OK) { - continue; - } - - res = totemip_parse (&node_addr, node_addr_str, totem_config->ip_version); - free(node_addr_str); - if (res == -1) { - continue ; - } - - if (totemip_equal(&bind_addr, &node_addr)) { - local_node_pos = node_pos; - } - } - icmap_iter_finalize(iter); - - return (local_node_pos); -} - /* * This needs to be done last of all. It would be nice to do it when reading the * interface params, but the totem params need to have them to be read first. We @@ -813,7 +1026,7 @@ static void reconfigure_links(struct totem_config *totem_config) char *addr_string; struct totem_ip_address local_ip; int err; - unsigned int local_node_pos = find_local_node_in_nodelist(totem_config); + int local_node_pos = find_local_node(0); for (i = 0; iinterfaces[i].configured) { @@ -890,8 +1103,7 @@ static void put_nodelist_members_to_config(struct totem_config *totem_config, in if (res != 2) { continue; } - - if (strcmp(tmp_key, "ring0_addr") != 0) { + if (strcmp(tmp_key, "nodeid") != 0) { continue; } @@ -914,14 +1126,12 @@ static void put_nodelist_members_to_config(struct totem_config *totem_config, in } member_count = totem_config->interfaces[linknumber].member_count; - res = totemip_parse(&totem_config->interfaces[linknumber].member_list[member_count], node_addr_str, totem_config->ip_version); if (res != -1) { totem_config->interfaces[linknumber].member_list[member_count].nodeid = nodeid; totem_config->interfaces[linknumber].member_count++; } - totem_config->interfaces[linknumber].configured = 1; free(node_addr_str); } @@ -977,112 +1187,11 @@ static void nodelist_dynamic_notify( } -/* - * Tries to find node (node_pos) in config nodelist which address matches any - * local interface. Address can be stored in ring0_addr or if ipaddr_key_prefix is not NULL - * key with prefix ipaddr_key is used (there can be multiuple of them) - * This function differs * from find_local_node_in_nodelist because it doesn't need bindnetaddr, - * but doesn't work when bind addr is network address (so IP must be exact - * match). - * - * Returns 1 on success (address was found, node_pos is then correctly set) or 0 on failure. - */ -int totem_config_find_local_addr_in_nodelist(struct totem_config *totem_config, const char *ipaddr_key_prefix, unsigned int *node_pos) -{ - struct qb_list_head addrs; - struct totem_ip_if_address *if_addr; - icmap_iter_t iter, iter2; - const char *iter_key, *iter_key2; - struct qb_list_head *list; - const char *ipaddr_key; - int ip_version; - struct totem_ip_address node_addr; - char *node_addr_str; - int node_found = 0; - int res = 0; - char tmp_key[ICMAP_KEYNAME_MAXLEN]; - - if (totemip_getifaddrs(&addrs) == -1) { - return 0; - } - - ip_version = totem_config_get_ip_version(totem_config); - - iter = icmap_iter_init("nodelist.node."); - - while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) { - res = sscanf(iter_key, "nodelist.node.%u.%s", node_pos, tmp_key); - if (res != 2) { - continue; - } - - if (strcmp(tmp_key, "ring0_addr") != 0) { - continue; - } - - if (icmap_get_string(iter_key, &node_addr_str) != CS_OK) { - continue ; - } - - free(node_addr_str); - - /* - * ring0_addr found -> let's iterate thru ipaddr_key_prefix - */ - snprintf(tmp_key, sizeof(tmp_key), "nodelist.node.%u.%s", *node_pos, - (ipaddr_key_prefix != NULL ? ipaddr_key_prefix : "ring0_addr")); - - iter2 = icmap_iter_init(tmp_key); - while ((iter_key2 = icmap_iter_next(iter2, NULL, NULL)) != NULL) { - /* - * ring0_addr must be exact match, not prefix - */ - ipaddr_key = (ipaddr_key_prefix != NULL ? iter_key2 : tmp_key); - if (icmap_get_string(ipaddr_key, &node_addr_str) != CS_OK) { - continue ; - } - - if (totemip_parse(&node_addr, node_addr_str, ip_version) == -1) { - free(node_addr_str); - continue ; - } - free(node_addr_str); - - /* - * Try to match ip with if_addrs - */ - node_found = 0; - qb_list_for_each(list, &(addrs)) { - if_addr = qb_list_entry(list, struct totem_ip_if_address, list); - - if (totemip_equal(&node_addr, &if_addr->ip_addr)) { - node_found = 1; - break; - } - } - - if (node_found) { - break ; - } - } - - icmap_iter_finalize(iter2); - - if (node_found) { - break ; - } - } - - icmap_iter_finalize(iter); - totemip_freeifaddrs(&addrs); - - return (node_found); -} static void config_convert_nodelist_to_interface(struct totem_config *totem_config) { int res = 0; - unsigned int node_pos; + int node_pos; char tmp_key[ICMAP_KEYNAME_MAXLEN]; char tmp_key2[ICMAP_KEYNAME_MAXLEN]; char *node_addr_str; @@ -1090,7 +1199,8 @@ static void config_convert_nodelist_to_interface(struct totem_config *totem_conf icmap_iter_t iter; const char *iter_key; - if (totem_config_find_local_addr_in_nodelist(totem_config, NULL, &node_pos)) { + node_pos = find_local_node(1); + if (node_pos > -1) { /* * We found node, so create interface section */ @@ -1180,7 +1290,9 @@ static int get_interface_params(struct totem_config *totem_config, /* * Get the bind net address */ - if (icmap_get_string(iter_key, &str) == CS_OK) { + snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.bindnetaddr", linknumber); + + if (icmap_get_string(tmp_key, &str) == CS_OK) { res = totemip_parse (&totem_config->interfaces[linknumber].bindnet, str, totem_config->ip_version); free(str); @@ -1462,14 +1574,13 @@ extern int totem_config_read ( /* * Check existence of nodelist */ - if (icmap_get_string("nodelist.node.0.ring0_addr", &str) == CS_OK) { + if (icmap_get_string("nodelist.node.0.name", &str) == CS_OK) { free(str); /* * find local node */ - local_node_pos = find_local_node_in_nodelist(totem_config); + local_node_pos = find_local_node(1); if (local_node_pos != -1) { - icmap_set_uint32("nodelist.local_node_pos", local_node_pos); snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.nodeid", local_node_pos); @@ -1482,13 +1593,7 @@ extern int totem_config_read ( return -1; } - /* - * Make localnode ring0_addr read only, so we can be sure that local - * node never changes. If rebinding to other IP would be in future - * supported, this must be changed and handled properly! - */ - snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.ring0_addr", local_node_pos); - icmap_set_ro_access(tmp_key, 0, 1); + /* Users must not change this */ icmap_set_ro_access("nodelist.local_node_pos", 0, 1); } @@ -1522,8 +1627,6 @@ int totem_config_validate ( int num_configured = 0; unsigned int interface_max = INTERFACE_MAX; - - for (i = 0; i < INTERFACE_MAX; i++) { if (totem_config->interfaces[i].configured) { num_configured++; @@ -1883,7 +1986,6 @@ static void totem_reload_notify( void *user_data) { struct totem_config *totem_config = (struct totem_config *)user_data; - uint32_t local_node_pos; const char *error_string; uint64_t warnings; @@ -1911,10 +2013,7 @@ static void totem_reload_notify( } /* Reinstate the local_node_pos */ - local_node_pos = find_local_node_in_nodelist(totem_config); - if (local_node_pos != -1) { - icmap_set_uint32("nodelist.local_node_pos", local_node_pos); - } + (void)find_local_node(0); /* Reconfigure network params as appropriate */ totempg_reconfigure(); diff --git a/exec/totemknet.c b/exec/totemknet.c index dd3feb9a..e52fc956 100644 --- a/exec/totemknet.c +++ b/exec/totemknet.c @@ -145,6 +145,8 @@ struct totemknet_instance { int our_nodeid; + int loopback_link; + struct totem_config *totem_config; struct totem_ip_address token_target; @@ -832,6 +834,8 @@ int totemknet_initialize ( instance->totemknet_target_set_completed = target_set_completed; + instance->loopback_link = -1; + res = pipe(instance->logpipes); if (res == -1) { KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to create pipe for instance->logpipes"); @@ -1043,13 +1047,12 @@ extern void totemknet_net_mtu_adjust (void *knet_context, struct totem_config *t int totemknet_token_target_set ( void *knet_context, - const struct totem_ip_address *token_target) + unsigned int nodeid) { struct totemknet_instance *instance = (struct totemknet_instance *)knet_context; int res = 0; - memcpy (&instance->token_target, token_target, - sizeof (struct totem_ip_address)); + instance->token_target.nodeid = nodeid; instance->totemknet_target_set_completed (instance->context); @@ -1139,18 +1142,23 @@ int totemknet_member_add ( int addrlen; /* Only create 1 loopback link */ - if (member->nodeid == instance->our_nodeid && link_no > 0) { + // NOTE: THis depends on member_remove being run before member_add when reeconfiguring + // otherwise we could be left with no loopback. + if (member->nodeid == instance->our_nodeid && instance->loopback_link > -1) { return 0; } knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: %d (%s), link=%d", member->nodeid, totemip_print(member), link_no); knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: local: %d (%s)", local->nodeid, totemip_print(local)); - if (link_no == 0) { - if (knet_host_add(instance->knet_handle, member->nodeid)) { - KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_add"); - return -1; - } + // TODO FIXME - prints errors when host already exists + err = knet_host_add(instance->knet_handle, member->nodeid); + if (err != 0 && errno != EEXIST) { + KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_add"); + return -1; + } + + if (err == 0) { if (knet_host_set_policy(instance->knet_handle, member->nodeid, instance->link_mode)) { KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_set_policy failed"); return -1; @@ -1161,7 +1169,11 @@ int totemknet_member_add ( /* Casts to remove const */ totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)member, port, &remote_ss, &addrlen); totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)local, port, &local_ss, &addrlen); + if (member->nodeid == instance->our_nodeid) { + knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: loopback link is %d\n", link_no); + + instance->loopback_link = link_no; err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no, KNET_TRANSPORT_LOOPBACK, &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO); @@ -1221,9 +1233,9 @@ int totemknet_member_remove ( knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_remove: %d, link=%d", token_target->nodeid, link_no); - /* Only link 0 is valid for localhost */ - if (token_target->nodeid == instance->our_nodeid && link_no > 0) { - return 0; + /* Removing link with the loopback on it */ + if (token_target->nodeid == instance->our_nodeid && link_no == instance->loopback_link) { + instance->loopback_link= -1; } /* Tidy stats */ diff --git a/exec/totemknet.h b/exec/totemknet.h index 22a77918..0f54d6cc 100644 --- a/exec/totemknet.h +++ b/exec/totemknet.h @@ -112,7 +112,7 @@ extern int totemknet_iface_set (void *net_context, extern int totemknet_token_target_set ( void *knet_context, - const struct totem_ip_address *token_target); + unsigned int nodeid); extern int totemknet_crypto_set ( void *knet_context, diff --git a/exec/totemnet.c b/exec/totemnet.c index 47ef546d..bdcd5c43 100644 --- a/exec/totemnet.c +++ b/exec/totemnet.c @@ -116,7 +116,7 @@ struct transport { int (*token_target_set) ( void *transport_context, - const struct totem_ip_address *token_target); + unsigned int nodeid); int (*crypto_set) ( void *transport_context, @@ -176,6 +176,8 @@ struct transport transport_entries[] = { .token_target_set = totemudp_token_target_set, .crypto_set = totemudp_crypto_set, .recv_mcast_empty = totemudp_recv_mcast_empty, + .member_add = totemudp_member_add, + .member_remove = totemudp_member_remove, .reconfigure = totemudp_reconfigure }, { @@ -476,12 +478,12 @@ int totemnet_ifaces_get ( int totemnet_token_target_set ( void *net_context, - const struct totem_ip_address *token_target) + unsigned int nodeid) { struct totemnet_instance *instance = (struct totemnet_instance *)net_context; unsigned int res; - res = instance->transport->token_target_set (instance->transport_context, token_target); + res = instance->transport->token_target_set (instance->transport_context, nodeid); return (res); } diff --git a/exec/totemnet.h b/exec/totemnet.h index 17a3d9a8..9b54177f 100644 --- a/exec/totemnet.h +++ b/exec/totemnet.h @@ -129,7 +129,7 @@ extern int totemnet_ifaces_get ( extern int totemnet_token_target_set ( void *net_context, - const struct totem_ip_address *token_target); + unsigned int target_nodeid); extern int totemnet_crypto_set ( void *net_context, diff --git a/exec/totemsrp.c b/exec/totemsrp.c index 2439f466..83388a6e 100644 --- a/exec/totemsrp.c +++ b/exec/totemsrp.c @@ -102,11 +102,9 @@ /* * SRP address. - * CC: TODO: Can we remove IP address from this and just use nodeids? */ struct srp_addr { - uint8_t no_addrs; - struct totem_ip_address addr[INTERFACE_MAX]; + unsigned int nodeid; }; /* @@ -300,8 +298,12 @@ struct totemsrp_instance { int consensus_list_entries; + int lowest_active_if; + struct srp_addr my_id; + struct totem_ip_address my_addrs[INTERFACE_MAX]; + struct srp_addr my_proc_list[PROCESSOR_COUNT_MAX]; struct srp_addr my_failed_list[PROCESSOR_COUNT_MAX]; @@ -466,11 +468,11 @@ struct totemsrp_instance { void (*memb_ring_id_create_or_load) ( struct memb_ring_id *memb_ring_id, - const struct totem_ip_address *addr); + unsigned int nodeid); void (*memb_ring_id_store) ( const struct memb_ring_id *memb_ring_id, - const struct totem_ip_address *addr); + unsigned int nodeid); int global_seqno; @@ -617,6 +619,7 @@ static void totemsrp_instance_initialize (struct totemsrp_instance *instance); static void srp_addr_copy (struct srp_addr *dest, const struct srp_addr *src); static void srp_addr_to_nodeid ( + struct totemsrp_instance *instance, unsigned int *nodeid_out, struct srp_addr *srp_addr_in, unsigned int entries); @@ -736,8 +739,6 @@ static void totemsrp_instance_initialize (struct totemsrp_instance *instance) instance->commit_token = (struct memb_commit_token *)instance->commit_token_storage; - instance->my_id.no_addrs = INTERFACE_MAX; - instance->waiting_trans_ack = 1; } @@ -873,7 +874,7 @@ int totemsrp_initialize ( /* * Initialize local variables for totemsrp */ - totemip_copy (&instance->mcast_address, &totem_config->interfaces[0].mcast_addr); + totemip_copy (&instance->mcast_address, &totem_config->interfaces[instance->lowest_active_if].mcast_addr); /* * Display totem configuration @@ -974,6 +975,8 @@ int totemsrp_initialize ( goto error_exit; } + instance->my_id.nodeid = instance->totem_config->interfaces[instance->lowest_active_if].boundto.nodeid; + /* * Must have net_mtu adjusted by totemnet_initialize first */ @@ -1037,54 +1040,18 @@ int totemsrp_ifaces_get ( { struct totemsrp_instance *instance = (struct totemsrp_instance *)srp_context; int res = 0; - unsigned int found = 0; - int i; memset(interfaces, 0, sizeof(struct totem_ip_address) * interfaces_size); - for (i = 0; i < instance->my_memb_entries; i++) { - if (instance->my_memb_list[i].addr[0].nodeid == nodeid) { - found = 1; - break; - } - } - - if (found) { - *iface_count = INTERFACE_MAX; - - if (interfaces_size >= *iface_count) { - memcpy (interfaces, instance->my_memb_list[i].addr, - sizeof (struct totem_ip_address) * *iface_count); - } else { - res = -2; - } - - goto finish; - } - - for (i = 0; i < instance->my_left_memb_entries; i++) { - if (instance->my_left_memb_list[i].addr[0].nodeid == nodeid) { - found = 1; - break; - } - } - - if (found) { - *iface_count = INTERFACE_MAX; - - if (interfaces_size >= *iface_count) { - memcpy (interfaces, instance->my_left_memb_list[i].addr, - sizeof (struct totem_ip_address) * *iface_count); - } else { - res = -2; - } + if (interfaces_size >= *iface_count) { + memcpy (interfaces, instance->my_addrs, + sizeof (struct totem_ip_address) * *iface_count); } else { - res = -1; + res = -2; } *iface_count = INTERFACE_MAX; -finish: totemnet_ifaces_get(instance->totemnet_context, status, iface_count); return (res); } @@ -1109,7 +1076,7 @@ unsigned int totemsrp_my_nodeid_get ( struct totemsrp_instance *instance = (struct totemsrp_instance *)srp_context; unsigned int res; - res = instance->totem_config->interfaces[0].boundto.nodeid; + res = instance->my_id.nodeid; return (res); } @@ -1120,7 +1087,7 @@ int totemsrp_my_family_get ( struct totemsrp_instance *instance = (struct totemsrp_instance *)srp_context; int res; - res = instance->totem_config->interfaces[0].boundto.family; + res = instance->totem_config->interfaces[instance->lowest_active_if].boundto.family; return (res); } @@ -1131,30 +1098,19 @@ int totemsrp_my_family_get ( */ static int srp_addr_equal (const struct srp_addr *a, const struct srp_addr *b) { - unsigned int i; - unsigned int res; - - for (i = 0; i < 1; i++) { - res = totemip_equal (&a->addr[i], &b->addr[i]); - if (res == 0) { - return (0); - } + if (a->nodeid == b->nodeid) { + return 1; } - return (1); + return 0; } static void srp_addr_copy (struct srp_addr *dest, const struct srp_addr *src) { - unsigned int i; - - dest->no_addrs = src->no_addrs; - - for (i = 0; i < INTERFACE_MAX; i++) { - totemip_copy (&dest->addr[i], &src->addr[i]); - } + dest->nodeid = src->nodeid; } static void srp_addr_to_nodeid ( + struct totemsrp_instance *instance, unsigned int *nodeid_out, struct srp_addr *srp_addr_in, unsigned int entries) @@ -1162,17 +1118,13 @@ static void srp_addr_to_nodeid ( unsigned int i; for (i = 0; i < entries; i++) { - nodeid_out[i] = srp_addr_in[i].addr[0].nodeid; + nodeid_out[i] = srp_addr_in[i].nodeid; } } static void srp_addr_copy_endian_convert (struct srp_addr *out, const struct srp_addr *in) { - int i; - - for (i = 0; i < INTERFACE_MAX; i++) { - totemip_copy_endian_convert (&out->addr[i], &in->addr[i]); - } + out->nodeid = swab32 (in->nodeid); } static void memb_consensus_reset (struct totemsrp_instance *instance) @@ -1216,7 +1168,7 @@ static void memb_consensus_set ( int found = 0; int i; - if (addr->addr[0].nodeid == LEAVE_DUMMY_NODEID) + if (addr->nodeid == LEAVE_DUMMY_NODEID) return; for (i = 0; i < instance->consensus_list_entries; i++) { @@ -1927,9 +1879,9 @@ static void memb_state_operational_enter (struct totemsrp_instance *instance) /* * Deliver transitional configuration to application */ - srp_addr_to_nodeid (left_list, instance->my_left_memb_list, + srp_addr_to_nodeid (instance, left_list, instance->my_left_memb_list, instance->my_left_memb_entries); - srp_addr_to_nodeid (trans_memb_list_totemip, + srp_addr_to_nodeid (instance, trans_memb_list_totemip, instance->my_trans_memb_list, instance->my_trans_memb_entries); instance->totemsrp_confchg_fn (TOTEM_CONFIGURATION_TRANSITIONAL, trans_memb_list_totemip, instance->my_trans_memb_entries, @@ -1947,9 +1899,9 @@ static void memb_state_operational_enter (struct totemsrp_instance *instance) /* * Deliver regular configuration to application */ - srp_addr_to_nodeid (new_memb_list_totemip, + srp_addr_to_nodeid (instance, new_memb_list_totemip, instance->my_new_memb_list, instance->my_new_memb_entries); - srp_addr_to_nodeid (joined_list_totemip, joined_list, + srp_addr_to_nodeid (instance, joined_list_totemip, joined_list, joined_list_entries); instance->totemsrp_confchg_fn (TOTEM_CONFIGURATION_REGULAR, new_memb_list_totemip, instance->my_new_memb_entries, @@ -2063,8 +2015,8 @@ static void memb_state_operational_enter (struct totemsrp_instance *instance) log_printf (instance->totemsrp_log_level_debug, "entering OPERATIONAL state."); log_printf (instance->totemsrp_log_level_notice, - "A new membership (%s:%lld) was formed. Members%s%s", - totemip_print (&instance->my_ring_id.rep), + "A new membership (%u:%lld) was formed. Members%s%s", + instance->my_ring_id.rep, instance->my_ring_id.seq, joined_node_msg, left_node_msg); @@ -2199,7 +2151,8 @@ static void memb_state_commit_enter ( instance->memb_timer_state_gather_consensus_timeout = 0; memb_ring_id_set (instance, &instance->commit_token->ring_id); - instance->memb_ring_id_store (&instance->my_ring_id, &instance->my_id.addr[0]); + + instance->memb_ring_id_store (&instance->my_ring_id, instance->my_id.nodeid); instance->token_ring_id_seq = instance->my_ring_id.seq; @@ -2283,15 +2236,15 @@ static void memb_state_recovery_enter ( for (i = 0; i < instance->my_trans_memb_entries; i++) { log_printf (instance->totemsrp_log_level_debug, - "TRANS [%d] member %s:", i, totemip_print (&instance->my_trans_memb_list[i].addr[0])); + "TRANS [%d] member %u:", i, instance->my_trans_memb_list[i].nodeid); } for (i = 0; i < instance->my_new_memb_entries; i++) { log_printf (instance->totemsrp_log_level_debug, - "position [%d] member %s:", i, totemip_print (&addr[i].addr[0])); + "position [%d] member %u:", i, addr[i].nodeid); log_printf (instance->totemsrp_log_level_debug, - "previous ring seq %llx rep %s", + "previous ring seq %llx rep %u", memb_list[i].ring_id.seq, - totemip_print (&memb_list[i].ring_id.rep)); + memb_list[i].ring_id.rep); log_printf (instance->totemsrp_log_level_debug, "aru %x high delivered %x received flag %d", @@ -2378,7 +2331,8 @@ static void memb_state_recovery_enter ( message_item.mcast->header.type = MESSAGE_TYPE_MCAST; srp_addr_copy (&message_item.mcast->system_from, &instance->my_id); message_item.mcast->header.encapsulated = MESSAGE_ENCAPSULATED; - message_item.mcast->header.nodeid = instance->my_id.addr[0].nodeid; + + message_item.mcast->header.nodeid = instance->my_id.nodeid; assert (message_item.mcast->header.nodeid); message_item.mcast->header.endian_detector = ENDIAN_LOCAL; memcpy (&message_item.mcast->ring_id, &instance->my_ring_id, @@ -2465,7 +2419,8 @@ int totemsrp_mcast ( message_item.mcast->header.type = MESSAGE_TYPE_MCAST; message_item.mcast->header.endian_detector = ENDIAN_LOCAL; message_item.mcast->header.encapsulated = MESSAGE_NOT_ENCAPSULATED; - message_item.mcast->header.nodeid = instance->my_id.addr[0].nodeid; + + message_item.mcast->header.nodeid = instance->my_id.nodeid; assert (message_item.mcast->header.nodeid); message_item.mcast->guarantee = guarantee; @@ -2911,7 +2866,7 @@ static void timer_function_merge_detect_timeout(void *data) switch (instance->memb_state) { case MEMB_STATE_OPERATIONAL: - if (totemip_equal(&instance->my_ring_id.rep, &instance->my_id.addr[0])) { + if (instance->my_ring_id.rep == instance->my_id.nodeid) { memb_merge_detect_transmit (instance); } break; @@ -2936,7 +2891,7 @@ static int token_send ( orf_token_size = sizeof (struct orf_token) + (orf_token->rtr_list_entries * sizeof (struct rtr_item)); - orf_token->header.nodeid = instance->my_id.addr[0].nodeid; + orf_token->header.nodeid = instance->my_id.nodeid; memcpy (instance->orf_token_retransmit, orf_token, orf_token_size); instance->orf_token_retransmit_size = orf_token_size; assert (orf_token->header.nodeid); @@ -2970,7 +2925,7 @@ static int token_hold_cancel_send (struct totemsrp_instance *instance) token_hold_cancel.header.type = MESSAGE_TYPE_TOKEN_HOLD_CANCEL; token_hold_cancel.header.endian_detector = ENDIAN_LOCAL; token_hold_cancel.header.encapsulated = 0; - token_hold_cancel.header.nodeid = instance->my_id.addr[0].nodeid; + token_hold_cancel.header.nodeid = instance->my_id.nodeid; memcpy (&token_hold_cancel.ring_id, &instance->my_ring_id, sizeof (struct memb_ring_id)); assert (token_hold_cancel.header.nodeid); @@ -2991,7 +2946,7 @@ static int orf_token_send_initial (struct totemsrp_instance *instance) orf_token.header.type = MESSAGE_TYPE_ORF_TOKEN; orf_token.header.endian_detector = ENDIAN_LOCAL; orf_token.header.encapsulated = 0; - orf_token.header.nodeid = instance->my_id.addr[0].nodeid; + orf_token.header.nodeid = instance->my_id.nodeid; assert (orf_token.header.nodeid); orf_token.seq = SEQNO_START_MSG; orf_token.token_seq = SEQNO_START_TOKEN; @@ -3009,7 +2964,7 @@ static int orf_token_send_initial (struct totemsrp_instance *instance) orf_token.aru = 0; orf_token.aru = SEQNO_START_MSG - 1; - orf_token.aru_addr = instance->my_id.addr[0].nodeid; + orf_token.aru_addr = instance->my_id.nodeid; memcpy (&orf_token.ring_id, &instance->my_ring_id, sizeof (struct memb_ring_id)); orf_token.fcc = 0; @@ -3083,7 +3038,7 @@ static void memb_state_commit_token_update ( } } - instance->commit_token->header.nodeid = instance->my_id.addr[0].nodeid; + instance->commit_token->header.nodeid = instance->my_id.nodeid; instance->commit_token->memb_index += 1; assert (instance->commit_token->memb_index <= instance->commit_token->addr_entries); assert (instance->commit_token->header.nodeid); @@ -3099,8 +3054,8 @@ static void memb_state_commit_token_target_set ( /* Totemnet just looks at the node id */ totemnet_token_target_set ( instance->totemnet_context, - &addr[instance->commit_token->memb_index % - instance->commit_token->addr_entries].addr[0]); + addr[instance->commit_token->memb_index % + instance->commit_token->addr_entries].nodeid); } static int memb_state_commit_token_send_recovery ( @@ -3110,7 +3065,7 @@ static int memb_state_commit_token_send_recovery ( unsigned int commit_token_size; commit_token->token_seq++; - commit_token->header.nodeid = instance->my_id.addr[0].nodeid; + commit_token->header.nodeid = instance->my_id.nodeid; commit_token_size = sizeof (struct memb_commit_token) + ((sizeof (struct srp_addr) + sizeof (struct memb_commit_token_memb_entry)) * commit_token->addr_entries); @@ -3139,7 +3094,7 @@ static int memb_state_commit_token_send ( unsigned int commit_token_size; instance->commit_token->token_seq++; - instance->commit_token->header.nodeid = instance->my_id.addr[0].nodeid; + instance->commit_token->header.nodeid = instance->my_id.nodeid; commit_token_size = sizeof (struct memb_commit_token) + ((sizeof (struct srp_addr) + sizeof (struct memb_commit_token_memb_entry)) * instance->commit_token->addr_entries); @@ -3168,7 +3123,7 @@ static int memb_lowest_in_config (struct totemsrp_instance *instance) struct srp_addr token_memb[PROCESSOR_COUNT_MAX]; int token_memb_entries = 0; int i; - struct totem_ip_address *lowest_addr; + unsigned int lowest_nodeid; memb_set_subtract (token_memb, &token_memb_entries, instance->my_proc_list, instance->my_proc_list_entries, @@ -3178,13 +3133,13 @@ static int memb_lowest_in_config (struct totemsrp_instance *instance) * find representative by searching for smallest identifier */ - lowest_addr = &token_memb[0].addr[0]; + lowest_nodeid = token_memb[0].nodeid; for (i = 1; i < token_memb_entries; i++) { - if (totemip_compare(lowest_addr, &token_memb[i].addr[0]) > 0) { - totemip_copy (lowest_addr, &token_memb[i].addr[0]); + if (lowest_nodeid > token_memb[i].nodeid) { + lowest_nodeid = token_memb[i].nodeid; } } - return (totemip_compare (lowest_addr, &instance->my_id.addr[0]) == 0); + return (lowest_nodeid == instance->my_id.nodeid); } static int srp_addr_compare (const void *a, const void *b) @@ -3192,7 +3147,7 @@ static int srp_addr_compare (const void *a, const void *b) const struct srp_addr *srp_a = (const struct srp_addr *)a; const struct srp_addr *srp_b = (const struct srp_addr *)b; - return (totemip_compare (&srp_a->addr[0], &srp_b->addr[0])); + return (srp_a->nodeid == srp_b->nodeid); } static void memb_state_commit_token_create ( @@ -3214,11 +3169,10 @@ static void memb_state_commit_token_create ( instance->commit_token->header.type = MESSAGE_TYPE_MEMB_COMMIT_TOKEN; instance->commit_token->header.endian_detector = ENDIAN_LOCAL; instance->commit_token->header.encapsulated = 0; - instance->commit_token->header.nodeid = instance->my_id.addr[0].nodeid; + instance->commit_token->header.nodeid = instance->my_id.nodeid; assert (instance->commit_token->header.nodeid); - totemip_copy(&instance->commit_token->ring_id.rep, &instance->my_id.addr[0]); - + instance->commit_token->ring_id.rep = instance->my_id.nodeid; instance->commit_token->ring_id.seq = instance->token_ring_id_seq + 4; /* @@ -3250,7 +3204,7 @@ static void memb_join_message_send (struct totemsrp_instance *instance) memb_join->header.type = MESSAGE_TYPE_MEMB_JOIN; memb_join->header.endian_detector = ENDIAN_LOCAL; memb_join->header.encapsulated = 0; - memb_join->header.nodeid = instance->my_id.addr[0].nodeid; + memb_join->header.nodeid = instance->my_id.nodeid; assert (memb_join->header.nodeid); memb_join->ring_seq = instance->my_ring_id.seq; @@ -3326,7 +3280,7 @@ static void memb_leave_message_send (struct totemsrp_instance *instance) memb_join->proc_list_entries = active_memb_entries; memb_join->failed_list_entries = instance->my_failed_list_entries; srp_addr_copy (&memb_join->system_from, &instance->my_id); - memb_join->system_from.addr[0].nodeid = LEAVE_DUMMY_NODEID; + memb_join->system_from.nodeid = LEAVE_DUMMY_NODEID; // TODO: CC Maybe use the actual join send routine. /* @@ -3369,7 +3323,7 @@ static void memb_merge_detect_transmit (struct totemsrp_instance *instance) memb_merge_detect.header.type = MESSAGE_TYPE_MEMB_MERGE_DETECT; memb_merge_detect.header.endian_detector = ENDIAN_LOCAL; memb_merge_detect.header.encapsulated = 0; - memb_merge_detect.header.nodeid = instance->my_id.addr[0].nodeid; + memb_merge_detect.header.nodeid = instance->my_id.nodeid; srp_addr_copy (&memb_merge_detect.system_from, &instance->my_id); memcpy (&memb_merge_detect.ring_id, &instance->my_ring_id, sizeof (struct memb_ring_id)); @@ -3655,13 +3609,14 @@ static int message_handler_orf_token ( * Determine if we should hold (in reality drop) the token */ instance->my_token_held = 0; - if (totemip_equal(&instance->my_ring_id.rep, &instance->my_id.addr[0]) && + if (instance->my_ring_id.rep == instance->my_id.nodeid && instance->my_seq_unchanged > instance->totem_config->seqno_unchanged_const) { instance->my_token_held = 1; - } else - if (!totemip_equal(&instance->my_ring_id.rep, &instance->my_id.addr[0]) && - instance->my_seq_unchanged >= instance->totem_config->seqno_unchanged_const) { - instance->my_token_held = 1; + } else { + if (instance->my_ring_id.rep != instance->my_id.nodeid && + instance->my_seq_unchanged >= instance->totem_config->seqno_unchanged_const) { + instance->my_token_held = 1; + } } /* @@ -3669,7 +3624,7 @@ static int message_handler_orf_token ( * this processor is the ring rep */ forward_token = 1; - if (totemip_equal(&instance->my_ring_id.rep, &instance->my_id.addr[0])) { + if (instance->my_ring_id.rep == instance->my_id.nodeid) { if (instance->my_token_held) { forward_token = 0; } @@ -3741,14 +3696,14 @@ printf ("token seq %d\n", token->seq); mcasted_regular); if (sq_lt_compare (instance->my_aru, token->aru) || - instance->my_id.addr[0].nodeid == token->aru_addr || + instance->my_id.nodeid == token->aru_addr || token->aru_addr == 0) { token->aru = instance->my_aru; if (token->aru == token->seq) { token->aru_addr = 0; } else { - token->aru_addr = instance->my_id.addr[0].nodeid; + token->aru_addr = instance->my_id.nodeid; } } if (token->aru == last_aru && token->aru_addr != 0) { @@ -3763,7 +3718,7 @@ printf ("token seq %d\n", token->seq); * to failed list (so node never mark itself as failed) */ if (instance->my_aru_count > instance->totem_config->fail_to_recv_const && - token->aru_addr == instance->my_id.addr[0].nodeid) { + token->aru_addr == instance->my_id.nodeid) { log_printf (instance->totemsrp_log_level_error, "FAILED TO RECEIVE"); @@ -3860,7 +3815,7 @@ printf ("token seq %d\n", token->seq); */ reset_token_timeout (instance); // REVIEWED reset_token_retransmit_timeout (instance); // REVIEWED - if (totemip_equal(&instance->my_id.addr[0], &instance->my_ring_id.rep) && + if (instance->my_id.nodeid == instance->my_ring_id.rep && instance->my_token_held == 1) { start_token_hold_retransmit_timeout (instance); @@ -4060,8 +4015,8 @@ static int message_handler_mcast ( } log_printf (instance->totemsrp_log_level_trace, - "Received ringid(%s:%lld) seq %x", - totemip_print (&mcast_header.ring_id.rep), + "Received ringid(%u:%lld) seq %x", + mcast_header.ring_id.rep, mcast_header.ring_id.seq, mcast_header.seq); @@ -4186,9 +4141,9 @@ static void memb_join_process ( if (memb_join->header.nodeid == LEAVE_DUMMY_NODEID) { log_printf (instance->totemsrp_log_level_warning, "Discarding LEAVE message during flush, nodeid=%u", - memb_join->failed_list_entries > 0 ? failed_list[memb_join->failed_list_entries - 1 ].addr[0].nodeid : LEAVE_DUMMY_NODEID); + memb_join->failed_list_entries > 0 ? failed_list[memb_join->failed_list_entries - 1 ].nodeid : LEAVE_DUMMY_NODEID); if (memb_join->failed_list_entries > 0) { - my_leave_memb_set(instance, failed_list[memb_join->failed_list_entries - 1 ].addr[0].nodeid); + my_leave_memb_set(instance, failed_list[memb_join->failed_list_entries - 1 ].nodeid); } } else { log_printf (instance->totemsrp_log_level_warning, @@ -4198,9 +4153,9 @@ static void memb_join_process ( } else { if (memb_join->header.nodeid == LEAVE_DUMMY_NODEID) { log_printf (instance->totemsrp_log_level_debug, - "Received LEAVE message from %u", memb_join->failed_list_entries > 0 ? failed_list[memb_join->failed_list_entries - 1 ].addr[0].nodeid : LEAVE_DUMMY_NODEID); + "Received LEAVE message from %u", memb_join->failed_list_entries > 0 ? failed_list[memb_join->failed_list_entries - 1 ].nodeid : LEAVE_DUMMY_NODEID); if (memb_join->failed_list_entries > 0) { - my_leave_memb_set(instance, failed_list[memb_join->failed_list_entries - 1 ].addr[0].nodeid); + my_leave_memb_set(instance, failed_list[memb_join->failed_list_entries - 1 ].nodeid); } } } @@ -4351,7 +4306,7 @@ static void memb_commit_token_endian_convert (const struct memb_commit_token *in out->header.endian_detector = ENDIAN_LOCAL; out->header.nodeid = swab32 (in->header.nodeid); out->token_seq = swab32 (in->token_seq); - totemip_copy_endian_convert(&out->ring_id.rep, &in->ring_id.rep); + out->ring_id.rep = swab32(in->ring_id.rep); out->ring_id.seq = swab64 (in->ring_id.seq); out->retrans_flg = swab32 (in->retrans_flg); out->memb_index = swab32 (in->memb_index); @@ -4365,9 +4320,8 @@ static void memb_commit_token_endian_convert (const struct memb_commit_token *in /* * Only convert the memb entry if it has been set */ - if (in_memb_list[i].ring_id.rep.family != 0) { - totemip_copy_endian_convert (&out_memb_list[i].ring_id.rep, - &in_memb_list[i].ring_id.rep); + if (in_memb_list[i].ring_id.rep != 0) { + out_memb_list[i].ring_id.rep = swab32(in_memb_list[i].ring_id.rep); out_memb_list[i].ring_id.seq = swab64 (in_memb_list[i].ring_id.seq); @@ -4388,7 +4342,7 @@ static void orf_token_endian_convert (const struct orf_token *in, struct orf_tok out->seq = swab32 (in->seq); out->token_seq = swab32 (in->token_seq); out->aru = swab32 (in->aru); - totemip_copy_endian_convert(&out->ring_id.rep, &in->ring_id.rep); + out->ring_id.rep = swab32(in->ring_id.rep); out->aru_addr = swab32(in->aru_addr); out->ring_id.seq = swab64 (in->ring_id.seq); out->fcc = swab32 (in->fcc); @@ -4396,7 +4350,7 @@ static void orf_token_endian_convert (const struct orf_token *in, struct orf_tok out->retrans_flg = swab32 (in->retrans_flg); out->rtr_list_entries = swab32 (in->rtr_list_entries); for (i = 0; i < out->rtr_list_entries; i++) { - totemip_copy_endian_convert(&out->rtr_list[i].ring_id.rep, &in->rtr_list[i].ring_id.rep); + out->rtr_list[i].ring_id.rep = swab32(in->rtr_list[i].ring_id.rep); out->rtr_list[i].ring_id.seq = swab64 (in->rtr_list[i].ring_id.seq); out->rtr_list[i].seq = swab32 (in->rtr_list[i].seq); } @@ -4411,7 +4365,7 @@ static void mcast_endian_convert (const struct mcast *in, struct mcast *out) out->seq = swab32 (in->seq); out->this_seqno = swab32 (in->this_seqno); - totemip_copy_endian_convert(&out->ring_id.rep, &in->ring_id.rep); + out->ring_id.rep = swab32(in->ring_id.rep); out->ring_id.seq = swab64 (in->ring_id.seq); out->node_id = swab32 (in->node_id); out->guarantee = swab32 (in->guarantee); @@ -4425,7 +4379,7 @@ static void memb_merge_detect_endian_convert ( out->header.type = in->header.type; out->header.endian_detector = ENDIAN_LOCAL; out->header.nodeid = swab32 (in->header.nodeid); - totemip_copy_endian_convert(&out->ring_id.rep, &in->ring_id.rep); + out->ring_id.rep = swab32(in->ring_id.rep); out->ring_id.seq = swab64 (in->ring_id.seq); srp_addr_copy_endian_convert (&out->system_from, &in->system_from); } @@ -4594,7 +4548,7 @@ static int message_handler_memb_commit_token ( break; case MEMB_STATE_RECOVERY: - if (totemip_equal (&instance->my_id.addr[0], &instance->my_ring_id.rep)) { + if (instance->my_id.nodeid == instance->my_ring_id.rep) { /* Filter out duplicated tokens */ if (instance->originated_orf_token) { @@ -4628,7 +4582,7 @@ static int message_handler_token_hold_cancel ( sizeof (struct memb_ring_id)) == 0) { instance->my_seq_unchanged = 0; - if (totemip_equal(&instance->my_ring_id.rep, &instance->my_id.addr[0])) { + if (instance->my_ring_id.rep == instance->my_id.nodeid) { timer_function_token_retransmit_timeout (instance); } } @@ -4695,7 +4649,7 @@ int totemsrp_iface_set ( struct totemsrp_instance *instance = context; int res; - totemip_copy(&instance->my_id.addr[iface_no], interface_addr); + totemip_copy(&instance->my_addrs[iface_no], interface_addr); res = totemnet_iface_set ( instance->totemnet_context, @@ -4716,20 +4670,19 @@ void main_iface_change_fn ( int num_interfaces; int i; - totemip_copy (&instance->my_id.addr[iface_no], iface_addr); - assert (instance->my_id.addr[iface_no].nodeid); - - totemip_copy (&instance->my_memb_list[0].addr[iface_no], iface_addr); + if (!instance->my_id.nodeid) { + instance->my_id.nodeid = iface_addr->nodeid; + } + totemip_copy (&instance->my_addrs[iface_no], iface_addr); if (instance->iface_changes++ == 0) { - instance->memb_ring_id_create_or_load (&instance->my_ring_id, - &instance->my_id.addr[0]); + instance->memb_ring_id_create_or_load (&instance->my_ring_id, instance->my_id.nodeid); instance->token_ring_id_seq = instance->my_ring_id.seq; log_printf ( instance->totemsrp_log_level_debug, - "Created or loaded sequence id %llx.%s for this ring.", + "Created or loaded sequence id %llx.%u for this ring.", instance->my_ring_id.seq, - totemip_print (&instance->my_ring_id.rep)); + instance->my_ring_id.rep); if (instance->totemsrp_service_ready_fn) { instance->totemsrp_service_ready_fn (); @@ -4750,7 +4703,6 @@ void main_iface_change_fn ( } } - if (instance->iface_changes >= num_interfaces) { memb_state_gather_enter (instance, TOTEMSRP_GSFROM_INTERFACE_CHANGE); } @@ -4777,7 +4729,7 @@ int totemsrp_member_add ( struct totemsrp_instance *instance = (struct totemsrp_instance *)context; int res; - res = totemnet_member_add (instance->totemnet_context, &instance->my_id.addr[iface_no], member, iface_no); + res = totemnet_member_add (instance->totemnet_context, &instance->my_addrs[iface_no], member, iface_no); return (res); } diff --git a/exec/totemudp.c b/exec/totemudp.c index cb370f52..a022f1c3 100644 --- a/exec/totemudp.c +++ b/exec/totemudp.c @@ -81,6 +81,11 @@ #define BIND_STATE_REGULAR 1 #define BIND_STATE_LOOPBACK 2 +struct totemudp_member { + struct qb_list_head list; + struct totem_ip_address member; +}; + struct totemudp_socket { int mcast_recv; int mcast_send; @@ -143,6 +148,8 @@ struct totemudp_instance { void *udp_context; + struct qb_list_head member_list; + char iov_buffer[UDP_RECEIVE_FRAME_SIZE_MAX]; char iov_buffer_flush[UDP_RECEIVE_FRAME_SIZE_MAX]; @@ -216,6 +223,8 @@ static void totemudp_instance_initialize (struct totemudp_instance *instance) * There is always atleast 1 processor */ instance->my_memb_entries = 1; + + qb_list_init (&instance->member_list); } #define log_printf(level, format, args...) \ @@ -1344,16 +1353,26 @@ extern void totemudp_net_mtu_adjust (void *udp_context, struct totem_config *tot int totemudp_token_target_set ( void *udp_context, - const struct totem_ip_address *token_target) + unsigned int nodeid) { struct totemudp_instance *instance = (struct totemudp_instance *)udp_context; + struct qb_list_head *list; + struct totemudp_member *member; int res = 0; - memcpy (&instance->token_target, token_target, - sizeof (struct totem_ip_address)); + qb_list_for_each(list, &(instance->member_list)) { + member = qb_list_entry (list, + struct totemudp_member, + list); - instance->totemudp_target_set_completed (instance->context); + if (member->member.nodeid == nodeid) { + memcpy (&instance->token_target, &member->member, + sizeof (struct totem_ip_address)); + instance->totemudp_target_set_completed (instance->context); + break; + } + } return (res); } @@ -1421,6 +1440,65 @@ extern int totemudp_recv_mcast_empty ( return (msg_processed); } + +int totemudp_member_add ( + void *udp_context, + const struct totem_ip_address *local, + const struct totem_ip_address *member, + int ring_no) +{ + struct totemudp_instance *instance = (struct totemudp_instance *)udp_context; + + struct totemudp_member *new_member; + + new_member = malloc (sizeof (struct totemudp_member)); + if (new_member == NULL) { + return (-1); + } + + memset(new_member, 0, sizeof(*new_member)); + + qb_list_init (&new_member->list); + qb_list_add_tail (&new_member->list, &instance->member_list); + memcpy (&new_member->member, member, sizeof (struct totem_ip_address)); + + return (0); +} + +int totemudp_member_remove ( + void *udp_context, + const struct totem_ip_address *token_target, + int ring_no) +{ + int found = 0; + struct qb_list_head *list; + struct totemudp_member *member; + struct totemudp_instance *instance = (struct totemudp_instance *)udp_context; + + /* + * Find the member to remove and close its socket + */ + qb_list_for_each(list, &(instance->member_list)) { + member = qb_list_entry (list, + struct totemudp_member, + list); + + if (totemip_compare (token_target, &member->member)==0) { + found = 1; + break; + } + } + + /* + * Delete the member from the list + */ + if (found) { + qb_list_del (list); + } + + return (0); +} + int totemudp_iface_set (void *net_context, const struct totem_ip_address *local_addr, unsigned short ip_port, diff --git a/exec/totemudp.h b/exec/totemudp.h index 82fee566..135284f4 100644 --- a/exec/totemudp.h +++ b/exec/totemudp.h @@ -112,7 +112,7 @@ extern void totemudp_net_mtu_adjust (void *udp_context, struct totem_config *tot extern int totemudp_token_target_set ( void *udp_context, - const struct totem_ip_address *token_target); + unsigned int nodeid); extern int totemudp_crypto_set ( void *udp_context, @@ -122,6 +122,17 @@ extern int totemudp_crypto_set ( extern int totemudp_recv_mcast_empty ( void *udp_context); +extern int totemudp_member_add ( + void *udpu_context, + const struct totem_ip_address *local, + const struct totem_ip_address *member, + int ring_no); + +extern int totemudp_member_remove ( + void *udpu_context, + const struct totem_ip_address *member, + int ring_no); + extern int totemudp_reconfigure ( void *udp_context, struct totem_config *totem_config); diff --git a/exec/totemudpu.c b/exec/totemudpu.c index 8b49fc84..89d261b6 100644 --- a/exec/totemudpu.c +++ b/exec/totemudpu.c @@ -957,16 +957,27 @@ extern void totemudpu_net_mtu_adjust (void *udpu_context, struct totem_config *t int totemudpu_token_target_set ( void *udpu_context, - const struct totem_ip_address *token_target) + unsigned int nodeid) { + struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context; + struct qb_list_head *list; + struct totemudpu_member *member; int res = 0; - memcpy (&instance->token_target, token_target, - sizeof (struct totem_ip_address)); + qb_list_for_each(list, &(instance->member_list)) { + member = qb_list_entry (list, + struct totemudpu_member, + list); - instance->totemudpu_target_set_completed (instance->context); + if (member->member.nodeid == nodeid) { + memcpy (&instance->token_target, &member->member, + sizeof (struct totem_ip_address)); + instance->totemudpu_target_set_completed (instance->context); + break; + } + } return (res); } diff --git a/exec/totemudpu.h b/exec/totemudpu.h index 04b74598..e1f22722 100644 --- a/exec/totemudpu.h +++ b/exec/totemudpu.h @@ -112,7 +112,7 @@ extern void totemudpu_net_mtu_adjust (void *udpu_context, struct totem_config *t extern int totemudpu_token_target_set ( void *udpu_context, - const struct totem_ip_address *token_target); + unsigned int nodeid); extern int totemudpu_crypto_set ( void *udpu_context, diff --git a/exec/votequorum.c b/exec/votequorum.c index c912a346..246c5e0f 100644 --- a/exec/votequorum.c +++ b/exec/votequorum.c @@ -1172,7 +1172,7 @@ static int votequorum_read_nodelist_configuration(uint32_t *votes, continue; } - if (strcmp(tmp_key, "ring0_addr") != 0) { + if (strcmp(tmp_key, "nodeid") != 0) { continue; } @@ -1813,13 +1813,13 @@ static int votequorum_exec_send_nodelist_notification(void *conn, uint64_t conte ENTER(); - log_printf(LOGSYS_LEVEL_DEBUG, "Sending nodelist callback. ring_id = %d/%lld", quorum_ringid.rep.nodeid, quorum_ringid.seq); + log_printf(LOGSYS_LEVEL_DEBUG, "Sending nodelist callback. ring_id = %d/%lld", quorum_ringid.nodeid, quorum_ringid.seq); size = sizeof(struct res_lib_votequorum_nodelist_notification) + sizeof(uint32_t) * quorum_members_entries; res_lib_votequorum_notification = (struct res_lib_votequorum_nodelist_notification *)&buf; res_lib_votequorum_notification->node_list_entries = quorum_members_entries; - res_lib_votequorum_notification->ring_id.nodeid = quorum_ringid.rep.nodeid; + res_lib_votequorum_notification->ring_id.nodeid = quorum_ringid.nodeid; res_lib_votequorum_notification->ring_id.seq = quorum_ringid.seq; res_lib_votequorum_notification->context = context; @@ -2945,12 +2945,12 @@ static void message_handler_req_lib_votequorum_qdevice_poll (void *conn, } if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) { - if (!(req_lib_votequorum_qdevice_poll->ring_id.nodeid == quorum_ringid.rep.nodeid && + if (!(req_lib_votequorum_qdevice_poll->ring_id.nodeid == quorum_ringid.nodeid && req_lib_votequorum_qdevice_poll->ring_id.seq == quorum_ringid.seq)) { log_printf(LOGSYS_LEVEL_DEBUG, "Received poll ring id (%u.%"PRIu64") != last sync " "ring id (%u.%"PRIu64"). Ignoring poll call.", req_lib_votequorum_qdevice_poll->ring_id.nodeid, req_lib_votequorum_qdevice_poll->ring_id.seq, - quorum_ringid.rep.nodeid, quorum_ringid.seq); + quorum_ringid.nodeid, quorum_ringid.seq); error = CS_ERR_MESSAGE_ERROR; goto out; } diff --git a/include/corosync/coroapi.h b/include/corosync/coroapi.h index cd24d26b..cf259e69 100644 --- a/include/corosync/coroapi.h +++ b/include/corosync/coroapi.h @@ -120,7 +120,7 @@ struct totem_ip_address { * @brief The memb_ring_id struct */ struct memb_ring_id { - struct totem_ip_address rep; + unsigned int nodeid; unsigned long long seq; } __attribute__((packed)); #endif diff --git a/include/corosync/totem/totem.h b/include/corosync/totem/totem.h index 3b7ff77f..c9d561ad 100644 --- a/include/corosync/totem/totem.h +++ b/include/corosync/totem/totem.h @@ -137,7 +137,7 @@ typedef enum { #define MEMB_RING_ID struct memb_ring_id { - struct totem_ip_address rep; + unsigned int rep; unsigned long long seq; } __attribute__((packed)); @@ -225,11 +225,11 @@ struct totem_config { void (*totem_memb_ring_id_create_or_load) ( struct memb_ring_id *memb_ring_id, - const struct totem_ip_address *addr); + unsigned int nodeid); void (*totem_memb_ring_id_store) ( const struct memb_ring_id *memb_ring_id, - const struct totem_ip_address *addr); + unsigned int nodeid); }; #define TOTEM_CONFIGURATION_TYPE diff --git a/tools/corosync-cfgtool.c b/tools/corosync-cfgtool.c index 6d5cce29..fe76aab7 100644 --- a/tools/corosync-cfgtool.c +++ b/tools/corosync-cfgtool.c @@ -110,6 +110,7 @@ linkstatusget_do (char *interface_name, int brief) printf ("Could not get the link status, the error is: %d\n", result); } else { for (i = 0; i < interface_count; i++) { + s = 0; if ( (interface_name && interface_names[i][0] != '\0' && (interface_name[0]=='\0' || diff --git a/tools/corosync-quorumtool.c b/tools/corosync-quorumtool.c index 9d0fae00..e57c64ec 100644 --- a/tools/corosync-quorumtool.c +++ b/tools/corosync-quorumtool.c @@ -279,7 +279,7 @@ static const char *node_name_by_nodelist(uint32_t nodeid) continue; } - if (strcmp(tmp_key, "ring0_addr") != 0) { + if (strcmp(tmp_key, "nodeid") != 0) { continue; } @@ -290,7 +290,7 @@ static const char *node_name_by_nodelist(uint32_t nodeid) if (cur_nodeid != nodeid) { continue; } - snprintf(tmp_key, CMAP_KEYNAME_MAXLEN, "nodelist.node.%u.ring0_addr", node_pos); + snprintf(tmp_key, CMAP_KEYNAME_MAXLEN, "nodelist.node.%u.name", node_pos); if (cmap_get_string(cmap_handle, tmp_key, &str) != CS_OK) { continue; }