diff --git a/exec/amf.c b/exec/amf.c index f8f3ae89..76f73255 100644 --- a/exec/amf.c +++ b/exec/amf.c @@ -244,7 +244,8 @@ static int amf_confchg_fn ( struct in_addr *left_list, void *left_list_private, int left_list_entries, struct in_addr *joined_list, void *joined_list_private, - int joined_list_entries); + int joined_list_entries, + struct memb_ring_id *ring_id); /*** static void amf_dump (void); @@ -1903,7 +1904,8 @@ static int amf_confchg_fn ( struct in_addr *left_list, void *left_list_private, int left_list_entries, struct in_addr *joined_list, void *joined_list_private, - int joined_list_entries) + int joined_list_entries, + struct memb_ring_id *ring_id) { int i; diff --git a/exec/ckpt.c b/exec/ckpt.c index 657edf24..7e3a0806 100644 --- a/exec/ckpt.c +++ b/exec/ckpt.c @@ -140,7 +140,8 @@ static int ckpt_confchg_fn ( struct in_addr *left_list, void *left_list_private, int left_list_entries, struct in_addr *joined_list, void *joined_list_private, - int joined_list_entries) { + int joined_list_entries, + struct memb_ring_id *ring_id) { #ifdef TODO if (configuration_type == TOTEMPG_CONFIGURATION_REGULAR) { @@ -272,6 +273,8 @@ struct service_handler ckpt_service_handler = { .exec_dump_fn = 0 }; +static struct memb_ring_id saved_ring_id; + static struct saCkptCheckpoint *ckpt_checkpoint_find_global (SaNameT *name) { struct list_head *checkpointList; @@ -401,9 +404,11 @@ int ckpt_checkpoint_close (struct saCkptCheckpoint *checkpoint) { static int ckpt_exec_init_fn (void) { + // Initialize the saved ring ID. + saved_ring_id.seq = 0; + saved_ring_id.rep.s_addr = this_ip.sin_addr.s_addr; #ifdef TODO int res; - res = totempg_recovery_plug_create (&ckpt_checkpoint_recovery_plug_handle); if (res != 0) { log_printf(LOG_LEVEL_ERROR, diff --git a/exec/ckpt.h b/exec/ckpt.h index 9a49d68f..88db529d 100644 --- a/exec/ckpt.h +++ b/exec/ckpt.h @@ -35,10 +35,16 @@ #include "../include/saCkpt.h" #include "aispoll.h" #include "parse.h" +#include "totemsrp.h" #ifndef CKPT_H_DEFINED #define CKPT_H_DEFINED +struct ckpt_refcnt { + int count; + struct in_addr addr; +}; + struct saCkptCheckpointSection { struct list_head list; SaCkptSectionDescriptorT sectionDescriptor; @@ -55,6 +61,7 @@ struct saCkptCheckpoint { int unlinked; poll_timer_handle retention_timer; int expired; + struct ckpt_refcnt ckpt_refcount[PROCESSOR_COUNT_MAX]; }; struct saCkptSectionIteratorEntry { diff --git a/exec/clm.c b/exec/clm.c index 7625ecd5..784c5f74 100644 --- a/exec/clm.c +++ b/exec/clm.c @@ -107,7 +107,8 @@ static int clm_confchg_fn ( struct in_addr *left_list, void *left_list_private, int left_list_entries, struct in_addr *joined_list, void *joined_list_private, - int joined_list_entries); + int joined_list_entries, + struct memb_ring_id *ring_id); static int message_handler_req_exec_clm_nodejoin (void *message, struct in_addr source_addr, int endian_conversion_required); @@ -393,7 +394,8 @@ static int clm_confchg_fn ( struct in_addr *left_list, void *left_list_private, int left_list_entries, struct in_addr *joined_list, void *joined_list_private, - int joined_list_entries) + int joined_list_entries, + struct memb_ring_id *ring_id) { int i; diff --git a/exec/evs.c b/exec/evs.c index 7fda9978..ab9bdaee 100644 --- a/exec/evs.c +++ b/exec/evs.c @@ -76,7 +76,8 @@ static int evs_confchg_fn ( struct in_addr *left_list, void *left_list_private, int left_list_entries, struct in_addr *joined_list, void *joined_list_private, - int joined_list_entries); + int joined_list_entries, + struct memb_ring_id *ring_id); static int message_handler_req_exec_mcast (void *message, struct in_addr source_addr, int endian_conversion_required); @@ -156,7 +157,8 @@ static int evs_confchg_fn ( struct in_addr *left_list, void *left_list_private, int left_list_entries, struct in_addr *joined_list, void *joined_list_private, - int joined_list_entries) + int joined_list_entries, + struct memb_ring_id *ring_id) { int i; diff --git a/exec/handlers.h b/exec/handlers.h index cde2bec0..9cfa88d0 100644 --- a/exec/handlers.h +++ b/exec/handlers.h @@ -37,6 +37,7 @@ #include #include "main.h" #include "totempg.h" +#include "totemsrp.h" struct libais_handler { int (*libais_handler_fn) (struct conn_info *conn_info, void *msg); @@ -56,7 +57,8 @@ struct service_handler { struct in_addr *left_list, void *left_list_private, int left_list_entries, struct in_addr *joined_list, void *joined_list_private, - int joined_list_entries); + int joined_list_entries, + struct memb_ring_id *ring_id); int (*libais_init_fn) (struct conn_info *conn_info, void *msg); int (*libais_exit_fn) (struct conn_info *conn_info); int (*exec_init_fn) (void); diff --git a/exec/main.c b/exec/main.c index 4808393d..2b88e463 100644 --- a/exec/main.c +++ b/exec/main.c @@ -60,6 +60,7 @@ #include "../include/queue.h" #include "poll.h" #include "totempg.h" +#include "totemsrp.h" #include "mempool.h" #include "parse.h" #include "main.h" @@ -707,7 +708,8 @@ static void confchg_fn ( struct in_addr *left_list, void *left_list_private, int left_list_entries, struct in_addr *joined_list, void *joined_list_private, - int joined_list_entries) + int joined_list_entries, + struct memb_ring_id *ring_id) { int i; @@ -719,7 +721,7 @@ static void confchg_fn ( ais_service_handlers[i]->confchg_fn (configuration_type, member_list, member_list_private, member_list_entries, left_list, left_list_private, left_list_entries, - joined_list, joined_list_private, joined_list_entries); + joined_list, joined_list_private, joined_list_entries, ring_id); } } } diff --git a/exec/totempg.c b/exec/totempg.c index dde5390b..a492289b 100644 --- a/exec/totempg.c +++ b/exec/totempg.c @@ -149,7 +149,8 @@ static void (*app_confchg_fn) ( struct in_addr *left_list, void *left_list_private, int left_list_entries, struct in_addr *joined_list, void *joined_list_private, - int joined_list_entries) = 0; + int joined_list_entries, + struct memb_ring_id *ring_id) = 0; struct assembly { struct in_addr addr; @@ -195,7 +196,8 @@ static void totempg_confchg_fn ( struct in_addr *left_list, void *left_list_private, int left_list_entries, struct in_addr *joined_list, void *joined_list_private, - int joined_list_entries) + int joined_list_entries, + struct memb_ring_id *ring_id) { int i; int j; @@ -239,7 +241,7 @@ static void totempg_confchg_fn ( app_confchg_fn (configuration_type, member_list, member_list_private, member_list_entries, left_list, left_list_private, left_list_entries, - joined_list, joined_list_private, joined_list_entries); + joined_list, joined_list_private, joined_list_entries, ring_id); } static void totempg_deliver_fn ( @@ -457,7 +459,8 @@ int totempg_initialize ( struct in_addr *left_list, void *left_list_private, int left_list_entries, struct in_addr *joined_list, void *joined_list_private, - int joined_list_entries)) + int joined_list_entries, + struct memb_ring_id *ring_id)) { int res; diff --git a/exec/totempg.h b/exec/totempg.h index cbbed573..b5fe6e3f 100644 --- a/exec/totempg.h +++ b/exec/totempg.h @@ -98,7 +98,8 @@ int totempg_initialize ( struct in_addr *left_list, void *left_list_private, int left_list_entries, struct in_addr *joined_list, void *joined_list_private, - int joined_list_entries)); + int joined_list_entries, + struct memb_ring_id *ring_id)); /* * Multicast a message diff --git a/exec/totemsrp.c b/exec/totemsrp.c index 90500138..241d1749 100644 --- a/exec/totemsrp.c +++ b/exec/totemsrp.c @@ -102,7 +102,6 @@ int totemsrp_brake; #define TIMEOUT_STATE_GATHER_CONSENSUS 200 #define TIMEOUT_TOKEN 1000 #define TIMEOUT_TOKEN_RETRANSMIT 200 -#define MAX_MEMBERS 16 #define PACKET_SIZE_MAX 2000 #define FAIL_TO_RECV_CONST 250 #define SEQNO_UNCHANGED_CONST 20 @@ -156,21 +155,21 @@ struct consensus_list_item { int set; }; -static struct consensus_list_item consensus_list[MAX_MEMBERS]; +static struct consensus_list_item consensus_list[PROCESSOR_COUNT_MAX]; static int consensus_list_entries; -static struct in_addr my_proc_list[MAX_MEMBERS]; +static struct in_addr my_proc_list[PROCESSOR_COUNT_MAX]; -static struct in_addr my_failed_list[MAX_MEMBERS]; +static struct in_addr my_failed_list[PROCESSOR_COUNT_MAX]; -static struct in_addr my_new_memb_list[MAX_MEMBERS]; +static struct in_addr my_new_memb_list[PROCESSOR_COUNT_MAX]; -static struct in_addr my_trans_memb_list[MAX_MEMBERS]; +static struct in_addr my_trans_memb_list[PROCESSOR_COUNT_MAX]; -static struct in_addr my_memb_list[MAX_MEMBERS]; +static struct in_addr my_memb_list[PROCESSOR_COUNT_MAX]; -static struct in_addr my_deliver_memb_list[MAX_MEMBERS]; +static struct in_addr my_deliver_memb_list[PROCESSOR_COUNT_MAX]; static int my_proc_list_entries = 0; @@ -301,11 +300,6 @@ struct message_header { unsigned short endian_detector; } __attribute__((packed)); -struct memb_ring_id { - struct in_addr rep; - unsigned long long seq; -} __attribute__((packed)); - struct mcast { struct message_header header; int seq; @@ -344,9 +338,9 @@ struct orf_token { struct memb_join { struct message_header header; - struct in_addr proc_list[MAX_MEMBERS]; + struct in_addr proc_list[PROCESSOR_COUNT_MAX]; int proc_list_entries; - struct in_addr failed_list[MAX_MEMBERS]; + struct in_addr failed_list[PROCESSOR_COUNT_MAX]; int failed_list_entries; unsigned long long ring_seq; } __attribute__((packed)); @@ -365,8 +359,8 @@ struct memb_commit_token { unsigned int retrans_flg; int memb_index; int addr_entries; - struct in_addr addr[MAX_MEMBERS]; - struct memb_commit_token_memb_entry memb_list[MAX_MEMBERS]; + struct in_addr addr[PROCESSOR_COUNT_MAX]; + struct memb_commit_token_memb_entry memb_list[PROCESSOR_COUNT_MAX]; }__attribute__((packed)); struct message_item { @@ -429,7 +423,8 @@ void (*totemsrp_confchg_fn) ( struct in_addr *left_list, void *left_list_private, int left_list_entries, struct in_addr *joined_list, void *joined_list_private, - int joined_list_entries) = 0; + int joined_list_entries, + struct memb_ring_id *ring_id) = 0; /* * forward decls @@ -542,7 +537,8 @@ int totemsrp_initialize ( struct in_addr *left_list, void *left_list_private, int left_list_entries, struct in_addr *joined_list, void *joined_list_private, - int joined_list_entries)) + int joined_list_entries, + struct memb_ring_id *ring_id)) { int res; @@ -696,7 +692,7 @@ static int memb_consensus_isset (struct in_addr *addr) */ static int memb_consensus_agreed (void) { - struct in_addr token_memb[MAX_MEMBERS]; + struct in_addr token_memb[PROCESSOR_COUNT_MAX]; int token_memb_entries = 0; int agreed = 1; int i; @@ -880,7 +876,7 @@ void cancel_token_retransmit_timeout (void) { static void memb_state_consensus_timeout_expired (void) { - struct in_addr no_consensus_list[MAX_MEMBERS]; + struct in_addr no_consensus_list[PROCESSOR_COUNT_MAX]; int no_consensus_list_entries; if (memb_consensus_agreed ()) { @@ -1015,9 +1011,9 @@ printf ("item not present in recovery sort queue\n"); */ static void memb_state_operational_enter (void) { - struct in_addr joined_list[MAX_MEMBERS]; + struct in_addr joined_list[PROCESSOR_COUNT_MAX]; int joined_list_entries = 0; - struct in_addr left_list[MAX_MEMBERS]; + struct in_addr left_list[PROCESSOR_COUNT_MAX]; int left_list_entries = 0; deliver_messages_from_recovery_to_regular (); @@ -1043,7 +1039,7 @@ static void memb_state_operational_enter (void) totemsrp_confchg_fn (TOTEMSRP_CONFIGURATION_TRANSITIONAL, my_trans_memb_list, 0, my_trans_memb_entries, left_list, 0, left_list_entries, - 0, 0, 0); + 0, 0, 0, &my_ring_id); // TODO we need to filter to ensure we only deliver those // messages which are part of my_deliver_memb @@ -1055,7 +1051,7 @@ static void memb_state_operational_enter (void) totemsrp_confchg_fn (TOTEMSRP_CONFIGURATION_REGULAR, my_new_memb_list, 0, my_new_memb_entries, 0, 0, 0, - joined_list, 0, joined_list_entries); + joined_list, 0, joined_list_entries, &my_ring_id); /* * Install new membership @@ -2302,7 +2298,7 @@ static int memb_state_commit_token_send (struct memb_commit_token *memb_commit_t int memb_lowest_in_config (void) { - struct in_addr token_memb[MAX_MEMBERS]; + struct in_addr token_memb[PROCESSOR_COUNT_MAX]; int token_memb_entries = 0; struct in_addr lowest_addr; int i; @@ -2326,7 +2322,7 @@ int memb_lowest_in_config (void) static void memb_state_commit_token_create (struct memb_commit_token *commit_token) { - struct in_addr token_memb[MAX_MEMBERS]; + struct in_addr token_memb[PROCESSOR_COUNT_MAX]; int token_memb_entries = 0; totemsrp_log_printf (totemsrp_log_level_notice, @@ -2349,7 +2345,7 @@ static void memb_state_commit_token_create (struct memb_commit_token *commit_tok memcpy (commit_token->addr, token_memb, token_memb_entries * sizeof (struct in_addr)); memset (commit_token->memb_list, 0, - sizeof (struct memb_commit_token_memb_entry) * MAX_MEMBERS); + sizeof (struct memb_commit_token_memb_entry) * PROCESSOR_COUNT_MAX); commit_token->memb_index = token_memb_entries - 1; commit_token->addr_entries = token_memb_entries; } @@ -3223,7 +3219,7 @@ static int message_handler_memb_commit_token ( { struct memb_commit_token memb_commit_token_convert; struct memb_commit_token *memb_commit_token; - struct in_addr sub[MAX_MEMBERS]; + struct in_addr sub[PROCESSOR_COUNT_MAX]; int sub_entries; diff --git a/exec/totemsrp.h b/exec/totemsrp.h index 2765df61..26a97798 100644 --- a/exec/totemsrp.h +++ b/exec/totemsrp.h @@ -37,12 +37,18 @@ #include "aispoll.h" #define TOTEMSRP_PACKET_SIZE_MAX 1408 +#define PROCESSOR_COUNT_MAX 16 enum totemsrp_configuration_type { TOTEMSRP_CONFIGURATION_REGULAR, TOTEMSRP_CONFIGURATION_TRANSITIONAL }; +struct memb_ring_id { + struct in_addr rep; + unsigned long long seq; +} __attribute__((packed)); + /* * This represents an interface that TOTEMSRP binds to */ @@ -93,7 +99,8 @@ int totemsrp_initialize ( struct in_addr *left_list, void *left_list_private, int left_list_entries, struct in_addr *joined_list, void *joined_list_private, - int joined_list_entries)); + int joined_list_entries, + struct memb_ring_id *ring_id)); /* * Multicast a message