defect 1170

Fix recovery protocol to avoid assertion in memb_state_recovery_enter


git-svn-id: http://svn.fedorahosted.org/svn/corosync/trunk@967 fd59a12c-fef9-0310-b244-a6a79926bd2f
This commit is contained in:
Steven Dake 2006-04-06 07:32:21 +00:00
parent 74952a56a3
commit 29c19bf56e

View File

@ -83,10 +83,10 @@
#include "crypto.h"
#define LOCALHOST_IP inet_addr("127.0.0.1")
#define QUEUE_RTR_ITEMS_SIZE_MAX 256 /* allow 512 retransmit items */
#define QUEUE_RTR_ITEMS_SIZE_MAX 256 /* allow 256 retransmit items */
#define RETRANS_MESSAGE_QUEUE_SIZE_MAX 500 /* allow 500 messages to be queued */
#define RECEIVED_MESSAGE_QUEUE_SIZE_MAX 500 /* allow 500 messages to be queued */
#define MAXIOVS 5
#define MAXIOVS 5
#define RETRANSMIT_ENTRIES_MAX 30
#define MISSING_MCAST_WINDOW 128
@ -106,8 +106,20 @@
*/
#define SEQNO_START_MSG 0x0
#define SEQNO_START_TOKEN 0x0
//#define SEQNO_START_MSG 0xfffffe00
//#define SEQNO_START_TOKEN 0xfffffe00
/*
* These can be used ot test different rollover points
* #define SEQNO_START_MSG 0xfffffe00
* #define SEQNO_START_TOKEN 0xfffffe00
*/
/*
* These can be used to test the error recovery algorithms
* #define TEST_DROP_ORF_TOKEN_PERCENTAGE 30
* #define TEST_DROP_COMMIT_TOKEN_PERCENTAGE 30
* #define TEST_DROP_MCAST_PERCENTAGE 50
* #define TEST_RECOVERY_MSG_COUNT 300
*/
/*
* we compare incoming messages to determine if their endian is
@ -217,7 +229,7 @@ struct memb_commit_token_memb_entry {
struct memb_ring_id ring_id;
unsigned int aru;
unsigned int high_delivered;
int received_flg;
unsigned int received_flg;
}__attribute__((packed));
struct memb_commit_token {
@ -575,7 +587,7 @@ void totemsrp_instance_initialize (struct totemsrp_instance *instance)
list_init (&instance->token_callback_sent_listhead);
instance->my_received_flg = 1;
instance->my_received_flg = 0;
instance->my_token_seq = SEQNO_START_TOKEN - 1;
@ -1093,7 +1105,7 @@ static void ring_state_restore (struct totemsrp_instance *instance)
totemip_zero_set(&instance->my_ring_id.rep);
instance->my_aru = instance->old_ring_state_aru;
instance->my_high_seq_received = instance->old_ring_state_high_seq_received;
instance->totemsrp_log_printf (instance->totemsrp_log_level_debug,
instance->totemsrp_log_printf (instance->totemsrp_log_level_notice,
"Restoring instance->my_aru %x my high seq received %x\n",
instance->my_aru, instance->my_high_seq_received);
}
@ -1425,6 +1437,7 @@ static void memb_state_operational_enter (struct totemsrp_instance *instance)
"entering OPERATIONAL state.\n");
instance->memb_state = MEMB_STATE_OPERATIONAL;
instance->my_received_flg = 0;
return;
}
@ -1522,9 +1535,7 @@ static void memb_state_recovery_enter (
struct memb_commit_token *commit_token)
{
int i;
#ifdef COMPILE_OUT
int local_received_flg = 1;
#endif
unsigned int low_ring_aru;
unsigned int range = 0;
unsigned int messages_originated = 0;
@ -1532,6 +1543,9 @@ static void memb_state_recovery_enter (
char not_originated[4096];
char seqno_string_hex[10];
instance->totemsrp_log_printf (instance->totemsrp_log_level_notice,
"entering RECOVERY state.\n");
instance->my_high_ring_delivered = 0;
sq_reinit (&instance->recovery_sort_queue, SEQNO_START_MSG);
@ -1581,23 +1595,19 @@ static void memb_state_recovery_enter (
/*
* Determine if any received flag is false
*/
#ifdef COMPILE_OUT
for (i = 0; i < commit_token->addr_entries; i++) {
if (memb_set_subset (&instance->my_new_memb_list[i], 1,
instance->my_trans_memb_list, instance->my_trans_memb_entries) &&
commit_token->memb_list[i].received_flg == 0) {
#endif
instance->my_deliver_memb_entries = instance->my_trans_memb_entries;
memcpy (instance->my_deliver_memb_list, instance->my_trans_memb_list,
sizeof (struct totem_ip_address) * instance->my_trans_memb_entries);
#ifdef COMPILE_OUT
local_received_flg = 0;
break;
}
}
#endif
// if (local_received_flg == 0) {
if (local_received_flg == 0) {
/*
* Calculate my_low_ring_aru, instance->my_high_ring_delivered for the transitional membership
*/
@ -1672,7 +1682,10 @@ static void memb_state_recovery_enter (
strcat (is_originated, "\n");
instance->totemsrp_log_printf (instance->totemsrp_log_level_notice, is_originated);
instance->totemsrp_log_printf (instance->totemsrp_log_level_notice, not_originated);
// }
} else {
instance->totemsrp_log_printf (instance->totemsrp_log_level_notice,
"Did not need to originate any messages in recovery.\n");
}
instance->my_aru = SEQNO_START_MSG;
instance->my_aru_count = 0;
@ -1958,18 +1971,6 @@ static void update_aru (
}
}
instance->my_aru += i - 1;
// instance->totemsrp_log_printf (instance->totemsrp_log_level_debug,
// "setting received flag to FALSE %d %d\n",
// instance->my_aru, instance->my_high_seq_received);
instance->my_received_flg = 0;
if (instance->my_aru == instance->my_high_seq_received) {
// instance->totemsrp_log_printf (instance->totemsrp_log_level_debug,
// "setting received flag to TRUE %d %d\n",
// instance->my_aru, instance->my_high_seq_received);
instance->my_received_flg = 1;
}
}
/*
@ -2091,6 +2092,7 @@ static int orf_token_rtr (
}
rtr_list = &orf_token->rtr_list[0];
strcpy (retransmit_msg, "Retransmit List: ");
if (orf_token->rtr_list_entries) {
instance->totemsrp_log_printf (instance->totemsrp_log_level_debug,
@ -2783,8 +2785,8 @@ static int message_handler_orf_token (
}
#endif
#ifdef RANDOM_DROP
if (random()%100 < 10) {
#ifdef TEST_DROP_ORF_TOKEN_PERCENTAGE
if (random()%100 < TEST_DROP_ORF_TOKEN_PERCENTAGE) {
return (0);
}
#endif
@ -2819,6 +2821,12 @@ static int message_handler_orf_token (
instance->my_last_seq = token->seq;
#ifdef TEST_RECOVERY_MSG_COUNT
if (instance->memb_state == MEMB_STATE_OPERATIONAL && token->seq > TEST_RECOVERY_MSG_COUNT) {
return (0);
}
#endif
totemrrp_recv_flush (instance->totemrrp_handle);
/*
@ -2936,6 +2944,7 @@ static int message_handler_orf_token (
instance->totemsrp_log_printf (instance->totemsrp_log_level_error,
"FAILED TO RECEIVE\n");
exit (1);
// TODO if we fail to receive, it may be possible to end with a gather
// state of proc == failed = 0 entries
memb_set_merge (&token->aru_addr, 1,
@ -3164,14 +3173,6 @@ static void messages_deliver_to_app (
}
//TODO instance->stats_delv += 1;
}
instance->my_received_flg = 0;
if (instance->my_aru == instance->my_high_seq_received) {
// instance->totemsrp_log_printf (instance->totemsrp_log_level_debug,
// "setting received flag to TRUE %d %d\n",
// instance->my_aru, instance->my_high_seq_received);
instance->my_received_flg = 1;
}
}
/*
@ -3208,11 +3209,16 @@ static int message_handler_mcast (
sort_queue = &instance->regular_sort_queue;
}
assert (msg_len < FRAME_SIZE_MAX);
#ifdef RANDOM_DROP
if (random()%100 < 50) {
return (0);
}
#ifdef TEST_DROP_MCAST_PERCENTAGE
if (random()%100 < TEST_DROP_MCAST_PERCENTAGE) {
printf ("dropping message %d\n", mcast_header.seq);
return (0);
} else {
printf ("accepting message %d\n", mcast_header.seq);
}
#endif
if (!totemip_equal(system_from, &instance->my_id)) {
cancel_token_retransmit_timeout (instance);
}
@ -3291,8 +3297,8 @@ if (random()%100 < 50) {
sq_item_add (sort_queue, &sort_queue_item, mcast_header.seq);
}
update_aru (instance);
if (instance->memb_state == MEMB_STATE_OPERATIONAL) {
update_aru (instance);
messages_deliver_to_app (instance, 0, instance->my_high_seq_received);
}
@ -3622,8 +3628,9 @@ static int message_handler_memb_commit_token (
return (0);
}
*/
#ifdef RANDOM_DROP
if (random()%100 < 10) {
#ifdef TEST_DROP_COMMIT_TOKEN_PERCENTAGE
if (random()%100 < TEST_DROP_COMMIT_TOKEN_PERCENTAGE) {
return (0);
}
#endif