From c50a6bd065f72d81a25a0d750bf49d9eea827c78 Mon Sep 17 00:00:00 2001 From: Jan Friesse Date: Mon, 27 Jul 2009 10:12:55 +0000 Subject: [PATCH] Support for monotime timer This patch should solve problems with corosync and ntp, by using clock_gettime where it make sense. git-svn-id: http://svn.fedorahosted.org/svn/corosync/trunk@2373 fd59a12c-fef9-0310-b244-a6a79926bd2f --- exec/timer.h | 2 +- exec/tlist.h | 69 +++++++++++++++++++++++++++++++++++-------- exec/totemsrp.c | 37 +++++++++++------------ services/pload.c | 20 +++++++------ services/votequorum.c | 14 +++++---- 5 files changed, 93 insertions(+), 49 deletions(-) diff --git a/exec/timer.h b/exec/timer.h index 3df9d362..7d6826ac 100644 --- a/exec/timer.h +++ b/exec/timer.h @@ -43,7 +43,7 @@ extern int corosync_timer_init ( int sched_priority); extern int corosync_timer_add_duration ( - unsigned long long nanoseconds_in_future, + unsigned long long nanosec_duration, void *data, void (*timer_fn) (void *data), corosync_timer_handle *handle); diff --git a/exec/tlist.h b/exec/tlist.h index 263c8d74..2bad3b68 100644 --- a/exec/tlist.h +++ b/exec/tlist.h @@ -54,6 +54,13 @@ typedef void * timer_handle; #define TIMER_HANDLE #endif +#define TIMERLIST_MS_IN_SEC 1000ULL +#define TIMERLIST_US_IN_SEC 1000000ULL +#define TIMERLIST_NS_IN_SEC 1000000000ULL +#define TIMERLIST_US_IN_MSEC 1000ULL +#define TIMERLIST_NS_IN_MSEC 1000000ULL +#define TIMERLIST_NS_IN_USEC 1000ULL + struct timerlist { struct list_head timer_head; struct list_head *timer_iter; @@ -61,7 +68,8 @@ struct timerlist { struct timerlist_timer { struct list_head list; - unsigned long long nano_from_epoch; + unsigned long long expire_time; + int is_absolute_timer; void (*timer_fn)(void *data); void *data; timer_handle handle_addr; @@ -78,10 +86,34 @@ static inline unsigned long long timerlist_nano_from_epoch (void) struct timeval time_from_epoch; gettimeofday (&time_from_epoch, 0); - nano_from_epoch = ((time_from_epoch.tv_sec * 1000000000ULL) + (time_from_epoch.tv_usec * 1000ULL)); + nano_from_epoch = ((time_from_epoch.tv_sec * TIMERLIST_NS_IN_SEC) + + (time_from_epoch.tv_usec * TIMERLIST_NS_IN_USEC)); + return (nano_from_epoch); } +static inline unsigned long long timerlist_nano_current_get (void) +{ + unsigned long long nano_monotonic; + struct timespec ts; + + clock_gettime (CLOCK_MONOTONIC, &ts); + + nano_monotonic = (ts.tv_sec * TIMERLIST_NS_IN_SEC) + (unsigned long long )ts.tv_nsec; + return (nano_monotonic); +} + +static inline unsigned long long timerlist_nano_monotonic_hz (void) { + unsigned long long nano_monotonic_hz; + struct timespec ts; + + clock_getres (CLOCK_MONOTONIC, &ts); + + nano_monotonic_hz = TIMERLIST_NS_IN_SEC / ((ts.tv_sec * TIMERLIST_NS_IN_SEC) + ts.tv_nsec); + + return (nano_monotonic_hz); +} + static inline void timerlist_add (struct timerlist *timerlist, struct timerlist_timer *timer) { struct list_head *timer_list = 0; @@ -95,7 +127,7 @@ static inline void timerlist_add (struct timerlist *timerlist, struct timerlist_ timer_from_list = list_entry (timer_list, struct timerlist_timer, list); - if (timer_from_list->nano_from_epoch > timer->nano_from_epoch) { + if (timer_from_list->expire_time > timer->expire_time) { list_add (&timer->list, timer_list->prev); found = 1; break; /* for timer iteration */ @@ -120,7 +152,8 @@ static inline int timerlist_add_absolute (struct timerlist *timerlist, return (-1); } - timer->nano_from_epoch = nano_from_epoch; + timer->expire_time = nano_from_epoch; + timer->is_absolute_timer = 1; timer->data = data; timer->timer_fn = timer_fn; timer->handle_addr = handle; @@ -144,7 +177,8 @@ static inline int timerlist_add_duration (struct timerlist *timerlist, return (-1); } - timer->nano_from_epoch = timerlist_nano_from_epoch() + nano_duration; + timer->expire_time = timerlist_nano_current_get () + nano_duration; + timer->is_absolute_timer = 0; timer->data = data; timer->timer_fn = timer_fn; timer->handle_addr = handle; @@ -177,7 +211,7 @@ static inline unsigned long long timerlist_expire_time (struct timerlist *timerl { struct timerlist_timer *timer = (struct timerlist_timer *)_timer_handle; - return (timer->nano_from_epoch); + return (timer->expire_time); } static inline void timerlist_pre_dispatch (struct timerlist *timerlist, timer_handle _timer_handle) @@ -202,7 +236,7 @@ static inline void timerlist_post_dispatch (struct timerlist *timerlist, timer_h static inline unsigned long long timerlist_msec_duration_to_expire (struct timerlist *timerlist) { struct timerlist_timer *timer_from_list; - volatile unsigned long long nano_from_epoch; + volatile unsigned long long current_time; volatile unsigned long long msec_duration_to_expire; /* @@ -215,16 +249,20 @@ static inline unsigned long long timerlist_msec_duration_to_expire (struct timer timer_from_list = list_entry (timerlist->timer_head.next, struct timerlist_timer, list); - nano_from_epoch = timerlist_nano_from_epoch(); + if (timer_from_list->is_absolute_timer) { + current_time = timerlist_nano_from_epoch (); + } else { + current_time = timerlist_nano_current_get (); + } /* * timer at head of list is expired, zero msecs required */ - if (timer_from_list->nano_from_epoch < nano_from_epoch) { + if (timer_from_list->expire_time < current_time) { return (0); } - msec_duration_to_expire = ((timer_from_list->nano_from_epoch - nano_from_epoch) / 1000000ULL) + + msec_duration_to_expire = ((timer_from_list->expire_time - current_time) / TIMERLIST_NS_IN_MSEC) + (1000 / HZ); return (msec_duration_to_expire); } @@ -235,9 +273,12 @@ static inline unsigned long long timerlist_msec_duration_to_expire (struct timer static inline void timerlist_expire (struct timerlist *timerlist) { struct timerlist_timer *timer_from_list; - unsigned long long nano_from_epoch; + unsigned long long current_time_from_epoch; + unsigned long long current_monotonic_time; + unsigned long long current_time; - nano_from_epoch = timerlist_nano_from_epoch(); + current_monotonic_time = timerlist_nano_current_get (); + current_time_from_epoch = current_time = timerlist_nano_from_epoch (); for (timerlist->timer_iter = timerlist->timer_head.next; timerlist->timer_iter != &timerlist->timer_head;) { @@ -245,7 +286,9 @@ static inline void timerlist_expire (struct timerlist *timerlist) timer_from_list = list_entry (timerlist->timer_iter, struct timerlist_timer, list); - if (timer_from_list->nano_from_epoch < nano_from_epoch) { + current_time = (timer_from_list->is_absolute_timer ? current_time_from_epoch : current_monotonic_time); + + if (timer_from_list->expire_time < current_time) { timerlist->timer_iter = timerlist->timer_iter->next; timerlist_pre_dispatch (timerlist, timer_from_list); diff --git a/exec/totemsrp.c b/exec/totemsrp.c index 2194411e..60190ce6 100644 --- a/exec/totemsrp.c +++ b/exec/totemsrp.c @@ -89,6 +89,7 @@ #include "wthread.h" #include "crypto.h" +#include "tlist.h" #define LOCALHOST_IP inet_addr("127.0.0.1") #define QUEUE_RTR_ITEMS_SIZE_MAX 256 /* allow 256 retransmit items */ @@ -506,7 +507,7 @@ struct totemsrp_instance { unsigned int my_cbl; - struct timeval pause_timestamp; + unsigned long long int pause_timestamp; struct memb_commit_token *commit_token; @@ -696,14 +697,12 @@ static unsigned int main_msgs_missing (void) static int pause_flush (struct totemsrp_instance *instance) { - struct timeval now; uint64_t now_msec; uint64_t timestamp_msec; int res = 0; - gettimeofday (&now, NULL); - now_msec = ((now.tv_sec * 1000ULL) + (now.tv_usec / 1000ULL)); - timestamp_msec = ((instance->pause_timestamp.tv_sec * 1000ULL) + (instance->pause_timestamp.tv_usec/1000ULL)); + now_msec = (timerlist_nano_current_get () / TIMERLIST_NS_IN_MSEC); + timestamp_msec = instance->pause_timestamp / TIMERLIST_NS_IN_MSEC; if ((now_msec - timestamp_msec) > (instance->totem_config->token_timeout / 2)) { log_printf (instance->totemsrp_log_level_notice, @@ -844,7 +843,7 @@ int totemsrp_initialize ( instance->totemsrp_confchg_fn = confchg_fn; instance->use_heartbeat = 1; - gettimeofday (&instance->pause_timestamp, NULL); + instance->pause_timestamp = timerlist_nano_current_get (); if ( totem_config->heartbeat_failures_allowed == 0 ) { log_printf (instance->totemsrp_log_level_debug, @@ -1474,7 +1473,7 @@ static void timer_function_pause_timeout (void *data) { struct totemsrp_instance *instance = data; - gettimeofday (&instance->pause_timestamp, NULL); + instance->pause_timestamp = timerlist_nano_current_get (); reset_pause_timeout (instance); } @@ -3231,7 +3230,7 @@ static void fcc_token_update ( * Message Handlers */ -struct timeval tv_old; +unsigned long long int tv_old; /* * message handler called when TOKEN message type received */ @@ -3251,17 +3250,15 @@ static int message_handler_orf_token ( unsigned int last_aru; #ifdef GIVEINFO - struct timeval tv_current; - struct timeval tv_diff; + unsigned long long tv_current; + unsigned long long tv_diff; - gettimeofday (&tv_current, NULL); - timersub (&tv_current, &tv_old, &tv_diff); - memcpy (&tv_old, &tv_current, sizeof (struct timeval)); + tv_current = timerlist_nano_current_get (); + tv_diff = tv_current - tv_old; + tv_old = tv_current; log_printf (instance->totemsrp_log_level_debug, - "Time since last token %0.4f ms\n", - (((float)tv_diff.tv_sec) * 1000) + ((float)tv_diff.tv_usec) - / 1000.0); + "Time since last token %0.4f ms\n", ((float)tv_diff) / 1000000.0); #endif #ifdef TEST_DROP_ORF_TOKEN_PERCENTAGE @@ -3504,12 +3501,12 @@ printf ("token seq %d\n", token->seq); token_send (instance, token, forward_token); #ifdef GIVEINFO - gettimeofday (&tv_current, NULL); - timersub (&tv_current, &tv_old, &tv_diff); - memcpy (&tv_old, &tv_current, sizeof (struct timeval)); + tv_current = timerlist_nano_current_get (); + tv_diff = tv_current - tv_old; + tv_old = tv_current; log_printf (instance->totemsrp_log_level_debug, "I held %0.4f ms\n", - ((float)tv_diff.tv_usec) / 1000.0); + ((float)tv_diff) / 1000000.0); #endif if (instance->memb_state == MEMB_STATE_OPERATIONAL) { messages_deliver_to_app (instance, 0, diff --git a/services/pload.c b/services/pload.c index cf82bbc3..19bc79a9 100644 --- a/services/pload.c +++ b/services/pload.c @@ -62,6 +62,8 @@ #include #include +#include "../exec/tlist.h" + LOGSYS_DECLARE_SUBSYS ("PLOAD"); enum pload_exec_message_req_types { @@ -344,9 +346,9 @@ static void message_handler_req_exec_pload_start ( } while (0) #endif -struct timeval tv1; -struct timeval tv2; -struct timeval tv_elapsed; +unsigned long long int tv1; +unsigned long long int tv2; +unsigned long long int tv_elapsed; int last_msg_no = 0; static void message_handler_req_exec_pload_mcast ( @@ -357,19 +359,19 @@ static void message_handler_req_exec_pload_mcast ( last_msg_no = pload_mcast->msg_code; if (msgs_delivered == 0) { - gettimeofday (&tv1, NULL); + tv1 = timerlist_nano_current_get (); } msgs_delivered += 1; if (msgs_delivered == msgs_wanted) { - gettimeofday (&tv2, NULL); - timersub (&tv2, &tv1, &tv_elapsed); + tv2 = timerlist_nano_current_get (); + tv_elapsed = tv2 - tv1; printf ("%5d Writes ", msgs_delivered); printf ("%5d bytes per write ", msg_size); printf ("%7.3f Seconds runtime ", - (tv_elapsed.tv_sec + (tv_elapsed.tv_usec / 1000000.0))); + (tv_elapsed / 1000000000.0)); printf ("%9.3f TP/s ", - ((float)msgs_delivered) / (tv_elapsed.tv_sec + (tv_elapsed.tv_usec / 1000000.0))); + ((float)msgs_delivered) / (tv_elapsed / 1000000000.0)); printf ("%7.3f MB/s.\n", - ((float)msgs_delivered) * ((float)msg_size) / ((tv_elapsed.tv_sec + (tv_elapsed.tv_usec / 1000000.0)) * 1000000.0)); + ((float)msgs_delivered) * ((float)msg_size) / ((tv_elapsed / 1000000000.0)) * 1000000.0); } } diff --git a/services/votequorum.c b/services/votequorum.c index 79855bf5..273d50e1 100644 --- a/services/votequorum.c +++ b/services/votequorum.c @@ -68,6 +68,8 @@ #include #include +#include "../exec/tlist.h" + #define VOTEQUORUM_MAJOR_VERSION 7 #define VOTEQUORUM_MINOR_VERSION 0 #define VOTEQUORUM_PATCH_VERSION 0 @@ -107,7 +109,7 @@ struct cluster_node { nodestate_t state; - struct timeval last_hello; /* Only used for quorum devices */ + unsigned long long int last_hello; /* Only used for quorum devices */ struct list_head list; }; @@ -1307,13 +1309,13 @@ static void message_handler_req_lib_votequorum_leaving (void *conn, const void * static void quorum_device_timer_fn(void *arg) { - struct timeval now; - ENTER(); if (!quorum_device || quorum_device->state == NODESTATE_DEAD) return; - gettimeofday(&now, NULL); - if (quorum_device->last_hello.tv_sec + quorumdev_poll/1000 < now.tv_sec) { + + if ( (quorum_device->last_hello / TIMERLIST_NS_IN_SEC) + quorumdev_poll/1000 < + (timerlist_nano_current_get () / TIMERLIST_NS_IN_SEC)) { + quorum_device->state = NODESTATE_DEAD; log_printf(LOGSYS_LEVEL_INFO, "lost contact with quorum device\n"); recalculate_quorum(0, 0); @@ -1395,7 +1397,7 @@ static void message_handler_req_lib_votequorum_qdisk_poll (void *conn, if (quorum_device) { if (req_lib_votequorum_qdisk_poll->state) { - gettimeofday(&quorum_device->last_hello, NULL); + quorum_device->last_hello = timerlist_nano_current_get (); if (quorum_device->state == NODESTATE_DEAD) { quorum_device->state = NODESTATE_MEMBER; recalculate_quorum(0, 0);