[lib] Bug #134: threads should be more robust against backward time jumps

2006-08-25 Paul Jakma <paul.jakma@sun.com>

	* thread.c: (general) Add support for monotonic clock, it may still
	  jump forward by huge amounts, but should be immune to going
	  backwards. Fixes bug #134.
	  (quagga_gettimeofday_relative_adjust) helper, does what name
	  says - adjusts gettimeofday based relative timer.
	  (quagga_gettimeofday) helper to keep recent_time up to date.
	  (quagga_get_relative) helper, update and getch the relative
	  timer using gettimeofday(). POSIX CLOCK_MONOTONIC is also
	  supported, but the code is not enabled yet nor tested.
	  (quagga_real_stabilised) helper, retrieve absolute time but
	  stabilised so as to never decrease.
	  (quagga_gettime) Exported interface, analogous to POSIX
	  clock_gettime() in interface, supporting several clocks.
	  (quagga_time) Exported interface, analogous to traditional
	  time(), will never decrease.
	  (recent_relative_time) Convenience function to retrieve
	  relative_time timeval, similar to existing recent_time absolute
	  timeval, for when an approximately recent value will do.
	  (remainder) Update to use above helpers.
	  (thread_getrusage) Previously was a macro, but needs to be
	  a function to twiddle with thread.c private stuff.
	* thread.c: Point the GETRUSAGE macro at previous function.
	  Export quagga_gettime, quagga_time and recent_relative_time for
	  general use.

lib/ChangeLog

@@ -1,3 +1,30 @@
+2006-08-25 Paul Jakma <paul.jakma@sun.com>
+
+	* thread.c: (general) Add support for monotonic clock, it may still
+	  jump forward by huge amounts, but should be immune to going
+	  backwards. Fixes bug #134.
+	  (quagga_gettimeofday_relative_adjust) helper, does what name
+	  says - adjusts gettimeofday based relative timer.
+	  (quagga_gettimeofday) helper to keep recent_time up to date.
+	  (quagga_get_relative) helper, update and getch the relative
+	  timer using gettimeofday(). POSIX CLOCK_MONOTONIC is also
+	  supported, but the code is not enabled yet nor tested.
+	  (quagga_real_stabilised) helper, retrieve absolute time but
+	  stabilised so as to never decrease.
+	  (quagga_gettime) Exported interface, analogous to POSIX
+	  clock_gettime() in interface, supporting several clocks.
+	  (quagga_time) Exported interface, analogous to traditional
+	  time(), will never decrease.
+	  (recent_relative_time) Convenience function to retrieve
+	  relative_time timeval, similar to existing recent_time absolute
+	  timeval, for when an approximately recent value will do.
+	  (remainder) Update to use above helpers.
+	  (thread_getrusage) Previously was a macro, but needs to be
+	  a function to twiddle with thread.c private stuff.
+	* thread.c: Point the GETRUSAGE macro at previous function.
+	  Export quagga_gettime, quagga_time and recent_relative_time for 
+	  general use.
+
 2006-07-25 Paul Jakma <paul.jakma@sun.com>
 
 	* thread.h: (struct thread) Add a cache pointer to the struct

lib/thread.c

@@ -30,7 +30,15 @@
 #include "command.h"
 #include "sigevent.h"
 
+/* Recent absolute time of day */
 struct timeval recent_time;
+static struct timeval last_recent_time;
+/* Relative time, since startup */
+static struct timeval relative_time;
+static struct timeval relative_time_base;
+/* init flag */
+static unsigned short timers_inited;
+
 static struct hash *cpu_record = NULL;
 
 /* Struct timeval's tv_usec one second value.  */
@@ -85,6 +93,129 @@ timeval_elapsed (struct timeval a, struct timeval b)
 	  + (a.tv_usec - b.tv_usec));
 }
 
+#ifndef HAVE_CLOCK_MONOTONIC
+static void
+quagga_gettimeofday_relative_adjust (void)
+{
+  struct timeval diff;
+  if (timeval_cmp (recent_time, last_recent_time) < 0)
+    {
+      relative_time.tv_sec++;
+      relative_time.tv_usec = 0;
+    }
+  else
+    {
+      diff = timeval_subtract (recent_time, last_recent_time);
+      relative_time.tv_sec += diff.tv_sec;
+      relative_time.tv_usec += diff.tv_usec;
+      relative_time = timeval_adjust (relative_time);
+    }
+  last_recent_time = recent_time;
+}
+#endif /* !HAVE_CLOCK_MONOTONIC */
+
+/* gettimeofday wrapper, to keep recent_time updated */
+static int
+quagga_gettimeofday (struct timeval *tv)
+{
+  int ret;
+  
+  assert (tv);
+  
+  if (!(ret = gettimeofday (&recent_time, NULL)))
+    {
+      /* init... */
+      if (!timers_inited)
+        {
+          relative_time_base = last_recent_time = recent_time;
+          timers_inited = 1;
+        }
+      /* avoid copy if user passed recent_time pointer.. */
+      if (tv != &recent_time)
+        *tv = recent_time;
+      return 0;
+    }
+  return ret;
+}
+
+static int
+quagga_get_relative (struct timeval *tv)
+{
+  int ret;
+
+#ifdef HAVE_CLOCK_MONOTONIC
+  {
+    struct timespec tp;
+    if (!(ret = clock_gettime (CLOCK_MONOTONIC, &tp)))
+      {
+        relative_time.tv_sec = tp.tv_sec;
+        relative_time.tv_usec = tp.tv_nsec / 1000;
+      }
+  }
+#else /* !HAVE_CLOCK_MONOTONIC */
+  if (!(ret = quagga_gettimeofday (&recent_time)))
+    quagga_gettimeofday_relative_adjust();
+#endif /* HAVE_CLOCK_MONOTONIC */
+
+  if (tv)
+    *tv = relative_time;
+
+  return ret;
+}
+
+/* Get absolute time stamp, but in terms of the internal timer
+ * Could be wrong, but at least won't go back.
+ */
+static void
+quagga_real_stabilised (struct timeval *tv)
+{
+  *tv = relative_time_base;
+  tv->tv_sec += relative_time.tv_sec;
+  tv->tv_usec += relative_time.tv_usec;
+  *tv = timeval_adjust (*tv);
+}
+
+/* Exported Quagga timestamp function.
+ * Modelled on POSIX clock_gettime.
+ */
+int
+quagga_gettime (enum quagga_clkid clkid, struct timeval *tv)
+{
+  switch (clkid)
+    {
+      case QUAGGA_CLK_REALTIME:
+        return quagga_gettimeofday (tv);
+      case QUAGGA_CLK_MONOTONIC:
+        return quagga_get_relative (tv);
+      case QUAGGA_CLK_REALTIME_STABILISED:
+        quagga_real_stabilised (tv);
+        return 0;
+      default:
+        errno = EINVAL;
+        return -1;
+    }
+}
+
+/* time_t value in terms of stabilised absolute time. 
+ * replacement for POSIX time()
+ */
+time_t
+quagga_time (time_t *t)
+{
+  struct timeval tv;
+  quagga_real_stabilised (&tv);
+  if (t)
+    *t = tv.tv_sec;
+  return tv.tv_sec;
+}
+
+/* Public export of recent_relative_time by value */
+struct timeval
+recent_relative_time (void)
+{
+  return relative_time;
+}
+
 static unsigned int
 cpu_record_hash_key (struct cpu_thread_history *a)
 {
@@ -396,10 +527,10 @@ thread_trim_head (struct thread_list *list)
 unsigned long
 thread_timer_remain_second (struct thread *thread)
 {
-  gettimeofday (&recent_time, NULL);
+  quagga_get_relative (NULL);
-
+  
-  if (thread->u.sands.tv_sec - recent_time.tv_sec > 0)
+  if (thread->u.sands.tv_sec - relative_time.tv_sec > 0)
-    return thread->u.sands.tv_sec - recent_time.tv_sec;
+    return thread->u.sands.tv_sec - relative_time.tv_sec;
   else
     return 0;
 }
@@ -515,8 +646,8 @@ funcname_thread_add_timer_timeval (struct thread_master *m,
                                   const char* funcname)
 {
   struct thread *thread;
-  struct timeval alarm_time;
   struct thread_list *list;
+  struct timeval alarm_time;
   struct thread *tt;
 
   assert (m != NULL);
@@ -528,9 +659,9 @@ funcname_thread_add_timer_timeval (struct thread_master *m,
   thread = thread_get (m, type, func, arg, funcname);
 
   /* Do we need jitter here? */
-  gettimeofday (&recent_time, NULL);
+  quagga_gettimeofday (&recent_time);
-  alarm_time.tv_sec = recent_time.tv_sec + time_relative->tv_sec;
+  alarm_time.tv_sec = relative_time.tv_sec + time_relative->tv_sec;
-  alarm_time.tv_usec = recent_time.tv_usec + time_relative->tv_usec;
+  alarm_time.tv_usec = relative_time.tv_usec + time_relative->tv_usec;
   thread->u.sands = timeval_adjust(alarm_time);
 
   /* Sort by timeval. */
@@ -693,7 +824,7 @@ thread_timer_wait (struct thread_list *tlist, struct timeval *timer_val)
 {
   if (!thread_empty (tlist))
     {
-      *timer_val = timeval_subtract (tlist->head->u.sands, recent_time);
+      *timer_val = timeval_subtract (tlist->head->u.sands, relative_time);
       return timer_val;
     }
   return NULL;
@@ -790,7 +921,7 @@ thread_fetch (struct thread_master *m, struct thread *fetch)
       exceptfd = m->exceptfd;
       
       /* Calculate select wait timer if nothing else to do */
-      gettimeofday (&recent_time, NULL);
+      quagga_get_relative (NULL);
       timer_wait = thread_timer_wait (&m->timer, &timer_val);
       timer_wait_bg = thread_timer_wait (&m->background, &timer_val_bg);
       
@@ -812,8 +943,8 @@ thread_fetch (struct thread_master *m, struct thread *fetch)
       /* Check foreground timers.  Historically, they have had higher
          priority than I/O threads, so let's push them onto the ready
 	 list in front of the I/O threads. */
-      gettimeofday (&recent_time, NULL);
+      quagga_get_relative (NULL);
-      thread_timer_process (&m->timer, &recent_time);
+      thread_timer_process (&m->timer, &relative_time);
       
       /* Got IO, process it */
       if (num > 0)
@@ -834,7 +965,7 @@ thread_fetch (struct thread_master *m, struct thread *fetch)
 #endif
 
       /* Background timer/events, lowest priority */
-      thread_timer_process (&m->background, &recent_time);
+      thread_timer_process (&m->background, &relative_time);
       
       if ((thread = thread_trim_head (&m->ready)) != NULL)
         return thread_run (m, thread, fetch);
@@ -866,11 +997,29 @@ thread_consumed_time (RUSAGE_T *now, RUSAGE_T *start, unsigned long *cputime)
 int
 thread_should_yield (struct thread *thread)
 {
-  gettimeofday(&recent_time, NULL);
+  quagga_get_relative (NULL);
-  return (timeval_elapsed(recent_time, thread->ru.real) >
+  return (timeval_elapsed(relative_time, thread->ru.real) >
   	  THREAD_YIELD_TIME_SLOT);
 }
 
+void
+thread_getrusage (RUSAGE_T *r)
+{
+  quagga_get_relative (NULL);
+#ifdef HAVE_RUSAGE
+  getrusage(RUSAGE_SELF, &(r->cpu));
+#endif
+  r->real = relative_time;
+
+#ifdef HAVE_CLOCK_MONOTONIC
+  /* quagga_get_relative() only updates recent_time if gettimeofday
+   * based, not when using CLOCK_MONOTONIC. As we export recent_time
+   * and guarantee to update it before threads are run...
+   */
+  quagga_gettimeofday(&recent_time);
+#endif /* HAVE_CLOCK_MONOTONIC */
+}
+
 /* We check thread consumed time. If the system has getrusage, we'll
    use that to get in-depth stats on the performance of the thread in addition
    to wall clock time stats from gettimeofday. */

lib/thread.h

@@ -31,14 +31,7 @@ struct rusage_t
 };
 #define RUSAGE_T        struct rusage_t
 
-#ifdef HAVE_RUSAGE
+#define GETRUSAGE(X) thread_getrusage(X)
-#define GETRUSAGE(X)	\
-  getrusage(RUSAGE_SELF, &((X)->cpu));	\
-  gettimeofday(&recent_time, NULL); (X)->real = recent_time
-#else
-#define GETRUSAGE(X)	\
-  gettimeofday(&recent_time, NULL); (X)->real = recent_time
-#endif /* HAVE_RUSAGE */
 
 /* Linked list of thread. */
 struct thread_list
@@ -99,6 +92,13 @@ struct cpu_thread_history
   unsigned char types;
 };
 
+/* Clocks supported by Quagga */
+enum quagga_clkid {
+  QUAGGA_CLK_REALTIME = 0,	/* ala gettimeofday() */
+  QUAGGA_CLK_MONOTONIC,		/* monotonic, against an indeterminate base */
+  QUAGGA_CLK_REALTIME_STABILISED, /* like realtime, but non-decrementing */
+};
+
 /* Thread types. */
 #define THREAD_READ           0
 #define THREAD_WRITE          1
@@ -192,8 +192,17 @@ extern void thread_call (struct thread *);
 extern unsigned long thread_timer_remain_second (struct thread *);
 extern int thread_should_yield (struct thread *);
 
+/* Internal libzebra exports */
+extern void thread_getrusage (RUSAGE_T *);
 extern struct cmd_element show_thread_cpu_cmd;
 
+/* replacements for the system gettimeofday(), clock_gettime() and
+ * time() functions, providing support for non-decrementing clock on
+ * all systems, and fully monotonic on /some/ systems.
+ */
+extern int quagga_gettime (enum quagga_clkid, struct timeval *);
+extern time_t quagga_time (time_t *);
+
 /* Returns elapsed real (wall clock) time. */
 extern unsigned long thread_consumed_time(RUSAGE_T *after, RUSAGE_T *before,
 					  unsigned long *cpu_time_elapsed);
@@ -202,5 +211,6 @@ extern unsigned long thread_consumed_time(RUSAGE_T *after, RUSAGE_T *before,
    be used instead of calling gettimeofday if a recent value is sufficient.
    This is guaranteed to be refreshed before a thread is called. */
 extern struct timeval recent_time;
-
+/* Similar to recent_time, but a monotonically increasing time value */
+extern struct timeval recent_relative_time (void);
 #endif /* _ZEBRA_THREAD_H */