diff --git a/lib/frr_pthread.c b/lib/frr_pthread.c index de522e5ef9..72b47ae5c3 100644 --- a/lib/frr_pthread.c +++ b/lib/frr_pthread.c @@ -1,5 +1,5 @@ /* - * Utilities and interfaces for managing POSIX threads + * Utilities and interfaces for managing POSIX threads within FRR. * Copyright (C) 2017 Cumulus Networks * * This program is free software; you can redistribute it and/or modify @@ -25,79 +25,99 @@ #include "memory.h" #include "hash.h" -DEFINE_MTYPE_STATIC(LIB, FRR_PTHREAD, "FRR POSIX Thread"); +DEFINE_MTYPE(LIB, FRR_PTHREAD, "FRR POSIX Thread"); DEFINE_MTYPE(LIB, PTHREAD_PRIM, "POSIX synchronization primitives"); +/* id for next created pthread */ static unsigned int next_id = 0; -/* Hash table of all frr_pthreads along with synchronization primitive(s) and - * hash table callbacks. - * ------------------------------------------------------------------------ */ -static struct hash *pthread_table; -static pthread_mutex_t pthread_table_mtx = PTHREAD_MUTEX_INITIALIZER; +/* default frr_pthread start/stop routine prototypes */ +static void *fpt_run(void *arg); +static int fpt_halt(struct frr_pthread *fpt, void **res); -/* pthread_table->hash_cmp */ -static int pthread_table_hash_cmp(const void *value1, const void *value2) +/* default frr_pthread attributes */ +struct frr_pthread_attr frr_pthread_attr_default = { + .id = 0, + .start = fpt_run, + .stop = fpt_halt, + .name = "Anonymous", +}; + +/* hash table to keep track of all frr_pthreads */ +static struct hash *frr_pthread_hash; +static pthread_mutex_t frr_pthread_hash_mtx = PTHREAD_MUTEX_INITIALIZER; + +/* frr_pthread_hash->hash_cmp */ +static int frr_pthread_hash_cmp(const void *value1, const void *value2) { const struct frr_pthread *tq1 = value1; const struct frr_pthread *tq2 = value2; - return (tq1->id == tq2->id); + return (tq1->attr.id == tq2->attr.id); } -/* pthread_table->hash_key */ -static unsigned int pthread_table_hash_key(void *value) +/* frr_pthread_hash->hash_key */ +static unsigned int frr_pthread_hash_key(void *value) { - return ((struct frr_pthread *)value)->id; + return ((struct frr_pthread *)value)->attr.id; } + /* ------------------------------------------------------------------------ */ void frr_pthread_init() { - pthread_mutex_lock(&pthread_table_mtx); + pthread_mutex_lock(&frr_pthread_hash_mtx); { - pthread_table = hash_create(pthread_table_hash_key, - pthread_table_hash_cmp, NULL); + frr_pthread_hash = hash_create(frr_pthread_hash_key, + frr_pthread_hash_cmp, NULL); } - pthread_mutex_unlock(&pthread_table_mtx); + pthread_mutex_unlock(&frr_pthread_hash_mtx); } void frr_pthread_finish() { - pthread_mutex_lock(&pthread_table_mtx); + pthread_mutex_lock(&frr_pthread_hash_mtx); { - hash_clean(pthread_table, + hash_clean(frr_pthread_hash, (void (*)(void *))frr_pthread_destroy); - hash_free(pthread_table); + hash_free(frr_pthread_hash); } - pthread_mutex_unlock(&pthread_table_mtx); + pthread_mutex_unlock(&frr_pthread_hash_mtx); } -struct frr_pthread *frr_pthread_new(const char *name, unsigned int id, - void *(*start_routine)(void *), - int (*stop_routine)(void **, - struct frr_pthread *)) +struct frr_pthread *frr_pthread_new(struct frr_pthread_attr *attr) { static struct frr_pthread holder = {0}; struct frr_pthread *fpt = NULL; - pthread_mutex_lock(&pthread_table_mtx); + attr = attr ? attr : &frr_pthread_attr_default; + + pthread_mutex_lock(&frr_pthread_hash_mtx); { - holder.id = id; + holder.attr.id = attr->id; - if (!hash_lookup(pthread_table, &holder)) { - struct frr_pthread *fpt = XCALLOC( - MTYPE_FRR_PTHREAD, sizeof(struct frr_pthread)); - fpt->id = id; - fpt->master = thread_master_create(name); - fpt->start_routine = start_routine; - fpt->stop_routine = stop_routine; - fpt->name = XSTRDUP(MTYPE_FRR_PTHREAD, name); + if (!hash_lookup(frr_pthread_hash, &holder)) { + fpt = XCALLOC(MTYPE_FRR_PTHREAD, + sizeof(struct frr_pthread)); + /* create new thread master */ + fpt->master = thread_master_create(attr->name); + /* set attributes */ + fpt->attr = *attr; + if (attr == &frr_pthread_attr_default) + fpt->attr.id = frr_pthread_get_id(); + /* initialize startup synchronization primitives */ + fpt->running_cond_mtx = XCALLOC( + MTYPE_PTHREAD_PRIM, sizeof(pthread_mutex_t)); + fpt->running_cond = XCALLOC(MTYPE_PTHREAD_PRIM, + sizeof(pthread_cond_t)); + pthread_mutex_init(fpt->running_cond_mtx, NULL); + pthread_cond_init(fpt->running_cond, NULL); - hash_get(pthread_table, fpt, hash_alloc_intern); + /* insert into global thread hash */ + hash_get(frr_pthread_hash, fpt, hash_alloc_intern); } } - pthread_mutex_unlock(&pthread_table_mtx); + pthread_mutex_unlock(&frr_pthread_hash_mtx); return fpt; } @@ -105,7 +125,11 @@ struct frr_pthread *frr_pthread_new(const char *name, unsigned int id, void frr_pthread_destroy(struct frr_pthread *fpt) { thread_master_free(fpt->master); - XFREE(MTYPE_FRR_PTHREAD, fpt->name); + + pthread_mutex_destroy(fpt->running_cond_mtx); + pthread_cond_destroy(fpt->running_cond); + XFREE(MTYPE_PTHREAD_PRIM, fpt->running_cond_mtx); + XFREE(MTYPE_PTHREAD_PRIM, fpt->running_cond); XFREE(MTYPE_FRR_PTHREAD, fpt); } @@ -114,74 +138,82 @@ struct frr_pthread *frr_pthread_get(unsigned int id) static struct frr_pthread holder = {0}; struct frr_pthread *fpt; - pthread_mutex_lock(&pthread_table_mtx); + pthread_mutex_lock(&frr_pthread_hash_mtx); { - holder.id = id; - fpt = hash_lookup(pthread_table, &holder); + holder.attr.id = id; + fpt = hash_lookup(frr_pthread_hash, &holder); } - pthread_mutex_unlock(&pthread_table_mtx); + pthread_mutex_unlock(&frr_pthread_hash_mtx); return fpt; } -int frr_pthread_run(unsigned int id, const pthread_attr_t *attr, void *arg) +int frr_pthread_run(struct frr_pthread *fpt, const pthread_attr_t *attr) { - struct frr_pthread *fpt = frr_pthread_get(id); int ret; - if (!fpt) - return -1; + ret = pthread_create(&fpt->thread, attr, fpt->attr.start, fpt); - ret = pthread_create(&fpt->thread, attr, fpt->start_routine, arg); - - /* Per pthread_create(3), the contents of fpt->thread are undefined if - * pthread_create() did not succeed. Reset this value to zero. */ + /* + * Per pthread_create(3), the contents of fpt->thread are undefined if + * pthread_create() did not succeed. Reset this value to zero. + */ if (ret < 0) memset(&fpt->thread, 0x00, sizeof(fpt->thread)); return ret; } -/** - * Calls the stop routine for the frr_pthread and resets any relevant fields. - * - * @param fpt - the frr_pthread to stop - * @param result - pointer to result pointer - * @return the return code from the stop routine - */ -static int frr_pthread_stop_actual(struct frr_pthread *fpt, void **result) +void frr_pthread_wait_running(struct frr_pthread *fpt) { - int ret = (*fpt->stop_routine)(result, fpt); + pthread_mutex_lock(fpt->running_cond_mtx); + { + while (!fpt->running) + pthread_cond_wait(fpt->running_cond, + fpt->running_cond_mtx); + } + pthread_mutex_unlock(fpt->running_cond_mtx); +} + +void frr_pthread_notify_running(struct frr_pthread *fpt) +{ + pthread_mutex_lock(fpt->running_cond_mtx); + { + fpt->running = true; + pthread_cond_signal(fpt->running_cond); + } + pthread_mutex_unlock(fpt->running_cond_mtx); +} + +int frr_pthread_stop(struct frr_pthread *fpt, void **result) +{ + int ret = (*fpt->attr.stop)(fpt, result); memset(&fpt->thread, 0x00, sizeof(fpt->thread)); return ret; } -int frr_pthread_stop(unsigned int id, void **result) -{ - struct frr_pthread *fpt = frr_pthread_get(id); - return frr_pthread_stop_actual(fpt, result); -} - -/** +/* * Callback for hash_iterate to stop all frr_pthread's. */ static void frr_pthread_stop_all_iter(struct hash_backet *hb, void *arg) { struct frr_pthread *fpt = hb->data; - frr_pthread_stop_actual(fpt, NULL); + frr_pthread_stop(fpt, NULL); } void frr_pthread_stop_all() { - pthread_mutex_lock(&pthread_table_mtx); + pthread_mutex_lock(&frr_pthread_hash_mtx); { - hash_iterate(pthread_table, frr_pthread_stop_all_iter, NULL); + hash_iterate(frr_pthread_hash, frr_pthread_stop_all_iter, NULL); } - pthread_mutex_unlock(&pthread_table_mtx); + pthread_mutex_unlock(&frr_pthread_hash_mtx); } unsigned int frr_pthread_get_id() { + /* just a sanity check, this should never happen */ + assert(next_id <= INT_MAX - 1); return next_id++; } @@ -189,3 +221,60 @@ void frr_pthread_yield(void) { (void)sched_yield(); } + +/* + * ---------------------------------------------------------------------------- + * Default Event Loop + * ---------------------------------------------------------------------------- + */ + +/* dummy task for sleeper pipe */ +static int fpt_dummy(struct thread *thread) +{ + return 0; +} + +/* poison pill task to end event loop */ +static int fpt_finish(struct thread *thread) +{ + struct frr_pthread *fpt = THREAD_ARG(thread); + atomic_store_explicit(&fpt->running, false, memory_order_relaxed); + return 0; +} + +/* stop function, called from other threads to halt this one */ +static int fpt_halt(struct frr_pthread *fpt, void **res) +{ + thread_add_event(fpt->master, &fpt_finish, fpt, 0, NULL); + pthread_join(fpt->thread, res); + fpt = NULL; + + return 0; +} + +/* entry pthread function & main event loop */ +static void *fpt_run(void *arg) +{ + struct frr_pthread *fpt = arg; + fpt->master->owner = pthread_self(); + + int sleeper[2]; + pipe(sleeper); + thread_add_read(fpt->master, &fpt_dummy, NULL, sleeper[0], NULL); + + fpt->master->handle_signals = false; + + frr_pthread_notify_running(fpt); + + struct thread task; + while (atomic_load_explicit(&fpt->running, memory_order_relaxed)) { + if (thread_fetch(fpt->master, &task)) { + thread_call(&task); + } + } + + close(sleeper[1]); + close(sleeper[0]); + + return NULL; +} diff --git a/lib/frr_pthread.h b/lib/frr_pthread.h index 7915b43a46..2cc50196a8 100644 --- a/lib/frr_pthread.h +++ b/lib/frr_pthread.h @@ -1,5 +1,5 @@ /* - * Utilities and interfaces for managing POSIX threads + * Utilities and interfaces for managing POSIX threads within FRR. * Copyright (C) 2017 Cumulus Networks * * This program is free software; you can redistribute it and/or modify @@ -21,39 +21,73 @@ #define _FRR_PTHREAD_H #include +#include "frratomic.h" #include "memory.h" #include "thread.h" +DECLARE_MTYPE(FRR_PTHREAD); DECLARE_MTYPE(PTHREAD_PRIM); +struct frr_pthread; +struct frr_pthread_attr; + +struct frr_pthread_attr { + int id; + void *(*start)(void *); + int (*stop)(struct frr_pthread *, void **); + const char *name; +}; + struct frr_pthread { /* pthread id */ pthread_t thread; - /* frr thread identifier */ - unsigned int id; - /* thread master for this pthread's thread.c event loop */ struct thread_master *master; - /* start routine */ - void *(*start_routine)(void *); + /* caller-specified data; start & stop funcs, name, id */ + struct frr_pthread_attr attr; - /* stop routine */ - int (*stop_routine)(void **, struct frr_pthread *); + /* + * Notification mechanism for allowing pthreads to notify their parents + * when they are ready to do work. This mechanism has two associated + * functions: + * + * - frr_pthread_wait_running() + * This function should be called by the spawning thread after + * frr_pthread_run(). It safely waits until the spawned thread + * indicates that is ready to do work by posting to the condition + * variable. + * + * - frr_pthread_notify_running() + * This function should be called by the spawned thread when it is + * ready to do work. It will wake up any threads waiting on the + * previously described condition. + */ + pthread_cond_t *running_cond; + pthread_mutex_t *running_cond_mtx; + _Atomic bool running; - /* the (hopefully descriptive) name of this thread */ - char *name; + /* + * Fake thread-specific storage. No constraints on usage. Helpful when + * creating reentrant pthread implementations. Can be used to pass + * argument to pthread entry function. + */ + void *data; }; -/* Initializes this module. +extern struct frr_pthread_attr frr_pthread_attr_default; + +/* + * Initializes this module. * * Must be called before using any of the other functions. */ void frr_pthread_init(void); -/* Uninitializes this module. +/* + * Uninitializes this module. * * Destroys all registered frr_pthread's and internal data structures. * @@ -62,34 +96,23 @@ void frr_pthread_init(void); */ void frr_pthread_finish(void); -/* Creates a new frr_pthread. +/* + * Creates a new frr_pthread with the given attributes. * - * If the provided ID is already assigned to an existing frr_pthread, the - * return value will be NULL. - * - * @param name - the name of the thread. Doesn't have to be unique, but it - * probably should be. This value is copied and may be safely free'd upon - * return. - * - * @param id - the integral ID of the thread. MUST be unique. The caller may - * use this id to retrieve the thread. - * - * @param start_routine - start routine for the pthread, will be passed to - * pthread_create (see those docs for details) - * - * @param stop_routine - stop routine for the pthread, called to terminate the - * thread. This function should gracefully stop the pthread and clean up any - * thread-specific resources. The passed pointer is used to return a data - * result. + * The 'attr' argument should be filled out with the desired attributes, + * including ID, start and stop functions and the desired name. Alternatively, + * if attr is NULL, the default attributes will be used. The pthread will be + * set up to run a basic threadmaster loop and the name will be "Anonymous". + * Scheduling tasks onto the threadmaster in the 'master' field of the returned + * frr_pthread will cause them to run on that pthread. * + * @param attr - the thread attributes * @return the created frr_pthread upon success, or NULL upon failure */ -struct frr_pthread *frr_pthread_new(const char *name, unsigned int id, - void *(*start_routine)(void *), - int (*stop_routine)(void **, - struct frr_pthread *)); +struct frr_pthread *frr_pthread_new(struct frr_pthread_attr *attr); -/* Destroys an frr_pthread. +/* + * Destroys an frr_pthread. * * Assumes that the associated pthread, if any, has already terminated. * @@ -97,38 +120,66 @@ struct frr_pthread *frr_pthread_new(const char *name, unsigned int id, */ void frr_pthread_destroy(struct frr_pthread *fpt); -/* Gets an existing frr_pthread by its id. +/* + * Gets an existing frr_pthread by its id. * * @return frr_thread associated with the provided id, or NULL on error */ struct frr_pthread *frr_pthread_get(unsigned int id); -/* Creates a new pthread and binds it to a frr_pthread. +/* + * Creates a new pthread and binds it to a frr_pthread. * * This function is a wrapper for pthread_create. The first parameter is the * frr_pthread to bind the created pthread to. All subsequent arguments are - * passed unmodified to pthread_create(). + * passed unmodified to pthread_create(). The frr_pthread * provided will be + * used as the argument to the pthread entry function. If it is necessary to + * pass additional data, the 'data' field in the frr_pthread may be used. * * This function returns the same code as pthread_create(). If the value is * zero, the provided frr_pthread is bound to a running POSIX thread. If the * value is less than zero, the provided frr_pthread is guaranteed to be a * clean instance that may be susbsequently passed to frr_pthread_run(). * - * @param id - frr_pthread to bind the created pthread to + * @param fpt - frr_pthread * to run * @param attr - see pthread_create(3) - * @param arg - see pthread_create(3) * * @return see pthread_create(3) */ -int frr_pthread_run(unsigned int id, const pthread_attr_t *attr, void *arg); +int frr_pthread_run(struct frr_pthread *fpt, const pthread_attr_t *attr); -/* Stops an frr_pthread with a result. +/* + * Waits until the specified pthread has finished setting up and is ready to + * begin work. * - * @param id - frr_pthread to stop + * If the pthread's code makes use of the startup synchronization mechanism, + * this function should be called before attempting to use the functionality + * exposed by the pthread. It waits until the 'running' condition is satisfied + * (see struct definition of frr_pthread). + * + * @param fpt - the frr_pthread * to wait on + */ +void frr_pthread_wait_running(struct frr_pthread *fpt); + +/* + * Notifies other pthreads that the calling thread has finished setting up and + * is ready to begin work. + * + * This will allow any other pthreads waiting in 'frr_pthread_wait_running' to + * proceed. + * + * @param fpt - the frr_pthread * that has finished setting up + */ +void frr_pthread_notify_running(struct frr_pthread *fpt); + +/* + * Stops a frr_pthread with a result. + * + * @param fpt - frr_pthread * to stop * @param result - where to store the thread's result, if any. May be NULL if a * result is not needed. */ -int frr_pthread_stop(unsigned int id, void **result); +int frr_pthread_stop(struct frr_pthread *fpt, void **result); /* Stops all frr_pthread's. */ void frr_pthread_stop_all(void); @@ -136,7 +187,8 @@ void frr_pthread_stop_all(void); /* Yields the current thread of execution */ void frr_pthread_yield(void); -/* Returns a unique identifier for use with frr_pthread_new(). +/* + * Returns a unique identifier for use with frr_pthread_new(). * * Internally, this is an integer that increments after each call to this * function. Because the number of pthreads created should never exceed INT_MAX