Merge pull request #3526 from mjstapp/dplane_lists

zebra: pass lists of results from dataplane to zebra

zebra/zebra_dplane.c

@@ -157,9 +157,9 @@ struct zebra_dplane_provider {
 	/* Flags */
 	int dp_flags;
 
-	dplane_provider_process_fp dp_fp;
+	int (*dp_fp)(struct zebra_dplane_provider *prov);
 
-	dplane_provider_fini_fp dp_fini;
+	int (*dp_fini)(struct zebra_dplane_provider *prov, bool early_p);
 
 	_Atomic uint32_t dp_in_counter;
 	_Atomic uint32_t dp_in_queued;
@@ -189,7 +189,7 @@ static struct zebra_dplane_globals {
 	pthread_mutex_t dg_mutex;
 
 	/* Results callback registered by zebra 'core' */
-	dplane_results_fp dg_results_cb;
+	int (*dg_results_cb)(struct dplane_ctx_q *ctxlist);
 
 	/* Sentinel for beginning of shutdown */
 	volatile bool dg_is_shutdown;
@@ -988,8 +988,9 @@ int dplane_show_provs_helper(struct vty *vty, bool detailed)
 int dplane_provider_register(const char *name,
 			     enum dplane_provider_prio prio,
 			     int flags,
-			     dplane_provider_process_fp fp,
+			     int (*fp)(struct zebra_dplane_provider *),
-			     dplane_provider_fini_fp fini_fp,
+			     int (*fini_fp)(struct zebra_dplane_provider *,
+					    bool early),
 			     void *data)
 {
 	int ret = 0;
@@ -1209,15 +1210,6 @@ int dplane_provider_work_ready(void)
 	return AOK;
 }
 
-/*
- * Zebra registers a results callback with the dataplane system
- */
-int dplane_results_register(dplane_results_fp fp)
-{
-	zdplane_info.dg_results_cb = fp;
-	return AOK;
-}
-
 /*
  * Kernel dataplane provider
  */
@@ -1677,27 +1669,20 @@ static int dplane_thread_loop(struct thread *event)
 			   counter, error_counter);
 
 	/*
-	 * TODO -- I'd rather hand lists through the api to zebra main,
+	 * Hand lists through the api to zebra main,
 	 * to reduce the number of lock/unlock cycles
 	 */
-	for (ctx = TAILQ_FIRST(&error_list); ctx; ) {
-		TAILQ_REMOVE(&error_list, ctx, zd_q_entries);
 
-		/* Call through to zebra main */
+	/* Call through to zebra main */
-		(*zdplane_info.dg_results_cb)(ctx);
+	(zdplane_info.dg_results_cb)(&error_list);
 
-		ctx = TAILQ_FIRST(&error_list);
+	TAILQ_INIT(&error_list);
-	}
 
 
-	for (ctx = TAILQ_FIRST(&work_list); ctx; ) {
+	/* Call through to zebra main */
-		TAILQ_REMOVE(&work_list, ctx, zd_q_entries);
+	(zdplane_info.dg_results_cb)(&work_list);
 
-		/* Call through to zebra main */
+	TAILQ_INIT(&work_list);
-		(*zdplane_info.dg_results_cb)(ctx);
-
-		ctx = TAILQ_FIRST(&work_list);
-	}
 
 done:
 	return 0;
@@ -1782,7 +1767,8 @@ void zebra_dplane_start(void)
 /*
  * Initialize the dataplane module at startup; called by zebra rib_init()
  */
-void zebra_dplane_init(void)
+void zebra_dplane_init(int (*results_fp)(struct dplane_ctx_q *))
 {
 	zebra_dplane_init_internal(&zebrad);
+	zdplane_info.dg_results_cb = results_fp;
 }

zebra/zebra_dplane.h

@@ -250,21 +250,6 @@ enum dplane_provider_prio {
 	DPLANE_PRIO_LAST
 };
 
-/* Provider's entry-point for incoming work, called in the context of the
- * dataplane pthread. The dataplane pthread enqueues any new work to the
- * provider's 'inbound' queue, then calls the callback. The dataplane
- * then checks the provider's outbound queue.
- */
-typedef int (*dplane_provider_process_fp)(struct zebra_dplane_provider *prov);
-
-/* Provider's entry-point for shutdown and cleanup. Called with 'early'
- * during shutdown, to indicate that the dataplane subsystem is allowing
- * work to move through the providers and finish. When called without 'early',
- * the provider should release all resources (if it has any allocated).
- */
-typedef int (*dplane_provider_fini_fp)(struct zebra_dplane_provider *prov,
-				       bool early);
-
 /* Flags values used during provider registration. */
 #define DPLANE_PROV_FLAGS_DEFAULT  0x0
 
@@ -275,11 +260,25 @@ typedef int (*dplane_provider_fini_fp)(struct zebra_dplane_provider *prov,
 /* Provider registration: ordering or priority value, callbacks, and optional
  * opaque data value.
  */
+
+/* Providers offer an entry-point for incoming work, called in the context of
+ * the dataplane pthread. The dataplane pthread enqueues any new work to the
+ * provider's 'inbound' queue, then calls the callback. The dataplane
+ * then checks the provider's outbound queue for completed work.
+ */
+
+/* Providers offer an entry-point for shutdown and cleanup. This is called
+ * with 'early' during shutdown, to indicate that the dataplane subsystem
+ * is allowing work to move through the providers and finish.
+ * When called without 'early', the provider should release
+ * all resources (if it has any allocated).
+ */
 int dplane_provider_register(const char *name,
 			     enum dplane_provider_prio prio,
 			     int flags,
-			     dplane_provider_process_fp fp,
+			     int (*fp)(struct zebra_dplane_provider *),
-			     dplane_provider_fini_fp fini_fp,
+			     int (*fini_fp)(struct zebra_dplane_provider *,
+					    bool early),
 			     void *data);
 
 /* Accessors for provider attributes */
@@ -317,21 +316,14 @@ int dplane_provider_dequeue_in_list(struct zebra_dplane_provider *prov,
 void dplane_provider_enqueue_out_ctx(struct zebra_dplane_provider *prov,
 				     struct zebra_dplane_ctx *ctx);
 
-/*
- * Zebra registers a results callback with the dataplane. The callback is
- * called in the dataplane pthread context, so the expectation is that the
- * context is queued for the zebra main pthread or that processing
- * is very limited.
- */
-typedef int (*dplane_results_fp)(struct zebra_dplane_ctx *ctx);
-
-int dplane_results_register(dplane_results_fp fp);
-
 /*
  * Initialize the dataplane modules at zebra startup. This is currently called
- * by the rib module.
+ * by the rib module. Zebra registers a results callback with the dataplane.
+ * The callback is called in the dataplane pthread context,
+ * so the expectation is that the contexts are queued for the zebra
+ * main pthread.
  */
-void zebra_dplane_init(void);
+void zebra_dplane_init(int (*) (struct dplane_ctx_q *));
 
 /*
  * Start the dataplane pthread. This step needs to be run later than the

zebra/zebra_rib.c

@@ -3165,21 +3165,34 @@ void rib_close_table(struct route_table *table)
 static int rib_process_dplane_results(struct thread *thread)
 {
 	struct zebra_dplane_ctx *ctx;
+	struct dplane_ctx_q ctxlist;
+
+	/* Dequeue a list of completed updates with one lock/unlock cycle */
 
 	do {
+		TAILQ_INIT(&ctxlist);
+
 		/* Take lock controlling queue of results */
 		pthread_mutex_lock(&dplane_mutex);
 		{
 			/* Dequeue context block */
-			ctx = dplane_ctx_dequeue(&rib_dplane_q);
+			dplane_ctx_list_append(&ctxlist, &rib_dplane_q);
 		}
 		pthread_mutex_unlock(&dplane_mutex);
 
-		if (ctx)
+		/* Dequeue context block */
-			rib_process_after(ctx);
+		ctx = dplane_ctx_dequeue(&ctxlist);
-		else
+
+		/* If we've emptied the results queue, we're done */
+		if (ctx == NULL)
 			break;
 
+		while (ctx) {
+			rib_process_after(ctx);
+
+			ctx = dplane_ctx_dequeue(&ctxlist);
+		}
+
 	} while (1);
 
 	/* Check for nexthop tracking processing after finishing with results */
@@ -3193,17 +3206,17 @@ static int rib_process_dplane_results(struct thread *thread)
  * the dataplane pthread. We enqueue the results here for processing by
  * the main thread later.
  */
-static int rib_dplane_results(struct zebra_dplane_ctx *ctx)
+static int rib_dplane_results(struct dplane_ctx_q *ctxlist)
 {
 	/* Take lock controlling queue of results */
 	pthread_mutex_lock(&dplane_mutex);
 	{
-		/* Enqueue context block */
+		/* Enqueue context blocks */
-		dplane_ctx_enqueue_tail(&rib_dplane_q, ctx);
+		dplane_ctx_list_append(&rib_dplane_q, ctxlist);
 	}
 	pthread_mutex_unlock(&dplane_mutex);
 
-	/* Ensure event is signalled to zebra main thread */
+	/* Ensure event is signalled to zebra main pthread */
 	thread_add_event(zebrad.master, rib_process_dplane_results, NULL, 0,
 			 &t_dplane);
 
@@ -3218,8 +3231,7 @@ void rib_init(void)
 	/* Init dataplane, and register for results */
 	pthread_mutex_init(&dplane_mutex, NULL);
 	TAILQ_INIT(&rib_dplane_q);
-	zebra_dplane_init();
+	zebra_dplane_init(rib_dplane_results);
-	dplane_results_register(rib_dplane_results);
 }
 
 /*