zebra: improve dataplane shutdown checks

Update the dataplane shutdown checks to include the providers.
Also revise the typedef for provider structs to make const
work.

Signed-off-by: Mark Stapp <mjs@voltanet.io>

zebra/zebra_dplane.c

@@ -162,8 +162,10 @@ struct zebra_dplane_provider {
 	dplane_provider_fini_fp dp_fini;
 
 	_Atomic uint32_t dp_in_counter;
+	_Atomic uint32_t dp_in_queued;
 	_Atomic uint32_t dp_in_max;
 	_Atomic uint32_t dp_out_counter;
+	_Atomic uint32_t dp_out_queued;
 	_Atomic uint32_t dp_out_max;
 	_Atomic uint32_t dp_error_counter;
 
@@ -959,8 +961,8 @@ int dplane_show_provs_helper(struct vty *vty, bool detailed)
 		out_max = atomic_load_explicit(&prov->dp_out_max,
 					       memory_order_relaxed);
 
-		vty_out(vty, "%s (%u): in: %"PRIu64", max: %"PRIu64", "
+		vty_out(vty, "%s (%u): in: %"PRIu64", q_max: %"PRIu64", "
-			"out: %"PRIu64", max: %"PRIu64"\n",
+			"out: %"PRIu64", q_max: %"PRIu64"\n",
 			prov->dp_name, prov->dp_id, in, in_max, out, out_max);
 
 		DPLANE_LOCK();
@@ -1081,6 +1083,9 @@ struct zebra_dplane_ctx *dplane_provider_dequeue_in_ctx(
 	ctx = TAILQ_FIRST(&(prov->dp_ctx_in_q));
 	if (ctx) {
 		TAILQ_REMOVE(&(prov->dp_ctx_in_q), ctx, zd_q_entries);
+
+		atomic_fetch_sub_explicit(&prov->dp_in_queued, 1,
+					  memory_order_relaxed);
 	}
 
 	if (dplane_provider_is_threaded(prov))
@@ -1114,6 +1119,10 @@ int dplane_provider_dequeue_in_list(struct zebra_dplane_provider *prov,
 		}
 	}
 
+	if (ret > 0)
+		atomic_fetch_sub_explicit(&prov->dp_in_queued, ret,
+					  memory_order_relaxed);
+
 	if (dplane_provider_is_threaded(prov))
 		DPLANE_PROV_UNLOCK(prov);
 
@@ -1272,6 +1281,10 @@ static int test_dplane_process_func(struct zebra_dplane_provider *prov)
 		dplane_provider_enqueue_out_ctx(prov, ctx);
 	}
 
+	if (IS_ZEBRA_DEBUG_DPLANE_DETAIL)
+		zlog_debug("dplane provider '%s': processed %d",
+			   dplane_provider_get_name(prov), counter);
+
 	/* Ensure that we'll run the work loop again if there's still
 	 * more work to do.
 	 */
@@ -1342,7 +1355,7 @@ bool dplane_is_in_shutdown(void)
  * early during zebra shutdown, as a signal to stop new work and prepare
  * for updates generated by shutdown/cleanup activity, as zebra tries to
  * remove everything it's responsible for.
- * NB: This runs in the main zebra thread context.
+ * NB: This runs in the main zebra pthread context.
  */
 void zebra_dplane_pre_finish(void)
 {
@@ -1351,7 +1364,7 @@ void zebra_dplane_pre_finish(void)
 
 	zdplane_info.dg_is_shutdown = true;
 
-	/* Notify provider(s) of pending shutdown */
+	/* TODO -- Notify provider(s) of pending shutdown */
 }
 
 /*
@@ -1360,7 +1373,9 @@ void zebra_dplane_pre_finish(void)
  */
 static bool dplane_work_pending(void)
 {
+	bool ret = false;
 	struct zebra_dplane_ctx *ctx;
+	struct zebra_dplane_provider *prov;
 
 	/* TODO -- just checking incoming/pending work for now, must check
 	 * providers
@@ -1368,10 +1383,40 @@ static bool dplane_work_pending(void)
 	DPLANE_LOCK();
 	{
 		ctx = TAILQ_FIRST(&zdplane_info.dg_route_ctx_q);
+		prov = TAILQ_FIRST(&zdplane_info.dg_providers_q);
 	}
 	DPLANE_UNLOCK();
 
-	return (ctx != NULL);
+	if (ctx != NULL) {
+		ret = true;
+		goto done;
+	}
+
+	while (prov) {
+
+		if (dplane_provider_is_threaded(prov))
+			DPLANE_PROV_LOCK(prov);
+
+		ctx = TAILQ_FIRST(&(prov->dp_ctx_in_q));
+		if (ctx == NULL)
+			ctx = TAILQ_FIRST(&(prov->dp_ctx_out_q));
+
+		if (dplane_provider_is_threaded(prov))
+			DPLANE_PROV_UNLOCK(prov);
+
+		if (ctx != NULL)
+			break;
+
+		DPLANE_LOCK();
+		prov = TAILQ_NEXT(prov, dp_prov_link);
+		DPLANE_UNLOCK();
+	}
+
+	if (ctx != NULL)
+		ret = true;
+
+done:
+	return ret;
 }
 
 /*
@@ -1443,7 +1488,7 @@ static int dplane_thread_loop(struct thread *event)
 	struct zebra_dplane_provider *prov;
 	struct zebra_dplane_ctx *ctx, *tctx;
 	int limit, counter, error_counter;
-	uint64_t lval;
+	uint64_t curr, high;
 
 	/* Capture work limit per cycle */
 	limit = zdplane_info.dg_updates_per_cycle;
@@ -1534,10 +1579,16 @@ static int dplane_thread_loop(struct thread *event)
 			TAILQ_CONCAT(&(prov->dp_ctx_in_q), &work_list,
 				     zd_q_entries);
 
-		lval = atomic_add_fetch_explicit(&prov->dp_in_counter, counter,
+		atomic_fetch_add_explicit(&prov->dp_in_counter, counter,
-						 memory_order_relaxed);
+					  memory_order_relaxed);
-		if (lval > prov->dp_in_max)
+		atomic_fetch_add_explicit(&prov->dp_in_queued, counter,
-			atomic_store_explicit(&prov->dp_in_max, lval,
+					  memory_order_relaxed);
+		curr = atomic_load_explicit(&prov->dp_in_queued,
+					    memory_order_relaxed);
+		high = atomic_load_explicit(&prov->dp_in_max,
+					    memory_order_relaxed);
+		if (curr > high)
+			atomic_store_explicit(&prov->dp_in_max, curr,
 					      memory_order_relaxed);
 
 		if (dplane_provider_is_threaded(prov))