cfg: Reinstate cfg tracking

CFG tracking was removed in 815375411e,
probably as a mistake, as part of the tidy up of cfg and the removal of
dynamic loading. This means that shutdown tracking (using
cfg_try_shutdown()) stopped working.

This patch restores the trackstart & trackstop API calls (renamed to be
more consistent with the exiting libraries) so that shutdown tracking
can be used again.

Change cfg.shutdown_timeout to be in milliseconds rather than seconds
nd use libqb macros for conversion.

Add --force option to corosync-cfgtool -H

Signed-off-by: Christine Caulfield <ccaulfie@redhat.com>
Reviewed-by: Jan Friesse <jfriesse@redhat.com>

exec/cfg.c

@@ -55,6 +55,7 @@
 #include <qb/qbipc_common.h>
 #include <corosync/cfg.h>
 #include <qb/qblist.h>
+#include <qb/qbutil.h>
 #include <corosync/mar_gen.h>
 #include <corosync/totem/totemip.h>
 #include <corosync/totem/totem.h>
@@ -79,7 +80,8 @@ enum cfg_message_req_types {
 	MESSAGE_REQ_EXEC_CFG_CRYPTO_RECONFIG = 4
 };
 
-#define DEFAULT_SHUTDOWN_TIMEOUT 5
+/* in milliseconds */
+#define DEFAULT_SHUTDOWN_TIMEOUT 5000
 
 static struct qb_list_head trackers_list;
 
@@ -162,6 +164,14 @@ static void message_handler_req_lib_cfg_replytoshutdown (
 	void *conn,
 	const void *msg);
 
+static void message_handler_req_lib_cfg_trackstart (
+	void *conn,
+	const void *msg);
+
+static void message_handler_req_lib_cfg_trackstop (
+	void *conn,
+	const void *msg);
+
 static void message_handler_req_lib_cfg_get_node_addrs (
 	void *conn,
 	const void *msg);
@@ -222,7 +232,16 @@ static struct corosync_lib_handler cfg_lib_engine[] =
 	{ /* 9 */
 		.lib_handler_fn		= message_handler_req_lib_cfg_nodestatusget,
 		.flow_control		= CS_LIB_FLOW_CONTROL_NOT_REQUIRED
-	}
+	},
+	{ /* 10 */
+		.lib_handler_fn		= message_handler_req_lib_cfg_trackstart,
+		.flow_control		= CS_LIB_FLOW_CONTROL_REQUIRED
+	},
+	{ /* 11 */
+		.lib_handler_fn		= message_handler_req_lib_cfg_trackstop,
+		.flow_control		= CS_LIB_FLOW_CONTROL_REQUIRED
+	},
+
 };
 
 static struct corosync_exec_handler cfg_exec_engine[] =
@@ -1045,6 +1064,52 @@ ipc_response_send:
 	LEAVE();
 }
 
+static void message_handler_req_lib_cfg_trackstart (
+	void *conn,
+	const void *msg)
+{
+	struct cfg_info *ci = (struct cfg_info *)api->ipc_private_data_get (conn);
+	struct res_lib_cfg_trackstart res_lib_cfg_trackstart;
+
+	ENTER();
+
+	/*
+	 * We only do shutdown tracking at the moment
+	 */
+	if (qb_list_empty(&ci->list)) {
+		qb_list_add(&ci->list, &trackers_list);
+		ci->tracker_conn = conn;
+
+		if (shutdown_con) {
+			/*
+			 * Shutdown already in progress, ask the newcomer's opinion
+			 */
+			ci->shutdown_reply = SHUTDOWN_REPLY_UNKNOWN;
+			shutdown_expected++;
+			send_test_shutdown(conn, NULL, CS_OK);
+		}
+	}
+
+	res_lib_cfg_trackstart.header.size = sizeof(struct res_lib_cfg_trackstart);
+	res_lib_cfg_trackstart.header.id = MESSAGE_RES_CFG_STATETRACKSTART;
+	res_lib_cfg_trackstart.header.error = CS_OK;
+
+	api->ipc_response_send(conn, &res_lib_cfg_trackstart,
+				    sizeof(res_lib_cfg_trackstart));
+
+	LEAVE();
+}
+
+static void message_handler_req_lib_cfg_trackstop (
+	void *conn,
+	const void *msg)
+{
+	struct cfg_info *ci = (struct cfg_info *)api->ipc_private_data_get (conn);
+
+	ENTER();
+	remove_ci_from_shutdown(ci);
+	LEAVE();
+}
 
 static void message_handler_req_lib_cfg_ringreenable (
 	void *conn,
@@ -1240,7 +1305,7 @@ static void message_handler_req_lib_cfg_tryshutdown (
 		 * Start the timer. If we don't get a full set of replies before this goes
 		 * off we'll cancel the shutdown
 		 */
-		api->timer_add_duration((unsigned long long)shutdown_timeout*1000000000, NULL,
+		api->timer_add_duration((unsigned long long)shutdown_timeout*QB_TIME_NS_IN_MSEC, NULL,
 					shutdown_timer_fn, &shutdown_timer);
 
 		/*

include/corosync/cfg.h

@@ -217,6 +217,30 @@ corosync_cfg_kill_node (
 	unsigned int nodeid,
 	const char *reason);
 
+/**
+ * @brief corosync_cfg_trackstart
+ * Track CFG for shutdown requests
+ * @param cfg_handle
+ * @param track_flags (none currently supported)
+ * @param reason
+ * @return
+ */
+cs_error_t
+corosync_cfg_trackstart (
+        corosync_cfg_handle_t cfg_handle,
+        uint8_t track_flags);
+
+/**
+ * @brief corosync_cfg_trackstop
+ * Stop tracking CFG for shutdown requests
+ * @param cfg_handle
+ * @param reason
+ * @return
+ */
+cs_error_t
+corosync_cfg_trackstop (
+        corosync_cfg_handle_t cfg_handle);
+
 /**
  * @brief corosync_cfg_try_shutdown
  * @param cfg_handle

include/corosync/ipc_cfg.h

@@ -60,7 +60,9 @@ enum req_lib_cfg_types {
 	MESSAGE_REQ_CFG_LOCAL_GET = 6,
 	MESSAGE_REQ_CFG_RELOAD_CONFIG = 7,
 	MESSAGE_REQ_CFG_REOPEN_LOG_FILES = 8,
-	MESSAGE_REQ_CFG_NODESTATUSGET = 9
+	MESSAGE_REQ_CFG_NODESTATUSGET = 9,
+	MESSAGE_REQ_CFG_TRACKSTART = 10,
+	MESSAGE_REQ_CFG_TRACKSTOP = 11
 };
 
 /**
@@ -255,6 +257,24 @@ struct res_lib_cfg_reopen_log_files {
 	struct qb_ipc_response_header header __attribute__((aligned(8)));
 };
 
+struct req_lib_cfg_trackstart {
+	struct qb_ipc_request_header header;
+	uint8_t track_flags;
+};
+
+struct res_lib_cfg_trackstart {
+	struct qb_ipc_response_header header;
+};
+
+struct req_lib_cfg_trackstop {
+	struct qb_ipc_request_header header;
+};
+
+struct res_lib_cfg_trackstop {
+	struct qb_ipc_response_header header;
+};
+
+
 /**
  * @brief corosync_administrative_target_t enum
  */

lib/cfg.c

@@ -447,6 +447,75 @@ error_put:
 	return (error);
 }
 
+
+cs_error_t
+corosync_cfg_trackstart (
+	corosync_cfg_handle_t cfg_handle,
+	uint8_t track_flags)
+{
+	struct cfg_inst *cfg_inst;
+	struct req_lib_cfg_trackstart req_lib_cfg_trackstart;
+	struct res_lib_cfg_trackstart res_lib_cfg_trackstart;
+	cs_error_t error;
+	struct iovec iov;
+
+	req_lib_cfg_trackstart.header.size = sizeof (struct req_lib_cfg_trackstart);
+	req_lib_cfg_trackstart.header.id = MESSAGE_REQ_CFG_TRACKSTART;
+	req_lib_cfg_trackstart.track_flags = track_flags;
+
+	error = hdb_error_to_cs(hdb_handle_get (&cfg_hdb, cfg_handle,
+		(void *)&cfg_inst));
+	if (error != CS_OK) {
+		return (error);
+	}
+
+	iov.iov_base = (void *)&req_lib_cfg_trackstart,
+	iov.iov_len = sizeof (struct req_lib_cfg_trackstart),
+
+	error = qb_to_cs_error (qb_ipcc_sendv_recv (cfg_inst->c,
+		&iov,
+		1,
+		&res_lib_cfg_trackstart,
+		sizeof (struct res_lib_cfg_trackstart), CS_IPC_TIMEOUT_MS));
+
+	(void)hdb_handle_put (&cfg_hdb, cfg_handle);
+
+	return (error == CS_OK ? res_lib_cfg_trackstart.header.error : error);
+}
+
+cs_error_t
+corosync_cfg_trackstop (
+	corosync_cfg_handle_t cfg_handle)
+{
+	struct cfg_inst *cfg_inst;
+	struct req_lib_cfg_trackstop req_lib_cfg_trackstop;
+	struct res_lib_cfg_trackstop res_lib_cfg_trackstop;
+	cs_error_t error;
+	struct iovec iov;
+
+	error = hdb_error_to_cs (hdb_handle_get (&cfg_hdb, cfg_handle,
+		(void *)&cfg_inst));
+	if (error != CS_OK) {
+		return (error);
+	}
+
+	req_lib_cfg_trackstop.header.size = sizeof (struct req_lib_cfg_trackstop);
+	req_lib_cfg_trackstop.header.id = MESSAGE_REQ_CFG_TRACKSTOP;
+
+	iov.iov_base = (void *)&req_lib_cfg_trackstop,
+	iov.iov_len = sizeof (struct req_lib_cfg_trackstop),
+
+	error = qb_to_cs_error (qb_ipcc_sendv_recv (cfg_inst->c,
+		&iov,
+		1,
+		&res_lib_cfg_trackstop,
+		sizeof (struct res_lib_cfg_trackstop), CS_IPC_TIMEOUT_MS));
+
+	(void)hdb_handle_put (&cfg_hdb, cfg_handle);
+
+	return (error == CS_OK ? res_lib_cfg_trackstop.header.error : error);
+}
+
 cs_error_t
 corosync_cfg_kill_node (
 	corosync_cfg_handle_t cfg_handle,
@@ -487,7 +556,7 @@ corosync_cfg_kill_node (
 
 	(void)hdb_handle_put (&cfg_hdb, cfg_handle);
 
-        return (error == CS_OK ? res_lib_cfg_killnode.header.error : error);
+	return (error == CS_OK ? res_lib_cfg_killnode.header.error : error);
 }
 
 cs_error_t
@@ -522,7 +591,7 @@ corosync_cfg_try_shutdown (
 
 	(void)hdb_handle_put (&cfg_hdb, cfg_handle);
 
-        return (error == CS_OK ? res_lib_cfg_tryshutdown.header.error : error);
+	return (error == CS_OK ? res_lib_cfg_tryshutdown.header.error : error);
 }
 
 cs_error_t

lib/libcfg.versions

@@ -13,6 +13,6 @@ COROSYNC_CFG_0.82 {
 		corosync_cfg_ring_status_get;
 		corosync_cfg_node_status_get;
 		corosync_cfg_ring_reenable;
-		corosync_cfg_service_load;
-		corosync_cfg_service_unload;
+		corosync_cfg_trackstart;
+		corosync_cfg_trackstop;
 };

lib/libcfg.verso

@@ -1 +1 @@
-7.2.0
+7.3.0

man/cmap_keys.7

@@ -147,6 +147,11 @@ quorum.cancel_wait_for_all
 Tells votequorum to cancel waiting for all nodes at cluster startup. Can be used
 to unblock quorum if notes are known to be down. For pcs use only.
 
+.TP
+cfg.shutdown_timeout
+Sets the timeout within which daemons that are registered for cfg callbacks must respond
+to a corosync_cfg_try_shutdown() request. the default is 5000 mS
+
 .TP
 config.reload_in_progress
 This value will be set to 1 (or created) when a corosync.conf reload is started,

man/corosync-cfgtool.8

@@ -35,7 +35,7 @@
 .SH "NAME"
 corosync-cfgtool \- An administrative tool for corosync.
 .SH "SYNOPSIS"
-.B corosync\-cfgtool [[\-i IP_address] [\-b] [\-s] [\-n] [\-R] [\-L] [\-k nodeid] [\-a nodeid] [\-h] [\-H]
+.B corosync\-cfgtool [[\-i IP_address] [\-b] [\-s] [\-n] [\-R] [\-L] [\-k nodeid] [\-a nodeid] [\-h] [\-H] [\--force]
 .SH "DESCRIPTION"
 .B corosync\-cfgtool
 A tool for displaying and configuring active parameters within corosync.
@@ -123,6 +123,12 @@ Print basic usage.
 .TP
 .B -H
 Shutdown corosync cleanly on this node.
+corosync-cfgtool -H will request a shutdown from corosync, which means it will
+consult any interested daemons before shutting down and the shutdown maybe vetoed if a
+daemon regards the shutdown as inappropriate.
+If --force is added to the command line then corosync will shutdown regardless
+of the daemons' opinions on the matter.
+
 .SH "SEE ALSO"
 .BR corosync_overview (7),
 .SH "AUTHOR"

tools/corosync-cfgtool.c

@@ -47,6 +47,7 @@
 #include <netinet/in.h>
 #include <arpa/inet.h>
 #include <limits.h>
+#include <getopt.h>
 
 #include <corosync/corotypes.h>
 #include <corosync/totem/totem.h>
@@ -341,13 +342,19 @@ static int reopen_log_files_do (void)
 	return (rc);
 }
 
-static void shutdown_do(void)
+static void shutdown_do(int force)
 {
 	cs_error_t result;
 	corosync_cfg_handle_t handle;
 	corosync_cfg_callbacks_t callbacks;
+	int flag;
 
 	callbacks.corosync_cfg_shutdown_callback = NULL;
+	if (force) {
+		flag = COROSYNC_CFG_SHUTDOWN_FLAG_REGARDLESS;
+	} else {
+		flag = COROSYNC_CFG_SHUTDOWN_FLAG_REQUEST;
+	}
 
 	result = corosync_cfg_initialize (&handle, &callbacks);
 	if (result != CS_OK) {
@@ -356,7 +363,7 @@ static void shutdown_do(void)
 	}
 
 	printf ("Shutting down corosync\n");
-	cs_repeat(result, 30, corosync_cfg_try_shutdown (handle, COROSYNC_CFG_SHUTDOWN_FLAG_REQUEST));
+	cs_repeat(result, 30, corosync_cfg_try_shutdown (handle, flag));
 	if (result != CS_OK) {
 		fprintf (stderr, "Could not shutdown (error = %d)\n", result);
 	}
@@ -450,10 +457,10 @@ static void usage_do (void)
 	printf ("\t-a\tDisplay the IP address(es) of a node\n");
 	printf ("\t-h\tPrint basic usage.\n");
 	printf ("\t-H\tShutdown corosync cleanly on this node.\n");
+	printf ("\t\t--force will shut down corosync regardless of daemon vetos\n");
 }
 
 int main (int argc, char *argv[]) {
-	const char *options = "i:snbrRLk:a:hH";
 	int opt;
 	unsigned int nodeid = 0;
 	char interface_name[128] = "";
@@ -461,9 +468,30 @@ int main (int argc, char *argv[]) {
 	enum user_action action = ACTION_NOOP;
 	int brief = 0;
 	long long int l;
+	int option_index = 0;
+	int force_shutdown  = 0;
+	const char *options = "i:snbrRLk:a:hH";
+	struct option long_options[] = {
+		{"if",       required_argument, 0, 'i'},
+		{"status",   no_argument,       0, 's'},
+		{"nodes",    no_argument,       0, 'n'},
+		{"brief",    no_argument,       0, 'b'},
+		{"reload",   no_argument,       0, 'R'},
+		{"reopen",   no_argument,       0, 'L'},
+		{"kill",     required_argument, 0, 'k'},
+		{"address",  required_argument, 0, 'a'},
+		{"shutdown", no_argument,       0, 'H'},
+		{"force",    no_argument,       0, 0},
+		{0,          0,                 0, 0}
+	};
 
-	while ( (opt = getopt(argc, argv, options)) != -1 ) {
+		while ( (opt = getopt_long(argc, argv, options, long_options, &option_index)) != -1 ) {
 		switch (opt) {
+			case 0: // options with no short equivalent - just --force ATM
+			if (strcmp(long_options[option_index].name, "force") == 0) {
+				force_shutdown = 1;
+			}
+			break;
 		case 'i':
 			strncpy(interface_name, optarg, sizeof(interface_name));
 			interface_name[sizeof(interface_name) - 1] = '\0';
@@ -527,7 +555,7 @@ int main (int argc, char *argv[]) {
 		killnode_do(nodeid);
 		break;
 	case ACTION_SHUTDOW:
-		shutdown_do();
+		shutdown_do(force_shutdown);
 		break;
 	case ACTION_SHOWADDR:
 		rc = showaddrs_do(nodeid);