From ed7d054e552b4cb2a0cb502b65f84310ce6da844 Mon Sep 17 00:00:00 2001
From: Steven Dake <sdake@redhat.com>
Date: Thu, 23 Apr 2009 15:24:29 +0000
Subject: [PATCH] Remove priority inversion in logsys.

git-svn-id: http://svn.fedorahosted.org/svn/corosync/trunk@2130 fd59a12c-fef9-0310-b244-a6a79926bd2f
---
 exec/logsys.c                    | 11 +++++++++
 exec/main.c                      | 39 ++++++++++++++++++++------------
 exec/mainconfig.c                |  2 --
 include/corosync/engine/logsys.h |  5 ++++
 4 files changed, 41 insertions(+), 16 deletions(-)

diff --git a/exec/logsys.c b/exec/logsys.c
index 994cf21d..b3b831c9 100644
--- a/exec/logsys.c
+++ b/exec/logsys.c
@@ -1493,6 +1493,17 @@ const char *logsys_tag_name_get (unsigned int tag)
 	return (NULL);
 }
 
+int logsys_thread_priority_set (
+	int policy,
+	const struct sched_param *param)
+{
+	int res;
+
+	res = pthread_setschedparam (logsys_thread_id, policy, param);
+	return (res);
+
+}
+
 int logsys_log_rec_store (const char *filename)
 {
 	int fd;
diff --git a/exec/main.c b/exec/main.c
index 87820e46..68af8c63 100644
--- a/exec/main.c
+++ b/exec/main.c
@@ -128,6 +128,8 @@ unsigned long long *(*main_clm_get_by_nodeid) (unsigned int node_id);
 
 hdb_handle_t corosync_poll_handle;
 
+struct sched_param global_sched_param;
+
 static void sigusr2_handler (int num)
 {
 	int i;
@@ -369,16 +371,15 @@ static void corosync_tty_detach (void)
 static void corosync_setscheduler (void)
 {
 #if ! defined(TS_CLASS) && (defined(COROSYNC_BSD) || defined(COROSYNC_LINUX) || defined(COROSYNC_SOLARIS))
-	struct sched_param sched_param;
 	int res;
 
 	sched_priority = sched_get_priority_max (SCHED_RR);
 	if (sched_priority != -1) {
-		sched_param.sched_priority = sched_priority;
-		res = sched_setscheduler (0, SCHED_RR, &sched_param);
+		global_sched_param.sched_priority = sched_priority;
+		res = sched_setscheduler (0, SCHED_RR, &global_sched_param);
 		if (res == -1) {
 			log_printf (LOGSYS_LEVEL_WARNING, "Could not set SCHED_RR at priority %d: %s\n",
-				sched_param.sched_priority, strerror (errno));
+				global_sched_param.sched_priority, strerror (errno));
 		}
 	} else {
 		log_printf (LOGSYS_LEVEL_WARNING, "Could not get maximum scheduler priority: %s\n", strerror (errno));
@@ -734,6 +735,17 @@ int main (int argc, char **argv)
 	if (background)
 		corosync_tty_detach ();
 
+	/*
+	 * Set round robin realtime scheduling with priority 99
+	 * Lock all memory to avoid page faults which may interrupt
+	 * application healthchecking
+	 */
+	if (setprio) {
+		corosync_setscheduler ();
+	}
+
+	corosync_mlockall ();
+
 	log_printf (LOGSYS_LEVEL_NOTICE, "Corosync Executive Service RELEASE '%s'\n", RELEASE_VERSION);
 	log_printf (LOGSYS_LEVEL_NOTICE, "Copyright (C) 2002-2006 MontaVista Software, Inc and contributors.\n");
 	log_printf (LOGSYS_LEVEL_NOTICE, "Copyright (C) 2006-2008 Red Hat, Inc.\n");
@@ -840,6 +852,15 @@ int main (int argc, char **argv)
 		syslog (LOGSYS_LEVEL_ERROR, "%s", error_string);
 		corosync_exit_error (AIS_DONE_MAINCONFIGREAD);
 	}
+	logsys_fork_completed();
+	if (setprio) {
+		res = logsys_thread_priority_set (SCHED_RR, &global_sched_param);
+		if (res == -1) {
+			log_printf (LOGSYS_LEVEL_ERROR,
+				"Could not set logsys thread priority.  Can't continue because of priority inversions.");
+			corosync_exit_error (AIS_DONE_LOGSETUP);
+		}
+	}
 
 	res = totem_config_read (objdb, &totem_config, &error_string);
 	if (res == -1) {
@@ -859,16 +880,6 @@ int main (int argc, char **argv)
 		corosync_exit_error (AIS_DONE_MAINCONFIGREAD);
 	}
 
-	/*
-	 * Set round robin realtime scheduling with priority 99
-	 * Lock all memory to avoid page faults which may interrupt
-	 * application healthchecking
-	 */
-	if (setprio)
-		corosync_setscheduler ();
-
-	corosync_mlockall ();
-
 	totem_config.totem_logging_configuration = totem_logging_configuration;
 	totem_config.totem_logging_configuration.log_subsys_id =
 		_logsys_subsys_create ("TOTEM");
diff --git a/exec/mainconfig.c b/exec/mainconfig.c
index 09aee818..f7b3990e 100644
--- a/exec/mainconfig.c
+++ b/exec/mainconfig.c
@@ -722,8 +722,6 @@ int corosync_main_config_read (
 
 	add_logsys_config_notification(objdb);
 
-	logsys_fork_completed ();
-
 	return 0;
 
 parse_error:
diff --git a/include/corosync/engine/logsys.h b/include/corosync/engine/logsys.h
index 916c1388..12de1a2b 100644
--- a/include/corosync/engine/logsys.h
+++ b/include/corosync/engine/logsys.h
@@ -40,6 +40,7 @@
 #include <stdarg.h>
 #include <stdlib.h>
 #include <syslog.h>
+#include <pthread.h>
 
 /*
  * All of the LOGSYS_MODE's can be ORed together for combined behavior
@@ -251,6 +252,10 @@ extern int logsys_tag_id_get (
 extern const char *logsys_tag_name_get (
 	unsigned int tag);
 
+extern int logsys_thread_priority_set (
+	int policy,
+	const struct sched_param *param);
+
 /*
  * External definitions
  */