From 14aee7cd745cbb6d2f3d5203dbb878dbe966dbc5 Mon Sep 17 00:00:00 2001
From: Frediano Ziglio <fziglio@redhat.com>
Date: Fri, 19 May 2017 12:21:48 +0100
Subject: [PATCH] gstreamer: Check if ORC library can work

ORC library is used internally by GStreamer to generate code
dynamically.
If ORC cannot allocate executable memory, the failure causes
an abort(3) to be called.
This happens on some SELinux configurations that disable executable
memory allocation (execmem boolean).
Check that ORC could work before attempting to use GStreamer to
avoid crashes.
While this check is done, the ORC library outputs an error which will
be well visible in Qemu output.

Signed-off-by: Frediano Ziglio <fziglio@redhat.com>
Acked-by: Christophe Fergeau <cfergeau@redhat.com>
---
 configure.ac               |  6 ++++++
 server/Makefile.am         |  2 ++
 server/gstreamer-encoder.c | 30 ++++++++++++++++++++++++++++++
 3 files changed, 38 insertions(+)

diff --git a/configure.ac b/configure.ac
index a840eebd..f80193b3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -116,6 +116,12 @@ AS_IF([test x"$missing_gstreamer_elements" = xyes],
     [SPICE_WARNING([The GStreamer video encoder can be built but may not work.])
 ])
 
+if test "x$have_gstreamer_0_10" = "xyes" -o "x$have_gstreamer_1_0" = "xyes"; then
+    PKG_CHECK_MODULES(ORC, orc-0.4)
+    AC_SUBST(ORC_CFLAGS)
+    AC_SUBST(ORC_LIBS)
+fi
+
 AC_ARG_ENABLE([automated_tests],
               AS_HELP_STRING([--enable-automated-tests], [Enable automated tests using spicy-screenshot (part of spice-gtk)]),,
               [enable_automated_tests="no"])
diff --git a/server/Makefile.am b/server/Makefile.am
index ef8d31fc..5d5590af 100644
--- a/server/Makefile.am
+++ b/server/Makefile.am
@@ -20,6 +20,7 @@ AM_CPPFLAGS =					\
 	$(SSL_CFLAGS)				\
 	$(VISIBILITY_HIDDEN_CFLAGS)		\
 	$(WARN_CFLAGS)				\
+	$(ORC_CFLAGS)				\
 	$(NULL)
 
 noinst_LTLIBRARIES = libserver.la
@@ -54,6 +55,7 @@ libserver_la_LIBADD =							\
 	$(SSL_LIBS)							\
 	$(Z_LIBS)							\
 	$(SPICE_NONPKGCONFIG_LIBS)					\
+	$(ORC_LIBS)							\
 	$(NULL)
 
 libspice_serverincludedir = $(includedir)/spice-server
diff --git a/server/gstreamer-encoder.c b/server/gstreamer-encoder.c
index bb4f27ba..b2ddfea9 100644
--- a/server/gstreamer-encoder.c
+++ b/server/gstreamer-encoder.c
@@ -27,6 +27,7 @@
 #include <gst/app/gstappsrc.h>
 #include <gst/app/gstappsink.h>
 #include <gst/video/video.h>
+#include <orc/orcprogram.h>
 
 #include "red-common.h"
 #include "video-encoder.h"
@@ -1702,6 +1703,30 @@ static void spice_gst_encoder_get_stats(VideoEncoder *video_encoder,
     }
 }
 
+/* Check if ORC library can work.
+ * ORC library is used quite extensively by GStreamer
+ * to generate code dynamically. If ORC cannot work, GStreamer
+ * will abort(3) the entire process.
+ */
+static bool orc_check(void)
+{
+    static bool orc_checked = false;
+    static bool orc_dynamic_code_ok = false;
+
+    if (SPICE_UNLIKELY(!orc_checked)) {
+        OrcCode *code = orc_code_new();
+        if (code) {
+            /* allocating 0 byte for code makes the function not crash
+             * but it does all the initializations and checks */
+            orc_code_allocate_codemem(code, 0);
+            orc_dynamic_code_ok = code->code != NULL;
+            orc_code_free(code);
+        }
+        orc_checked = true;
+    }
+    return orc_dynamic_code_ok;
+}
+
 VideoEncoder *gstreamer_encoder_new(SpiceVideoCodecType codec_type,
                                     uint64_t starting_bit_rate,
                                     VideoEncoderRateControlCbs *cbs,
@@ -1721,6 +1746,11 @@ VideoEncoder *gstreamer_encoder_new(SpiceVideoCodecType codec_type,
         return NULL;
     }
 
+    // avoid aborting the process
+    if (!orc_check()) {
+        return NULL;
+    }
+
     SpiceGstEncoder *encoder = spice_new0(SpiceGstEncoder, 1);
     encoder->base.destroy = spice_gst_encoder_destroy;
     encoder->base.encode_frame = spice_gst_encoder_encode_frame;