Adding support to automated tests

As suggested by Alon, a simple automated test to try to find
    regressions in Spice code.
    To use this, compile Spice with --enable-automated-tests and
    run test_display_streaming passing --automated-tests as parameter.

Makefile.am

@@ -12,4 +12,4 @@ DISTCLEANFILES =                                \
 
 EXTRA_DIST = spice.proto spice1.proto spice_codegen.py
 
-DISTCHECK_CONFIGURE_FLAGS=--enable-opengl --enable-gui --enable-tunnel --enable-smartcard --with-sasl
+DISTCHECK_CONFIGURE_FLAGS=--enable-opengl --enable-gui --enable-tunnel --enable-smartcard --with-sasl --enable-automated-tests

configure.ac

@@ -129,6 +129,13 @@ AC_ARG_ENABLE(client,
 AS_IF([test x"$enable_client" != "xno"], [enable_client="yes"])
 AM_CONDITIONAL(SUPPORT_CLIENT, test "x$enable_client" = "xyes")
 
+AC_ARG_ENABLE(automated_tests,
+[  --enable-automated-tests     Enable automated tests using snappy (part of spice--gtk)],,
+[enable_automated_tests="no"])
+AS_IF([test x"$enable_automated_tests" != "xno"], [enable_automated_tests="yes"])
+AM_CONDITIONAL(SUPPORT_AUTOMATED_TESTS, test "x$enable_automated_tests" != "xno")
+
+
 dnl =========================================================================
 dnl Check deps
 
@@ -372,6 +379,16 @@ AM_CONDITIONAL([HAVE_SASL], [test "x$with_sasl2" = "xyes" || test "x$with_sasl"
 AC_SUBST([SASL_CFLAGS])
 AC_SUBST([SASL_LIBS])
 
+if test "x$enable_automated_tests" = "xyes"; then
+    AC_MSG_CHECKING([for snappy])
+    snappy --help >/dev/null 2>&1
+    if test $? -ne 0 ; then
+        AC_MSG_RESULT([not found])
+        AC_MSG_ERROR([snappy was not found, this module is part of spice-gtk andis required to compile this package])
+    fi
+    AC_MSG_RESULT([found])
+fi
+
 dnl ===========================================================================
 dnl check compiler flags
 
@@ -549,6 +566,8 @@ echo "
         Smartcard:                ${enable_smartcard}
 
         SASL support:             ${enable_sasl}
+
+        Automated tests:          ${enable_automated_tests}
 "
 
 if test $os_win32 == "yes" ; then

server/tests/Makefile.am

@@ -9,6 +9,10 @@ INCLUDES =                              \
 	$(SPICE_NONPKGCONFIG_CFLAGS)    \
 	$(NULL)
 
+if SUPPORT_AUTOMATED_TESTS
+INCLUDES += -DAUTOMATED_TESTS
+endif
+
 AM_LDFLAGS = $(top_builddir)/server/libspice-server.la
 
 COMMON_BASE=basic_event_loop.c basic_event_loop.h test_util.h ../../common/backtrace.c

server/tests/README

@@ -21,3 +21,9 @@ test_fail_on_null_core_interface
 
 basic_event_loop.c
  used by test_just_sockets_no_ssl, can be used by other tests. very crude event loop. Should probably use libevent for better tests, but this is self contained.
+
+Automated tests
+===============
+
+test_display_streaming.c
+ this test can be used to check regressions. For this, Spice needs to be compiled with --enable-automated-tests and test_display_streaming needs to be called passing --automated-tests as parameter

server/tests/regression_test.py

@@ -0,0 +1,25 @@
+#!/usr/bin/python
+from subprocess import PIPE, Popen
+import Image
+import ImageChops
+
+
+def snappy():
+    cmd = "snappy -h localhost -p 5912 -o output.ppm"
+    p = Popen(cmd, shell=True)
+    p.wait()
+
+def verify():
+    base = Image.open("base_test.ppm")
+    output = Image.open("output.ppm")
+    return ImageChops.difference(base, output).getbbox()
+
+if __name__ == "__main__":
+    snappy()
+    diff = verify()
+
+    if diff is None:
+        print("\033[1;32mSUCCESS: No regressions were found!\033[1;m")
+    else:
+        print("\033[1;31mFAIL: Regressions were found!\n\033[1;m"
+              "\033[1;31m      Please, take a look in your code and go fix it!\033[1;m")

server/tests/test_display_base.c

@@ -1,9 +1,14 @@
+
 #include <config.h>
 #include <stdlib.h>
 #include <math.h>
 #include <string.h>
 #include <stdio.h>
+#include <unistd.h>
+#include <signal.h>
+#include <wait.h>
 #include <sys/select.h>
+#include <sys/types.h>
 #include <spice/qxl_dev.h>
 #include "test_display_base.h"
 #include "red_channel.h"
@@ -34,15 +39,53 @@ static void test_spice_destroy_update(SimpleSpiceUpdate *update)
     free(update);
 }
 
-#define WIDTH 320
+#define WIDTH 640
 #define HEIGHT 320
 
-#define SINGLE_PART 8
+#define SINGLE_PART 4
 static const int angle_parts = 64 / SINGLE_PART;
 static int unique = 1;
 static int color = -1;
 static int c_i = 0;
 
+/* Used for automated tests */
+static int control = 3; //used to know when we can take a screenshot
+static int rects = 16; //number of rects that will be draw
+static int has_automated_tests = 0; //automated test flag
+
+static void sigchld_handler(int signal_num) // wait for the child process and exit
+{
+    int status;
+    wait(&status);
+    exit(0);
+}
+
+static void regression_test(void)
+{
+    pid_t pid;
+
+    if (--rects != 0) {
+        return;
+    }
+
+    rects = 16;
+
+    if (--control != 0) {
+        return;
+    }
+
+    pid = fork();
+    if (pid == 0) {
+        char buf[PATH_MAX];
+        char *envp[] = {buf, NULL};
+
+        snprintf(buf, sizeof(buf), "PATH=%s", getenv("PATH"));
+        execve("regression_test.py", NULL, envp);
+    } else if (pid > 0) {
+        return;
+    }
+}
+
 static void set_cmd(QXLCommandExt *ext, uint32_t type, QXLPHYSICAL data)
 {
     ext->cmd.type = type;
@@ -107,7 +150,13 @@ static SimpleSpiceUpdate *test_spice_create_update_draw(uint32_t surface_id, int
     if ((t % angle_parts) == 0) {
         c_i++;
     }
-    color = (color + 1) % 2;
+
+    if(surface_id != 0) {
+        color = (color + 1) % 2;
+    } else {
+        color = surface_id;
+    }
+
     unique++;
 
     update   = calloc(sizeof(*update), 1);
@@ -116,6 +165,7 @@ static SimpleSpiceUpdate *test_spice_create_update_draw(uint32_t surface_id, int
 
     bw       = WIDTH/SINGLE_PART;
     bh       = 48;
+
     bbox.right = bbox.left + bw;
     bbox.bottom = bbox.top + bh;
     update->bitmap = malloc(bw * bh * 4);
@@ -303,6 +353,7 @@ int cursor_notify = NOTIFY_CURSOR_BATCH;
 #define SURF_WIDTH 320
 #define SURF_HEIGHT 240
 uint8_t secondary_surface[SURF_WIDTH * SURF_HEIGHT * 4];
+int has_secondary;
 
 // We shall now have a ring of commands, so that we can update
 // it from a separate thread - since get_command is called from
@@ -354,6 +405,10 @@ static void produce_command(void)
     static int target_surface = 0;
     static int cmd_index = 0;
 
+
+    if (has_secondary)
+        target_surface = 1;
+
     ASSERT(num_simple_commands);
 
     switch (simple_commands[cmd_index]) {
@@ -361,35 +416,50 @@ static void produce_command(void)
             path_progress(&path);
             break;
         case SIMPLE_UPDATE: {
-            QXLRect rect = {.left = 0, .right = WIDTH,
-                            .top = 0, .bottom = HEIGHT};
-            qxl_worker->update_area(qxl_worker, 0, &rect, NULL, 0, 1);
+            QXLRect rect = {.left = 0, .right = SURF_WIDTH,
+                            .top = 0, .bottom = SURF_HEIGHT};
+            qxl_worker->update_area(qxl_worker, target_surface, &rect, NULL, 0, 1);
             break;
         }
+
         case SIMPLE_COPY_BITS:
         case SIMPLE_DRAW: {
             SimpleSpiceUpdate *update;
+
+            if (has_automated_tests)
+            {
+                if (control == 0) {
+                     return;
+                }
+
+                regression_test();
+            }
+
             switch (simple_commands[cmd_index]) {
                 case SIMPLE_COPY_BITS:
-                    update = test_spice_create_update_copy_bits(target_surface);
+                    update = test_spice_create_update_copy_bits(0);
                     break;
                 case SIMPLE_DRAW:
-                    update = test_spice_create_update_draw(target_surface, path.t);
+                    update = test_spice_create_update_draw(0, path.t);
                     break;
             }
             push_command(&update->ext);
             break;
         }
+
         case SIMPLE_CREATE_SURFACE: {
             SimpleSurfaceCmd *update;
             target_surface = MAX_SURFACE_NUM - 1;
             update = create_surface(target_surface, SURF_WIDTH, SURF_HEIGHT,
                                     secondary_surface);
             push_command(&update->ext);
+            has_secondary = 1;
             break;
         }
+
         case SIMPLE_DESTROY_SURFACE: {
             SimpleSurfaceCmd *update;
+            has_secondary = 0;
             update = destroy_surface(target_surface);
             target_surface = 0;
             push_command(&update->ext);
@@ -529,6 +599,8 @@ QXLInterface display_sif = {
     .set_compression_level = set_compression_level,
     .set_mm_time = set_mm_time,
     .get_init_info = get_init_info,
+
+ /* the callbacks below are called from spice server thread context */
     .get_command = get_command,
     .req_cmd_notification = req_cmd_notification,
     .release_resource = release_resource,
@@ -572,6 +644,37 @@ SpiceServer* test_init(SpiceCoreInterface *core)
     path_init(&path, 0, angle_parts);
     bzero(primary_surface, sizeof(primary_surface));
     bzero(secondary_surface, sizeof(secondary_surface));
+    has_secondary = 0;
     wakeup_timer = core->timer_add(do_wakeup, NULL);
     return server;
 }
+
+void check_automated(int argc, char **argv)
+{
+    struct sigaction sa;
+
+    if (argc == 1) {
+        return;
+    }
+
+    if (argc > 2) {
+        goto invalid_option;
+    }
+
+    if (strcmp(argv[1], "--automated-tests") != 0) {
+        goto invalid_option;
+    }
+
+    has_automated_tests = 1;
+
+    memset(&sa, 0, sizeof sa);
+    sa.sa_handler = &sigchld_handler;
+    sigaction(SIGCHLD, &sa, NULL);
+
+    return;
+
+invalid_option:
+    printf("Invalid option!\n"
+           "Please, check README before run tests!\n" );
+    exit(0);
+}

server/tests/test_display_base.h

@@ -10,6 +10,9 @@ void test_set_simple_command_list(int* commands, int num_commands);
 void test_add_display_interface(SpiceServer *server);
 SpiceServer* test_init(SpiceCoreInterface* core);
 
+/* Used for automated tests */
+void check_automated(int argc, char **argv);
+
 // simple queue for commands
 enum {
     PATH_PROGRESS,

server/tests/test_display_streaming.c

@@ -10,13 +10,19 @@
 int simple_commands[] = {
     SIMPLE_DRAW,
     SIMPLE_UPDATE,
+    PATH_PROGRESS,
+    SIMPLE_CREATE_SURFACE,
+    SIMPLE_DESTROY_SURFACE,
 };
 
 SpiceCoreInterface *core;
 SpiceServer *server;
 
-int main(void)
+int main(int argc, char **argv)
 {
+#ifdef AUTOMATED_TESTS
+    check_automated(argc, argv);
+#endif
     core = basic_event_loop_init();
     server = test_init(core);
     spice_server_set_streaming_video(server, SPICE_STREAM_VIDEO_ALL);