From 2a4015bf8d2d55f26d73fde397a388ffa3be41b7 Mon Sep 17 00:00:00 2001
From: Alexander Larsson <alexl@redhat.com>
Date: Wed, 28 Apr 2010 12:15:24 +0200
Subject: [PATCH] Clear alpha in xRGB destination to avoid pixman setting it to
 0xff

Pixman sometimes sets the ignored high byte to 0xff during alpha
blending. This is correct according to pixman specs, as the high
byte is ignored. However its not what windows expects, and it causes
unnecessary regions with non-zero high byte, causing us to
send rgba data instead of rgb which compresses worse.

So, we detect this and clear the high byte.
---
 cairo_canvas.c | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)

diff --git a/cairo_canvas.c b/cairo_canvas.c
index 6498142..48f14a9 100644
--- a/cairo_canvas.c
+++ b/cairo_canvas.c
@@ -292,6 +292,57 @@ static void fill_tiled_rects_rop_from_surface(SpiceCanvas *spice_canvas,
                            offset_y, rop);
 }
 
+/* Some pixman implementations of OP_OVER on xRGB32 sets
+   the high bit to 0xff (which is the right value if the
+   destination was ARGB32, and it should be ignored for
+   xRGB32. However, this fills our alpha bits with
+   data that is not wanted or expected by windows, and its
+   causing us to send rgba images rather than rgb images to
+   the client. So, we manually clear these bytes. */
+static void clear_dest_alpha(pixman_image_t *dest,
+                             int x, int y,
+                             int width, int height)
+{
+    uint32_t *data;
+    int stride;
+    int w, h;
+
+    w = pixman_image_get_width(dest);
+    h = pixman_image_get_height(dest);
+
+    if (x + width <= 0 || x >= w ||
+        y + height <= 0 || y >= h ||
+        width == 0 || height == 0) {
+        return;
+    }
+
+    if (x < 0) {
+        width += x;
+        x = 0;
+    }
+    if (x + width > w) {
+        width = w - x;
+    }
+
+    if (y < 0) {
+        height += y;
+        y = 0;
+    }
+    if (y + height > h) {
+        height = h - y;
+    }
+
+    stride = pixman_image_get_stride(dest);
+    data = (uint32_t *) (
+        (uint8_t *)pixman_image_get_data(dest) + y * stride + 4 * x);
+
+    if ((*data & 0xff000000U) == 0xff000000U) {
+        spice_pixman_fill_rect_rop(dest,
+                                   x, y, width, height,
+                                   0x00ffffff, SPICE_ROP_AND);
+    }
+}
+
 static void __blit_image(SpiceCanvas *spice_canvas,
                          pixman_region32_t *region,
                          pixman_image_t *src_image,
@@ -611,6 +662,11 @@ static void __blend_image(SpiceCanvas *spice_canvas,
                              width,
                              height);
 
+    if (canvas->base.format == SPICE_SURFACE_FMT_32_xRGB &&
+        !dest_has_alpha) {
+        clear_dest_alpha(dest, dest_x, dest_y, width, height);
+    }
+
     if (mask) {
         pixman_image_unref(mask);
     }
@@ -704,6 +760,11 @@ static void __blend_scale_image(SpiceCanvas *spice_canvas,
                              dest_x, dest_y, /* dst */
                              dest_width, dest_height);
 
+    if (canvas->base.format == SPICE_SURFACE_FMT_32_xRGB &&
+        !dest_has_alpha) {
+        clear_dest_alpha(dest, dest_x, dest_y, dest_width, dest_height);
+    }
+
     pixman_transform_init_identity(&transform);
     pixman_image_set_transform(src, &transform);
 
@@ -1035,6 +1096,11 @@ static void canvas_draw_text(SpiceCanvas *spice_canvas, SpiceRect *bbox,
                                  pos.x, pos.y,
                                  pixman_image_get_width(str_mask),
                                  pixman_image_get_height(str_mask));
+        if (canvas->base.format == SPICE_SURFACE_FMT_32_xRGB) {
+            clear_dest_alpha(canvas->image, pos.x, pos.y,
+                             pixman_image_get_width(str_mask),
+                             pixman_image_get_height(str_mask));
+        }
         pixman_image_unref(brush);
 
         pixman_image_set_clip_region32(canvas->image, NULL);