pxcloud/spice
5
0
Fork 0
mirror of https://gitlab.uni-freiburg.de/opensourcevdi/spice synced 2025-12-29 08:47:13 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix integer overflow computing glyph_size in red_get_string

Browse Source 
If bpp is int the formula can lead to weird overflows. width and height
are uint16_t so the formula is:

  size_t = u16 * (u16 * int + const_int) / const_int;

so it became

  size_t = (int) u16 * ((int) u16 * int + const_int) / const_int;

However the (int) u16 * (int) u16 can then became negative to overflow.
Under 64 bit architectures size_t is 64 and int usually 32 so converting
this negative 32 bit number to a unsigned 64 bit lead to a very big
number as the signed is extended and then converted to unsigned.
Using unsigned arithmetic prevent extending the sign.

Signed-off-by: Frediano Ziglio <fziglio@redhat.com>
Acked-by: Christophe Fergeau <cfergeau@redhat.com>
This commit is contained in:
Frediano Ziglio 2015-09-08 10:13:24 +01:00
parent dfaedec789
commit caec52dc77
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
server/red_parse_qxl.c
View File

@ -804,7 +804,9 @@ static SpiceString *red_get_string(RedMemSlotInfo *slots, int group_id,
uint8_t *data;
bool free_data;
size_t chunk_size, qxl_size, red_size, glyph_size;
int glyphs, bpp = 0, i;
int glyphs, i;
/* use unsigned to prevent integer overflow in multiplication below */
unsigned int bpp = 0;
int error;
uint16_t qxl_flags, qxl_length;
@ -843,7 +845,7 @@ static SpiceString *red_get_string(RedMemSlotInfo *slots, int group_id,
while (start < end) {
spice_assert((QXLRasterGlyph*)(&start->data[0]) <= end);
glyphs++;
glyph_size = start->height * ((start->width * bpp + 7) / 8);
glyph_size = start->height * ((start->width * bpp + 7u) / 8u);
red_size += sizeof(SpiceRasterGlyph *) + SPICE_ALIGN(sizeof(SpiceRasterGlyph) + glyph_size, 4);
start = (QXLRasterGlyph*)(&start->data[glyph_size]);
}
@ -864,7 +866,7 @@ static SpiceString *red_get_string(RedMemSlotInfo *slots, int group_id,
glyph->height = start->height;
red_get_point_ptr(&glyph->render_pos, &start->render_pos);
red_get_point_ptr(&glyph->glyph_origin, &start->glyph_origin);
glyph_size = glyph->height * ((glyph->width * bpp + 7) / 8);
glyph_size = glyph->height * ((glyph->width * bpp + 7u) / 8u);
spice_assert((QXLRasterGlyph*)(&start->data[glyph_size]) <= end);
memcpy(glyph->data, start->data, glyph_size);
start = (QXLRasterGlyph*)(&start->data[glyph_size]);