Fix segfault in CVE-2023-39355.patch

(and use posix_memalign instead of memalign)
This commit is contained in:
Tobias Frost 2023-10-07 17:08:05 +02:00
parent f1641af0fd
commit bb305bcdf7

View File

@ -13,7 +13,16 @@ Subject: [PATCH] [codec,planar] fix reset
--- a/libfreerdp/codec/planar.c
+++ b/libfreerdp/codec/planar.c
@@ -847,6 +847,9 @@
@@ -31,6 +31,8 @@
#include <freerdp/codec/bitmap.h>
#include <freerdp/codec/planar.h>
+#include <stdlib.h>
+
#define TAG FREERDP_TAG("codec")
static INLINE UINT32 planar_invert_format(BITMAP_PLANAR_CONTEXT* planar, BOOL alpha,
@@ -847,6 +849,9 @@
{
BYTE* rleBuffer[4] = { 0 };
@ -23,19 +32,14 @@ Subject: [PATCH] [codec,planar] fix reset
rleBuffer[3] = planar->rlePlanesBuffer; /* AlphaPlane */
rleBuffer[0] = rleBuffer[3] + planeSize; /* LumaOrRedPlane */
rleBuffer[1] = rleBuffer[0] + planeSize; /* OrangeChromaOrGreenPlane */
@@ -1486,27 +1489,41 @@
@@ -1486,27 +1491,49 @@
context->maxHeight = height;
context->maxPlaneSize = context->maxWidth * context->maxHeight;
context->nTempStep = context->maxWidth * 4;
+
+ memset(context->planes, 0, sizeof(context->planes));
+ memset(context->rlePlanes, 0, sizeof(context->rlePlanes));
+ memset(context->deltaPlanes, 0, sizeof(context->deltaPlanes));
+
free(context->planesBuffer);
free(context->pTempData);
free(context->deltaPlanesBuffer);
free(context->rlePlanesBuffer);
- free(context->planesBuffer);
- free(context->pTempData);
- free(context->deltaPlanesBuffer);
- free(context->rlePlanesBuffer);
- context->planesBuffer = calloc(context->maxPlaneSize, 4);
- context->pTempData = calloc(context->maxPlaneSize, 6);
- context->deltaPlanesBuffer = calloc(context->maxPlaneSize, 4);
@ -44,16 +48,38 @@ Subject: [PATCH] [codec,planar] fix reset
- if (!context->planesBuffer || !context->pTempData || !context->deltaPlanesBuffer ||
- !context->rlePlanesBuffer)
- return FALSE;
+ memset(context->planes, 0, sizeof(context->planes));
+ memset(context->rlePlanes, 0, sizeof(context->rlePlanes));
+ memset(context->deltaPlanes, 0, sizeof(context->deltaPlanes));
+
+ free(context->planesBuffer); context->planesBuffer = NULL;
+ free(context->pTempData); context->pTempData = NULL;
+ free(context->deltaPlanesBuffer); context->deltaPlanesBuffer = NULL;
+ free(context->rlePlanesBuffer); context->rlePlanesBuffer = NULL;
+
+ if (context->maxPlaneSize > 0)
+ {
+ context->planesBuffer = memalign(32, context->maxPlaneSize * 4);
+ context->pTempData = memalign(32,context->maxPlaneSize * 6);
+ context->deltaPlanesBuffer = memalign(32, context->maxPlaneSize * 4);
+ context->rlePlanesBuffer = memalign(32, context->maxPlaneSize * 4);
+ unsigned int err = 0;
+ err = posix_memalign(&context->planesBuffer, 32, context->maxPlaneSize * 4);
+ memset(context->planesBuffer, 0, context->maxPlaneSize * 4);
+
+ if (!context->planesBuffer || !context->pTempData || !context->deltaPlanesBuffer ||
+ !context->rlePlanesBuffer)
+ err |= posix_memalign(&context->pTempData, 32, context->maxPlaneSize * 6);
+ memset(context->pTempData, 0, context->maxPlaneSize * 6);
+
+ err |= posix_memalign(&context->deltaPlanesBuffer, 32, context->maxPlaneSize * 4);
+ memset(context->deltaPlanesBuffer, 0, context->maxPlaneSize * 4);
+
+ err |= posix_memalign(&context->rlePlanesBuffer, 32, context->maxPlaneSize * 4);
+ memset(context->rlePlanesBuffer, 0, context->maxPlaneSize * 4);
+
+ if (err || !context->planesBuffer || !context->pTempData || !context->deltaPlanesBuffer ||
+ !context->rlePlanesBuffer){
+ free(context->planesBuffer); context->planesBuffer = NULL;
+ free(context->pTempData); context->pTempData = NULL;
+ free(context->deltaPlanesBuffer); context->deltaPlanesBuffer = NULL;
+ free(context->rlePlanesBuffer); context->rlePlanesBuffer = NULL;
+ return FALSE;
+ }
- context->planes[0] = &context->planesBuffer[context->maxPlaneSize * 0];
- context->planes[1] = &context->planesBuffer[context->maxPlaneSize * 1];
@ -63,11 +89,6 @@ Subject: [PATCH] [codec,planar] fix reset
- context->deltaPlanes[1] = &context->deltaPlanesBuffer[context->maxPlaneSize * 1];
- context->deltaPlanes[2] = &context->deltaPlanesBuffer[context->maxPlaneSize * 2];
- context->deltaPlanes[3] = &context->deltaPlanesBuffer[context->maxPlaneSize * 3];
+ memset(context->planesBuffer , 0, context->maxPlaneSize * 4);
+ memset(context->pTempData , 0, context->maxPlaneSize * 4);
+ memset(context->deltaPlanesBuffer , 0, context->maxPlaneSize * 4);
+ memset(context->rlePlanesBuffer , 0, context->maxPlaneSize * 4);
+
+ context->planes[0] = &context->planesBuffer[context->maxPlaneSize * 0];
+ context->planes[1] = &context->planesBuffer[context->maxPlaneSize * 1];
+ context->planes[2] = &context->planesBuffer[context->maxPlaneSize * 2];