diff --git a/server/red-replay-qxl.c b/server/red-replay-qxl.c
index aeaa5452..8c52e51c 100644
--- a/server/red-replay-qxl.c
+++ b/server/red-replay-qxl.c
@@ -1256,6 +1256,7 @@ static void replay_handle_create_primary(QXLWorker *worker, SpiceReplay *replay)
     read_binary(replay, "data", &size, &mem, 0);
     surface.group_id = 0;
     free(replay->primary_mem);
+    replay->allocated = g_list_remove(replay->allocated, mem);
     replay->primary_mem = mem;
     surface.mem = QXLPHYSICAL_FROM_PTR(mem);
     worker->create_primary_surface(worker, 0, &surface);