pxcloud/spice
5
0
Fork 0
mirror of https://gitlab.uni-freiburg.de/opensourcevdi/spice synced 2026-01-05 14:21:52 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

gstreamer-encoder: Add an encoder function that takes dmabuf fd as input

Browse Source 
This patch adds a new function to enable the creation of Gst memory with
the dmabuf fd as the source by using a dmabuf allocator. And, it also
adds a mechanism to register and invoke any callbacks once the Gst memory
object is no longer used by the pipeline.

This patch also ensures that the source_fps value is always non-zero.

add dmabuf encoding if `drm/drm_fourcc.h` is present and
gstreamer is at least 1.24 due to
`gst_video_dma_drm_fourcc_to_format()`.

Cc: Frediano Ziglio <freddy77@gmail.com>
Cc: Gerd Hoffmann <kraxel@redhat.com>
Cc: Marc-André Lureau <marcandre.lureau@redhat.com>
Cc: Dongwon Kim <dongwon.kim@intel.com>
Signed-off-by: Vivek Kasireddy <vivek.kasireddy@intel.com>
Signed-off-by: Michael Scherle <michael.scherle@rz.uni-freiburg.de>
Acked-by: Frediano Ziglio <freddy77@gmail.com>
This commit is contained in:
Vivek Kasireddy 2023-03-15 14:10:47 -07:00 committed by Frediano Ziglio
parent b2fcf9b3fe
commit 63d82dad8e
3 changed files with 156 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
configure.ac
View File

@ -99,7 +99,7 @@ AC_ARG_ENABLE(gstreamer,
[enable_gstreamer="auto"])
if test "x$enable_gstreamer" != "xno" && test "x$enable_gstreamer" != "x0.10"; then
SPICE_CHECK_GSTREAMER(GSTREAMER_1_0, 1.0, [gstreamer-1.0 gstreamer-base-1.0 gstreamer-app-1.0 gstreamer-video-1.0],
SPICE_CHECK_GSTREAMER(GSTREAMER_1_0, 1.0, [gstreamer-1.0 gstreamer-base-1.0 gstreamer-app-1.0 gstreamer-video-1.0 gstreamer-allocators-1.0],
[enable_gstreamer="1.0"
SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_1_0, [gst-plugins-base 1.0], [appsrc videoconvert appsink])
SPICE_CHECK_GSTREAMER_ELEMENTS($GST_INSPECT_1_0, [gstreamer-libav 1.0], [avenc_mjpeg])

12
meson.build
View File

@ -131,15 +131,23 @@ endforeach
spice_server_has_gstreamer = false
spice_server_gst_version = get_option('gstreamer')
if spice_server_gst_version != 'no'
gst_deps = ['gstreamer', 'gstreamer-base', 'gstreamer-app', 'gstreamer-video']
gstreamer_version = ''
gst_deps = ['gstreamer', 'gstreamer-base', 'gstreamer-app', 'gstreamer-video', 'gstreamer-allocators']
foreach dep : gst_deps
dep = '@0@-@1@'.format(dep, spice_server_gst_version)
spice_server_deps += dependency(dep)
dep = dependency(dep)
if gstreamer_version == '' or dep.version().version_compare('< ' + gstreamer_version)
gstreamer_version = dep.version()
endif
spice_server_deps += dep
endforeach
spice_server_deps += dependency('orc-0.4')
gst_def = 'HAVE_GSTREAMER_1_0'
spice_server_config_data.set(gst_def, '1')
if compiler.has_header('drm/drm_fourcc.h') and gstreamer_version.version_compare('>= 1.24')
spice_server_config_data.set('HAVE_GSTREAMER_DMABUF_ENCODING', '1')
endif
spice_server_has_gstreamer = true
endif

177
server/gstreamer-encoder.c
View File

@ -30,6 +30,9 @@
#include <gst/app/gstappsrc.h>
#include <gst/app/gstappsink.h>
#include <gst/video/video.h>
#ifdef HAVE_GSTREAMER_DMABUF_ENCODING
#include <gst/allocators/gstdmabuf.h>
#endif
#include <orc/orcprogram.h>
#if defined(__GNUC__) && (__GNUC__ >= 6)
# pragma GCC diagnostic pop
@ -283,6 +286,7 @@ typedef struct SpiceGstEncoder {
/* How many frames were dropped by the server since the last encoded frame. */
uint32_t server_drops;
GstAllocator *allocator;
} SpiceGstEncoder;
@ -318,8 +322,12 @@ static inline double get_mbps(uint64_t bit_rate)
*/
static uint32_t get_source_fps(const SpiceGstEncoder *encoder)
{
return encoder->cbs.get_source_fps ?
encoder->cbs.get_source_fps(encoder->cbs.opaque) : SPICE_GST_DEFAULT_FPS;
uint32_t source_fps = 0;
if (encoder->cbs.get_source_fps) {
source_fps = encoder->cbs.get_source_fps(encoder->cbs.opaque);
}
return source_fps ? source_fps : SPICE_GST_DEFAULT_FPS;
}
static uint32_t get_network_latency(const SpiceGstEncoder *encoder)
@ -1523,6 +1531,9 @@ static void spice_gst_encoder_destroy(VideoEncoder *video_encoder)
{
SpiceGstEncoder *encoder = (SpiceGstEncoder*)video_encoder;
#ifdef HAVE_GSTREAMER_DMABUF_ENCODING
gst_object_unref(encoder->allocator);
#endif
free_pipeline(encoder);
pthread_mutex_destroy(&encoder->outbuf_mutex);
pthread_cond_destroy(&encoder->outbuf_cond);
@ -1533,35 +1544,45 @@ static void spice_gst_encoder_destroy(VideoEncoder *video_encoder)
g_free(encoder);
}
static VideoEncodeResults
spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
uint32_t frame_mm_time,
const SpiceBitmap *bitmap,
const SpiceRect *src, int top_down,
gpointer bitmap_opaque,
VideoBuffer **outbuf)
static void spice_gst_encoder_add_frame(SpiceGstEncoder *encoder,
VideoBuffer **outbuf,
uint64_t start,
uint32_t frame_mm_time)
{
SpiceGstEncoder *encoder = (SpiceGstEncoder*)video_encoder;
g_return_val_if_fail(outbuf != NULL, VIDEO_ENCODER_FRAME_UNSUPPORTED);
*outbuf = NULL;
uint32_t last_mm_time = get_last_frame_mm_time(encoder);
/* Unref the last frame's bitmap_opaque structures if any */
clear_zero_copy_queue(encoder, FALSE);
add_frame(encoder, frame_mm_time, spice_get_monotonic_time_ns() - start,
(*outbuf)->size);
int32_t refill = encoder->bit_rate * (frame_mm_time - last_mm_time) / MSEC_PER_SEC / 8;
encoder->vbuffer_free = MIN(encoder->vbuffer_free + refill,
encoder->vbuffer_size) - (*outbuf)->size;
server_increase_bit_rate(encoder, frame_mm_time);
update_next_frame_mm_time(encoder);
}
static VideoEncodeResults
spice_gst_encoder_configure_pipeline(SpiceGstEncoder *encoder,
uint32_t width, uint32_t height,
const SpiceBitmap *bitmap,
uint32_t frame_mm_time)
{
SpiceBitmapFmt format = bitmap ? (SpiceBitmapFmt) bitmap->format :
SPICE_BITMAP_FMT_32BIT;
uint32_t width = src->right - src->left;
uint32_t height = src->bottom - src->top;
if (width != encoder->width || height != encoder->height ||
encoder->spice_format != bitmap->format) {
encoder->spice_format != format) {
spice_debug("video format change: width %d -> %d, height %d -> %d, format %d -> %d",
encoder->width, width, encoder->height, height,
encoder->spice_format, bitmap->format);
encoder->format = map_format((SpiceBitmapFmt) bitmap->format);
encoder->spice_format, format);
encoder->format = map_format(format);
if (encoder->format == GSTREAMER_FORMAT_INVALID) {
spice_warning("unable to map format type %d", bitmap->format);
spice_warning("unable to map format type %d", format);
encoder->errors = 4;
return VIDEO_ENCODER_FRAME_UNSUPPORTED;
}
encoder->spice_format = (SpiceBitmapFmt) bitmap->format;
encoder->spice_format = format;
encoder->width = width;
encoder->height = height;
if (encoder->bit_rate == 0) {
@ -1600,8 +1621,36 @@ spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
return VIDEO_ENCODER_FRAME_UNSUPPORTED;
}
return VIDEO_ENCODER_FRAME_ENCODE_DONE;
}
static VideoEncodeResults
spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
uint32_t frame_mm_time,
const SpiceBitmap *bitmap,
const SpiceRect *src, int top_down,
gpointer bitmap_opaque,
VideoBuffer **outbuf)
{
SpiceGstEncoder *encoder = (SpiceGstEncoder*)video_encoder;
g_return_val_if_fail(outbuf != NULL, VIDEO_ENCODER_FRAME_UNSUPPORTED);
VideoEncodeResults rc;
*outbuf = NULL;
/* Unref the last frame's bitmap_opaque structures if any */
clear_zero_copy_queue(encoder, FALSE);
uint32_t width = src->right - src->left;
uint32_t height = src->bottom - src->top;
rc = spice_gst_encoder_configure_pipeline(encoder, width, height,
bitmap, frame_mm_time);
if (rc != VIDEO_ENCODER_FRAME_ENCODE_DONE) {
return rc;
}
uint64_t start = spice_get_monotonic_time_ns();
VideoEncodeResults rc = push_raw_frame(encoder, bitmap, src, top_down, bitmap_opaque);
rc = push_raw_frame(encoder, bitmap, src, top_down, bitmap_opaque);
if (rc == VIDEO_ENCODER_FRAME_ENCODE_DONE) {
rc = pull_compressed_buffer(encoder, outbuf);
if (rc != VIDEO_ENCODER_FRAME_ENCODE_DONE) {
@ -1621,20 +1670,80 @@ spice_gst_encoder_encode_frame(VideoEncoder *video_encoder,
if (rc != VIDEO_ENCODER_FRAME_ENCODE_DONE) {
return rc;
}
uint32_t last_mm_time = get_last_frame_mm_time(encoder);
add_frame(encoder, frame_mm_time, spice_get_monotonic_time_ns() - start,
(*outbuf)->size);
int32_t refill = encoder->bit_rate * (frame_mm_time - last_mm_time) / MSEC_PER_SEC / 8;
encoder->vbuffer_free = MIN(encoder->vbuffer_free + refill,
encoder->vbuffer_size) - (*outbuf)->size;
server_increase_bit_rate(encoder, frame_mm_time);
update_next_frame_mm_time(encoder);
spice_gst_encoder_add_frame(encoder, outbuf, start, frame_mm_time);
return rc;
}
#ifdef HAVE_GSTREAMER_DMABUF_ENCODING
static void
spice_gst_mem_free_cb(VideoEncoderDmabufData *dmabuf_data, GstMiniObject *obj)
{
if (dmabuf_data->free) {
dmabuf_data->free(dmabuf_data);
}
}
static VideoEncodeResults
spice_gst_encoder_encode_dmabuf(VideoEncoder *video_encoder,
uint32_t frame_mm_time,
VideoEncoderDmabufData *dmabuf_data,
VideoBuffer **outbuf)
{
SpiceGstEncoder *encoder = (SpiceGstEncoder*)video_encoder;
g_return_val_if_fail(outbuf != NULL, VIDEO_ENCODER_FRAME_UNSUPPORTED);
g_return_val_if_fail(dmabuf_data != NULL, VIDEO_ENCODER_FRAME_UNSUPPORTED);
VideoEncodeResults rc;
rc = spice_gst_encoder_configure_pipeline(encoder, dmabuf_data->width,
dmabuf_data->height, NULL,
frame_mm_time);
if (rc != VIDEO_ENCODER_FRAME_ENCODE_DONE) {
return rc;
}
gsize size = dmabuf_data->stride * dmabuf_data->height;
uint64_t start = spice_get_monotonic_time_ns();
GstBuffer *buffer;
GstMemory *mem;
*outbuf = NULL;
rc = VIDEO_ENCODER_FRAME_UNSUPPORTED;
mem = gst_dmabuf_allocator_alloc_with_flags(encoder->allocator,
dmabuf_data->drm_dma_buf_fd,
size,
GST_FD_MEMORY_FLAG_DONT_CLOSE);
if (!mem) {
return rc;
}
buffer = gst_buffer_new();
gst_buffer_append_memory(buffer, mem);
gst_mini_object_weak_ref(GST_MINI_OBJECT(mem),
(GstMiniObjectNotify)spice_gst_mem_free_cb,
dmabuf_data);
GstFlowReturn ret = gst_app_src_push_buffer(encoder->appsrc, buffer);
if (ret != GST_FLOW_OK) {
spice_warning("GStreamer error: unable to push source buffer (%d)", ret);
return rc;
}
rc = pull_compressed_buffer(encoder, outbuf);
if (rc != VIDEO_ENCODER_FRAME_ENCODE_DONE) {
/* The input buffer will be stuck in the pipeline, preventing
* later ones from being processed. Furthermore something went
* wrong with this pipeline, so it may be safer to rebuild it
* from scratch.
*/
free_pipeline(encoder);
encoder->errors++;
return rc;
}
spice_gst_encoder_add_frame(encoder, outbuf, start, frame_mm_time);
return rc;
}
#endif
static void spice_gst_encoder_client_stream_report(VideoEncoder *video_encoder,
uint32_t num_frames,
uint32_t num_drops,
@ -1825,6 +1934,10 @@ VideoEncoder *gstreamer_encoder_new(SpiceVideoCodecType codec_type,
SpiceGstEncoder *encoder = g_new0(SpiceGstEncoder, 1);
encoder->base.destroy = spice_gst_encoder_destroy;
encoder->base.encode_frame = spice_gst_encoder_encode_frame;
#ifdef HAVE_GSTREAMER_DMABUF_ENCODING
encoder->base.encode_dmabuf = spice_gst_encoder_encode_dmabuf;
encoder->allocator = gst_dmabuf_allocator_new();
#endif
encoder->base.client_stream_report = spice_gst_encoder_client_stream_report;
encoder->base.notify_server_frame_drop = spice_gst_encoder_notify_server_frame_drop;
encoder->base.get_bit_rate = spice_gst_encoder_get_bit_rate;