pxcloud/spice
6
0
Fork 0
mirror of https://gitlab.uni-freiburg.de/opensourcevdi/spice synced 2026-08-08 09:04:33 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

4:4:4 & upsampling hw encoding

Browse Source
This commit is contained in:
Michael Scherle 2024-07-29 12:58:31 +00:00
parent c8d2012a5a
commit 173a25fbb8
4 changed files with 161 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
server/display-channel.cpp
View File

@ -2230,7 +2230,21 @@ DisplayChannel::DisplayChannel(RedsState *reds,
set_cap(SPICE_DISPLAY_CAP_PREF_VIDEO_CODEC_TYPE);
set_cap(SPICE_DISPLAY_CAP_STREAM_REPORT);
//TODO actualy check
set_cap(SPICE_DISPLAY_CAP_CODEC_H264);
//set_cap(SPICE_DISPLAY_CAP_CODEC_VP9);
set_cap(SPICE_DISPLAY_CAP_CODEC_H265);
set_cap(SPICE_DISPLAY_CAP_CODEC_AV1);
// set_cap(SPICE_DISPLAY_CAP_CODEC_VP9_444);
set_cap(SPICE_DISPLAY_CAP_CODEC_H265_444);
// set_cap(SPICE_DISPLAY_CAP_CODEC_VP9_U);
set_cap(SPICE_DISPLAY_CAP_CODEC_H264_U);
set_cap(SPICE_DISPLAY_CAP_CODEC_H265_U);
set_cap(SPICE_DISPLAY_CAP_CODEC_AV1_U);
reds_register_channel(reds, this);
}
void display_channel_process_surface_cmd(DisplayChannel *display,

146
server/gstreamer-encoder.c
View File

@ -102,7 +102,7 @@ typedef struct SpiceGstEncoder {
const SpiceFormatForGStreamer *format;
SpiceBitmapFmt spice_format;
uint32_t drm_format;
float scaling;
/* Number of consecutive frame encoding errors. */
uint32_t errors;
@ -907,10 +907,23 @@ static const gchar* get_gst_codec_name(const SpiceGstEncoder *encoder)
return "avenc_mjpeg";
case SPICE_VIDEO_CODEC_TYPE_VP8:
return "vp8enc";
case SPICE_VIDEO_CODEC_TYPE_H264_U:
case SPICE_VIDEO_CODEC_TYPE_H264_444:
case SPICE_VIDEO_CODEC_TYPE_H264:
return "x264enc";
case SPICE_VIDEO_CODEC_TYPE_VP9_U:
case SPICE_VIDEO_CODEC_TYPE_VP9_444:
case SPICE_VIDEO_CODEC_TYPE_VP9:
return "vp9enc";
case SPICE_VIDEO_CODEC_TYPE_H265_U:
case SPICE_VIDEO_CODEC_TYPE_H265_444:
case SPICE_VIDEO_CODEC_TYPE_H265:
return "x265enc";
case SPICE_VIDEO_CODEC_TYPE_AV1_U:
case SPICE_VIDEO_CODEC_TYPE_AV1_444:
case SPICE_VIDEO_CODEC_TYPE_AV1:
return "avenc_av1";
default:
/* gstreamer_encoder_new() should have rejected this codec type */
spice_warning("unsupported codec type %d", encoder->base.codec_type);
@ -918,13 +931,22 @@ static const gchar* get_gst_codec_name(const SpiceGstEncoder *encoder)
}
}
static const char video_codecs[][8] = {
static const char video_codecs[][10] = {
{ "" },
{ "mjpeg" },
{ "vp8" },
{ "h264" },
{ "vp9" },
{ "h265" },
{ "av1" },
{ "h264" },
{ "vp9" },
{ "h265" },
{ "av1" },
{ "h264"},
{ "vp9"},
{ "h265"},
{ "av1"},
};
static bool gst_features_lookup(const gchar *feature_name)
@ -948,13 +970,13 @@ static bool gst_features_lookup(const gchar *feature_name)
static gchar *find_best_hw_plugin(const gchar *codec_name)
{
static const char plugins[][8] = {"msdk", "va", "vaapi"};
static const char plugins[][16] = {"varenderD130","msdk", "vaapi"};
gchar *feature_name;
int i;
for (i = 0; i < G_N_ELEMENTS(plugins); i++) {
feature_name = !codec_name ? g_strconcat(plugins[i], "postproc", NULL) :
g_strconcat(plugins[i], codec_name, "enc", NULL);
feature_name = !codec_name ? g_strconcat(plugins[i], "vpp", NULL) :
g_strconcat(plugins[i], codec_name, "lpenc", NULL);
if (!gst_features_lookup(feature_name)) {
g_free(feature_name);
feature_name = NULL;
@ -988,6 +1010,8 @@ static gchar *get_hw_gstenc_opts(gchar *encoder, const gchar *codec_name)
} else {
if (strcmp(codec_name, "vp9") == 0) {
gstenc_opts = g_strdup("min-qp=15 max-qp=35 rate-control=16 ref-frames=0 target-usage=7");
} else if (strcmp(codec_name, "av1") == 0){
gstenc_opts = g_strdup("rate-control=16 target-usage=7 min-qp=15 max-qp=35");
} else {
gstenc_opts = g_strdup("rate-control=16 b-frames=0 target-usage=7 min-qp=15 max-qp=35");
}
@ -996,10 +1020,10 @@ static gchar *get_hw_gstenc_opts(gchar *encoder, const gchar *codec_name)
}
static void try_intel_hw_plugins(const gchar *codec_name, gchar **converter,
gchar **gstenc_name, gchar **gstenc_opts)
gchar **gstenc_name, gchar **gstenc_opts, SpiceGstEncoder* gst_encoder)
{
gchar *encoder, *vpp;
SpiceVideoCodecType codec_type = gst_encoder->base.codec_type;
if (strcmp(codec_name, "vp8") == 0) {
return;
}
@ -1020,11 +1044,22 @@ static void try_intel_hw_plugins(const gchar *codec_name, gchar **converter,
*gstenc_name = encoder;
*gstenc_opts = get_hw_gstenc_opts(encoder, codec_name);
if (g_str_has_prefix(vpp, "vaapi")) {
*converter = g_strconcat(vpp, " ! video/x-raw(memory:VASurface),format=NV12", NULL);
} else {
*converter = g_strconcat(vpp, " ! video/x-raw(memory:VAMemory),format=NV12", NULL);
char format[5] = "NV12\0";
if( codec_type == SPICE_VIDEO_CODEC_TYPE_AV1_444 ||
codec_type == SPICE_VIDEO_CODEC_TYPE_H264_444 ||
codec_type == SPICE_VIDEO_CODEC_TYPE_H265_444 ||
codec_type == SPICE_VIDEO_CODEC_TYPE_VP9_444 ) {
g_strlcpy(format,"VUYA\0",5);
}
char memory[11] = "VASurface\0";
if(!g_str_has_prefix(vpp, "vaapi")){
g_strlcpy(memory,"VAMemory\0",11);
}
//scaling-mode=1 interpolation-method=1
*converter = g_strdup_printf(" %s %s! capsfilter name=vpp caps=\"video/x-raw(memory:%s),format=%s\"",
vpp, gst_encoder->scaling==1 ? "":"scaling-mode=1 interpolation-method=1 ", memory,format);
g_free(vpp);
}
@ -1079,6 +1114,8 @@ static gboolean create_pipeline(SpiceGstEncoder *encoder)
gstenc_opts = g_strdup("");
break;
case SPICE_VIDEO_CODEC_TYPE_VP9:
case SPICE_VIDEO_CODEC_TYPE_VP9_U:
case SPICE_VIDEO_CODEC_TYPE_VP9_444:
case SPICE_VIDEO_CODEC_TYPE_VP8: {
/* See http://www.webmproject.org/docs/encoder-parameters/
* - Set mode/end-usage to get a constant bitrate to help with
@ -1099,6 +1136,8 @@ static gboolean create_pipeline(SpiceGstEncoder *encoder)
gstenc_opts = g_strdup_printf("end-usage=cbr min-quantizer=10 error-resilient=default lag-in-frames=0 deadline=1 cpu-used=4");
break;
}
case SPICE_VIDEO_CODEC_TYPE_H264_U:
case SPICE_VIDEO_CODEC_TYPE_H264_444:
case SPICE_VIDEO_CODEC_TYPE_H264:
/* - Set tune and sliced-threads to ensure a zero-frame latency
* - qp-min ensures the bitrate does not get needlessly high.
@ -1111,6 +1150,25 @@ static gboolean create_pipeline(SpiceGstEncoder *encoder)
*/
gstenc_opts = g_strdup("byte-stream=true aud=true qp-min=15 qp-max=35 tune=4 sliced-threads=true speed-preset=ultrafast intra-refresh=true");
break;
case SPICE_VIDEO_CODEC_TYPE_H265_U:
case SPICE_VIDEO_CODEC_TYPE_H265_444:
case SPICE_VIDEO_CODEC_TYPE_H265:
/* - Set tune and sliced-threads to ensure a zero-frame latency
* - qp-min ensures the bitrate does not get needlessly high.
* - qp-max ensures the compression does not go so high that the video
* is unrecognizable. When that threshold is reached it is better to
* drop frames to lower the bit rate further.
* - Set speed-preset to get realtime speed.
* - Set intra-refresh to get more uniform compressed frame sizes,
* thus helping with streaming.
*/
gstenc_opts = g_strdup("qp=15 tune=4 speed-preset=ultrafast ");
break;
case SPICE_VIDEO_CODEC_TYPE_AV1_U:
case SPICE_VIDEO_CODEC_TYPE_AV1_444:
case SPICE_VIDEO_CODEC_TYPE_AV1:
gstenc_opts = g_strdup("");
break;
default:
/* gstreamer_encoder_new() should have rejected this codec type */
spice_warning("unsupported codec type %d", encoder->base.codec_type);
@ -1118,18 +1176,28 @@ static gboolean create_pipeline(SpiceGstEncoder *encoder)
g_free(converter);
return FALSE;
}
if(encoder->base.codec_type == SPICE_VIDEO_CODEC_TYPE_H264_U ||
encoder->base.codec_type == SPICE_VIDEO_CODEC_TYPE_H265_U ||
encoder->base.codec_type == SPICE_VIDEO_CODEC_TYPE_VP9_U ||
encoder->base.codec_type == SPICE_VIDEO_CODEC_TYPE_AV1_U) {
encoder->scaling = 2;
}
const char *codec_name = video_codecs[encoder->base.codec_type];
GpuVendor vendor = spice_udev_detect_gpu(INTEL_VENDOR_ID);
if (vendor == VENDOR_GPU_DETECTED) {
try_intel_hw_plugins(codec_name, &converter, &gstenc_name,
&gstenc_opts);
&gstenc_opts, encoder);
}
GError *err = NULL;
gchar *desc = g_strdup_printf("appsrc is-live=true format=time do-timestamp=true name=src !"
" %s ! %s name=encoder %s ! appsink name=sink",
converter, gstenc_name, gstenc_opts);
converter, gstenc_name, gstenc_opts);
// gchar *desc = g_strdup_printf("appsrc is-live=true format=time do-timestamp=true name=src ! vapostproc ! video/x-raw(memory:VAMemory),format=NV12 ! msdkh264enc name=encoder async-depth=1 b-frames=0 rate-control=3 target-usage=7 ! appsink name=sink");
//desc = g_strdup_printf("appsrc is-live=true format=time do-timestamp=true name=src ! varenderD130postproc ! video/x-raw(memory:VAMemory),format=NV12 ! varenderD130h264lpenc name=encoder rate-control=16 b-frames=0 target-usage=7 min-qp=15 max-qp=35 ! appsink name=sink");
printf("gst-pipeline: %s\n",desc);
spice_debug("GStreamer pipeline: %s", desc);
encoder->pipeline = gst_parse_launch_full(desc, NULL, GST_PARSE_FLAG_FATAL_ERRORS, &err);
g_free(converter);
@ -1273,6 +1341,22 @@ static gboolean configure_pipeline(SpiceGstEncoder *encoder)
set_appsrc_caps(encoder);
}
/* set scaling */
if(encoder->set_pipeline && encoder->scaling != 1){
GstElement * vpp = gst_bin_get_by_name(GST_BIN(encoder->pipeline), "vpp");
if(vpp == NULL){
printf("no vpp \n");
} else {
GstCaps *vpp_caps = NULL;
g_object_get (G_OBJECT (vpp), "caps", &vpp_caps, NULL);
vpp_caps = gst_caps_make_writable (vpp_caps);
gst_caps_set_simple (vpp_caps, "width", G_TYPE_INT, (uint32_t)(encoder->width*encoder->scaling),
"height", G_TYPE_INT, (uint32_t) (encoder->height*encoder->scaling), NULL);
g_object_set(G_OBJECT(vpp), "caps", vpp_caps, NULL);
gst_caps_unref (vpp_caps);
}
}
/* Start playing */
if (gst_element_set_state(encoder->pipeline, GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) {
spice_warning("GStreamer error: unable to set the pipeline to the playing state");
@ -1772,7 +1856,7 @@ spice_gst_encoder_encode_dmabuf(VideoEncoder *video_encoder,
GstMemory *mem;
*outbuf = NULL;
rc = VIDEO_ENCODER_FRAME_UNSUPPORTED;
mem = gst_dmabuf_allocator_alloc_with_flags(encoder->allocator,
dmabuf_data->drm_dma_buf_fd,
size,
@ -1782,6 +1866,19 @@ spice_gst_encoder_encode_dmabuf(VideoEncoder *video_encoder,
}
buffer = gst_buffer_new();
gst_buffer_append_memory(buffer, mem);
/*
const gsize psize[2] = {0,0};
const gint stride[2] = {dmabuf_data->stride,0};
gst_buffer_add_video_meta_full(buffer,
GST_VIDEO_FRAME_FLAG_NONE,
encoder->format->drm_format,
(guint)dmabuf_data->width,
(guint)dmabuf_data->height,
(guint)1,
psize,
stride );
*/
gst_mini_object_weak_ref(GST_MINI_OBJECT(mem),
(GstMiniObjectNotify)spice_gst_mem_free_cb,
dmabuf_data);
@ -1980,8 +2077,20 @@ VideoEncoder *gstreamer_encoder_new(SpiceVideoCodecType codec_type,
spice_return_val_if_fail(codec_type == SPICE_VIDEO_CODEC_TYPE_MJPEG ||
codec_type == SPICE_VIDEO_CODEC_TYPE_VP8 ||
codec_type == SPICE_VIDEO_CODEC_TYPE_VP9 ||
codec_type == SPICE_VIDEO_CODEC_TYPE_H264, NULL);
codec_type == SPICE_VIDEO_CODEC_TYPE_H264 ||
codec_type == SPICE_VIDEO_CODEC_TYPE_H265 ||
codec_type == SPICE_VIDEO_CODEC_TYPE_AV1 ||
codec_type == SPICE_VIDEO_CODEC_TYPE_VP9_444 ||
codec_type == SPICE_VIDEO_CODEC_TYPE_H264_444 ||
codec_type == SPICE_VIDEO_CODEC_TYPE_H265_444 ||
codec_type == SPICE_VIDEO_CODEC_TYPE_VP9_U ||
codec_type == SPICE_VIDEO_CODEC_TYPE_H264_U ||
codec_type == SPICE_VIDEO_CODEC_TYPE_H265_U ||
codec_type == SPICE_VIDEO_CODEC_TYPE_AV1 ||
codec_type == SPICE_VIDEO_CODEC_TYPE_AV1_444 ||
codec_type == SPICE_VIDEO_CODEC_TYPE_AV1_U,
NULL);
GError *err = NULL;
if (!gst_init_check(NULL, NULL, &err)) {
spice_warning("GStreamer error: %s", err->message);
@ -2013,7 +2122,8 @@ VideoEncoder *gstreamer_encoder_new(SpiceVideoCodecType codec_type,
encoder->allocator = gst_dmabuf_allocator_new();
pthread_mutex_init(&encoder->outbuf_mutex, NULL);
pthread_cond_init(&encoder->outbuf_cond, NULL);
encoder->scaling = 1;
printf("Trying Codec: %s", get_gst_codec_name(encoder));
/* All the other fields are initialized to zero by g_new0(). */
if (!create_pipeline(encoder)) {

20
server/reds.cpp
View File

@ -3479,11 +3479,11 @@ err:
static const char default_renderer[] = "sw";
#if defined(HAVE_GSTREAMER_1_0)
#define GSTREAMER_CODECS "gstreamer:mjpeg;gstreamer:h264;gstreamer:vp8;gstreamer:vp9;"
#define GSTREAMER_CODECS "gstreamer:h264;gstreamer:h265;gstreamer:vp9;gstreamer:h264_444;gstreamer:h265_444;gstreamer:vp9_444;gstreamer:av1;gstreamer:av1_444;gstreamer:h264_u;gstreamer:h265_u;gstreamer:vp9_u;gstreamer:av1_u;"
#else
#define GSTREAMER_CODECS ""
#endif
static const char default_video_codecs[] = "spice:mjpeg;" GSTREAMER_CODECS;
static const char default_video_codecs[] = GSTREAMER_CODECS;
/* new interface */
SPICE_GNUC_VISIBLE SpiceServer *spice_server_new(void)
@ -3601,6 +3601,16 @@ static const EnumNames video_codec_names[] = {
{SPICE_VIDEO_CODEC_TYPE_VP8, "vp8"},
{SPICE_VIDEO_CODEC_TYPE_H264, "h264"},
{SPICE_VIDEO_CODEC_TYPE_VP9, "vp9"},
{SPICE_VIDEO_CODEC_TYPE_H265, "h265"},
{SPICE_VIDEO_CODEC_TYPE_AV1, "av1"},
{SPICE_VIDEO_CODEC_TYPE_H264_444, "h264_444"},
{SPICE_VIDEO_CODEC_TYPE_VP9_444, "vp9_444"},
{SPICE_VIDEO_CODEC_TYPE_H265_444, "h265_444"},
{SPICE_VIDEO_CODEC_TYPE_AV1_444, "av1_444"},
{SPICE_VIDEO_CODEC_TYPE_H264_U, "h264_u"},
{SPICE_VIDEO_CODEC_TYPE_VP9_U, "vp9_u"},
{SPICE_VIDEO_CODEC_TYPE_H265_U, "h265_u"},
{SPICE_VIDEO_CODEC_TYPE_AV1_U, "av1_u"},
{0, nullptr},
};
@ -3609,6 +3619,12 @@ static const int video_codec_caps[] = {
SPICE_DISPLAY_CAP_CODEC_VP8,
SPICE_DISPLAY_CAP_CODEC_H264,
SPICE_DISPLAY_CAP_CODEC_VP9,
SPICE_DISPLAY_CAP_CODEC_H265,
SPICE_DISPLAY_CAP_CODEC_AV1,
SPICE_DISPLAY_CAP_CODEC_H264_444,
SPICE_DISPLAY_CAP_CODEC_VP9_444,
SPICE_DISPLAY_CAP_CODEC_H265_444,
SPICE_DISPLAY_CAP_CODEC_AV1_444,
};
char *reds_get_video_codec_fullname(RedVideoCodec *codec)

2
subprojects/spice-common

@ -1 +1 @@
Subproject commit 58d375e5eadc6fb9e587e99fd81adcb95d01e8d6
Subproject commit 8c0319e31df967e41c74f4121cbdb3b785fe114e