From fdb7754043282dbe26f6f8d58d7f1add30f97bdb Mon Sep 17 00:00:00 2001 From: loki Date: Sun, 15 Aug 2021 20:38:30 +0200 Subject: [PATCH] Attempt to render cursor when X11 is available --- CMakeLists.txt | 1 + sunshine/platform/common.h | 4 -- sunshine/platform/linux/graphics.cpp | 86 ++++++++++++++++++++++------ sunshine/platform/linux/graphics.h | 27 ++++++++- sunshine/platform/linux/kmsgrab.cpp | 35 +++++++++-- sunshine/platform/linux/vaapi.cpp | 40 +++---------- sunshine/platform/linux/x11grab.cpp | 77 +++++++++++++++++++++++-- sunshine/platform/linux/x11grab.h | 48 ++++++++++++++++ 8 files changed, 251 insertions(+), 67 deletions(-) create mode 100644 sunshine/platform/linux/x11grab.h diff --git a/CMakeLists.txt b/CMakeLists.txt index f4f17e47..45d4bf74 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -140,6 +140,7 @@ else() sunshine/platform/linux/misc.cpp sunshine/platform/linux/audio.cpp sunshine/platform/linux/input.cpp + sunshine/platform/linux/x11grab.h third-party/glad/src/egl.c third-party/glad/src/gl.c third-party/glad/include/EGL/eglplatform.h diff --git a/sunshine/platform/common.h b/sunshine/platform/common.h index 1f6b8498..4fe87b7c 100644 --- a/sunshine/platform/common.h +++ b/sunshine/platform/common.h @@ -147,10 +147,6 @@ public: std::int32_t pixel_pitch {}; std::int32_t row_pitch {}; - img_t() = default; - img_t(const img_t &) = delete; - img_t(img_t &&) = delete; - virtual ~img_t() = default; }; diff --git a/sunshine/platform/linux/graphics.cpp b/sunshine/platform/linux/graphics.cpp index 2f9e157f..2799e4af 100644 --- a/sunshine/platform/linux/graphics.cpp +++ b/sunshine/platform/linux/graphics.cpp @@ -485,11 +485,16 @@ void sws_t::set_colorspace(std::uint32_t colorspace, std::uint32_t color_range) }; color_matrix.update(members, sizeof(members) / sizeof(decltype(members[0]))); + + program[0].bind(color_matrix); + program[1].bind(color_matrix); } std::optional sws_t::make(int in_width, int in_height, int out_width, int out_heigth, gl::tex_t &&tex) { sws_t sws; + sws.serial = std::numeric_limits::max(); + // Ensure aspect ratio is maintained auto scalar = std::fminf(out_width / (float)in_width, out_heigth / (float)in_height); auto out_width_f = in_width * scalar; @@ -499,13 +504,16 @@ std::optional sws_t::make(int in_width, int in_height, int out_width, int auto offsetX_f = (out_width - out_width_f) / 2; auto offsetY_f = (out_heigth - out_height_f) / 2; - sws.width = out_width_f; - sws.height = out_height_f; + sws.out_width = out_width_f; + sws.out_height = out_height_f; + + sws.in_width = in_width; + sws.in_height = in_height; sws.offsetX = offsetX_f; sws.offsetY = offsetY_f; - auto width_i = 1.0f / sws.width; + auto width_i = 1.0f / sws.out_width; { const char *sources[] { @@ -542,7 +550,16 @@ std::optional sws_t::make(int in_width, int in_height, int out_width, int return std::nullopt; } - auto program = gl::program_t::link(compiled_sources[1].left(), compiled_sources[0].left()); + auto program = gl::program_t::link(compiled_sources[3].left(), compiled_sources[4].left()); + if(program.has_right()) { + BOOST_LOG(error) << "GL linker: "sv << program.right(); + return std::nullopt; + } + + // Cursor - shader + sws.program[2] = std::move(program.left()); + + program = gl::program_t::link(compiled_sources[1].left(), compiled_sources[0].left()); if(program.has_right()) { BOOST_LOG(error) << "GL linker: "sv << program.right(); return std::nullopt; @@ -588,13 +605,21 @@ std::optional sws_t::make(int in_width, int in_height, int out_width, int sws.tex = std::move(tex); + sws.cursor_framebuffer = gl::frame_buf_t::make(1); + sws.cursor_framebuffer.bind(&sws.tex[0], &sws.tex[1]); + + sws.program[0].bind(sws.color_matrix); + sws.program[1].bind(sws.color_matrix); + + gl::ctx.BlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); + gl_drain_errors; return std::move(sws); } std::optional sws_t::make(int in_width, int in_height, int out_width, int out_heigth) { - auto tex = gl::tex_t::make(1); + auto tex = gl::tex_t::make(2); gl::ctx.BindTexture(GL_TEXTURE_2D, tex[0]); gl::ctx.TexStorage2D(GL_TEXTURE_2D, 1, GL_RGBA8, in_width, in_height); @@ -606,19 +631,46 @@ void sws_t::load_ram(platf::img_t &img) { gl::ctx.TexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, img.width, img.height, GL_BGRA, GL_UNSIGNED_BYTE, img.data); } -void sws_t::load_vram(platf::img_t &img, int offset_x, int offset_y, int framebuffer) { +void sws_t::load_vram(cursor_t &img, int offset_x, int offset_y, int framebuffer) { gl::ctx.BindFramebuffer(GL_FRAMEBUFFER, framebuffer); gl::ctx.ReadBuffer(GL_COLOR_ATTACHMENT0); gl::ctx.BindTexture(GL_TEXTURE_2D, tex[0]); - gl::ctx.CopyTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, offset_x, offset_y, img.width, img.height); + gl::ctx.CopyTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, offset_x, offset_y, in_width, in_height); - gl::ctx.Flush(); + if(img.data) { + gl::ctx.BindTexture(GL_TEXTURE_2D, tex[1]); + if(serial != img.serial) { + serial = img.serial; + + gl::ctx.TexStorage2D(GL_TEXTURE_2D, 1, GL_RGBA8, img.width, img.height); + gl::ctx.TexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, img.width, img.height, GL_BGRA, GL_UNSIGNED_BYTE, img.data); + } + + gl::ctx.Enable(GL_BLEND); + GLenum attachment = GL_COLOR_ATTACHMENT0; + gl::ctx.BindFramebuffer(GL_FRAMEBUFFER, cursor_framebuffer[0]); + gl::ctx.DrawBuffers(1, &attachment); + +#ifndef NDEBUG + auto status = gl::ctx.CheckFramebufferStatus(GL_FRAMEBUFFER); + if(status != GL_FRAMEBUFFER_COMPLETE) { + BOOST_LOG(error) << "Pass Cursor: CheckFramebufferStatus() --> [0x"sv << util::hex(status).to_string_view() << ']'; + return; + } +#endif + + gl::ctx.UseProgram(program[2].handle()); + gl::ctx.Viewport(img.x, img.y, img.width, img.height); + gl::ctx.DrawArrays(GL_TRIANGLES, 0, 3); + + gl::ctx.Disable(GL_BLEND); + } + + gl::ctx.BindTexture(GL_TEXTURE_2D, 0); } int sws_t::convert(nv12_t &nv12) { - auto texture = tex[0]; - - gl::ctx.BindTexture(GL_TEXTURE_2D, texture); + gl::ctx.BindTexture(GL_TEXTURE_2D, tex[0]); GLenum attachments[] { GL_COLOR_ATTACHMENT0, @@ -629,21 +681,23 @@ int sws_t::convert(nv12_t &nv12) { gl::ctx.BindFramebuffer(GL_FRAMEBUFFER, nv12->buf[x]); gl::ctx.DrawBuffers(1, &attachments[x]); +#ifndef NDEBUG auto status = gl::ctx.CheckFramebufferStatus(GL_FRAMEBUFFER); if(status != GL_FRAMEBUFFER_COMPLETE) { BOOST_LOG(error) << "Pass "sv << x << ": CheckFramebufferStatus() --> [0x"sv << util::hex(status).to_string_view() << ']'; return -1; } - - gl::ctx.BindTexture(GL_TEXTURE_2D, texture); +#endif gl::ctx.UseProgram(program[x].handle()); - program[x].bind(color_matrix); - - gl::ctx.Viewport(offsetX / (x + 1), offsetY / (x + 1), width / (x + 1), height / (x + 1)); + gl::ctx.Viewport(offsetX / (x + 1), offsetY / (x + 1), out_width / (x + 1), out_height / (x + 1)); gl::ctx.DrawArrays(GL_TRIANGLES, 0, 3); } + gl::ctx.BindTexture(GL_TEXTURE_2D, 0); + + gl::ctx.Flush(); + return 0; } } // namespace egl diff --git a/sunshine/platform/linux/graphics.h b/sunshine/platform/linux/graphics.h index 74c9ef49..a42b6257 100644 --- a/sunshine/platform/linux/graphics.h +++ b/sunshine/platform/linux/graphics.h @@ -221,6 +221,16 @@ std::optional import_target( std::array &&fds, const surface_descriptor_t &r8, const surface_descriptor_t &gr88); +class cursor_t : public platf::img_t { +public: + int x, y; + int xhot, yhot; + + unsigned long serial; + + std::vector buffer; +}; + class sws_t { public: static std::optional make(int in_width, int in_height, int out_width, int out_heigth, gl::tex_t &&tex); @@ -229,16 +239,27 @@ public: int convert(nv12_t &nv12); void load_ram(platf::img_t &img); - void load_vram(platf::img_t &img, int offset_x, int offset_y, int framebuffer); + void load_vram(cursor_t &img, int offset_x, int offset_y, int framebuffer); void set_colorspace(std::uint32_t colorspace, std::uint32_t color_range); + // The first texture is the monitor image. + // The second texture is the cursor image gl::tex_t tex; - gl::program_t program[2]; + + // The cursor image will be blended into this framebuffer + gl::frame_buf_t cursor_framebuffer; + + // Y - shader, UV - shader, Cursor - shader + gl::program_t program[3]; gl::buffer_t color_matrix; - int width, height; + int out_width, out_height; + int in_width, in_height; int offsetX, offsetY; + + // Store latest cursor for load_vram + std::uint64_t serial; }; bool fail(); diff --git a/sunshine/platform/linux/kmsgrab.cpp b/sunshine/platform/linux/kmsgrab.cpp index 8848d862..bcaad026 100644 --- a/sunshine/platform/linux/kmsgrab.cpp +++ b/sunshine/platform/linux/kmsgrab.cpp @@ -12,8 +12,10 @@ #include "sunshine/round_robin.h" #include "sunshine/utility.h" +// Cursor rendering support through x11 #include "graphics.h" #include "vaapi.h" +#include "x11grab.h" using namespace std::literals; namespace fs = std::filesystem; @@ -261,7 +263,8 @@ public: auto end = std::end(card); for(auto plane = std::begin(card); plane != end; ++plane) { - bool cursor; + bool cursor = false; + auto props = card.plane_props(plane->plane_id); for(auto &[prop, val] : props) { if(prop->name == "type"sv) { @@ -334,6 +337,8 @@ public: return -1; } + cursor_opt = x11::cursor_t::make(); + return 0; } @@ -342,6 +347,8 @@ public: card_t card; file_t fb_fd; + + std::optional cursor_opt; }; class display_ram_t : public display_t { @@ -440,6 +447,10 @@ public: gl::ctx.BindTexture(GL_TEXTURE_2D, rgb->tex[0]); gl::ctx.GetTextureSubImage(rgb->tex[0], 0, offset_x, offset_y, 0, width, height, 1, GL_BGRA, GL_UNSIGNED_BYTE, img_out_base->height * img_out_base->row_pitch, img_out_base->data); + if(cursor_opt && cursor) { + cursor_opt->blend(*img_out_base, offset_x, offset_y); + } + return capture_e::ok; } @@ -487,12 +498,11 @@ public: } std::shared_ptr alloc_img() override { - auto img = std::make_shared(); + auto img = std::make_shared(); - img->width = width; - img->height = height; + img->serial = std::numeric_limitsserial)>::max(); + img->data = nullptr; img->pixel_pitch = 4; - img->row_pitch = img->pixel_pitch * width; return img; } @@ -535,7 +545,20 @@ public: return capture_e::ok; } - capture_e snapshot(img_t * /*img_out_base */, std::chrono::milliseconds /* timeout */, bool /* cursor */) { + capture_e snapshot(img_t *img_out_base, std::chrono::milliseconds /* timeout */, bool cursor) { + if(!cursor || !cursor_opt) { + img_out_base->data = nullptr; + return capture_e::ok; + } + + auto img = (egl::cursor_t *)img_out_base; + cursor_opt->capture(*img); + + img->x -= offset_x; + img->xhot -= offset_x; + img->yhot -= offset_y; + img->y -= offset_y; + return capture_e::ok; } }; diff --git a/sunshine/platform/linux/vaapi.cpp b/sunshine/platform/linux/vaapi.cpp index d7961705..4f2304ba 100644 --- a/sunshine/platform/linux/vaapi.cpp +++ b/sunshine/platform/linux/vaapi.cpp @@ -198,7 +198,7 @@ public: return 0; } - int _set_frame(AVFrame *frame) { + int set_frame(AVFrame *frame) override { this->hwframe.reset(frame); this->frame = frame; @@ -252,6 +252,12 @@ public: return -1; } + auto sws_opt = egl::sws_t::make(width, height, frame->width, frame->height); + if(!sws_opt) { + return -1; + } + + this->sws = std::move(*sws_opt); this->nv12 = std::move(*nv12_opt); return 0; @@ -284,27 +290,12 @@ public: sws.convert(nv12); return 0; } - - int set_frame(AVFrame *frame) override { - if(_set_frame(frame)) { - return -1; - } - - auto sws_opt = egl::sws_t::make(width, height, frame->width, frame->height); - if(!sws_opt) { - return -1; - } - - this->sws = std::move(*sws_opt); - - return 0; - } }; class va_vram_t : public va_t { public: int convert(platf::img_t &img) override { - sws.load_vram(img, offset_x, offset_y, framebuffer[0]); + sws.load_vram((egl::cursor_t &)img, offset_x, offset_y, framebuffer[0]); sws.convert(nv12); return 0; @@ -331,21 +322,6 @@ public: return 0; } - int set_frame(AVFrame *frame) override { - if(_set_frame(frame)) { - return -1; - } - - auto sws_opt = egl::sws_t::make(width, height, frame->width, frame->height); - if(!sws_opt) { - return -1; - } - - this->sws = std::move(*sws_opt); - - return 0; - } - file_t fb_fd; egl::rgb_t rgb; diff --git a/sunshine/platform/linux/x11grab.cpp b/sunshine/platform/linux/x11grab.cpp index d10cfa88..434bbf3d 100644 --- a/sunshine/platform/linux/x11grab.cpp +++ b/sunshine/platform/linux/x11grab.cpp @@ -20,22 +20,22 @@ #include "sunshine/main.h" #include "sunshine/task_pool.h" +#include "graphics.h" #include "misc.h" #include "vaapi.h" +#include "x11grab.h" using namespace std::literals; namespace platf { +int load_xcb(); +int load_x11(); namespace x11 { #define _FN(x, ret, args) \ typedef ret(*x##_fn) args; \ static x##_fn x -using XID = unsigned long; -using Time = unsigned long; -using Rotation = unsigned short; - _FN(GetImage, XImage *, ( Display * display, @@ -313,7 +313,7 @@ struct shm_img_t : public img_t { } }; -void blend_cursor(Display *display, img_t &img, int offsetX, int offsetY) { +static void blend_cursor(Display *display, img_t &img, int offsetX, int offsetY) { xcursor_t overlay { x11::fix::GetCursorImage(display) }; if(!overlay) { @@ -705,7 +705,7 @@ std::shared_ptr x11_display(platf::mem_type_e hwdevice_type, const st } std::vector x11_display_names() { - if(xcb::init_shm() || xcb::init() || x11::init() || x11::rr::init() || x11::fix::init()) { + if(load_x11() || load_xcb()) { BOOST_LOG(error) << "Couldn't init x11 libraries"sv; return {}; @@ -746,4 +746,69 @@ void freeImage(XImage *p) { void freeX(XFixesCursorImage *p) { x11::Free(p); } + +int load_xcb() { + // This will be called once only + static int xcb_status = xcb::init_shm() || xcb::init(); + + return xcb_status; +} + +int load_x11() { + // This will be called once only + static int x11_status = x11::init() || x11::rr::init() || x11::fix::init(); + + return x11_status; +} + +namespace x11 { +std::optional cursor_t::make() { + if(load_x11()) { + return std::nullopt; + } + + cursor_t cursor; + + cursor.ctx.reset((cursor_ctx_t::pointer)x11::OpenDisplay(nullptr)); + + return cursor; +} + +void cursor_t::capture(egl::cursor_t &img) { + auto display = (xdisplay_t::pointer)ctx.get(); + + xcursor_t xcursor = fix::GetCursorImage(display); + + if(img.serial != xcursor->cursor_serial) { + auto buf_size = xcursor->width * xcursor->height * sizeof(int); + + if(img.buffer.size() < buf_size) { + img.buffer.resize(buf_size); + } + + std::transform(xcursor->pixels, xcursor->pixels + buf_size / 4, (int *)img.buffer.data(), [](long pixel) -> int { + return pixel; + }); + } + + img.data = img.buffer.data(); + img.width = xcursor->width; + img.height = xcursor->height; + img.xhot = xcursor->xhot; + img.yhot = xcursor->yhot; + img.x = xcursor->x; + img.y = xcursor->y; + img.pixel_pitch = 4; + img.row_pitch = img.pixel_pitch * img.width; + img.serial = xcursor->cursor_serial; +} + +void cursor_t::blend(img_t &img, int offsetX, int offsetY) { + blend_cursor((xdisplay_t::pointer)ctx.get(), img, offsetX, offsetY); +} + +void freeCursorCtx(cursor_ctx_t::pointer ctx) { + x11::CloseDisplay((xdisplay_t::pointer)ctx); +} +} // namespace x11 } // namespace platf diff --git a/sunshine/platform/linux/x11grab.h b/sunshine/platform/linux/x11grab.h new file mode 100644 index 00000000..1440ae76 --- /dev/null +++ b/sunshine/platform/linux/x11grab.h @@ -0,0 +1,48 @@ +#ifndef SUNSHINE_X11_GRAB +#define SUNSHINE_X11_GRAB + +#include + +#include "sunshine/platform/common.h" +#include "sunshine/utility.h" + +namespace egl { +class cursor_t; +} + +namespace platf::x11 { + +#ifdef SUNSHINE_BUILD_X11 +struct cursor_ctx_raw_t; +void freeCursorCtx(cursor_ctx_raw_t *ctx); + +using cursor_ctx_t = util::safe_ptr; + +class cursor_t { +public: + static std::optional make(); + + void capture(egl::cursor_t &img); + + /** + * Capture and blend the cursor into the image + * + * img <-- destination image + * offsetX, offsetY <--- Top left corner of the virtual screen + */ + void blend(img_t &img, int offsetX, int offsetY); + + cursor_ctx_t ctx; +}; +#else +class cursor_t { +public: + static std::optional make() { return std::nullopt; } + + void capture(egl::cursor_t &) {} + void blend(img_t &, int, int) {} +}; +#endif +} // namespace platf::x11 + +#endif \ No newline at end of file