From 9181028bcffa3efe6c7780e8ffdb6b7dcba56e42 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Mon, 23 Jan 2023 20:54:08 -0600 Subject: [PATCH] HDR prep work (#808) --- src/main.h | 1 + src/platform/common.h | 20 +++++- src/platform/linux/cuda.cpp | 11 ++-- src/platform/linux/kmsgrab.cpp | 19 +++--- src/platform/linux/misc.cpp | 18 +++--- src/platform/linux/wlgrab.cpp | 15 ++--- src/platform/linux/x11grab.cpp | 15 ++--- src/platform/macos/display.mm | 9 ++- src/platform/windows/display.h | 15 +++-- src/platform/windows/display_base.cpp | 76 +++++++++++++++++++--- src/platform/windows/display_ram.cpp | 11 +++- src/platform/windows/display_vram.cpp | 10 ++- src/stream.cpp | 50 +++++++++++++++ src/video.cpp | 91 ++++++++++++++++++++++----- src/video.h | 10 +++ 15 files changed, 294 insertions(+), 77 deletions(-) diff --git a/src/main.h b/src/main.h index 6169fb98..a684ee6d 100644 --- a/src/main.h +++ b/src/main.h @@ -51,6 +51,7 @@ MAIL(switch_display); MAIL(touch_port); MAIL(idr); MAIL(rumble); +MAIL(hdr); #undef MAIL } // namespace mail #endif // SUNSHINE_MAIN_H diff --git a/src/platform/common.h b/src/platform/common.h index 1d91c2f7..48c7ba23 100644 --- a/src/platform/common.h +++ b/src/platform/common.h @@ -15,6 +15,10 @@ #include "src/thread_safe.h" #include "src/utility.h" +extern "C" { +#include +} + struct sockaddr; struct AVFrame; struct AVBufferRef; @@ -39,6 +43,9 @@ class basic_environment; typedef basic_environment environment; } // namespace process } // namespace boost +namespace video { +struct config_t; +} namespace platf { constexpr auto MAX_GAMEPADS = 32; @@ -270,6 +277,15 @@ public: return std::make_shared(); } + virtual bool is_hdr() { + return false; + } + + virtual bool get_hdr_metadata(SS_HDR_METADATA &metadata) { + std::memset(&metadata, 0, sizeof(metadata)); + return false; + } + virtual ~display_t() = default; // Offsets for when streaming a specific monitor. By default, they are 0. @@ -315,11 +331,11 @@ std::unique_ptr audio_control(); * If display_name is empty --> Use the first monitor that's compatible you can find * If you require to use this parameter in a seperate thread --> make a copy of it. * - * framerate --> The peak number of images per second + * config --> Stream configuration * * Returns display_t based on hwdevice_type */ -std::shared_ptr display(mem_type_e hwdevice_type, const std::string &display_name, int framerate); +std::shared_ptr display(mem_type_e hwdevice_type, const std::string &display_name, const video::config_t &config); // A list of names of displays accepted as display_name with the mem_type_e std::vector display_names(mem_type_e hwdevice_type); diff --git a/src/platform/linux/cuda.cpp b/src/platform/linux/cuda.cpp index 9e3c95b4..845046d7 100644 --- a/src/platform/linux/cuda.cpp +++ b/src/platform/linux/cuda.cpp @@ -13,6 +13,7 @@ extern "C" { #include "graphics.h" #include "src/main.h" #include "src/utility.h" +#include "src/video.h" #include "wayland.h" #define SUNSHINE_STRINGVIEW_HELPER(x) x##sv @@ -414,7 +415,7 @@ public: class display_t : public platf::display_t { public: - int init(const std::string_view &display_name, int framerate) { + int init(const std::string_view &display_name, const ::video::config_t &config) { auto handle = handle_t::make(); if(!handle) { return -1; @@ -444,14 +445,14 @@ public: } } - delay = std::chrono::nanoseconds { 1s } / framerate; + delay = std::chrono::nanoseconds { 1s } / config.framerate; capture_params = NVFBC_CREATE_CAPTURE_SESSION_PARAMS { NVFBC_CREATE_CAPTURE_SESSION_PARAMS_VER }; capture_params.eCaptureType = NVFBC_CAPTURE_SHARED_CUDA; capture_params.bDisableAutoModesetRecovery = nv_bool(true); - capture_params.dwSamplingRateMs = 1000 /* ms */ / framerate; + capture_params.dwSamplingRateMs = 1000 /* ms */ / config.framerate; if(streamedMonitor != -1) { auto &output = status_params->outputs[streamedMonitor]; @@ -663,7 +664,7 @@ public: } // namespace cuda namespace platf { -std::shared_ptr nvfbc_display(mem_type_e hwdevice_type, const std::string &display_name, int framerate) { +std::shared_ptr nvfbc_display(mem_type_e hwdevice_type, const std::string &display_name, const video::config_t &config) { if(hwdevice_type != mem_type_e::cuda) { BOOST_LOG(error) << "Could not initialize nvfbc display with the given hw device type"sv; return nullptr; @@ -671,7 +672,7 @@ std::shared_ptr nvfbc_display(mem_type_e hwdevice_type, const std::st auto display = std::make_shared(); - if(display->init(display_name, framerate)) { + if(display->init(display_name, config)) { return nullptr; } diff --git a/src/platform/linux/kmsgrab.cpp b/src/platform/linux/kmsgrab.cpp index 2c84baea..214387b9 100644 --- a/src/platform/linux/kmsgrab.cpp +++ b/src/platform/linux/kmsgrab.cpp @@ -12,6 +12,7 @@ #include "src/platform/common.h" #include "src/round_robin.h" #include "src/utility.h" +#include "src/video.h" // Cursor rendering support through x11 #include "graphics.h" @@ -444,8 +445,8 @@ class display_t : public platf::display_t { public: display_t(mem_type_e mem_type) : platf::display_t(), mem_type { mem_type } {} - int init(const std::string &display_name, int framerate) { - delay = std::chrono::nanoseconds { 1s } / framerate; + int init(const std::string &display_name, const ::video::config_t &config) { + delay = std::chrono::nanoseconds { 1s } / config.framerate; int monitor_index = util::from_view(display_name); int monitor = 0; @@ -632,13 +633,13 @@ class display_ram_t : public display_t { public: display_ram_t(mem_type_e mem_type) : display_t(mem_type) {} - int init(const std::string &display_name, int framerate) { + int init(const std::string &display_name, const ::video::config_t &config) { if(!gbm::create_device) { BOOST_LOG(warning) << "libgbm not initialized"sv; return -1; } - if(display_t::init(display_name, framerate)) { + if(display_t::init(display_name, config)) { return -1; } @@ -852,8 +853,8 @@ public: return capture_e::ok; } - int init(const std::string &display_name, int framerate) { - if(display_t::init(display_name, framerate)) { + int init(const std::string &display_name, const ::video::config_t &config) { + if(display_t::init(display_name, config)) { return -1; } @@ -872,11 +873,11 @@ public: } // namespace kms -std::shared_ptr kms_display(mem_type_e hwdevice_type, const std::string &display_name, int framerate) { +std::shared_ptr kms_display(mem_type_e hwdevice_type, const std::string &display_name, const ::video::config_t &config) { if(hwdevice_type == mem_type_e::vaapi) { auto disp = std::make_shared(hwdevice_type); - if(!disp->init(display_name, framerate)) { + if(!disp->init(display_name, config)) { return disp; } @@ -885,7 +886,7 @@ std::shared_ptr kms_display(mem_type_e hwdevice_type, const std::stri auto disp = std::make_shared(hwdevice_type); - if(disp->init(display_name, framerate)) { + if(disp->init(display_name, config)) { return nullptr; } diff --git a/src/platform/linux/misc.cpp b/src/platform/linux/misc.cpp index b5660fe9..081fb534 100644 --- a/src/platform/linux/misc.cpp +++ b/src/platform/linux/misc.cpp @@ -418,7 +418,7 @@ static std::bitset sources; #ifdef SUNSHINE_BUILD_CUDA std::vector nvfbc_display_names(); -std::shared_ptr nvfbc_display(mem_type_e hwdevice_type, const std::string &display_name, int framerate); +std::shared_ptr nvfbc_display(mem_type_e hwdevice_type, const std::string &display_name, const video::config_t &config); bool verify_nvfbc() { return !nvfbc_display_names().empty(); @@ -427,7 +427,7 @@ bool verify_nvfbc() { #ifdef SUNSHINE_BUILD_WAYLAND std::vector wl_display_names(); -std::shared_ptr wl_display(mem_type_e hwdevice_type, const std::string &display_name, int framerate); +std::shared_ptr wl_display(mem_type_e hwdevice_type, const std::string &display_name, const video::config_t &config); bool verify_wl() { return window_system == window_system_e::WAYLAND && !wl_display_names().empty(); @@ -436,7 +436,7 @@ bool verify_wl() { #ifdef SUNSHINE_BUILD_DRM std::vector kms_display_names(); -std::shared_ptr kms_display(mem_type_e hwdevice_type, const std::string &display_name, int framerate); +std::shared_ptr kms_display(mem_type_e hwdevice_type, const std::string &display_name, const video::config_t &config); bool verify_kms() { return !kms_display_names().empty(); @@ -445,7 +445,7 @@ bool verify_kms() { #ifdef SUNSHINE_BUILD_X11 std::vector x11_display_names(); -std::shared_ptr x11_display(mem_type_e hwdevice_type, const std::string &display_name, int framerate); +std::shared_ptr x11_display(mem_type_e hwdevice_type, const std::string &display_name, const video::config_t &config); bool verify_x11() { return window_system == window_system_e::X11 && !x11_display_names().empty(); @@ -469,29 +469,29 @@ std::vector display_names(mem_type_e hwdevice_type) { return {}; } -std::shared_ptr display(mem_type_e hwdevice_type, const std::string &display_name, int framerate) { +std::shared_ptr display(mem_type_e hwdevice_type, const std::string &display_name, const video::config_t &config) { #ifdef SUNSHINE_BUILD_CUDA if(sources[source::NVFBC] && hwdevice_type == mem_type_e::cuda) { BOOST_LOG(info) << "Screencasting with NvFBC"sv; - return nvfbc_display(hwdevice_type, display_name, framerate); + return nvfbc_display(hwdevice_type, display_name, config); } #endif #ifdef SUNSHINE_BUILD_WAYLAND if(sources[source::WAYLAND]) { BOOST_LOG(info) << "Screencasting with Wayland's protocol"sv; - return wl_display(hwdevice_type, display_name, framerate); + return wl_display(hwdevice_type, display_name, config); } #endif #ifdef SUNSHINE_BUILD_DRM if(sources[source::KMS]) { BOOST_LOG(info) << "Screencasting with KMS"sv; - return kms_display(hwdevice_type, display_name, framerate); + return kms_display(hwdevice_type, display_name, config); } #endif #ifdef SUNSHINE_BUILD_X11 if(sources[source::X11]) { BOOST_LOG(info) << "Screencasting with X11"sv; - return x11_display(hwdevice_type, display_name, framerate); + return x11_display(hwdevice_type, display_name, config); } #endif diff --git a/src/platform/linux/wlgrab.cpp b/src/platform/linux/wlgrab.cpp index ce596e8c..ea4adc8d 100644 --- a/src/platform/linux/wlgrab.cpp +++ b/src/platform/linux/wlgrab.cpp @@ -1,6 +1,7 @@ #include "src/platform/common.h" #include "src/main.h" +#include "src/video.h" #include "vaapi.h" #include "wayland.h" @@ -18,8 +19,8 @@ struct img_t : public platf::img_t { class wlr_t : public platf::display_t { public: - int init(platf::mem_type_e hwdevice_type, const std::string &display_name, int framerate) { - delay = std::chrono::nanoseconds { 1s } / framerate; + int init(platf::mem_type_e hwdevice_type, const std::string &display_name, const ::video::config_t &config) { + delay = std::chrono::nanoseconds { 1s } / config.framerate; mem_type = hwdevice_type; if(display.init()) { @@ -175,8 +176,8 @@ public: return platf::capture_e::ok; } - int init(platf::mem_type_e hwdevice_type, const std::string &display_name, int framerate) { - if(wlr_t::init(hwdevice_type, display_name, framerate)) { + int init(platf::mem_type_e hwdevice_type, const std::string &display_name, const ::video::config_t &config) { + if(wlr_t::init(hwdevice_type, display_name, config)) { return -1; } @@ -307,7 +308,7 @@ public: } // namespace wl namespace platf { -std::shared_ptr wl_display(mem_type_e hwdevice_type, const std::string &display_name, int framerate) { +std::shared_ptr wl_display(mem_type_e hwdevice_type, const std::string &display_name, const video::config_t &config) { if(hwdevice_type != platf::mem_type_e::system && hwdevice_type != platf::mem_type_e::vaapi && hwdevice_type != platf::mem_type_e::cuda) { BOOST_LOG(error) << "Could not initialize display with the given hw device type."sv; return nullptr; @@ -315,7 +316,7 @@ std::shared_ptr wl_display(mem_type_e hwdevice_type, const std::strin if(hwdevice_type == platf::mem_type_e::vaapi) { auto wlr = std::make_shared(); - if(wlr->init(hwdevice_type, display_name, framerate)) { + if(wlr->init(hwdevice_type, display_name, config)) { return nullptr; } @@ -323,7 +324,7 @@ std::shared_ptr wl_display(mem_type_e hwdevice_type, const std::strin } auto wlr = std::make_shared(); - if(wlr->init(hwdevice_type, display_name, framerate)) { + if(wlr->init(hwdevice_type, display_name, config)) { return nullptr; } diff --git a/src/platform/linux/x11grab.cpp b/src/platform/linux/x11grab.cpp index 1f8f5b73..e18de69d 100644 --- a/src/platform/linux/x11grab.cpp +++ b/src/platform/linux/x11grab.cpp @@ -19,6 +19,7 @@ #include "src/config.h" #include "src/main.h" #include "src/task_pool.h" +#include "src/video.h" #include "cuda.h" #include "graphics.h" @@ -382,13 +383,13 @@ struct x11_attr_t : public display_t { x11::InitThreads(); } - int init(const std::string &display_name, int framerate) { + int init(const std::string &display_name, const ::video::config_t &config) { if(!xdisplay) { BOOST_LOG(error) << "Could not open X11 display"sv; return -1; } - delay = std::chrono::nanoseconds { 1s } / framerate; + delay = std::chrono::nanoseconds { 1s } / config.framerate; xwindow = DefaultRootWindow(xdisplay.get()); @@ -641,8 +642,8 @@ struct shm_attr_t : public x11_attr_t { return 0; } - int init(const std::string &display_name, int framerate) { - if(x11_attr_t::init(display_name, framerate)) { + int init(const std::string &display_name, const ::video::config_t &config) { + if(x11_attr_t::init(display_name, config)) { return 1; } @@ -685,7 +686,7 @@ struct shm_attr_t : public x11_attr_t { } }; -std::shared_ptr x11_display(platf::mem_type_e hwdevice_type, const std::string &display_name, int framerate) { +std::shared_ptr x11_display(platf::mem_type_e hwdevice_type, const std::string &display_name, const ::video::config_t &config) { if(hwdevice_type != platf::mem_type_e::system && hwdevice_type != platf::mem_type_e::vaapi && hwdevice_type != platf::mem_type_e::cuda) { BOOST_LOG(error) << "Could not initialize x11 display with the given hw device type"sv; return nullptr; @@ -700,7 +701,7 @@ std::shared_ptr x11_display(platf::mem_type_e hwdevice_type, const st // Attempt to use shared memory X11 to avoid copying the frame auto shm_disp = std::make_shared(hwdevice_type); - auto status = shm_disp->init(display_name, framerate); + auto status = shm_disp->init(display_name, config); if(status > 0) { // x11_attr_t::init() failed, don't bother trying again. return nullptr; @@ -712,7 +713,7 @@ std::shared_ptr x11_display(platf::mem_type_e hwdevice_type, const st // Fallback auto x11_disp = std::make_shared(hwdevice_type); - if(x11_disp->init(display_name, framerate)) { + if(x11_disp->init(display_name, config)) { return nullptr; } diff --git a/src/platform/macos/display.mm b/src/platform/macos/display.mm index a53ac0e6..c818235e 100644 --- a/src/platform/macos/display.mm +++ b/src/platform/macos/display.mm @@ -6,6 +6,11 @@ #include "src/config.h" #include "src/main.h" +// Avoid conflict between AVFoundation and libavutil both defining AVMediaType +#define AVMediaType AVMediaType_FFmpeg +#include "src/video.h" +#undef AVMediaType + namespace fs = std::filesystem; namespace platf { @@ -147,7 +152,7 @@ struct av_display_t : public display_t { } }; -std::shared_ptr display(platf::mem_type_e hwdevice_type, const std::string &display_name, int framerate) { +std::shared_ptr display(platf::mem_type_e hwdevice_type, const std::string &display_name, const video::config_t &config) { if(hwdevice_type != platf::mem_type_e::system) { BOOST_LOG(error) << "Could not initialize display with the given hw device type."sv; return nullptr; @@ -168,7 +173,7 @@ std::shared_ptr display(platf::mem_type_e hwdevice_type, const std::s } } - display->av_capture = [[AVVideo alloc] initWithDisplay:display->display_id frameRate:framerate]; + display->av_capture = [[AVVideo alloc] initWithDisplay:display->display_id frameRate:config.framerate]; if(!display->av_capture) { BOOST_LOG(error) << "Video setup failed."sv; diff --git a/src/platform/windows/display.h b/src/platform/windows/display.h index 2a3b6cc9..a72c94bd 100644 --- a/src/platform/windows/display.h +++ b/src/platform/windows/display.h @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include "src/platform/common.h" #include "src/utility.h" @@ -37,6 +37,7 @@ using adapter_t = util::safe_ptr>; using output1_t = util::safe_ptr>; using output5_t = util::safe_ptr>; +using output6_t = util::safe_ptr>; using dup_t = util::safe_ptr>; using texture2d_t = util::safe_ptr>; using texture1d_t = util::safe_ptr>; @@ -115,7 +116,7 @@ public: class display_base_t : public display_t { public: - int init(int framerate, const std::string &display_name); + int init(const ::video::config_t &config, const std::string &display_name); capture_e capture(snapshot_cb_t &&snapshot_cb, std::shared_ptr img, bool *cursor) override; std::chrono::nanoseconds delay; @@ -141,6 +142,9 @@ public: typedef NTSTATUS WINAPI (*PD3DKMTSetProcessSchedulingPriorityClass)(HANDLE, D3DKMT_SCHEDULINGPRIORITYCLASS); + virtual bool is_hdr() override; + virtual bool get_hdr_metadata(SS_HDR_METADATA &metadata) override; + protected: int get_pixel_pitch() { return (capture_format == DXGI_FORMAT_R16G16B16A16_FLOAT) ? 8 : 4; @@ -151,6 +155,7 @@ protected: virtual capture_e snapshot(img_t *img, std::chrono::milliseconds timeout, bool cursor_visible) = 0; virtual int complete_img(img_t *img, bool dummy) = 0; virtual std::vector get_supported_sdr_capture_formats() = 0; + virtual std::vector get_supported_hdr_capture_formats() = 0; }; class display_ram_t : public display_base_t { @@ -161,8 +166,9 @@ public: int dummy_img(img_t *img) override; int complete_img(img_t *img, bool dummy) override; std::vector get_supported_sdr_capture_formats() override; + std::vector get_supported_hdr_capture_formats() override; - int init(int framerate, const std::string &display_name); + int init(const ::video::config_t &config, const std::string &display_name); cursor_t cursor; D3D11_MAPPED_SUBRESOURCE img_info; @@ -177,8 +183,9 @@ public: int dummy_img(img_t *img_base) override; int complete_img(img_t *img_base, bool dummy) override; std::vector get_supported_sdr_capture_formats() override; + std::vector get_supported_hdr_capture_formats() override; - int init(int framerate, const std::string &display_name); + int init(const ::video::config_t &config, const std::string &display_name); std::shared_ptr make_hwdevice(pix_fmt_e pix_fmt) override; diff --git a/src/platform/windows/display_base.cpp b/src/platform/windows/display_base.cpp index 5a54aa43..5706f86c 100644 --- a/src/platform/windows/display_base.cpp +++ b/src/platform/windows/display_base.cpp @@ -17,6 +17,7 @@ typedef long NTSTATUS; #include "src/config.h" #include "src/main.h" #include "src/platform/common.h" +#include "src/video.h" namespace platf { using namespace std::literals; @@ -106,10 +107,16 @@ capture_e display_base_t::capture(snapshot_cb_t &&snapshot_cb, std::shared_ptr<: }); while(img) { - auto wait_time_us = std::chrono::duration_cast(next_frame - std::chrono::steady_clock::now()).count(); + // This will return false if the HDR state changes or for any number of other + // display or GPU changes. We should reinit to examine the updated state of + // the display subsystem. It is recommended to call this once per frame. + if(!factory->IsCurrent()) { + return platf::capture_e::reinit; + } // If the wait time is between 1 us and 1 second, wait the specified time // and offset the next frame time from the exact current frame time target. + auto wait_time_us = std::chrono::duration_cast(next_frame - std::chrono::steady_clock::now()).count(); if(wait_time_us > 0 && wait_time_us < 1000000) { LARGE_INTEGER due_time { .QuadPart = -10LL * wait_time_us }; SetWaitableTimer(timer, &due_time, 0, nullptr, nullptr, false); @@ -276,7 +283,7 @@ bool test_dxgi_duplication(adapter_t &adapter, output_t &output) { return false; } -int display_base_t::init(int framerate, const std::string &display_name) { +int display_base_t::init(const ::video::config_t &config, const std::string &display_name) { std::once_flag windows_cpp_once_flag; std::call_once(windows_cpp_once_flag, []() { @@ -296,7 +303,7 @@ int display_base_t::init(int framerate, const std::string &display_name) { // Ensure we can duplicate the current display syncThreadDesktop(); - delay = std::chrono::nanoseconds { 1s } / framerate; + delay = std::chrono::nanoseconds { 1s } / config.framerate; // Get rectangle of full desktop for absolute mouse coordinates env_width = GetSystemMetrics(SM_CXVIRTUALSCREEN); @@ -421,7 +428,7 @@ int display_base_t::init(int framerate, const std::string &display_name) { << "Virtual Desktop : "sv << env_width << 'x' << env_height; // Enable DwmFlush() only if the current refresh rate can match the client framerate. - auto refresh_rate = framerate; + auto refresh_rate = config.framerate; DWM_TIMING_INFO timing_info; timing_info.cbSize = sizeof(timing_info); @@ -433,7 +440,7 @@ int display_base_t::init(int framerate, const std::string &display_name) { refresh_rate = std::round((double)timing_info.rateRefresh.uiNumerator / (double)timing_info.rateRefresh.uiDenominator); } - dup.use_dwmflush = config::video.dwmflush && !(framerate > refresh_rate) ? true : false; + dup.use_dwmflush = config::video.dwmflush && !(config.framerate > refresh_rate) ? true : false; // Bump up thread priority { @@ -502,7 +509,7 @@ int display_base_t::init(int framerate, const std::string &display_name) { status = output->QueryInterface(IID_IDXGIOutput5, (void **)&output5); if(SUCCEEDED(status)) { // Ask the display implementation which formats it supports - auto supported_formats = get_supported_sdr_capture_formats(); + auto supported_formats = config.dynamicRange ? get_supported_hdr_capture_formats() : get_supported_sdr_capture_formats(); if(supported_formats.empty()) { BOOST_LOG(warning) << "No compatible capture formats for this encoder"sv; return -1; @@ -562,6 +569,57 @@ int display_base_t::init(int framerate, const std::string &display_name) { return 0; } +bool display_base_t::is_hdr() { + dxgi::output6_t output6 {}; + + auto status = output->QueryInterface(IID_IDXGIOutput6, (void **)&output6); + if(FAILED(status)) { + BOOST_LOG(warning) << "Failed to query IDXGIOutput6 from the output"sv; + return false; + } + + DXGI_OUTPUT_DESC1 desc1; + output6->GetDesc1(&desc1); + + return desc1.ColorSpace == DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020; +} + +bool display_base_t::get_hdr_metadata(SS_HDR_METADATA &metadata) { + dxgi::output6_t output6 {}; + + std::memset(&metadata, 0, sizeof(metadata)); + + auto status = output->QueryInterface(IID_IDXGIOutput6, (void **)&output6); + if(FAILED(status)) { + BOOST_LOG(warning) << "Failed to query IDXGIOutput6 from the output"sv; + return false; + } + + DXGI_OUTPUT_DESC1 desc1; + output6->GetDesc1(&desc1); + + metadata.displayPrimaries[0].x = desc1.RedPrimary[0] * 50000; + metadata.displayPrimaries[0].y = desc1.RedPrimary[1] * 50000; + metadata.displayPrimaries[1].x = desc1.GreenPrimary[0] * 50000; + metadata.displayPrimaries[1].y = desc1.GreenPrimary[1] * 50000; + metadata.displayPrimaries[2].x = desc1.BluePrimary[0] * 50000; + metadata.displayPrimaries[2].y = desc1.BluePrimary[1] * 50000; + + metadata.whitePoint.x = desc1.WhitePoint[0] * 50000; + metadata.whitePoint.y = desc1.WhitePoint[1] * 50000; + + metadata.maxDisplayLuminance = desc1.MaxLuminance; + metadata.minDisplayLuminance = desc1.MinLuminance * 10000; + + // These are content-specific metadata parameters that this interface doesn't give us + metadata.maxContentLightLevel = 0; + metadata.maxFrameAverageLightLevel = 0; + + metadata.maxFullFrameLuminance = desc1.MaxFullFrameLuminance; + + return true; +} + const char *format_str[] = { "DXGI_FORMAT_UNKNOWN", "DXGI_FORMAT_R32G32B32A32_TYPELESS", @@ -694,18 +752,18 @@ const char *display_base_t::dxgi_format_to_string(DXGI_FORMAT format) { } // namespace platf::dxgi namespace platf { -std::shared_ptr display(mem_type_e hwdevice_type, const std::string &display_name, int framerate) { +std::shared_ptr display(mem_type_e hwdevice_type, const std::string &display_name, const video::config_t &config) { if(hwdevice_type == mem_type_e::dxgi) { auto disp = std::make_shared(); - if(!disp->init(framerate, display_name)) { + if(!disp->init(config, display_name)) { return disp; } } else if(hwdevice_type == mem_type_e::system) { auto disp = std::make_shared(); - if(!disp->init(framerate, display_name)) { + if(!disp->init(config, display_name)) { return disp; } } diff --git a/src/platform/windows/display_ram.cpp b/src/platform/windows/display_ram.cpp index 29ebcefb..c3f64bdb 100644 --- a/src/platform/windows/display_ram.cpp +++ b/src/platform/windows/display_ram.cpp @@ -349,11 +349,16 @@ int display_ram_t::dummy_img(platf::img_t *img) { } std::vector display_ram_t::get_supported_sdr_capture_formats() { - return std::vector { DXGI_FORMAT_B8G8R8A8_UNORM }; + return { DXGI_FORMAT_B8G8R8A8_UNORM }; } -int display_ram_t::init(int framerate, const std::string &display_name) { - if(display_base_t::init(framerate, display_name)) { +std::vector display_ram_t::get_supported_hdr_capture_formats() { + // HDR is unsupported + return {}; +} + +int display_ram_t::init(const ::video::config_t &config, const std::string &display_name) { + if(display_base_t::init(config, display_name)) { return -1; } diff --git a/src/platform/windows/display_vram.cpp b/src/platform/windows/display_vram.cpp index 9b1af4ac..b50f2c24 100644 --- a/src/platform/windows/display_vram.cpp +++ b/src/platform/windows/display_vram.cpp @@ -952,8 +952,8 @@ capture_e display_vram_t::snapshot(platf::img_t *img_base, std::chrono::millisec return capture_e::ok; } -int display_vram_t::init(int framerate, const std::string &display_name) { - if(display_base_t::init(framerate, display_name)) { +int display_vram_t::init(const ::video::config_t &config, const std::string &display_name) { + if(display_base_t::init(config, display_name)) { return -1; } @@ -1104,7 +1104,11 @@ int display_vram_t::dummy_img(platf::img_t *img_base) { } std::vector display_vram_t::get_supported_sdr_capture_formats() { - return std::vector { DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_FORMAT_R8G8B8A8_UNORM }; + return { DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_FORMAT_R8G8B8A8_UNORM }; +} + +std::vector display_vram_t::get_supported_hdr_capture_formats() { + return { DXGI_FORMAT_R10G10B10A2_UNORM }; } std::shared_ptr display_vram_t::make_hwdevice(pix_fmt_e pix_fmt) { diff --git a/src/stream.cpp b/src/stream.cpp index 0bded57a..78e7a3d0 100644 --- a/src/stream.cpp +++ b/src/stream.cpp @@ -33,6 +33,7 @@ extern "C" { #define IDX_PERIODIC_PING 8 #define IDX_REQUEST_IDR_FRAME 9 #define IDX_ENCRYPTED 10 +#define IDX_HDR_MODE 11 static const short packetTypes[] = { 0x0305, // Start A @@ -46,6 +47,7 @@ static const short packetTypes[] = { 0x0200, // Periodic Ping 0x0302, // IDR frame 0x0001, // fully encrypted + 0x010e, // HDR mode }; namespace asio = boost::asio; @@ -131,6 +133,15 @@ struct control_rumble_t { std::uint16_t highfreq; }; +struct control_hdr_mode_t { + control_header_v2 header; + + std::uint8_t enabled; + + // Sunshine protocol extension + SS_HDR_METADATA metadata; +}; + typedef struct control_encrypted_t { std::uint16_t encryptedHeaderType; // Always LE 0x0001 std::uint16_t length; // sizeof(seq) + 16 byte tag + secondary header and data @@ -314,6 +325,7 @@ struct session_t { std::uint8_t seq; platf::rumble_queue_t rumble_queue; + safe::mail_raw_t::event_t hdr_queue; } control; safe::mail_raw_t::event_t shutdown_event; @@ -607,6 +619,36 @@ int send_rumble(session_t *session, std::uint16_t id, std::uint16_t lowfreq, std return 0; } +int send_hdr_mode(session_t *session, video::hdr_info_t hdr_info) { + if(!session->control.peer) { + BOOST_LOG(warning) << "Couldn't send HDR mode, still waiting for PING from Moonlight"sv; + // Still waiting for PING from Moonlight + return -1; + } + + control_hdr_mode_t plaintext {}; + plaintext.header.type = packetTypes[IDX_HDR_MODE]; + plaintext.header.payloadLength = sizeof(control_hdr_mode_t) - sizeof(control_header_v2); + + plaintext.enabled = hdr_info->enabled; + plaintext.metadata = hdr_info->metadata; + + std::array + encrypted_payload; + + auto payload = encode_control(session, util::view(plaintext), encrypted_payload); + if(session->broadcast_ref->control_server.send(payload, session->control.peer)) { + TUPLE_2D(port, addr, platf::from_sockaddr_ex((sockaddr *)&session->control.peer->address.address)); + BOOST_LOG(warning) << "Couldn't send HDR mode to ["sv << addr << ':' << port << ']'; + + return -1; + } + + BOOST_LOG(debug) << "Sent HDR mode: " << hdr_info->enabled; + return 0; +} + void controlBroadcastThread(control_server_t *server) { server->map(packetTypes[IDX_PERIODIC_PING], [](session_t *session, const std::string_view &payload) { BOOST_LOG(verbose) << "type [IDX_START_A]"sv; @@ -778,6 +820,13 @@ void controlBroadcastThread(control_server_t *server) { send_rumble(session, rumble->id, rumble->lowfreq, rumble->highfreq); } + auto &hdr_queue = session->control.hdr_queue; + while(hdr_queue->peek()) { + auto hdr_info = hdr_queue->pop(); + + send_hdr_mode(session, std::move(hdr_info)); + } + ++pos; }) } @@ -1513,6 +1562,7 @@ std::shared_ptr alloc(config_t &config, crypto::aes_t &gcm_key, crypt session->config = config; session->control.rumble_queue = mail->queue(mail::rumble); + session->control.hdr_queue = mail->event(mail::hdr); session->control.iv = iv; session->control.cipher = crypto::cipher::gcm_t { gcm_key, false diff --git a/src/video.cpp b/src/video.cpp index a3cf5b6a..91f9bfba 100644 --- a/src/video.cpp +++ b/src/video.cpp @@ -5,6 +5,7 @@ #include extern "C" { +#include #include } @@ -372,6 +373,7 @@ struct sync_session_ctx_t { safe::mail_raw_t::event_t shutdown_event; safe::mail_raw_t::queue_t packets; safe::mail_raw_t::event_t idr_events; + safe::mail_raw_t::event_t hdr_events; safe::mail_raw_t::event_t touch_port_events; config_t config; @@ -391,7 +393,7 @@ using encode_e = platf::capture_e; struct capture_ctx_t { img_event_t images; - int framerate; + config_t config; }; struct capture_thread_async_ctx_t { @@ -700,11 +702,11 @@ static std::vector encoders { software }; -void reset_display(std::shared_ptr &disp, AVHWDeviceType type, const std::string &display_name, int framerate) { +void reset_display(std::shared_ptr &disp, AVHWDeviceType type, const std::string &display_name, const config_t &config) { // We try this twice, in case we still get an error on reinitialization for(int x = 0; x < 2; ++x) { disp.reset(); - disp = platf::display(map_base_dev_type(type), display_name, framerate); + disp = platf::display(map_base_dev_type(type), display_name, config); if(disp) { break; } @@ -755,7 +757,7 @@ void captureThread( capture_ctxs.emplace_back(std::move(*capture_ctx)); } - auto disp = platf::display(map_base_dev_type(encoder.base_dev_type), display_names[display_p], capture_ctxs.front().framerate); + auto disp = platf::display(map_base_dev_type(encoder.base_dev_type), display_names[display_p], capture_ctxs.front().config); if(!disp) { return; } @@ -841,7 +843,7 @@ void captureThread( } while(capture_ctx_queue->running()) { - reset_display(disp, encoder.base_dev_type, display_names[display_p], capture_ctxs.front().framerate); + reset_display(disp, encoder.base_dev_type, display_names[display_p], capture_ctxs.front().config); if(disp) { break; @@ -939,7 +941,7 @@ int encode(int64_t frame_nr, session_t &session, frame_t::pointer frame, safe::m return 0; } -std::optional make_session(const encoder_t &encoder, const config_t &config, int width, int height, std::shared_ptr &&hwdevice) { +std::optional make_session(platf::display_t *disp, const encoder_t &encoder, const config_t &config, int width, int height, std::shared_ptr &&hwdevice) { bool hardware = encoder.base_dev_type != AV_HWDEVICE_TYPE_NONE; auto &video_format = config.videoFormat == 0 ? encoder.h264 : encoder.hevc; @@ -1037,6 +1039,15 @@ std::optional make_session(const encoder_t &encoder, const config_t & } else { sw_fmt = encoder.dynamic_pix_fmt; + + // When HDR is active, that overrides the colorspace the client requested + if(disp->is_hdr()) { + BOOST_LOG(info) << "HDR color coding override [SMPTE ST 2084 PQ]"sv; + ctx->color_primaries = AVCOL_PRI_BT2020; + ctx->color_trc = AVCOL_TRC_SMPTE2084; + ctx->colorspace = AVCOL_SPC_BT2020_NCL; + sws_color_space = SWS_CS_BT2020; + } } // Used by cbs::make_sps_hevc @@ -1169,6 +1180,37 @@ std::optional make_session(const encoder_t &encoder, const config_t & frame->width = ctx->width; frame->height = ctx->height; + // Attach HDR metadata to the AVFrame + if(config.dynamicRange && disp->is_hdr()) { + SS_HDR_METADATA hdr_metadata; + if(disp->get_hdr_metadata(hdr_metadata)) { + auto mdm = av_mastering_display_metadata_create_side_data(frame.get()); + + mdm->display_primaries[0][0] = av_make_q(hdr_metadata.displayPrimaries[0].x, 50000); + mdm->display_primaries[0][1] = av_make_q(hdr_metadata.displayPrimaries[0].y, 50000); + mdm->display_primaries[1][0] = av_make_q(hdr_metadata.displayPrimaries[1].x, 50000); + mdm->display_primaries[1][1] = av_make_q(hdr_metadata.displayPrimaries[1].y, 50000); + mdm->display_primaries[2][0] = av_make_q(hdr_metadata.displayPrimaries[2].x, 50000); + mdm->display_primaries[2][1] = av_make_q(hdr_metadata.displayPrimaries[2].y, 50000); + + mdm->white_point[0] = av_make_q(hdr_metadata.whitePoint.x, 50000); + mdm->white_point[1] = av_make_q(hdr_metadata.whitePoint.y, 50000); + + mdm->min_luminance = av_make_q(hdr_metadata.minDisplayLuminance, 10000); + mdm->max_luminance = av_make_q(hdr_metadata.maxDisplayLuminance, 1); + + mdm->has_luminance = hdr_metadata.maxDisplayLuminance != 0 ? 1 : 0; + mdm->has_primaries = hdr_metadata.displayPrimaries[0].x != 0 ? 1 : 0; + + if(hdr_metadata.maxContentLightLevel != 0 || hdr_metadata.maxFrameAverageLightLevel != 0) { + auto clm = av_content_light_metadata_create_side_data(frame.get()); + + clm->MaxCLL = hdr_metadata.maxContentLightLevel; + clm->MaxFALL = hdr_metadata.maxFrameAverageLightLevel; + } + } + } + std::shared_ptr device; if(!hwdevice->data) { @@ -1212,13 +1254,13 @@ void encode_run( safe::mail_t mail, img_event_t images, config_t config, - int width, int height, + std::shared_ptr disp, std::shared_ptr &&hwdevice, safe::signal_t &reinit_event, const encoder_t &encoder, void *channel_data) { - auto session = make_session(encoder, config, width, height, std::move(hwdevice)); + auto session = make_session(disp.get(), encoder, config, disp->width, disp->height, std::move(hwdevice)); if(!session) { return; } @@ -1303,7 +1345,15 @@ std::optional make_synced_session(platf::display_t *disp, const // absolute mouse coordinates require that the dimensions of the screen are known ctx.touch_port_events->raise(make_port(disp, ctx.config)); - auto session = make_session(encoder, ctx.config, img.width, img.height, std::move(hwdevice)); + // Update client with our current HDR display state + hdr_info_t hdr_info = std::make_unique(false); + if(ctx.config.dynamicRange && disp->is_hdr()) { + disp->get_hdr_metadata(hdr_info->metadata); + hdr_info->enabled = true; + } + ctx.hdr_events->raise(std::move(hdr_info)); + + auto session = make_session(disp, encoder, ctx.config, img.width, img.height, std::move(hwdevice)); if(!session) { return std::nullopt; } @@ -1346,10 +1396,8 @@ encode_e encode_run_sync( synced_session_ctxs.emplace_back(std::make_unique(std::move(*ctx))); } - int framerate = synced_session_ctxs.front()->config.framerate; - while(encode_session_ctx_queue.running()) { - reset_display(disp, encoder.base_dev_type, display_names[display_p], framerate); + reset_display(disp, encoder.base_dev_type, display_names[display_p], synced_session_ctxs.front()->config); if(disp) { break; } @@ -1509,8 +1557,7 @@ void capture_async( return; } - ref->capture_ctx_queue->raise(capture_ctx_t { - images, config.framerate }); + ref->capture_ctx_queue->raise(capture_ctx_t { images, config }); if(!ref->capture_ctx_queue->running()) { return; @@ -1519,6 +1566,7 @@ void capture_async( int frame_nr = 1; auto touch_port_event = mail->event(mail::touch_port); + auto hdr_event = mail->event(mail::hdr); // Encoding takes place on this thread platf::adjust_thread_priority(platf::thread_priority_e::high); @@ -1557,10 +1605,18 @@ void capture_async( // absolute mouse coordinates require that the dimensions of the screen are known touch_port_event->raise(make_port(display.get(), config)); + // Update client with our current HDR display state + hdr_info_t hdr_info = std::make_unique(false); + if(config.dynamicRange && display->is_hdr()) { + display->get_hdr_metadata(hdr_info->metadata); + hdr_info->enabled = true; + } + hdr_event->raise(std::move(hdr_info)); + encode_run( frame_nr, mail, images, - config, display->width, display->height, + config, display, std::move(hwdevice), ref->reinit_event, *ref->encoder_p, channel_data); @@ -1592,6 +1648,7 @@ void capture( mail->event(mail::shutdown), mail::man->queue(mail::video_packets), std::move(idr_events), + mail->event(mail::hdr), mail->event(mail::touch_port), config, 1, @@ -1609,7 +1666,7 @@ enum validate_flag_e { }; int validate_config(std::shared_ptr &disp, const encoder_t &encoder, const config_t &config) { - reset_display(disp, encoder.base_dev_type, config::video.output_name, config.framerate); + reset_display(disp, encoder.base_dev_type, config::video.output_name, config); if(!disp) { return -1; } @@ -1620,7 +1677,7 @@ int validate_config(std::shared_ptr &disp, const encoder_t &en return -1; } - auto session = make_session(encoder, config, disp->width, disp->height, std::move(hwdevice)); + auto session = make_session(disp.get(), encoder, config, disp->width, disp->height, std::move(hwdevice)); if(!session) { return -1; } diff --git a/src/video.h b/src/video.h index 3d99f855..3d48af14 100644 --- a/src/video.h +++ b/src/video.h @@ -48,6 +48,16 @@ struct packet_raw_t { using packet_t = std::unique_ptr; +struct hdr_info_raw_t { + explicit hdr_info_raw_t(bool enabled) : enabled { enabled }, metadata {} {}; + explicit hdr_info_raw_t(bool enabled, const SS_HDR_METADATA &metadata) : enabled { enabled }, metadata { metadata } {}; + + bool enabled; + SS_HDR_METADATA metadata; +}; + +using hdr_info_t = std::unique_ptr; + struct config_t { int width; int height;