d3v1l/matrixion
1
0
Fork 0
mirror of https://github.com/Nheko-Reborn/nheko.git synced 2024-11-21 18:50:47 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Windows screenshare/video call support, general call improvements (#1725)

Browse source 
* Initial support for d3d11 qml video item

* Windows screenshare support, D3D11

* misc fixes

* further window visibility checks

* preview updates

* fix qml preview

* fix compositor

* add libnice plugin dep

* re-run formatter

* final formatter fix [skip ci]

* fix tumbleweed build

---------

Co-authored-by: Joseph Donofry <rubberduckie3554@gmail.com>
This commit is contained in:
checkraisefold 2024-05-10 12:22:58 -07:00 committed by GitHub
parent cc75c930fc
commit e7d28b96da
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 252 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
CMakeLists.txt
View file

@ -25,8 +25,8 @@ set(CMAKE_OSX_DEPLOYMENT_TARGET "10.15" CACHE STRING "macos deployment target")
option(HUNTER_ENABLED "Enable Hunter package manager" OFF)
include("cmake/HunterGate.cmake")
HunterGate(
URL "https://github.com/cpp-pm/hunter/archive/v0.25.2.tar.gz"
SHA1 "560c000d5b6c972d41c2caf44f24189c868cc404"
URL "https://github.com/cpp-pm/hunter/archive/v0.25.5.tar.gz"
SHA1 "a20151e4c0740ee7d0f9994476856d813cdead29"
LOCAL
)
@ -618,6 +618,9 @@ endif()
if(VOIP)
include(FindPkgConfig)
pkg_check_modules(GSTREAMER REQUIRED IMPORTED_TARGET gstreamer-sdp-1.0>=1.18 gstreamer-webrtc-1.0>=1.18 gstreamer-gl-1.0)
if (WIN32)
pkg_check_modules(GSTREAMER REQUIRED IMPORTED_TARGET gstreamer-d3d11-1.0)
endif()
endif()
if(X11 AND NOT WIN32 AND NOT APPLE AND NOT HAIKU)
@ -807,6 +810,7 @@ set(QML_SOURCES
resources/qml/voip/PlaceCall.qml
resources/qml/voip/ScreenShare.qml
resources/qml/voip/VideoCall.qml
resources/qml/voip/VideoCallD3D11.qml
resources/qml/delegates/EncryptionEnabled.qml
resources/qml/ui/TimelineEffects.qml
)
@ -839,6 +843,7 @@ qt_add_translations(nheko RESOURCE_PREFIX "/translations" TS_FILES
if(WIN32)
target_compile_definitions(nheko PRIVATE WIN32_LEAN_AND_MEAN)
target_link_libraries(nheko PRIVATE Qt6::DBus)
if(MSVC)
target_compile_options(nheko PUBLIC "/Zc:__cplusplus")
endif()

2
resources/qml/TimelineView.qml
View file

@ -119,7 +119,7 @@ Item {
searchString: topBar.searchString
}
Loader {
source: CallManager.isOnCall && CallManager.callType != Voip.VOICE ? "voip/VideoCall.qml" : ""
source: CallManager.isOnCall && CallManager.callType != Voip.VOICE ? (Qt.platform.os != "windows" ? "voip/VideoCall.qml" : "voip/VideoCallD3D11.qml") : ""
onLoaded: TimelineManager.setVideoCallItem()
}

2
resources/qml/voip/ScreenShare.qml
View file

@ -63,7 +63,7 @@ Popup {
}
ComboBox {
visible: CallManager.screenShareType == Voip.X11
visible: CallManager.screenShareType == Voip.X11 || CallManager.screenShareType == Voip.D3D11
id: windowCombo
Layout.fillWidth: true

9
resources/qml/voip/VideoCallD3D11.qml Normal file
View file

@ -0,0 +1,9 @@
// SPDX-FileCopyrightText: Nheko Contributors
//
// SPDX-License-Identifier: GPL-3.0-or-later
import org.freedesktop.gstreamer.Qt6D3D11VideoItem 1.0
GstD3D11Qt6VideoItem {
objectName: "videoCallItem"
}

103
src/voip/CallManager.cpp
View file

@ -34,6 +34,10 @@
#include <xcb/xcb_ewmh.h>
#endif
#ifdef Q_OS_WINDOWS
#include <Windows.h>
#endif
#ifdef GSTREAMER_AVAILABLE
extern "C"
{
@ -82,12 +86,12 @@ CallManager::CallManager(QObject *parent)
{
#ifdef GSTREAMER_AVAILABLE
std::string errorMessage;
if (session_.havePlugins(true, true, ScreenShareType::XDP, &errorMessage)) {
screenShareTypes_.push_back(ScreenShareType::XDP);
screenShareType_ = ScreenShareType::XDP;
}
if (std::getenv("DISPLAY")) {
if (QGuiApplication::platformName() == QStringLiteral("windows") &&
session_.havePlugins(true, true, ScreenShareType::D3D11, &errorMessage)) {
screenShareType_ = ScreenShareType::D3D11;
screenShareTypes_.push_back(ScreenShareType::D3D11);
} else if (std::getenv("DISPLAY")) {
screenShareTypes_.push_back(ScreenShareType::X11);
if (QGuiApplication::platformName() != QStringLiteral("wayland")) {
// Selected by default
@ -96,6 +100,12 @@ CallManager::CallManager(QObject *parent)
std::swap(screenShareTypes_[0], screenShareTypes_[1]);
}
}
if (QGuiApplication::platformName() != QStringLiteral("windows") &&
session_.havePlugins(true, true, ScreenShareType::XDP, &errorMessage)) {
screenShareTypes_.push_back(ScreenShareType::XDP);
screenShareType_ = ScreenShareType::XDP;
}
#endif
connect(
@ -254,7 +264,8 @@ CallManager::sendInvite(const QString &roomid, CallType callType, unsigned int w
#ifdef GSTREAMER_AVAILABLE
if (callType == CallType::SCREEN) {
if (screenShareType_ == ScreenShareType::X11) {
if (screenShareType_ == ScreenShareType::X11 ||
screenShareType_ == ScreenShareType::D3D11) {
if (windows_.empty() || windowIndex >= windows_.size()) {
nhlog::ui()->error("WebRTC: window index out of range");
return;
@ -270,8 +281,8 @@ CallManager::sendInvite(const QString &roomid, CallType callType, unsigned int w
#endif
if (haveCallInvite_) {
nhlog::ui()->debug(
"WebRTC: Discarding outbound call for inbound call. localUser is polite party");
nhlog::ui()->debug("WebRTC: Discarding outbound call for inbound call. "
"localUser is polite party");
if (callParty_ == callee->user_id) {
if (callType == callType_)
acceptInvite();
@ -305,7 +316,8 @@ CallManager::sendInvite(const QString &roomid, CallType callType, unsigned int w
playRingtone(QUrl(QStringLiteral("qrc:/media/media/ringback.ogg")), true);
uint32_t shareWindowId =
callType == CallType::SCREEN && screenShareType_ == ScreenShareType::X11
callType == CallType::SCREEN &&
(screenShareType_ == ScreenShareType::X11 || screenShareType_ == ScreenShareType::D3D11)
? windows_[windowIndex].second
: 0;
if (!session_.createOffer(callType, screenShareType_, shareWindowId)) {
@ -337,7 +349,7 @@ callHangUpReasonString(CallHangUp::Reason reason)
return "User";
}
}
}
} // namespace
void
CallManager::hangUp(CallHangUp::Reason reason)
@ -511,8 +523,8 @@ CallManager::handleEvent(const RoomEvent<CallInvite> &callInviteEvent)
void
CallManager::acceptInvite()
{
// if call was accepted/rejected elsewhere and m.call.select_answer is received
// before acceptInvite
// if call was accepted/rejected elsewhere and m.call.select_answer is
// received before acceptInvite
if (!haveCallInvite_)
return;
@ -699,7 +711,8 @@ CallManager::handleEvent(const RoomEvent<CallReject> &callRejectEvent)
if (callRejectEvent.content.call_id == callid_) {
if (session_.state() == webrtc::State::OFFERSENT) {
// only accept reject if webrtc is in OFFERSENT state, else call has been accepted
// only accept reject if webrtc is in OFFERSENT state, else call has been
// accepted
emit newMessage(
roomid_,
CallSelectAnswer{
@ -861,7 +874,7 @@ bool
CallManager::screenShareReady() const
{
#ifdef GSTREAMER_AVAILABLE
if (screenShareType_ == ScreenShareType::X11) {
if (screenShareType_ == ScreenShareType::X11 || screenShareType_ == ScreenShareType::D3D11) {
return true;
} else {
return ScreenCastPortal::instance().ready();
@ -875,12 +888,15 @@ QStringList
CallManager::screenShareTypeList()
{
QStringList ret;
ret.reserve(2);
ret.reserve(3);
for (ScreenShareType type : screenShareTypes_) {
switch (type) {
case ScreenShareType::X11:
ret.append(tr("X11"));
break;
case ScreenShareType::D3D11:
ret.append("DirectX 11");
break;
case ScreenShareType::XDP:
ret.append(tr("PipeWire"));
break;
@ -893,8 +909,10 @@ CallManager::screenShareTypeList()
QStringList
CallManager::windowList()
{
if (!(std::find(screenShareTypes_.begin(), screenShareTypes_.end(), ScreenShareType::X11) !=
screenShareTypes_.end())) {
if (std::find(screenShareTypes_.begin(), screenShareTypes_.end(), ScreenShareType::X11) ==
screenShareTypes_.end() &&
std::find(screenShareTypes_.begin(), screenShareTypes_.end(), ScreenShareType::D3D11) ==
screenShareTypes_.end()) {
return {};
}
@ -949,6 +967,32 @@ CallManager::windowList()
}
xcb_ewmh_get_windows_reply_wipe(&clients);
}
#endif
#ifdef Q_OS_WINDOWS
for (HWND windowHandle = GetTopWindow(nullptr); windowHandle != nullptr;
windowHandle = GetNextWindow(windowHandle, GW_HWNDNEXT)) {
if (!IsWindowVisible(windowHandle))
continue;
int titleLength = GetWindowTextLengthW(windowHandle);
if (titleLength == 0)
continue;
if (GetWindowLong(windowHandle, GWL_EXSTYLE) & WS_EX_TOOLWINDOW)
continue;
TITLEBARINFO titleInfo;
titleInfo.cbSize = sizeof(titleInfo);
GetTitleBarInfo(windowHandle, &titleInfo);
if (titleInfo.rgstate[0] & STATE_SYSTEM_INVISIBLE)
continue;
wchar_t *windowTitle = new wchar_t[titleLength + 1];
GetWindowTextW(windowHandle, windowTitle, titleLength + 1);
windows_.push_back(
{QString::fromWCharArray(windowTitle), reinterpret_cast<uint64_t>(windowHandle)});
}
#endif
QStringList ret;
assert(windows_.size() < std::numeric_limits<int>::max());
@ -1007,11 +1051,13 @@ make_preview_sink()
{
if (QGuiApplication::platformName() == QStringLiteral("wayland")) {
return gst_element_factory_make("waylandsink", nullptr);
} else if (QGuiApplication::platformName() == QStringLiteral("windows")) {
return gst_element_factory_make("d3d11videosink", nullptr);
} else {
return gst_element_factory_make("ximagesink", nullptr);
}
}
}
} // namespace
#endif
void
@ -1032,6 +1078,12 @@ CallManager::previewWindow(unsigned int index) const
return;
}
if (screenShareType_ == ScreenShareType::D3D11 &&
(windows_.empty() || index >= windows_.size())) {
nhlog::ui()->error("D3D11 screencast not available");
return;
}
auto settings = ChatPage::instance()->userSettings();
pipe_ = gst_pipeline_new(nullptr);
@ -1066,6 +1118,19 @@ CallManager::previewWindow(unsigned int index) const
gst_bin_add(GST_BIN(pipe_), ximagesrc);
screencastsrc = ximagesrc;
} else if (screenShareType_ == ScreenShareType::D3D11) {
GstElement *d3d11screensrc = gst_element_factory_make("d3d11screencapturesrc", nullptr);
if (!d3d11screensrc) {
nhlog::ui()->error("Failed to create d3d11screencapturesrc");
gst_object_unref(pipe_);
pipe_ = nullptr;
return;
}
g_object_set(d3d11screensrc, "window-handle", windows_[index].second, nullptr);
g_object_set(d3d11screensrc, "show-cursor", !settings->screenShareHideCursor(), nullptr);
gst_bin_add(GST_BIN(pipe_), d3d11screensrc);
screencastsrc = d3d11screensrc;
} else {
ScreenCastPortal &sc_portal = ScreenCastPortal::instance();
const ScreenCastPortal::Stream *stream = sc_portal.getStream();
@ -1171,6 +1236,6 @@ getTurnURIs(const mtx::responses::TurnServer &turnServer)
}
return ret;
}
}
} // namespace
#include "moc_CallManager.cpp"

4
src/voip/CallManager.h
View file

@ -133,7 +133,11 @@ private:
QTimer turnServerTimer_;
QMediaPlayer player_;
std::vector<webrtc::ScreenShareType> screenShareTypes_;
#ifndef Q_OS_WINDOWS
std::vector<std::pair<QString, uint32_t>> windows_;
#else
std::vector<std::pair<QString, uint64_t>> windows_;
#endif
std::vector<std::string> rejectCallPartyIDs_;
template<typename T>

185
src/voip/WebRTCSession.cpp
View file

@ -23,6 +23,7 @@
#include "voip/ScreenCastPortal.h"
#ifdef GSTREAMER_AVAILABLE
#include "MainWindow.h"
extern "C"
{
#include "gst/gl/gstgldisplay.h"
@ -330,33 +331,66 @@ GstElement *
newVideoSinkChain(GstElement *pipe)
{
// use compositor for now; acceleration needs investigation
GstElement *queue = gst_element_factory_make("queue", nullptr);
GstElement *compositor = gst_element_factory_make("compositor", "compositor");
GstElement *glupload = gst_element_factory_make("glupload", nullptr);
GstElement *qmlglsink = gst_element_factory_make("qml6glsink", nullptr);
GstElement *glsinkbin = gst_element_factory_make("glsinkbin", nullptr);
GstElement *queue = gst_element_factory_make("queue", nullptr);
auto graphicsApi = MainWindow::instance()->graphicsApi();
GstElement *compositor = gst_element_factory_make(
graphicsApi == QSGRendererInterface::OpenGL ? "compositor" : "d3d11compositor", "compositor");
g_object_set(compositor, "background", 1, nullptr);
g_object_set(qmlglsink, "widget", WebRTCSession::instance().getVideoItem(), nullptr);
g_object_set(glsinkbin, "sink", qmlglsink, nullptr);
gst_bin_add_many(GST_BIN(pipe), queue, compositor, glupload, glsinkbin, nullptr);
gst_element_link_many(queue, compositor, glupload, glsinkbin, nullptr);
gst_element_sync_state_with_parent(queue);
gst_element_sync_state_with_parent(compositor);
gst_element_sync_state_with_parent(glupload);
gst_element_sync_state_with_parent(glsinkbin);
switch (graphicsApi) {
case QSGRendererInterface::OpenGL: {
GstElement *glupload = gst_element_factory_make("glupload", nullptr);
GstElement *glcolorconvert = gst_element_factory_make("glcolorconvert", nullptr);
GstElement *qmlglsink = gst_element_factory_make("qml6glsink", nullptr);
GstElement *glsinkbin = gst_element_factory_make("glsinkbin", nullptr);
// to propagate context (hopefully)
gst_element_set_state(qmlglsink, GST_STATE_READY);
g_object_set(qmlglsink, "widget", WebRTCSession::instance().getVideoItem(), nullptr);
g_object_set(glsinkbin, "sink", qmlglsink, nullptr);
gst_bin_add_many(
GST_BIN(pipe), queue, compositor, glupload, glcolorconvert, glsinkbin, nullptr);
gst_element_link_many(queue, compositor, glupload, glcolorconvert, glsinkbin, nullptr);
// Workaround: On wayland, when egl is used, gstreamer might terminate the display connection.
// Prevent that by "leaking" a reference to the display. See
// https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/3743
if (QGuiApplication::platformName() == QStringLiteral("wayland")) {
auto context = gst_element_get_context(qmlglsink, "gst.gl.GLDisplay");
if (context) {
GstGLDisplay *display;
gst_context_get_gl_display(context, &display);
gst_element_sync_state_with_parent(queue);
gst_element_sync_state_with_parent(compositor);
gst_element_sync_state_with_parent(glupload);
gst_element_sync_state_with_parent(glcolorconvert);
gst_element_sync_state_with_parent(glsinkbin);
// to propagate context (hopefully)
gst_element_set_state(qmlglsink, GST_STATE_READY);
// Workaround: On wayland, when egl is used, gstreamer might terminate the display
// connection. Prevent that by "leaking" a reference to the display. See
// https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/3743
if (QGuiApplication::platformName() == QStringLiteral("wayland")) {
auto context = gst_element_get_context(qmlglsink, "gst.gl.GLDisplay");
if (context) {
GstGLDisplay *display;
gst_context_get_gl_display(context, &display);
}
}
} break;
case QSGRendererInterface::Direct3D11: {
GstElement *d3d11upload = gst_element_factory_make("d3d11upload", nullptr);
GstElement *d3d11colorconvert = gst_element_factory_make("d3d11colorconvert", nullptr);
GstElement *qmld3d11sink = gst_element_factory_make("qml6d3d11sink", nullptr);
g_object_set(qmld3d11sink, "widget", WebRTCSession::instance().getVideoItem(), nullptr);
gst_bin_add_many(
GST_BIN(pipe), queue, d3d11upload, compositor, d3d11colorconvert, qmld3d11sink, nullptr);
gst_element_link_many(
queue, d3d11upload, compositor, d3d11colorconvert, qmld3d11sink, nullptr);
gst_element_sync_state_with_parent(queue);
gst_element_sync_state_with_parent(compositor);
gst_element_sync_state_with_parent(d3d11upload);
gst_element_sync_state_with_parent(d3d11colorconvert);
// to propagate context (hopefully)
gst_element_set_state(qmld3d11sink, GST_STATE_READY);
} break;
default:
break;
}
return queue;
@ -606,20 +640,20 @@ WebRTCSession::havePlugins(bool isVideo,
if (!initialised_ && !init(errorMessage))
return false;
static constexpr std::initializer_list<const char *> audio_elements = {
"audioconvert",
"audioresample",
"autoaudiosink",
"capsfilter",
"decodebin",
"opusenc",
"queue",
"rtpopuspay",
"volume",
"webrtcbin",
};
static constexpr std::initializer_list<const char *> audio_elements = {"audioconvert",
"audioresample",
"autoaudiosink",
"capsfilter",
"decodebin",
"opusenc",
"queue",
"rtpopuspay",
"volume",
"webrtcbin",
"nicesrc",
"nicesink"};
static constexpr std::initializer_list<const char *> video_elements = {
static constexpr std::initializer_list<const char *> gl_video_elements = {
"compositor",
"glsinkbin",
"glupload",
@ -631,6 +665,19 @@ WebRTCSession::havePlugins(bool isVideo,
"vp8enc",
};
static constexpr std::initializer_list<const char *> d3d11_video_elements = {
"compositor",
"d3d11colorconvert",
"d3d11videosink",
"d3d11upload",
"qml6d3d11sink",
"rtpvp8pay",
"tee",
"videoconvert",
"videoscale",
"vp8enc",
};
std::string strError("Missing GStreamer elements: ");
GstRegistry *registry = gst_registry_get();
@ -654,8 +701,19 @@ WebRTCSession::havePlugins(bool isVideo,
haveVoicePlugins_ = check_plugins(audio_elements);
// check both elements at once
if (isVideo)
haveVideoPlugins_ = check_plugins(video_elements);
if (isVideo) {
switch (MainWindow::instance()->graphicsApi()) {
case QSGRendererInterface::OpenGL:
haveVideoPlugins_ = check_plugins(gl_video_elements);
break;
case QSGRendererInterface::Direct3D11:
haveVideoPlugins_ = check_plugins(d3d11_video_elements);
break;
default:
haveVideoPlugins_ = false;
break;
}
}
bool haveScreensharePlugins = false;
if (isScreenshare) {
@ -663,6 +721,8 @@ WebRTCSession::havePlugins(bool isVideo,
if (haveScreensharePlugins) {
if (QGuiApplication::platformName() == QStringLiteral("wayland")) {
haveScreensharePlugins = check_plugins({"waylandsink"});
} else if (QGuiApplication::platformName() == QStringLiteral("windows")) {
haveScreensharePlugins = check_plugins({"d3d11videosink"});
} else {
haveScreensharePlugins = check_plugins({"ximagesink"});
}
@ -670,6 +730,9 @@ WebRTCSession::havePlugins(bool isVideo,
if (haveScreensharePlugins) {
if (screenShareType == ScreenShareType::X11) {
haveScreensharePlugins = check_plugins({"ximagesrc"});
} else if (screenShareType == ScreenShareType::D3D11) {
haveScreensharePlugins =
check_plugins({"d3d11screencapturesrc", "d3d11download", "d3d11convert"});
} else {
haveScreensharePlugins = check_plugins({"pipewiresrc"});
}
@ -685,9 +748,19 @@ WebRTCSession::havePlugins(bool isVideo,
}
if (isVideo || isScreenshare) {
// load qmlglsink to register GStreamer's GstGLVideoItem QML type
GstElement *qmlglsink = gst_element_factory_make("qml6glsink", nullptr);
gst_object_unref(qmlglsink);
switch (MainWindow::instance()->graphicsApi()) {
case QSGRendererInterface::OpenGL: {
// load qmlglsink to register GStreamer's GstGLVideoItem QML type
GstElement *qmlglsink = gst_element_factory_make("qml6glsink", nullptr);
gst_object_unref(qmlglsink);
} break;
case QSGRendererInterface::Direct3D11: {
GstElement *qmld3d11sink = gst_element_factory_make("qml6d3d11sink", nullptr);
gst_object_unref(qmld3d11sink);
} break;
default:
break;
}
}
return true;
}
@ -1027,6 +1100,36 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType)
gst_bin_add(GST_BIN(pipe_), ximagesrc);
screencastsrc = ximagesrc;
} else if (screenShareType_ == ScreenShareType::D3D11) {
GstElement *d3d11screensrc =
gst_element_factory_make("d3d11screencapturesrc", "screenshare");
if (!d3d11screensrc) {
nhlog::ui()->error("WebRTC: failed to create d3d11screencapturesrc");
gst_object_unref(pipe_);
pipe_ = nullptr;
return false;
}
g_object_set(
d3d11screensrc, "window-handle", static_cast<guint64>(shareWindowId_), nullptr);
g_object_set(
d3d11screensrc, "show-cursor", !settings->screenShareHideCursor(), nullptr);
g_object_set(d3d11screensrc, "do-timestamp", (gboolean)1, nullptr);
gst_bin_add(GST_BIN(pipe_), d3d11screensrc);
GstElement *d3d11convert = gst_element_factory_make("d3d11convert", nullptr);
gst_bin_add(GST_BIN(pipe_), d3d11convert);
if (!gst_element_link(d3d11screensrc, d3d11convert)) {
nhlog::ui()->error("WebRTC: failed to link d3d11screencapturesrc -> d3d11convert");
return false;
}
GstElement *d3d11download = gst_element_factory_make("d3d11download", nullptr);
gst_bin_add(GST_BIN(pipe_), d3d11download);
if (!gst_element_link(d3d11convert, d3d11download)) {
nhlog::ui()->error("WebRTC: failed to link d3d11convert -> d3d11download");
return false;
}
screencastsrc = d3d11download;
} else {
ScreenCastPortal &sc_portal = ScreenCastPortal::instance();
GstElement *pipewiresrc = gst_element_factory_make("pipewiresrc", "screenshare");

3
src/voip/WebRTCSession.h
View file

@ -31,7 +31,8 @@ Q_ENUM_NS(CallType)
enum class ScreenShareType
{
X11,
XDP
XDP,
D3D11
};
Q_ENUM_NS(ScreenShareType)