Windows screenshare/video call support, general call improvements (#1725)

* Initial support for d3d11 qml video item

* Windows screenshare support, D3D11

* misc fixes

* further window visibility checks

* preview updates

* fix qml preview

* fix compositor

* add libnice plugin dep

* re-run formatter

* final formatter fix [skip ci]

* fix tumbleweed build

---------

Co-authored-by: Joseph Donofry <rubberduckie3554@gmail.com>

CMakeLists.txt

@@ -25,8 +25,8 @@ set(CMAKE_OSX_DEPLOYMENT_TARGET "10.15" CACHE STRING "macos deployment target")
 option(HUNTER_ENABLED "Enable Hunter package manager" OFF)
 include("cmake/HunterGate.cmake")
 HunterGate(
-    URL "https://github.com/cpp-pm/hunter/archive/v0.25.2.tar.gz"
-    SHA1 "560c000d5b6c972d41c2caf44f24189c868cc404"
+    URL "https://github.com/cpp-pm/hunter/archive/v0.25.5.tar.gz"
+    SHA1 "a20151e4c0740ee7d0f9994476856d813cdead29"
     LOCAL
 )
 
@@ -618,6 +618,9 @@ endif()
 if(VOIP)
     include(FindPkgConfig)
     pkg_check_modules(GSTREAMER REQUIRED IMPORTED_TARGET gstreamer-sdp-1.0>=1.18 gstreamer-webrtc-1.0>=1.18 gstreamer-gl-1.0)
+    if (WIN32)
+        pkg_check_modules(GSTREAMER REQUIRED IMPORTED_TARGET gstreamer-d3d11-1.0)
+    endif()
 endif()
 
 if(X11 AND NOT WIN32 AND NOT APPLE AND NOT HAIKU)
@@ -807,6 +810,7 @@ set(QML_SOURCES
     resources/qml/voip/PlaceCall.qml
     resources/qml/voip/ScreenShare.qml
     resources/qml/voip/VideoCall.qml
+	resources/qml/voip/VideoCallD3D11.qml
     resources/qml/delegates/EncryptionEnabled.qml
     resources/qml/ui/TimelineEffects.qml
 )
@@ -839,6 +843,7 @@ qt_add_translations(nheko RESOURCE_PREFIX "/translations" TS_FILES
 
 if(WIN32)
     target_compile_definitions(nheko PRIVATE WIN32_LEAN_AND_MEAN)
+    target_link_libraries(nheko PRIVATE Qt6::DBus)
     if(MSVC)
         target_compile_options(nheko PUBLIC "/Zc:__cplusplus")
     endif()

resources/qml/TimelineView.qml

@@ -119,7 +119,7 @@ Item {
                         searchString: topBar.searchString
                     }
                     Loader {
-                        source: CallManager.isOnCall && CallManager.callType != Voip.VOICE ? "voip/VideoCall.qml" : ""
+                        source: CallManager.isOnCall && CallManager.callType != Voip.VOICE ? (Qt.platform.os != "windows" ? "voip/VideoCall.qml" : "voip/VideoCallD3D11.qml") : ""
 
                         onLoaded: TimelineManager.setVideoCallItem()
                     }

resources/qml/voip/ScreenShare.qml

@@ -63,7 +63,7 @@ Popup {
             }
 
             ComboBox {
-                visible: CallManager.screenShareType == Voip.X11
+                visible: CallManager.screenShareType == Voip.X11 || CallManager.screenShareType == Voip.D3D11
                 id: windowCombo
 
                 Layout.fillWidth: true

resources/qml/voip/VideoCallD3D11.qml

@@ -0,0 +1,9 @@
+// SPDX-FileCopyrightText: Nheko Contributors
+//
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+import org.freedesktop.gstreamer.Qt6D3D11VideoItem 1.0
+
+GstD3D11Qt6VideoItem {
+    objectName: "videoCallItem"
+}

src/voip/CallManager.cpp

@@ -34,6 +34,10 @@
 #include <xcb/xcb_ewmh.h>
 #endif
 
+#ifdef Q_OS_WINDOWS
+#include <Windows.h>
+#endif
+
 #ifdef GSTREAMER_AVAILABLE
 extern "C"
 {
@@ -82,12 +86,12 @@ CallManager::CallManager(QObject *parent)
 {
 #ifdef GSTREAMER_AVAILABLE
     std::string errorMessage;
-    if (session_.havePlugins(true, true, ScreenShareType::XDP, &errorMessage)) {
-        screenShareTypes_.push_back(ScreenShareType::XDP);
-        screenShareType_ = ScreenShareType::XDP;
-    }
 
-    if (std::getenv("DISPLAY")) {
+    if (QGuiApplication::platformName() == QStringLiteral("windows") &&
+        session_.havePlugins(true, true, ScreenShareType::D3D11, &errorMessage)) {
+        screenShareType_ = ScreenShareType::D3D11;
+        screenShareTypes_.push_back(ScreenShareType::D3D11);
+    } else if (std::getenv("DISPLAY")) {
         screenShareTypes_.push_back(ScreenShareType::X11);
         if (QGuiApplication::platformName() != QStringLiteral("wayland")) {
             // Selected by default
@@ -96,6 +100,12 @@ CallManager::CallManager(QObject *parent)
                 std::swap(screenShareTypes_[0], screenShareTypes_[1]);
         }
     }
+
+    if (QGuiApplication::platformName() != QStringLiteral("windows") &&
+        session_.havePlugins(true, true, ScreenShareType::XDP, &errorMessage)) {
+        screenShareTypes_.push_back(ScreenShareType::XDP);
+        screenShareType_ = ScreenShareType::XDP;
+    }
 #endif
 
     connect(
@@ -254,7 +264,8 @@ CallManager::sendInvite(const QString &roomid, CallType callType, unsigned int w
 
 #ifdef GSTREAMER_AVAILABLE
     if (callType == CallType::SCREEN) {
-        if (screenShareType_ == ScreenShareType::X11) {
+        if (screenShareType_ == ScreenShareType::X11 ||
+            screenShareType_ == ScreenShareType::D3D11) {
             if (windows_.empty() || windowIndex >= windows_.size()) {
                 nhlog::ui()->error("WebRTC: window index out of range");
                 return;
@@ -270,8 +281,8 @@ CallManager::sendInvite(const QString &roomid, CallType callType, unsigned int w
 #endif
 
     if (haveCallInvite_) {
-        nhlog::ui()->debug(
-          "WebRTC: Discarding outbound call for inbound call. localUser is polite party");
+        nhlog::ui()->debug("WebRTC: Discarding outbound call for inbound call. "
+                           "localUser is polite party");
         if (callParty_ == callee->user_id) {
             if (callType == callType_)
                 acceptInvite();
@@ -305,7 +316,8 @@ CallManager::sendInvite(const QString &roomid, CallType callType, unsigned int w
     playRingtone(QUrl(QStringLiteral("qrc:/media/media/ringback.ogg")), true);
 
     uint32_t shareWindowId =
-      callType == CallType::SCREEN && screenShareType_ == ScreenShareType::X11
+      callType == CallType::SCREEN &&
+          (screenShareType_ == ScreenShareType::X11 || screenShareType_ == ScreenShareType::D3D11)
         ? windows_[windowIndex].second
         : 0;
     if (!session_.createOffer(callType, screenShareType_, shareWindowId)) {
@@ -337,7 +349,7 @@ callHangUpReasonString(CallHangUp::Reason reason)
         return "User";
     }
 }
-}
+} // namespace
 
 void
 CallManager::hangUp(CallHangUp::Reason reason)
@@ -511,8 +523,8 @@ CallManager::handleEvent(const RoomEvent<CallInvite> &callInviteEvent)
 void
 CallManager::acceptInvite()
 {
-    // if call was accepted/rejected elsewhere and m.call.select_answer is received
-    // before acceptInvite
+    // if call was accepted/rejected elsewhere and m.call.select_answer is
+    // received before acceptInvite
     if (!haveCallInvite_)
         return;
 
@@ -699,7 +711,8 @@ CallManager::handleEvent(const RoomEvent<CallReject> &callRejectEvent)
 
     if (callRejectEvent.content.call_id == callid_) {
         if (session_.state() == webrtc::State::OFFERSENT) {
-            // only accept reject if webrtc is in OFFERSENT state, else call has been accepted
+            // only accept reject if webrtc is in OFFERSENT state, else call has been
+            // accepted
             emit newMessage(
               roomid_,
               CallSelectAnswer{
@@ -861,7 +874,7 @@ bool
 CallManager::screenShareReady() const
 {
 #ifdef GSTREAMER_AVAILABLE
-    if (screenShareType_ == ScreenShareType::X11) {
+    if (screenShareType_ == ScreenShareType::X11 || screenShareType_ == ScreenShareType::D3D11) {
         return true;
     } else {
         return ScreenCastPortal::instance().ready();
@@ -875,12 +888,15 @@ QStringList
 CallManager::screenShareTypeList()
 {
     QStringList ret;
-    ret.reserve(2);
+    ret.reserve(3);
     for (ScreenShareType type : screenShareTypes_) {
         switch (type) {
         case ScreenShareType::X11:
             ret.append(tr("X11"));
             break;
+        case ScreenShareType::D3D11:
+            ret.append("DirectX 11");
+            break;
         case ScreenShareType::XDP:
             ret.append(tr("PipeWire"));
             break;
@@ -893,8 +909,10 @@ CallManager::screenShareTypeList()
 QStringList
 CallManager::windowList()
 {
-    if (!(std::find(screenShareTypes_.begin(), screenShareTypes_.end(), ScreenShareType::X11) !=
-          screenShareTypes_.end())) {
+    if (std::find(screenShareTypes_.begin(), screenShareTypes_.end(), ScreenShareType::X11) ==
+          screenShareTypes_.end() &&
+        std::find(screenShareTypes_.begin(), screenShareTypes_.end(), ScreenShareType::D3D11) ==
+          screenShareTypes_.end()) {
         return {};
     }
 
@@ -949,6 +967,32 @@ CallManager::windowList()
         }
         xcb_ewmh_get_windows_reply_wipe(&clients);
     }
+#endif
+#ifdef Q_OS_WINDOWS
+    for (HWND windowHandle = GetTopWindow(nullptr); windowHandle != nullptr;
+         windowHandle      = GetNextWindow(windowHandle, GW_HWNDNEXT)) {
+        if (!IsWindowVisible(windowHandle))
+            continue;
+
+        int titleLength = GetWindowTextLengthW(windowHandle);
+        if (titleLength == 0)
+            continue;
+
+        if (GetWindowLong(windowHandle, GWL_EXSTYLE) & WS_EX_TOOLWINDOW)
+            continue;
+
+        TITLEBARINFO titleInfo;
+        titleInfo.cbSize = sizeof(titleInfo);
+        GetTitleBarInfo(windowHandle, &titleInfo);
+        if (titleInfo.rgstate[0] & STATE_SYSTEM_INVISIBLE)
+            continue;
+
+        wchar_t *windowTitle = new wchar_t[titleLength + 1];
+        GetWindowTextW(windowHandle, windowTitle, titleLength + 1);
+
+        windows_.push_back(
+          {QString::fromWCharArray(windowTitle), reinterpret_cast<uint64_t>(windowHandle)});
+    }
 #endif
     QStringList ret;
     assert(windows_.size() < std::numeric_limits<int>::max());
@@ -1007,11 +1051,13 @@ make_preview_sink()
 {
     if (QGuiApplication::platformName() == QStringLiteral("wayland")) {
         return gst_element_factory_make("waylandsink", nullptr);
+    } else if (QGuiApplication::platformName() == QStringLiteral("windows")) {
+        return gst_element_factory_make("d3d11videosink", nullptr);
     } else {
         return gst_element_factory_make("ximagesink", nullptr);
     }
 }
-}
+} // namespace
 #endif
 
 void
@@ -1032,6 +1078,12 @@ CallManager::previewWindow(unsigned int index) const
         return;
     }
 
+    if (screenShareType_ == ScreenShareType::D3D11 &&
+        (windows_.empty() || index >= windows_.size())) {
+        nhlog::ui()->error("D3D11 screencast not available");
+        return;
+    }
+
     auto settings = ChatPage::instance()->userSettings();
 
     pipe_ = gst_pipeline_new(nullptr);
@@ -1066,6 +1118,19 @@ CallManager::previewWindow(unsigned int index) const
 
         gst_bin_add(GST_BIN(pipe_), ximagesrc);
         screencastsrc = ximagesrc;
+    } else if (screenShareType_ == ScreenShareType::D3D11) {
+        GstElement *d3d11screensrc = gst_element_factory_make("d3d11screencapturesrc", nullptr);
+        if (!d3d11screensrc) {
+            nhlog::ui()->error("Failed to create d3d11screencapturesrc");
+            gst_object_unref(pipe_);
+            pipe_ = nullptr;
+            return;
+        }
+        g_object_set(d3d11screensrc, "window-handle", windows_[index].second, nullptr);
+        g_object_set(d3d11screensrc, "show-cursor", !settings->screenShareHideCursor(), nullptr);
+
+        gst_bin_add(GST_BIN(pipe_), d3d11screensrc);
+        screencastsrc = d3d11screensrc;
     } else {
         ScreenCastPortal &sc_portal            = ScreenCastPortal::instance();
         const ScreenCastPortal::Stream *stream = sc_portal.getStream();
@@ -1171,6 +1236,6 @@ getTurnURIs(const mtx::responses::TurnServer &turnServer)
     }
     return ret;
 }
-}
+} // namespace
 
 #include "moc_CallManager.cpp"

src/voip/CallManager.h

@@ -133,7 +133,11 @@ private:
     QTimer turnServerTimer_;
     QMediaPlayer player_;
     std::vector<webrtc::ScreenShareType> screenShareTypes_;
+#ifndef Q_OS_WINDOWS
     std::vector<std::pair<QString, uint32_t>> windows_;
+#else
+    std::vector<std::pair<QString, uint64_t>> windows_;
+#endif
     std::vector<std::string> rejectCallPartyIDs_;
 
     template<typename T>

src/voip/WebRTCSession.cpp

@@ -23,6 +23,7 @@
 #include "voip/ScreenCastPortal.h"
 
 #ifdef GSTREAMER_AVAILABLE
+#include "MainWindow.h"
 extern "C"
 {
 #include "gst/gl/gstgldisplay.h"
@@ -331,25 +332,35 @@ newVideoSinkChain(GstElement *pipe)
 {
     // use compositor for now; acceleration needs investigation
     GstElement *queue = gst_element_factory_make("queue", nullptr);
-    GstElement *compositor = gst_element_factory_make("compositor", "compositor");
+
+    auto graphicsApi       = MainWindow::instance()->graphicsApi();
+    GstElement *compositor = gst_element_factory_make(
+      graphicsApi == QSGRendererInterface::OpenGL ? "compositor" : "d3d11compositor", "compositor");
+    g_object_set(compositor, "background", 1, nullptr);
+    switch (graphicsApi) {
+    case QSGRendererInterface::OpenGL: {
         GstElement *glupload       = gst_element_factory_make("glupload", nullptr);
+        GstElement *glcolorconvert = gst_element_factory_make("glcolorconvert", nullptr);
         GstElement *qmlglsink      = gst_element_factory_make("qml6glsink", nullptr);
         GstElement *glsinkbin      = gst_element_factory_make("glsinkbin", nullptr);
-    g_object_set(compositor, "background", 1, nullptr);
+
         g_object_set(qmlglsink, "widget", WebRTCSession::instance().getVideoItem(), nullptr);
         g_object_set(glsinkbin, "sink", qmlglsink, nullptr);
-    gst_bin_add_many(GST_BIN(pipe), queue, compositor, glupload, glsinkbin, nullptr);
-    gst_element_link_many(queue, compositor, glupload, glsinkbin, nullptr);
+        gst_bin_add_many(
+          GST_BIN(pipe), queue, compositor, glupload, glcolorconvert, glsinkbin, nullptr);
+        gst_element_link_many(queue, compositor, glupload, glcolorconvert, glsinkbin, nullptr);
+
         gst_element_sync_state_with_parent(queue);
         gst_element_sync_state_with_parent(compositor);
         gst_element_sync_state_with_parent(glupload);
+        gst_element_sync_state_with_parent(glcolorconvert);
         gst_element_sync_state_with_parent(glsinkbin);
 
         // to propagate context (hopefully)
         gst_element_set_state(qmlglsink, GST_STATE_READY);
 
-    // Workaround: On wayland, when egl is used, gstreamer might terminate the display connection.
-    // Prevent that by "leaking" a reference to the display. See
+        // Workaround: On wayland, when egl is used, gstreamer might terminate the display
+        // connection. Prevent that by "leaking" a reference to the display. See
         // https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/3743
         if (QGuiApplication::platformName() == QStringLiteral("wayland")) {
             auto context = gst_element_get_context(qmlglsink, "gst.gl.GLDisplay");
@@ -358,6 +369,29 @@ newVideoSinkChain(GstElement *pipe)
                 gst_context_get_gl_display(context, &display);
             }
         }
+    } break;
+    case QSGRendererInterface::Direct3D11: {
+        GstElement *d3d11upload       = gst_element_factory_make("d3d11upload", nullptr);
+        GstElement *d3d11colorconvert = gst_element_factory_make("d3d11colorconvert", nullptr);
+        GstElement *qmld3d11sink      = gst_element_factory_make("qml6d3d11sink", nullptr);
+
+        g_object_set(qmld3d11sink, "widget", WebRTCSession::instance().getVideoItem(), nullptr);
+        gst_bin_add_many(
+          GST_BIN(pipe), queue, d3d11upload, compositor, d3d11colorconvert, qmld3d11sink, nullptr);
+        gst_element_link_many(
+          queue, d3d11upload, compositor, d3d11colorconvert, qmld3d11sink, nullptr);
+
+        gst_element_sync_state_with_parent(queue);
+        gst_element_sync_state_with_parent(compositor);
+        gst_element_sync_state_with_parent(d3d11upload);
+        gst_element_sync_state_with_parent(d3d11colorconvert);
+
+        // to propagate context (hopefully)
+        gst_element_set_state(qmld3d11sink, GST_STATE_READY);
+    } break;
+    default:
+        break;
+    }
 
     return queue;
 }
@@ -606,8 +640,7 @@ WebRTCSession::havePlugins(bool isVideo,
     if (!initialised_ && !init(errorMessage))
         return false;
 
-    static constexpr std::initializer_list<const char *> audio_elements = {
-      "audioconvert",
+    static constexpr std::initializer_list<const char *> audio_elements = {"audioconvert",
                                                                            "audioresample",
                                                                            "autoaudiosink",
                                                                            "capsfilter",
@@ -617,9 +650,10 @@ WebRTCSession::havePlugins(bool isVideo,
                                                                            "rtpopuspay",
                                                                            "volume",
                                                                            "webrtcbin",
-    };
+                                                                           "nicesrc",
+                                                                           "nicesink"};
 
-    static constexpr std::initializer_list<const char *> video_elements = {
+    static constexpr std::initializer_list<const char *> gl_video_elements = {
       "compositor",
       "glsinkbin",
       "glupload",
@@ -631,6 +665,19 @@ WebRTCSession::havePlugins(bool isVideo,
       "vp8enc",
     };
 
+    static constexpr std::initializer_list<const char *> d3d11_video_elements = {
+      "compositor",
+      "d3d11colorconvert",
+      "d3d11videosink",
+      "d3d11upload",
+      "qml6d3d11sink",
+      "rtpvp8pay",
+      "tee",
+      "videoconvert",
+      "videoscale",
+      "vp8enc",
+    };
+
     std::string strError("Missing GStreamer elements: ");
     GstRegistry *registry = gst_registry_get();
 
@@ -654,8 +701,19 @@ WebRTCSession::havePlugins(bool isVideo,
     haveVoicePlugins_ = check_plugins(audio_elements);
 
     // check both elements at once
-    if (isVideo)
-        haveVideoPlugins_ = check_plugins(video_elements);
+    if (isVideo) {
+        switch (MainWindow::instance()->graphicsApi()) {
+        case QSGRendererInterface::OpenGL:
+            haveVideoPlugins_ = check_plugins(gl_video_elements);
+            break;
+        case QSGRendererInterface::Direct3D11:
+            haveVideoPlugins_ = check_plugins(d3d11_video_elements);
+            break;
+        default:
+            haveVideoPlugins_ = false;
+            break;
+        }
+    }
 
     bool haveScreensharePlugins = false;
     if (isScreenshare) {
@@ -663,6 +721,8 @@ WebRTCSession::havePlugins(bool isVideo,
         if (haveScreensharePlugins) {
             if (QGuiApplication::platformName() == QStringLiteral("wayland")) {
                 haveScreensharePlugins = check_plugins({"waylandsink"});
+            } else if (QGuiApplication::platformName() == QStringLiteral("windows")) {
+                haveScreensharePlugins = check_plugins({"d3d11videosink"});
             } else {
                 haveScreensharePlugins = check_plugins({"ximagesink"});
             }
@@ -670,6 +730,9 @@ WebRTCSession::havePlugins(bool isVideo,
         if (haveScreensharePlugins) {
             if (screenShareType == ScreenShareType::X11) {
                 haveScreensharePlugins = check_plugins({"ximagesrc"});
+            } else if (screenShareType == ScreenShareType::D3D11) {
+                haveScreensharePlugins =
+                  check_plugins({"d3d11screencapturesrc", "d3d11download", "d3d11convert"});
             } else {
                 haveScreensharePlugins = check_plugins({"pipewiresrc"});
             }
@@ -685,9 +748,19 @@ WebRTCSession::havePlugins(bool isVideo,
     }
 
     if (isVideo || isScreenshare) {
+        switch (MainWindow::instance()->graphicsApi()) {
+        case QSGRendererInterface::OpenGL: {
             // load qmlglsink to register GStreamer's GstGLVideoItem QML type
             GstElement *qmlglsink = gst_element_factory_make("qml6glsink", nullptr);
             gst_object_unref(qmlglsink);
+        } break;
+        case QSGRendererInterface::Direct3D11: {
+            GstElement *qmld3d11sink = gst_element_factory_make("qml6d3d11sink", nullptr);
+            gst_object_unref(qmld3d11sink);
+        } break;
+        default:
+            break;
+        }
     }
     return true;
 }
@@ -1027,6 +1100,36 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType)
 
             gst_bin_add(GST_BIN(pipe_), ximagesrc);
             screencastsrc = ximagesrc;
+        } else if (screenShareType_ == ScreenShareType::D3D11) {
+            GstElement *d3d11screensrc =
+              gst_element_factory_make("d3d11screencapturesrc", "screenshare");
+            if (!d3d11screensrc) {
+                nhlog::ui()->error("WebRTC: failed to create d3d11screencapturesrc");
+                gst_object_unref(pipe_);
+                pipe_ = nullptr;
+                return false;
+            }
+            g_object_set(
+              d3d11screensrc, "window-handle", static_cast<guint64>(shareWindowId_), nullptr);
+            g_object_set(
+              d3d11screensrc, "show-cursor", !settings->screenShareHideCursor(), nullptr);
+            g_object_set(d3d11screensrc, "do-timestamp", (gboolean)1, nullptr);
+            gst_bin_add(GST_BIN(pipe_), d3d11screensrc);
+
+            GstElement *d3d11convert = gst_element_factory_make("d3d11convert", nullptr);
+            gst_bin_add(GST_BIN(pipe_), d3d11convert);
+            if (!gst_element_link(d3d11screensrc, d3d11convert)) {
+                nhlog::ui()->error("WebRTC: failed to link d3d11screencapturesrc -> d3d11convert");
+                return false;
+            }
+
+            GstElement *d3d11download = gst_element_factory_make("d3d11download", nullptr);
+            gst_bin_add(GST_BIN(pipe_), d3d11download);
+            if (!gst_element_link(d3d11convert, d3d11download)) {
+                nhlog::ui()->error("WebRTC: failed to link d3d11convert -> d3d11download");
+                return false;
+            }
+            screencastsrc = d3d11download;
         } else {
             ScreenCastPortal &sc_portal = ScreenCastPortal::instance();
             GstElement *pipewiresrc     = gst_element_factory_make("pipewiresrc", "screenshare");

src/voip/WebRTCSession.h

@@ -31,7 +31,8 @@ Q_ENUM_NS(CallType)
 enum class ScreenShareType
 {
     X11,
-    XDP
+    XDP,
+    D3D11
 };
 Q_ENUM_NS(ScreenShareType)