Screen sharing (X11): support picture-in-picture

resources/qml/voip/ScreenShare.qml

@@ -13,6 +13,7 @@ Popup {
             anchors.centerIn = parent;
 
         frameRateCombo.currentIndex = frameRateCombo.find(Settings.screenShareFrameRate);
+        pipCheckBox.checked = Settings.screenSharePiP;
         remoteVideoCheckBox.checked = Settings.screenShareRemoteVideo;
         hideCursorCheckBox.checked = Settings.screenShareHideCursor;
     }
@@ -45,6 +46,16 @@ Popup {
 
         }
 
+        CheckBox {
+            id: pipCheckBox
+
+            visible: CallManager.cameras.length > 0
+            Layout.alignment: Qt.AlignLeft
+            Layout.leftMargin: 8
+            Layout.rightMargin: 8
+            text: qsTr("Include your camera picture-in-picture")
+        }
+
         CheckBox {
             id: remoteVideoCheckBox
 
@@ -79,6 +90,7 @@ Popup {
                     if (buttonLayout.validateMic()) {
                         Settings.microphone = micCombo.currentText;
                         Settings.screenShareFrameRate = frameRateCombo.currentText;
+                        Settings.screenSharePiP = pipCheckBox.checked;
                         Settings.screenShareRemoteVideo = remoteVideoCheckBox.checked;
                         Settings.screenShareHideCursor = hideCursorCheckBox.checked;
                         CallManager.sendInvite(TimelineManager.timeline.roomId(), CallType.SCREEN);

src/UserSettingsPage.cpp

@@ -114,6 +114,7 @@ UserSettings::load(std::optional<QString> profile)
         cameraResolution_       = settings.value("user/camera_resolution", QString()).toString();
         cameraFrameRate_        = settings.value("user/camera_frame_rate", QString()).toString();
         screenShareFrameRate_   = settings.value("user/screen_share_frame_rate", 5).toInt();
+        screenSharePiP_         = settings.value("user/screen_share_pip", true).toBool();
         screenShareRemoteVideo_ = settings.value("user/screen_share_remote_video", false).toBool();
         screenShareHideCursor_  = settings.value("user/screen_share_hide_cursor", false).toBool();
         useStunServer_          = settings.value("user/use_stun_server", false).toBool();
@@ -457,6 +458,16 @@ UserSettings::setScreenShareFrameRate(int frameRate)
         save();
 }
 
+void
+UserSettings::setScreenSharePiP(bool state)
+{
+        if (state == screenSharePiP_)
+                return;
+        screenSharePiP_ = state;
+        emit screenSharePiPChanged(state);
+        save();
+}
+
 void
 UserSettings::setScreenShareRemoteVideo(bool state)
 {
@@ -627,6 +638,7 @@ UserSettings::save()
         settings.setValue("camera_resolution", cameraResolution_);
         settings.setValue("camera_frame_rate", cameraFrameRate_);
         settings.setValue("screen_share_frame_rate", screenShareFrameRate_);
+        settings.setValue("screen_share_pip", screenSharePiP_);
         settings.setValue("screen_share_remote_video", screenShareRemoteVideo_);
         settings.setValue("screen_share_hide_cursor", screenShareHideCursor_);
         settings.setValue("use_stun_server", useStunServer_);

src/UserSettingsPage.h

@@ -88,6 +88,8 @@ class UserSettings : public QObject
                      cameraFrameRateChanged)
         Q_PROPERTY(int screenShareFrameRate READ screenShareFrameRate WRITE setScreenShareFrameRate
                      NOTIFY screenShareFrameRateChanged)
+        Q_PROPERTY(bool screenSharePiP READ screenSharePiP WRITE setScreenSharePiP NOTIFY
+                     screenSharePiPChanged)
         Q_PROPERTY(bool screenShareRemoteVideo READ screenShareRemoteVideo WRITE
                      setScreenShareRemoteVideo NOTIFY screenShareRemoteVideoChanged)
         Q_PROPERTY(bool screenShareHideCursor READ screenShareHideCursor WRITE
@@ -150,6 +152,7 @@ public:
         void setCameraResolution(QString resolution);
         void setCameraFrameRate(QString frameRate);
         void setScreenShareFrameRate(int frameRate);
+        void setScreenSharePiP(bool state);
         void setScreenShareRemoteVideo(bool state);
         void setScreenShareHideCursor(bool state);
         void setUseStunServer(bool state);
@@ -201,6 +204,7 @@ public:
         QString cameraResolution() const { return cameraResolution_; }
         QString cameraFrameRate() const { return cameraFrameRate_; }
         int screenShareFrameRate() const { return screenShareFrameRate_; }
+        bool screenSharePiP() const { return screenSharePiP_; }
         bool screenShareRemoteVideo() const { return screenShareRemoteVideo_; }
         bool screenShareHideCursor() const { return screenShareHideCursor_; }
         bool useStunServer() const { return useStunServer_; }
@@ -242,6 +246,7 @@ signals:
         void cameraResolutionChanged(QString resolution);
         void cameraFrameRateChanged(QString frameRate);
         void screenShareFrameRateChanged(int frameRate);
+        void screenSharePiPChanged(bool state);
         void screenShareRemoteVideoChanged(bool state);
         void screenShareHideCursorChanged(bool state);
         void useStunServerChanged(bool state);
@@ -288,6 +293,7 @@ private:
         QString cameraResolution_;
         QString cameraFrameRate_;
         int screenShareFrameRate_;
+        bool screenSharePiP_;
         bool screenShareRemoteVideo_;
         bool screenShareHideCursor_;
         bool useStunServer_;

src/WebRTCSession.cpp

@@ -89,9 +89,10 @@ namespace {
 
 std::string localsdp_;
 std::vector<mtx::events::msg::CallCandidates::Candidate> localcandidates_;
-bool haveAudioStream_;
+bool haveAudioStream_     = false;
-bool haveVideoStream_;
+bool haveVideoStream_     = false;
-GstPad *insetSinkPad_ = nullptr;
+GstPad *localPiPSinkPad_  = nullptr;
+GstPad *remotePiPSinkPad_ = nullptr;
 
 gboolean
 newBusMessage(GstBus *bus G_GNUC_UNUSED, GstMessage *msg, gpointer user_data)
@@ -364,6 +365,7 @@ newVideoSinkChain(GstElement *pipe)
         GstElement *glcolorconvert = gst_element_factory_make("glcolorconvert", nullptr);
         GstElement *qmlglsink      = gst_element_factory_make("qmlglsink", nullptr);
         GstElement *glsinkbin      = gst_element_factory_make("glsinkbin", nullptr);
+        g_object_set(compositor, "background", 1, nullptr);
         g_object_set(qmlglsink, "widget", WebRTCSession::instance().getVideoItem(), nullptr);
         g_object_set(glsinkbin, "sink", qmlglsink, nullptr);
         gst_bin_add_many(
@@ -390,8 +392,9 @@ getResolution(GstPad *pad)
 }
 
 void
-addCameraView(GstElement *pipe, const std::pair<int, int> &videoCallSize)
+addLocalPiP(GstElement *pipe, const std::pair<int, int> &videoCallSize)
 {
+        // embed localUser's camera into received video
         GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe), "videosrctee");
         if (!tee)
                 return;
@@ -407,26 +410,56 @@ addCameraView(GstElement *pipe, const std::pair<int, int> &videoCallSize)
         GstElement *camerafilter = gst_bin_get_by_name(GST_BIN(pipe), "camerafilter");
         GstPad *filtersinkpad    = gst_element_get_static_pad(camerafilter, "sink");
         auto cameraResolution    = getResolution(filtersinkpad);
-        int insetWidth           = videoCallSize.first / 4;
+        int pipWidth             = videoCallSize.first / 4;
-        int insetHeight =
+        int pipHeight =
-          static_cast<double>(cameraResolution.second) / cameraResolution.first * insetWidth;
+          static_cast<double>(cameraResolution.second) / cameraResolution.first * pipWidth;
-        nhlog::ui()->debug("WebRTC: picture-in-picture size: {}x{}", insetWidth, insetHeight);
+        nhlog::ui()->debug("WebRTC: local picture-in-picture: {}x{}", pipWidth, pipHeight);
         gst_object_unref(filtersinkpad);
         gst_object_unref(camerafilter);
 
         GstPad *camerapad      = gst_element_get_static_pad(videorate, "src");
         GstElement *compositor = gst_bin_get_by_name(GST_BIN(pipe), "compositor");
-        insetSinkPad_          = gst_element_get_request_pad(compositor, "sink_%u");
+        localPiPSinkPad_       = gst_element_get_request_pad(compositor, "sink_%u");
-        g_object_set(insetSinkPad_, "zorder", 2, nullptr);
+        g_object_set(localPiPSinkPad_, "zorder", 2, nullptr);
-        g_object_set(insetSinkPad_, "width", insetWidth, "height", insetHeight, nullptr);
+        g_object_set(localPiPSinkPad_, "width", pipWidth, "height", pipHeight, nullptr);
         gint offset = videoCallSize.first / 80;
-        g_object_set(insetSinkPad_, "xpos", offset, "ypos", offset, nullptr);
+        g_object_set(localPiPSinkPad_, "xpos", offset, "ypos", offset, nullptr);
-        if (GST_PAD_LINK_FAILED(gst_pad_link(camerapad, insetSinkPad_)))
+        if (GST_PAD_LINK_FAILED(gst_pad_link(camerapad, localPiPSinkPad_)))
-                nhlog::ui()->error("WebRTC: failed to link camera view chain");
+                nhlog::ui()->error("WebRTC: failed to link local PiP elements");
         gst_object_unref(camerapad);
         gst_object_unref(compositor);
 }
 
+void
+addRemotePiP(GstElement *pipe)
+{
+        // embed localUser's camera into screen image being shared
+        if (remotePiPSinkPad_) {
+                GstElement *screen = gst_bin_get_by_name(GST_BIN(pipe), "screenshare");
+                GstPad *srcpad     = gst_element_get_static_pad(screen, "src");
+                auto resolution    = getResolution(srcpad);
+                nhlog::ui()->debug(
+                  "WebRTC: screen share: {}x{}", resolution.first, resolution.second);
+                gst_object_unref(srcpad);
+                gst_object_unref(screen);
+
+                int pipWidth = resolution.first / 5;
+                int pipHeight =
+                  static_cast<double>(resolution.second) / resolution.first * pipWidth;
+                nhlog::ui()->debug(
+                  "WebRTC: screen share picture-in-picture: {}x{}", pipWidth, pipHeight);
+                gint offset = resolution.first / 100;
+                g_object_set(remotePiPSinkPad_, "zorder", 2, nullptr);
+                g_object_set(remotePiPSinkPad_, "width", pipWidth, "height", pipHeight, nullptr);
+                g_object_set(remotePiPSinkPad_,
+                             "xpos",
+                             resolution.first - pipWidth - offset,
+                             "ypos",
+                             resolution.second - pipHeight - offset,
+                             nullptr);
+        }
+}
+
 void
 linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe)
 {
@@ -463,7 +496,7 @@ linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe)
                                   videoCallSize.first,
                                   videoCallSize.second);
                 if (session->callType() == CallType::VIDEO)
-                        addCameraView(pipe, videoCallSize);
+                        addLocalPiP(pipe, videoCallSize);
         } else {
                 g_free(mediaType);
                 nhlog::ui()->error("WebRTC: unknown pad type: {}", GST_PAD_NAME(newpad));
@@ -485,6 +518,7 @@ linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe)
                                         keyFrameRequestData_.timerid =
                                           g_timeout_add_seconds(3, testPacketLoss, nullptr);
                                 }
+                                addRemotePiP(pipe);
                         }
                 }
                 gst_object_unref(queuepad);
@@ -616,16 +650,9 @@ WebRTCSession::havePlugins(bool isVideo, std::string *errorMessage)
 bool
 WebRTCSession::createOffer(CallType callType)
 {
+        clear();
         isOffering_ = true;
         callType_   = callType;
-        isRemoteVideoRecvOnly_ = false;
-        isRemoteVideoSendOnly_ = false;
-        videoItem_             = nullptr;
-        haveAudioStream_       = false;
-        haveVideoStream_       = false;
-        insetSinkPad_          = nullptr;
-        localsdp_.clear();
-        localcandidates_.clear();
 
         // opus and vp8 rtp payload types must be defined dynamically
         // therefore from the range [96-127]
@@ -642,17 +669,7 @@ WebRTCSession::acceptOffer(const std::string &sdp)
         if (state_ != State::DISCONNECTED)
                 return false;
 
-        callType_              = webrtc::CallType::VOICE;
+        clear();
-        isOffering_            = false;
-        isRemoteVideoRecvOnly_ = false;
-        isRemoteVideoSendOnly_ = false;
-        videoItem_             = nullptr;
-        haveAudioStream_       = false;
-        haveVideoStream_       = false;
-        insetSinkPad_          = nullptr;
-        localsdp_.clear();
-        localcandidates_.clear();
-
         GstWebRTCSessionDescription *offer = parseSDP(sdp, GST_WEBRTC_SDP_TYPE_OFFER);
         if (!offer)
                 return false;
@@ -891,16 +908,23 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType)
         if (callType_ == CallType::VIDEO && !devices_.haveCamera())
                 return !isOffering_;
 
-        GstElement *source = nullptr;
+        auto settings = ChatPage::instance()->userSettings();
-        GstCaps *caps      = nullptr;
+        if (callType_ == CallType::SCREEN && settings->screenSharePiP() && !devices_.haveCamera())
-        if (callType_ == CallType::VIDEO) {
+                return false;
+
+        GstElement *camerafilter = nullptr;
+        GstElement *videoconvert = gst_element_factory_make("videoconvert", nullptr);
+        GstElement *tee          = gst_element_factory_make("tee", "videosrctee");
+        gst_bin_add_many(GST_BIN(pipe_), videoconvert, tee, nullptr);
+        if (callType_ == CallType::VIDEO || settings->screenSharePiP()) {
                 std::pair<int, int> resolution;
                 std::pair<int, int> frameRate;
                 GstDevice *device = devices_.videoDevice(resolution, frameRate);
                 if (!device)
                         return false;
-                source = gst_device_create_element(device, nullptr);
+
-                caps   = gst_caps_new_simple("video/x-raw",
+                GstElement *camera = gst_device_create_element(device, nullptr);
+                GstCaps *caps      = gst_caps_new_simple("video/x-raw",
                                                     "width",
                                                     G_TYPE_INT,
                                                     resolution.first,
@@ -912,30 +936,78 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType)
                                                     frameRate.first,
                                                     frameRate.second,
                                                     nullptr);
-        } else {
+                camerafilter       = gst_element_factory_make("capsfilter", "camerafilter");
-                source = gst_element_factory_make("ximagesrc", nullptr);
+                g_object_set(camerafilter, "caps", caps, nullptr);
-                if (!source) {
+                gst_caps_unref(caps);
+
+                gst_bin_add_many(GST_BIN(pipe_), camera, camerafilter, nullptr);
+                if (!gst_element_link_many(camera, videoconvert, camerafilter, nullptr)) {
+                        nhlog::ui()->error("WebRTC: failed to link camera elements");
+                        return false;
+                }
+                if (callType_ == CallType::VIDEO && !gst_element_link(camerafilter, tee)) {
+                        nhlog::ui()->error("WebRTC: failed to link camerafilter -> tee");
+                        return false;
+                }
+        }
+
+        if (callType_ == CallType::SCREEN) {
+                nhlog::ui()->debug("WebRTC: screen share frame rate: {} fps",
+                                   settings->screenShareFrameRate());
+                nhlog::ui()->debug("WebRTC: screen share picture-in-picture: {}",
+                                   settings->screenSharePiP());
+                nhlog::ui()->debug("WebRTC: screen share request remote camera: {}",
+                                   settings->screenShareRemoteVideo());
+                nhlog::ui()->debug("WebRTC: screen share hide mouse cursor: {}",
+                                   settings->screenShareHideCursor());
+
+                GstElement *ximagesrc = gst_element_factory_make("ximagesrc", "screenshare");
+                if (!ximagesrc) {
                         nhlog::ui()->error("WebRTC: failed to create ximagesrc");
                         return false;
                 }
-                g_object_set(source, "use-damage", FALSE, nullptr);
+                g_object_set(ximagesrc, "use-damage", FALSE, nullptr);
-                g_object_set(source, "xid", 0, nullptr);
+                g_object_set(ximagesrc, "xid", 0, nullptr);
-                auto settings = ChatPage::instance()->userSettings();
+                g_object_set(
-                g_object_set(source, "show-pointer", !settings->screenShareHideCursor(), nullptr);
+                  ximagesrc, "show-pointer", !settings->screenShareHideCursor(), nullptr);
-                nhlog::ui()->debug("WebRTC: screen share hide mouse cursor: {}",
-                                   settings->screenShareHideCursor());
-                int frameRate = settings->screenShareFrameRate();
-                caps          = gst_caps_new_simple(
-                  "video/x-raw", "framerate", GST_TYPE_FRACTION, frameRate, 1, nullptr);
-                nhlog::ui()->debug("WebRTC: screen share frame rate: {} fps", frameRate);
-        }
 
-        GstElement *videoconvert = gst_element_factory_make("videoconvert", nullptr);
+                GstCaps *caps          = gst_caps_new_simple("video/x-raw",
-        GstElement *capsfilter   = gst_element_factory_make("capsfilter", "camerafilter");
+                                                    "framerate",
+                                                    GST_TYPE_FRACTION,
+                                                    settings->screenShareFrameRate(),
+                                                    1,
+                                                    nullptr);
+                GstElement *capsfilter = gst_element_factory_make("capsfilter", nullptr);
                 g_object_set(capsfilter, "caps", caps, nullptr);
                 gst_caps_unref(caps);
+                gst_bin_add_many(GST_BIN(pipe_), ximagesrc, capsfilter, nullptr);
+
+                if (settings->screenSharePiP()) {
+                        GstElement *compositor = gst_element_factory_make("compositor", nullptr);
+                        g_object_set(compositor, "background", 1, nullptr);
+                        gst_bin_add(GST_BIN(pipe_), compositor);
+                        if (!gst_element_link_many(
+                              ximagesrc, compositor, capsfilter, tee, nullptr)) {
+                                nhlog::ui()->error("WebRTC: failed to link screen share elements");
+                                return false;
+                        }
+
+                        GstPad *srcpad    = gst_element_get_static_pad(camerafilter, "src");
+                        remotePiPSinkPad_ = gst_element_get_request_pad(compositor, "sink_%u");
+                        if (GST_PAD_LINK_FAILED(gst_pad_link(srcpad, remotePiPSinkPad_))) {
+                                nhlog::ui()->error(
+                                  "WebRTC: failed to link camerafilter -> compositor");
+                                gst_object_unref(srcpad);
+                                return false;
+                        }
+                        gst_object_unref(srcpad);
+                } else if (!gst_element_link_many(
+                             ximagesrc, videoconvert, capsfilter, tee, nullptr)) {
+                        nhlog::ui()->error("WebRTC: failed to link screen share elements");
+                        return false;
+                }
+        }
 
-        GstElement *tee    = gst_element_factory_make("tee", "videosrctee");
         GstElement *queue  = gst_element_factory_make("queue", nullptr);
         GstElement *vp8enc = gst_element_factory_make("vp8enc", nullptr);
         g_object_set(vp8enc, "deadline", 1, nullptr);
@@ -957,31 +1029,13 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType)
         g_object_set(rtpcapsfilter, "caps", rtpcaps, nullptr);
         gst_caps_unref(rtpcaps);
 
-        gst_bin_add_many(GST_BIN(pipe_),
+        gst_bin_add_many(
-                         source,
+          GST_BIN(pipe_), queue, vp8enc, rtpvp8pay, rtpqueue, rtpcapsfilter, nullptr);
-                         videoconvert,
-                         capsfilter,
-                         tee,
-                         queue,
-                         vp8enc,
-                         rtpvp8pay,
-                         rtpqueue,
-                         rtpcapsfilter,
-                         nullptr);
 
         GstElement *webrtcbin = gst_bin_get_by_name(GST_BIN(pipe_), "webrtcbin");
-        if (!gst_element_link_many(source,
+        if (!gst_element_link_many(
-                                   videoconvert,
+              tee, queue, vp8enc, rtpvp8pay, rtpqueue, rtpcapsfilter, webrtcbin, nullptr)) {
-                                   capsfilter,
+                nhlog::ui()->error("WebRTC: failed to link rtp video elements");
-                                   tee,
-                                   queue,
-                                   vp8enc,
-                                   rtpvp8pay,
-                                   rtpqueue,
-                                   rtpcapsfilter,
-                                   webrtcbin,
-                                   nullptr)) {
-                nhlog::ui()->error("WebRTC: failed to link video pipeline elements");
                 gst_object_unref(webrtcbin);
                 return false;
         }
@@ -1043,13 +1097,32 @@ WebRTCSession::toggleMicMute()
 void
 WebRTCSession::toggleCameraView()
 {
-        if (insetSinkPad_) {
+        if (localPiPSinkPad_) {
                 guint zorder;
-                g_object_get(insetSinkPad_, "zorder", &zorder, nullptr);
+                g_object_get(localPiPSinkPad_, "zorder", &zorder, nullptr);
-                g_object_set(insetSinkPad_, "zorder", zorder ? 0 : 2, nullptr);
+                g_object_set(localPiPSinkPad_, "zorder", zorder ? 0 : 2, nullptr);
         }
 }
 
+void
+WebRTCSession::clear()
+{
+        callType_              = webrtc::CallType::VOICE;
+        isOffering_            = false;
+        isRemoteVideoRecvOnly_ = false;
+        isRemoteVideoSendOnly_ = false;
+        videoItem_             = nullptr;
+        pipe_                  = nullptr;
+        webrtc_                = nullptr;
+        busWatchId_            = 0;
+        haveAudioStream_       = false;
+        haveVideoStream_       = false;
+        localPiPSinkPad_       = nullptr;
+        remotePiPSinkPad_      = nullptr;
+        localsdp_.clear();
+        localcandidates_.clear();
+}
+
 void
 WebRTCSession::end()
 {
@@ -1065,13 +1138,7 @@ WebRTCSession::end()
                 }
         }
 
-        webrtc_                = nullptr;
+        clear();
-        callType_              = CallType::VOICE;
-        isOffering_            = false;
-        isRemoteVideoRecvOnly_ = false;
-        isRemoteVideoSendOnly_ = false;
-        videoItem_             = nullptr;
-        insetSinkPad_          = nullptr;
         if (state_ != State::DISCONNECTED)
                 emit stateChanged(State::DISCONNECTED);
 }

src/WebRTCSession.h

@@ -105,6 +105,7 @@ private:
         bool startPipeline(int opusPayloadType, int vp8PayloadType);
         bool createPipeline(int opusPayloadType, int vp8PayloadType);
         bool addVideoPipeline(int vp8PayloadType);
+        void clear();
 
 public:
         WebRTCSession(WebRTCSession const &) = delete;