From 8df10eeecac15ddb45ed4e350d33814ac4690f89 Mon Sep 17 00:00:00 2001 From: trilene Date: Thu, 18 Feb 2021 15:55:29 -0500 Subject: [PATCH] Support desktop screen sharing on X11 --- resources/icons/ui/screen-share.png | Bin 0 -> 773 bytes resources/qml/TimelineView.qml | 2 +- resources/qml/voip/ActiveCallBar.qml | 52 ++++++++++- resources/qml/voip/CallDevices.qml | 2 +- resources/qml/voip/CallInvite.qml | 8 +- resources/qml/voip/CallInviteBar.qml | 8 +- resources/qml/voip/PlaceCall.qml | 23 ++++- resources/qml/voip/ScreenShare.qml | 95 +++++++++++++++++++ resources/res.qrc | 2 + src/CallManager.cpp | 45 ++++++--- src/CallManager.h | 25 +++-- src/UserSettingsPage.cpp | 38 ++++++-- src/UserSettingsPage.h | 12 +++ src/WebRTCSession.cpp | 134 +++++++++++++++++++-------- src/WebRTCSession.h | 20 +++- 15 files changed, 376 insertions(+), 90 deletions(-) create mode 100644 resources/icons/ui/screen-share.png create mode 100644 resources/qml/voip/ScreenShare.qml diff --git a/resources/icons/ui/screen-share.png b/resources/icons/ui/screen-share.png new file mode 100644 index 0000000000000000000000000000000000000000..d6cee4277f6ba194c43a627532c7b3666f029fab GIT binary patch literal 773 zcmV+g1N!`lP)n-*ntJ3vZ_0%2|4{mkpWFVHyzBH*oQ?1)Jham710r0 zWWboAlMMJ&MW>x)zzrM}X6-VF#_c2n;3y8@XS+J=Gy~u)cHvt)I_pFO;IdGOen}X! zD;|N7+{7k)D6^BQ4HyzOTc>b3^8H=fzGi;2Ksn zB4h+LJZ+)x3$dW>dIScs4_ok{$ruCJfP0BFfe9&t74TA+%w8vmHCT`JLZaPj*(2bI z@T;Xro}l-r=8}&tgJq6@ci1f53_cO(urpCSlxM&Y_DGlf6rniUk|^FE$}-?MZjIu) zxtNEk!b$G)$Tc%?73*8n)X*lYm1XcFW=CR9!gd_PePR1M^1lAaJ@bTBBgGQj7fZB^ z5pYD9t%vEwA?y}6rifl)i@ZFYw7*Ghz;~Rizn_T1!XAHegK?G#$uCMLElz3!uHZcu z7=_h5y-9Exy(E5DSZv9RCjkKH&_$3U&RQ`fE4vML0V=f&Ib`HA;_y z0%5F>>wu~SR82OZY5`T#Bk?hFCX>lzGMP*!(;fT0y70lMx00000NkvXXu0mjf D 0 + visible: CallManager.callType == CallType.VIDEO && CallManager.cameras.length > 0 Image { Layout.preferredWidth: 22 diff --git a/resources/qml/voip/CallInvite.qml b/resources/qml/voip/CallInvite.qml index 00dcc77f..df3343ed 100644 --- a/resources/qml/voip/CallInvite.qml +++ b/resources/qml/voip/CallInvite.qml @@ -53,7 +53,7 @@ Popup { Layout.bottomMargin: msgView.height / 25 Image { - property string image: CallManager.isVideo ? ":/icons/icons/ui/video-call.png" : ":/icons/icons/ui/place-call.png" + property string image: CallManager.callType == CallType.VIDEO ? ":/icons/icons/ui/video-call.png" : ":/icons/icons/ui/place-call.png" Layout.alignment: Qt.AlignCenter Layout.preferredWidth: msgView.height / 10 @@ -63,7 +63,7 @@ Popup { Label { Layout.alignment: Qt.AlignCenter - text: CallManager.isVideo ? qsTr("Video Call") : qsTr("Voice Call") + text: CallManager.callType == CallType.VIDEO ? qsTr("Video Call") : qsTr("Voice Call") font.pointSize: fontMetrics.font.pointSize * 2 color: colors.windowText } @@ -97,7 +97,7 @@ Popup { } RowLayout { - visible: CallManager.isVideo && CallManager.cameras.length > 0 + visible: CallManager.callType == CallType.VIDEO && CallManager.cameras.length > 0 Layout.alignment: Qt.AlignCenter Image { @@ -159,7 +159,7 @@ Popup { RoundButton { id: acceptButton - property string image: CallManager.isVideo ? ":/icons/icons/ui/video-call.png" : ":/icons/icons/ui/place-call.png" + property string image: CallManager.callType == CallType.VIDEO ? ":/icons/icons/ui/video-call.png" : ":/icons/icons/ui/place-call.png" implicitWidth: buttonLayout.buttonSize implicitHeight: buttonLayout.buttonSize diff --git a/resources/qml/voip/CallInviteBar.qml b/resources/qml/voip/CallInviteBar.qml index 65749c35..bf630e9e 100644 --- a/resources/qml/voip/CallInviteBar.qml +++ b/resources/qml/voip/CallInviteBar.qml @@ -52,12 +52,12 @@ Rectangle { Layout.leftMargin: 4 Layout.preferredWidth: 24 Layout.preferredHeight: 24 - source: CallManager.isVideo ? "qrc:/icons/icons/ui/video-call.png" : "qrc:/icons/icons/ui/place-call.png" + source: CallManager.callType == CallType.VIDEO ? "qrc:/icons/icons/ui/video-call.png" : "qrc:/icons/icons/ui/place-call.png" } Label { font.pointSize: fontMetrics.font.pointSize * 1.1 - text: CallManager.isVideo ? qsTr("Video Call") : qsTr("Voice Call") + text: CallManager.callType == CallType.VIDEO ? qsTr("Video Call") : qsTr("Voice Call") color: "#000000" } @@ -83,7 +83,7 @@ Rectangle { Button { Layout.rightMargin: 4 - icon.source: CallManager.isVideo ? "qrc:/icons/icons/ui/video-call.png" : "qrc:/icons/icons/ui/place-call.png" + icon.source: CallManager.callType == CallType.VIDEO ? "qrc:/icons/icons/ui/video-call.png" : "qrc:/icons/icons/ui/place-call.png" text: qsTr("Accept") palette: colors onClicked: { @@ -102,7 +102,7 @@ Rectangle { dialog.open(); return ; } - if (CallManager.isVideo && CallManager.cameras.length > 0 && !CallManager.cameras.includes(Settings.camera)) { + if (CallManager.callType == CallType.VIDEO && CallManager.cameras.length > 0 && !CallManager.cameras.includes(Settings.camera)) { var dialog = deviceError.createObject(timelineRoot, { "errorString": qsTr("Unknown camera: %1").arg(Settings.camera), "image": ":/icons/icons/ui/video-call.png" diff --git a/resources/qml/voip/PlaceCall.qml b/resources/qml/voip/PlaceCall.qml index 41cbd54c..5dbeb6e1 100644 --- a/resources/qml/voip/PlaceCall.qml +++ b/resources/qml/voip/PlaceCall.qml @@ -23,6 +23,14 @@ Popup { } + Component { + id: screenShareDialog + + ScreenShare { + } + + } + ColumnLayout { id: columnLayout @@ -76,7 +84,7 @@ Popup { onClicked: { if (buttonLayout.validateMic()) { Settings.microphone = micCombo.currentText; - CallManager.sendInvite(TimelineManager.timeline.roomId(), false); + CallManager.sendInvite(TimelineManager.timeline.roomId(), CallType.VOICE); close(); } } @@ -90,12 +98,23 @@ Popup { if (buttonLayout.validateMic()) { Settings.microphone = micCombo.currentText; Settings.camera = cameraCombo.currentText; - CallManager.sendInvite(TimelineManager.timeline.roomId(), true); + CallManager.sendInvite(TimelineManager.timeline.roomId(), CallType.VIDEO); close(); } } } + Button { + visible: CallManager.screenShareSupported + text: qsTr("Screen") + icon.source: "qrc:/icons/icons/ui/screen-share.png" + onClicked: { + var dialog = screenShareDialog.createObject(timelineRoot); + dialog.open(); + close(); + } + } + Button { text: qsTr("Cancel") onClicked: { diff --git a/resources/qml/voip/ScreenShare.qml b/resources/qml/voip/ScreenShare.qml new file mode 100644 index 00000000..b21a26fd --- /dev/null +++ b/resources/qml/voip/ScreenShare.qml @@ -0,0 +1,95 @@ +import "../" +import QtQuick 2.9 +import QtQuick.Controls 2.3 +import QtQuick.Layouts 1.2 +import im.nheko 1.0 + +Popup { + modal: true + // only set the anchors on Qt 5.12 or higher + // see https://doc.qt.io/qt-5/qml-qtquick-controls2-popup.html#anchors.centerIn-prop + Component.onCompleted: { + if (anchors) + anchors.centerIn = parent; + + frameRateCombo.currentIndex = frameRateCombo.find(Settings.screenShareFrameRate); + remoteVideoCheckBox.checked = Settings.screenShareRemoteVideo; + } + palette: colors + + ColumnLayout { + Label { + Layout.margins: 8 + Layout.alignment: Qt.AlignLeft + text: qsTr("Share desktop with %1?").arg(TimelineManager.timeline.roomName) + color: colors.windowText + } + + RowLayout { + Layout.leftMargin: 8 + Layout.rightMargin: 8 + + Label { + Layout.alignment: Qt.AlignLeft + text: qsTr("Frame rate:") + color: colors.windowText + } + + ComboBox { + id: frameRateCombo + + Layout.alignment: Qt.AlignRight + model: ["25", "20", "15", "10", "5", "2", "1"] + } + + } + + CheckBox { + id: remoteVideoCheckBox + + Layout.alignment: Qt.AlignLeft + Layout.leftMargin: 8 + Layout.rightMargin: 8 + text: qsTr("Request remote camera") + ToolTip.text: qsTr("View your callee's camera like a regular video call") + ToolTip.visible: hovered + } + + RowLayout { + Layout.margins: 8 + + Item { + Layout.fillWidth: true + } + + Button { + text: qsTr("Share") + icon.source: "qrc:/icons/icons/ui/screen-share.png" + onClicked: { + if (buttonLayout.validateMic()) { + Settings.microphone = micCombo.currentText; + Settings.screenShareFrameRate = frameRateCombo.currentText; + Settings.screenShareRemoteVideo = remoteVideoCheckBox.checked; + CallManager.sendInvite(TimelineManager.timeline.roomId(), CallType.SCREEN); + close(); + } + } + } + + Button { + text: qsTr("Cancel") + onClicked: { + close(); + } + } + + } + + } + + background: Rectangle { + color: colors.window + border.color: colors.windowText + } + +} diff --git a/resources/res.qrc b/resources/res.qrc index 308d81a6..2387fa75 100644 --- a/resources/res.qrc +++ b/resources/res.qrc @@ -74,6 +74,7 @@ icons/ui/end-call.png icons/ui/microphone-mute.png icons/ui/microphone-unmute.png + icons/ui/screen-share.png icons/ui/toggle-camera-view.png icons/ui/video-call.png @@ -165,6 +166,7 @@ qml/voip/CallInviteBar.qml qml/voip/DeviceError.qml qml/voip/PlaceCall.qml + qml/voip/ScreenShare.qml qml/voip/VideoCall.qml diff --git a/src/CallManager.cpp b/src/CallManager.cpp index 7acd9592..51bb7b33 100644 --- a/src/CallManager.cpp +++ b/src/CallManager.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -24,6 +25,8 @@ Q_DECLARE_METATYPE(mtx::responses::TurnServer) using namespace mtx::events; using namespace mtx::events::msg; +using webrtc::CallType; + namespace { std::vector getTurnURIs(const mtx::responses::TurnServer &turnServer); @@ -148,10 +151,12 @@ CallManager::CallManager(QObject *parent) } void -CallManager::sendInvite(const QString &roomid, bool isVideo) +CallManager::sendInvite(const QString &roomid, CallType callType) { if (isOnCall()) return; + if (callType == CallType::SCREEN && !screenShareSupported()) + return; auto roomInfo = cache::singleRoomInfo(roomid.toStdString()); if (roomInfo.member_count != 2) { @@ -161,17 +166,20 @@ CallManager::sendInvite(const QString &roomid, bool isVideo) std::string errorMessage; if (!session_.havePlugins(false, &errorMessage) || - (isVideo && !session_.havePlugins(true, &errorMessage))) { + ((callType == CallType::VIDEO || callType == CallType::SCREEN) && + !session_.havePlugins(true, &errorMessage))) { emit ChatPage::instance()->showNotification(QString::fromStdString(errorMessage)); return; } - isVideo_ = isVideo; - roomid_ = roomid; + callType_ = callType; + roomid_ = roomid; session_.setTurnServers(turnURIs_); generateCallID(); - nhlog::ui()->debug( - "WebRTC: call id: {} - creating {} invite", callid_, isVideo ? "video" : "voice"); + std::string strCallType = callType_ == CallType::VOICE + ? "voice" + : (callType_ == CallType::VIDEO ? "video" : "screen"); + nhlog::ui()->debug("WebRTC: call id: {} - creating {} invite", callid_, strCallType); std::vector members(cache::getMembers(roomid.toStdString())); const RoomMember &callee = members.front().user_id == utils::localUser() ? members.back() : members.front(); @@ -179,7 +187,7 @@ CallManager::sendInvite(const QString &roomid, bool isVideo) callPartyAvatarUrl_ = QString::fromStdString(roomInfo.avatar_url); emit newInviteState(); playRingtone(QUrl("qrc:/media/media/ringback.ogg"), true); - if (!session_.createOffer(isVideo)) { + if (!session_.createOffer(callType)) { emit ChatPage::instance()->showNotification("Problem setting up call."); endCall(); } @@ -280,7 +288,7 @@ CallManager::handleEvent(const RoomEvent &callInviteEvent) callPartyAvatarUrl_ = QString::fromStdString(roomInfo.avatar_url); haveCallInvite_ = true; - isVideo_ = isVideo; + callType_ = isVideo ? CallType::VIDEO : CallType::VOICE; inviteSDP_ = callInviteEvent.content.sdp; CallDevices::instance().refresh(); emit newInviteState(); @@ -295,7 +303,7 @@ CallManager::acceptInvite() stopRingtone(); std::string errorMessage; if (!session_.havePlugins(false, &errorMessage) || - (isVideo_ && !session_.havePlugins(true, &errorMessage))) { + (callType_ == CallType::VIDEO && !session_.havePlugins(true, &errorMessage))) { emit ChatPage::instance()->showNotification(QString::fromStdString(errorMessage)); hangUp(); return; @@ -383,7 +391,7 @@ CallManager::toggleMicMute() } bool -CallManager::callsSupported() const +CallManager::callsSupported() { #ifdef GSTREAMER_AVAILABLE return true; @@ -392,6 +400,21 @@ CallManager::callsSupported() const #endif } +bool +CallManager::screenShareSupported() +{ + return std::getenv("DISPLAY") != nullptr; +} + +bool +CallManager::haveVideo() const +{ + return callType() == CallType::VIDEO || + (callType() == CallType::SCREEN && + (ChatPage::instance()->userSettings()->screenShareRemoteVideo() && + !session_.isRemoteVideoRecvOnly())); +} + QStringList CallManager::devices(bool isVideo) const { @@ -424,7 +447,7 @@ CallManager::clear() callParty_.clear(); callPartyAvatarUrl_.clear(); callid_.clear(); - isVideo_ = false; + callType_ = CallType::VOICE; haveCallInvite_ = false; emit newInviteState(); inviteSDP_.clear(); diff --git a/src/CallManager.h b/src/CallManager.h index 97cffbc8..ed745b5b 100644 --- a/src/CallManager.h +++ b/src/CallManager.h @@ -25,34 +25,39 @@ class CallManager : public QObject Q_OBJECT Q_PROPERTY(bool haveCallInvite READ haveCallInvite NOTIFY newInviteState) Q_PROPERTY(bool isOnCall READ isOnCall NOTIFY newCallState) - Q_PROPERTY(bool isVideo READ isVideo NOTIFY newInviteState) - Q_PROPERTY(bool haveLocalVideo READ haveLocalVideo NOTIFY newCallState) + Q_PROPERTY(webrtc::CallType callType READ callType NOTIFY newInviteState) Q_PROPERTY(webrtc::State callState READ callState NOTIFY newCallState) Q_PROPERTY(QString callParty READ callParty NOTIFY newInviteState) Q_PROPERTY(QString callPartyAvatarUrl READ callPartyAvatarUrl NOTIFY newInviteState) Q_PROPERTY(bool isMicMuted READ isMicMuted NOTIFY micMuteChanged) - Q_PROPERTY(bool callsSupported READ callsSupported CONSTANT) + Q_PROPERTY(bool haveLocalCamera READ haveLocalCamera NOTIFY newCallState) + Q_PROPERTY(bool haveVideo READ haveVideo NOTIFY newInviteState) Q_PROPERTY(QStringList mics READ mics NOTIFY devicesChanged) Q_PROPERTY(QStringList cameras READ cameras NOTIFY devicesChanged) + Q_PROPERTY(bool callsSupported READ callsSupported CONSTANT) + Q_PROPERTY(bool screenShareSupported READ screenShareSupported CONSTANT) public: CallManager(QObject *); bool haveCallInvite() const { return haveCallInvite_; } bool isOnCall() const { return session_.state() != webrtc::State::DISCONNECTED; } - bool isVideo() const { return isVideo_; } - bool haveLocalVideo() const { return session_.haveLocalVideo(); } + webrtc::CallType callType() const { return callType_; } webrtc::State callState() const { return session_.state(); } QString callParty() const { return callParty_; } QString callPartyAvatarUrl() const { return callPartyAvatarUrl_; } bool isMicMuted() const { return session_.isMicMuted(); } - bool callsSupported() const; + bool haveLocalCamera() const { return session_.haveLocalCamera(); } + bool haveVideo() const; QStringList mics() const { return devices(false); } QStringList cameras() const { return devices(true); } void refreshTurnServer(); + static bool callsSupported(); + static bool screenShareSupported(); + public slots: - void sendInvite(const QString &roomid, bool isVideo); + void sendInvite(const QString &roomid, webrtc::CallType); void syncEvent(const mtx::events::collections::TimelineEvents &event); void refreshDevices() { CallDevices::instance().refresh(); } void toggleMicMute(); @@ -81,9 +86,9 @@ private: QString callParty_; QString callPartyAvatarUrl_; std::string callid_; - const uint32_t timeoutms_ = 120000; - bool isVideo_ = false; - bool haveCallInvite_ = false; + const uint32_t timeoutms_ = 120000; + webrtc::CallType callType_ = webrtc::CallType::VOICE; + bool haveCallInvite_ = false; std::string inviteSDP_; std::vector remoteICECandidates_; std::vector turnURIs_; diff --git a/src/UserSettingsPage.cpp b/src/UserSettingsPage.cpp index b6fdf504..186a03bb 100644 --- a/src/UserSettingsPage.cpp +++ b/src/UserSettingsPage.cpp @@ -107,13 +107,15 @@ UserSettings::load(std::optional profile) auto presenceValue = QMetaEnum::fromType().keyToValue(tempPresence.c_str()); if (presenceValue < 0) presenceValue = 0; - presence_ = static_cast(presenceValue); - ringtone_ = settings.value("user/ringtone", "Default").toString(); - microphone_ = settings.value("user/microphone", QString()).toString(); - camera_ = settings.value("user/camera", QString()).toString(); - cameraResolution_ = settings.value("user/camera_resolution", QString()).toString(); - cameraFrameRate_ = settings.value("user/camera_frame_rate", QString()).toString(); - useStunServer_ = settings.value("user/use_stun_server", false).toBool(); + presence_ = static_cast(presenceValue); + ringtone_ = settings.value("user/ringtone", "Default").toString(); + microphone_ = settings.value("user/microphone", QString()).toString(); + camera_ = settings.value("user/camera", QString()).toString(); + cameraResolution_ = settings.value("user/camera_resolution", QString()).toString(); + cameraFrameRate_ = settings.value("user/camera_frame_rate", QString()).toString(); + screenShareFrameRate_ = settings.value("user/screen_share_frame_rate", 5).toInt(); + screenShareRemoteVideo_ = settings.value("user/screen_share_remote_video", false).toBool(); + useStunServer_ = settings.value("user/use_stun_server", false).toBool(); if (profile) // set to "" if it's the default to maintain compatibility profile_ = (*profile == "default") ? "" : *profile; @@ -444,6 +446,26 @@ UserSettings::setCameraFrameRate(QString frameRate) save(); } +void +UserSettings::setScreenShareFrameRate(int frameRate) +{ + if (frameRate == screenShareFrameRate_) + return; + screenShareFrameRate_ = frameRate; + emit screenShareFrameRateChanged(frameRate); + save(); +} + +void +UserSettings::setScreenShareRemoteVideo(bool state) +{ + if (state == screenShareRemoteVideo_) + return; + screenShareRemoteVideo_ = state; + emit screenShareRemoteVideoChanged(state); + save(); +} + void UserSettings::setProfile(QString profile) { @@ -593,6 +615,8 @@ UserSettings::save() settings.setValue("camera", camera_); settings.setValue("camera_resolution", cameraResolution_); settings.setValue("camera_frame_rate", cameraFrameRate_); + settings.setValue("screen_share_frame_rate", screenShareFrameRate_); + settings.setValue("screen_share_remote_video", screenShareRemoteVideo_); settings.setValue("use_stun_server", useStunServer_); settings.setValue("currentProfile", profile_); diff --git a/src/UserSettingsPage.h b/src/UserSettingsPage.h index 49de94b3..4de9913a 100644 --- a/src/UserSettingsPage.h +++ b/src/UserSettingsPage.h @@ -86,6 +86,10 @@ class UserSettings : public QObject cameraResolutionChanged) Q_PROPERTY(QString cameraFrameRate READ cameraFrameRate WRITE setCameraFrameRate NOTIFY cameraFrameRateChanged) + Q_PROPERTY(int screenShareFrameRate READ screenShareFrameRate WRITE setScreenShareFrameRate + NOTIFY screenShareFrameRateChanged) + Q_PROPERTY(bool screenShareRemoteVideo READ screenShareRemoteVideo WRITE + setScreenShareRemoteVideo NOTIFY screenShareRemoteVideoChanged) Q_PROPERTY( bool useStunServer READ useStunServer WRITE setUseStunServer NOTIFY useStunServerChanged) Q_PROPERTY(bool shareKeysWithTrustedUsers READ shareKeysWithTrustedUsers WRITE @@ -143,6 +147,8 @@ public: void setCamera(QString camera); void setCameraResolution(QString resolution); void setCameraFrameRate(QString frameRate); + void setScreenShareFrameRate(int frameRate); + void setScreenShareRemoteVideo(bool state); void setUseStunServer(bool state); void setShareKeysWithTrustedUsers(bool state); void setProfile(QString profile); @@ -191,6 +197,8 @@ public: QString camera() const { return camera_; } QString cameraResolution() const { return cameraResolution_; } QString cameraFrameRate() const { return cameraFrameRate_; } + int screenShareFrameRate() const { return screenShareFrameRate_; } + bool screenShareRemoteVideo() const { return screenShareRemoteVideo_; } bool useStunServer() const { return useStunServer_; } bool shareKeysWithTrustedUsers() const { return shareKeysWithTrustedUsers_; } QString profile() const { return profile_; } @@ -229,6 +237,8 @@ signals: void cameraChanged(QString camera); void cameraResolutionChanged(QString resolution); void cameraFrameRateChanged(QString frameRate); + void screenShareFrameRateChanged(int frameRate); + void screenShareRemoteVideoChanged(bool state); void useStunServerChanged(bool state); void shareKeysWithTrustedUsersChanged(bool state); void profileChanged(QString profile); @@ -272,6 +282,8 @@ private: QString camera_; QString cameraResolution_; QString cameraFrameRate_; + int screenShareFrameRate_; + bool screenShareRemoteVideo_; bool useStunServer_; QString profile_; QString userId_; diff --git a/src/WebRTCSession.cpp b/src/WebRTCSession.cpp index b6d98058..9c01ddc4 100644 --- a/src/WebRTCSession.cpp +++ b/src/WebRTCSession.cpp @@ -10,6 +10,7 @@ #include #include +#include "CallDevices.h" #include "ChatPage.h" #include "Logging.h" #include "UserSettingsPage.h" @@ -29,14 +30,20 @@ extern "C" // https://github.com/vector-im/riot-web/issues/10173 #define STUN_SERVER "stun://turn.matrix.org:3478" +Q_DECLARE_METATYPE(webrtc::CallType) Q_DECLARE_METATYPE(webrtc::State) +using webrtc::CallType; using webrtc::State; WebRTCSession::WebRTCSession() : QObject() , devices_(CallDevices::instance()) { + qRegisterMetaType(); + qmlRegisterUncreatableMetaObject( + webrtc::staticMetaObject, "im.nheko", 1, 0, "CallType", "Can't instantiate enum"); + qRegisterMetaType(); qmlRegisterUncreatableMetaObject( webrtc::staticMetaObject, "im.nheko", 1, 0, "WebRTCState", "Can't instantiate enum"); @@ -455,7 +462,8 @@ linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe) nhlog::ui()->info("WebRTC: incoming video resolution: {}x{}", videoCallSize.first, videoCallSize.second); - addCameraView(pipe, videoCallSize); + if (session->callType() == CallType::VIDEO) + addCameraView(pipe, videoCallSize); } else { g_free(mediaType); nhlog::ui()->error("WebRTC: unknown pad type: {}", GST_PAD_NAME(newpad)); @@ -467,7 +475,7 @@ linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe) if (GST_PAD_LINK_FAILED(gst_pad_link(newpad, queuepad))) nhlog::ui()->error("WebRTC: unable to link new pad"); else { - if (!session->isVideo() || + if (session->callType() == CallType::VOICE || (haveAudioStream_ && (haveVideoStream_ || session->isRemoteVideoRecvOnly()))) { emit session->stateChanged(State::CONNECTED); @@ -523,14 +531,17 @@ getMediaAttributes(const GstSDPMessage *sdp, const char *mediaType, const char *encoding, int &payloadType, - bool &recvOnly) + bool &recvOnly, + bool &sendOnly) { payloadType = -1; recvOnly = false; + sendOnly = false; for (guint mlineIndex = 0; mlineIndex < gst_sdp_message_medias_len(sdp); ++mlineIndex) { const GstSDPMedia *media = gst_sdp_message_get_media(sdp, mlineIndex); if (!std::strcmp(gst_sdp_media_get_media(media), mediaType)) { recvOnly = gst_sdp_media_get_attribute_val(media, "recvonly") != nullptr; + sendOnly = gst_sdp_media_get_attribute_val(media, "sendonly") != nullptr; const gchar *rtpval = nullptr; for (guint n = 0; n == 0 || rtpval; ++n) { rtpval = gst_sdp_media_get_attribute_val_n(media, "rtpmap", n); @@ -603,11 +614,12 @@ WebRTCSession::havePlugins(bool isVideo, std::string *errorMessage) } bool -WebRTCSession::createOffer(bool isVideo) +WebRTCSession::createOffer(CallType callType) { isOffering_ = true; - isVideo_ = isVideo; + callType_ = callType; isRemoteVideoRecvOnly_ = false; + isRemoteVideoSendOnly_ = false; videoItem_ = nullptr; haveAudioStream_ = false; haveVideoStream_ = false; @@ -630,8 +642,10 @@ WebRTCSession::acceptOffer(const std::string &sdp) if (state_ != State::DISCONNECTED) return false; + callType_ = webrtc::CallType::VOICE; isOffering_ = false; isRemoteVideoRecvOnly_ = false; + isRemoteVideoSendOnly_ = false; videoItem_ = nullptr; haveAudioStream_ = false; haveVideoStream_ = false; @@ -645,7 +659,8 @@ WebRTCSession::acceptOffer(const std::string &sdp) int opusPayloadType; bool recvOnly; - if (getMediaAttributes(offer->sdp, "audio", "opus", opusPayloadType, recvOnly)) { + bool sendOnly; + if (getMediaAttributes(offer->sdp, "audio", "opus", opusPayloadType, recvOnly, sendOnly)) { if (opusPayloadType == -1) { nhlog::ui()->error("WebRTC: remote audio offer - no opus encoding"); gst_webrtc_session_description_free(offer); @@ -658,13 +673,18 @@ WebRTCSession::acceptOffer(const std::string &sdp) } int vp8PayloadType; - isVideo_ = - getMediaAttributes(offer->sdp, "video", "vp8", vp8PayloadType, isRemoteVideoRecvOnly_); - if (isVideo_ && vp8PayloadType == -1) { + bool isVideo = getMediaAttributes(offer->sdp, + "video", + "vp8", + vp8PayloadType, + isRemoteVideoRecvOnly_, + isRemoteVideoSendOnly_); + if (isVideo && vp8PayloadType == -1) { nhlog::ui()->error("WebRTC: remote video offer - no vp8 encoding"); gst_webrtc_session_description_free(offer); return false; } + callType_ = isVideo ? CallType::VIDEO : CallType::VOICE; if (!startPipeline(opusPayloadType, vp8PayloadType)) { gst_webrtc_session_description_free(offer); @@ -695,10 +715,14 @@ WebRTCSession::acceptAnswer(const std::string &sdp) return false; } - if (isVideo_) { + if (callType_ != CallType::VOICE) { int unused; - if (!getMediaAttributes( - answer->sdp, "video", "vp8", unused, isRemoteVideoRecvOnly_)) + if (!getMediaAttributes(answer->sdp, + "video", + "vp8", + unused, + isRemoteVideoRecvOnly_, + isRemoteVideoSendOnly_)) isRemoteVideoRecvOnly_ = true; } @@ -855,39 +879,59 @@ WebRTCSession::createPipeline(int opusPayloadType, int vp8PayloadType) return false; } - return isVideo_ ? addVideoPipeline(vp8PayloadType) : true; + return callType_ == CallType::VOICE || isRemoteVideoSendOnly_ + ? true + : addVideoPipeline(vp8PayloadType); } bool WebRTCSession::addVideoPipeline(int vp8PayloadType) { // allow incoming video calls despite localUser having no webcam - if (!devices_.haveCamera()) + if (callType_ == CallType::VIDEO && !devices_.haveCamera()) return !isOffering_; - std::pair resolution; - std::pair frameRate; - GstDevice *device = devices_.videoDevice(resolution, frameRate); - if (!device) - return false; + GstElement *source = nullptr; + GstCaps *caps = nullptr; + if (callType_ == CallType::VIDEO) { + std::pair resolution; + std::pair frameRate; + GstDevice *device = devices_.videoDevice(resolution, frameRate); + if (!device) + return false; + source = gst_device_create_element(device, nullptr); + caps = gst_caps_new_simple("video/x-raw", + "width", + G_TYPE_INT, + resolution.first, + "height", + G_TYPE_INT, + resolution.second, + "framerate", + GST_TYPE_FRACTION, + frameRate.first, + frameRate.second, + nullptr); + } else { + source = gst_element_factory_make("ximagesrc", nullptr); + if (!source) { + nhlog::ui()->error("WebRTC: failed to create ximagesrc"); + return false; + } + g_object_set(source, "use-damage", 0, nullptr); + g_object_set(source, "xid", 0, nullptr); + + int frameRate = ChatPage::instance()->userSettings()->screenShareFrameRate(); + caps = gst_caps_new_simple( + "video/x-raw", "framerate", GST_TYPE_FRACTION, frameRate, 1, nullptr); + nhlog::ui()->debug("WebRTC: screen share frame rate: {} fps", frameRate); + } - GstElement *source = gst_device_create_element(device, nullptr); GstElement *videoconvert = gst_element_factory_make("videoconvert", nullptr); GstElement *capsfilter = gst_element_factory_make("capsfilter", "camerafilter"); - GstCaps *caps = gst_caps_new_simple("video/x-raw", - "width", - G_TYPE_INT, - resolution.first, - "height", - G_TYPE_INT, - resolution.second, - "framerate", - GST_TYPE_FRACTION, - frameRate.first, - frameRate.second, - nullptr); g_object_set(capsfilter, "caps", caps, nullptr); gst_caps_unref(caps); + GstElement *tee = gst_element_factory_make("tee", "videosrctee"); GstElement *queue = gst_element_factory_make("queue", nullptr); GstElement *vp8enc = gst_element_factory_make("vp8enc", nullptr); @@ -938,14 +982,25 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType) gst_object_unref(webrtcbin); return false; } + + if (callType_ == CallType::SCREEN && + !ChatPage::instance()->userSettings()->screenShareRemoteVideo()) { + GArray *transceivers; + g_signal_emit_by_name(webrtcbin, "get-transceivers", &transceivers); + GstWebRTCRTPTransceiver *transceiver = + g_array_index(transceivers, GstWebRTCRTPTransceiver *, 1); + transceiver->direction = GST_WEBRTC_RTP_TRANSCEIVER_DIRECTION_SENDONLY; + g_array_unref(transceivers); + } + gst_object_unref(webrtcbin); return true; } bool -WebRTCSession::haveLocalVideo() const +WebRTCSession::haveLocalCamera() const { - if (isVideo_ && state_ >= State::INITIATED) { + if (callType_ == CallType::VIDEO && state_ >= State::INITIATED) { GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe_), "videosrctee"); if (tee) { gst_object_unref(tee); @@ -1008,9 +1063,10 @@ WebRTCSession::end() } webrtc_ = nullptr; - isVideo_ = false; + callType_ = CallType::VOICE; isOffering_ = false; isRemoteVideoRecvOnly_ = false; + isRemoteVideoSendOnly_ = false; videoItem_ = nullptr; insetSinkPad_ = nullptr; if (state_ != State::DISCONNECTED) @@ -1026,16 +1082,12 @@ WebRTCSession::havePlugins(bool, std::string *) } bool -WebRTCSession::haveLocalVideo() const +WebRTCSession::haveLocalCamera() const { return false; } -bool -WebRTCSession::createOffer(bool) -{ - return false; -} +bool WebRTCSession::createOffer(webrtc::CallType) { return false; } bool WebRTCSession::acceptOffer(const std::string &) diff --git a/src/WebRTCSession.h b/src/WebRTCSession.h index 0fe8a864..64eac706 100644 --- a/src/WebRTCSession.h +++ b/src/WebRTCSession.h @@ -5,15 +5,23 @@ #include -#include "CallDevices.h" #include "mtx/events/voip.hpp" typedef struct _GstElement GstElement; +class CallDevices; class QQuickItem; namespace webrtc { Q_NAMESPACE +enum class CallType +{ + VOICE, + VIDEO, + SCREEN // localUser is sharing screen +}; +Q_ENUM_NS(CallType) + enum class State { DISCONNECTED, @@ -42,13 +50,14 @@ public: } bool havePlugins(bool isVideo, std::string *errorMessage = nullptr); + webrtc::CallType callType() const { return callType_; } webrtc::State state() const { return state_; } - bool isVideo() const { return isVideo_; } - bool haveLocalVideo() const; + bool haveLocalCamera() const; bool isOffering() const { return isOffering_; } bool isRemoteVideoRecvOnly() const { return isRemoteVideoRecvOnly_; } + bool isRemoteVideoSendOnly() const { return isRemoteVideoSendOnly_; } - bool createOffer(bool isVideo); + bool createOffer(webrtc::CallType); bool acceptOffer(const std::string &sdp); bool acceptAnswer(const std::string &sdp); void acceptICECandidates(const std::vector &); @@ -81,10 +90,11 @@ private: bool initialised_ = false; bool haveVoicePlugins_ = false; bool haveVideoPlugins_ = false; + webrtc::CallType callType_ = webrtc::CallType::VOICE; webrtc::State state_ = webrtc::State::DISCONNECTED; - bool isVideo_ = false; bool isOffering_ = false; bool isRemoteVideoRecvOnly_ = false; + bool isRemoteVideoSendOnly_ = false; QQuickItem *videoItem_ = nullptr; GstElement *pipe_ = nullptr; GstElement *webrtc_ = nullptr;