From 43ec0c062467c05a66eac7a3fb992bc093315c89 Mon Sep 17 00:00:00 2001 From: trilene Date: Sun, 26 Jul 2020 10:59:50 -0400 Subject: [PATCH] Handle ICE failure --- src/ActiveCallBar.cpp | 7 +++ src/CallManager.cpp | 108 ++++++++++++++++++++++++++-------------- src/CallManager.h | 8 +-- src/ChatPage.cpp | 9 ---- src/TextInputWidget.cpp | 3 +- src/WebRTCSession.cpp | 65 ++++++++++++++++-------- src/WebRTCSession.h | 4 +- 7 files changed, 131 insertions(+), 73 deletions(-) diff --git a/src/ActiveCallBar.cpp b/src/ActiveCallBar.cpp index 564842da..e55b2e86 100644 --- a/src/ActiveCallBar.cpp +++ b/src/ActiveCallBar.cpp @@ -123,25 +123,32 @@ ActiveCallBar::update(WebRTCSession::State state) { switch (state) { case WebRTCSession::State::INITIATING: + show(); stateLabel_->setText("Initiating call..."); break; case WebRTCSession::State::INITIATED: + show(); stateLabel_->setText("Call initiated..."); break; case WebRTCSession::State::OFFERSENT: + show(); stateLabel_->setText("Calling..."); break; case WebRTCSession::State::CONNECTING: + show(); stateLabel_->setText("Connecting..."); break; case WebRTCSession::State::CONNECTED: + show(); callStartTime_ = QDateTime::currentSecsSinceEpoch(); timer_->start(1000); stateLabel_->setText("Voice call:"); durationLabel_->setText("00:00"); durationLabel_->show(); break; + case WebRTCSession::State::ICEFAILED: case WebRTCSession::State::DISCONNECTED: + hide(); timer_->stop(); callPartyLabel_->setText(QString()); stateLabel_->setText(QString()); diff --git a/src/CallManager.cpp b/src/CallManager.cpp index 3caa812d..b57ef1bb 100644 --- a/src/CallManager.cpp +++ b/src/CallManager.cpp @@ -11,9 +11,10 @@ #include "MatrixClient.h" #include "UserSettingsPage.h" #include "WebRTCSession.h" - #include "dialogs/AcceptCall.h" +#include "mtx/responses/turn_server.hpp" + Q_DECLARE_METATYPE(std::vector) Q_DECLARE_METATYPE(mtx::events::msg::CallCandidates::Candidate) Q_DECLARE_METATYPE(mtx::responses::TurnServer) @@ -24,6 +25,11 @@ using namespace mtx::events::msg; // https://github.com/vector-im/riot-web/issues/10173 #define STUN_SERVER "stun://turn.matrix.org:3478" +namespace { +std::vector +getTurnURIs(const mtx::responses::TurnServer &turnServer); +} + CallManager::CallManager(QSharedPointer userSettings) : QObject(), session_(WebRTCSession::instance()), @@ -80,15 +86,23 @@ CallManager::CallManager(QSharedPointer userSettings) // Request new credentials close to expiry // See https://tools.ietf.org/html/draft-uberti-behave-turn-rest-00 - turnServer_ = res; + turnURIs_ = getTurnURIs(res); turnServerTimer_.setInterval(res.ttl * 1000 * 0.9); }); connect(&session_, &WebRTCSession::stateChanged, this, [this](WebRTCSession::State state) { - if (state == WebRTCSession::State::DISCONNECTED) + if (state == WebRTCSession::State::DISCONNECTED) { playRingtone("qrc:/media/media/callend.ogg", false); - }); + } + else if (state == WebRTCSession::State::ICEFAILED) { + QString error("Call connection failed."); + if (turnURIs_.empty()) + error += " Your homeserver has no configured TURN server."; + emit ChatPage::instance()->showNotification(error); + hangUp(CallHangUp::Reason::ICEFailed); + } + }); connect(&player_, &QMediaPlayer::mediaStatusChanged, this, [this](QMediaPlayer::MediaStatus status) { @@ -116,8 +130,8 @@ CallManager::sendInvite(const QString &roomid) } roomid_ = roomid; - setTurnServers(); session_.setStunServer(settings_->useStunServer() ? STUN_SERVER : ""); + session_.setTurnServers(turnURIs_); generateCallID(); nhlog::ui()->debug("WebRTC: call id: {} - creating invite", callid_); @@ -132,11 +146,26 @@ CallManager::sendInvite(const QString &roomid) } } +namespace { +std::string callHangUpReasonString(CallHangUp::Reason reason) +{ + switch (reason) { + case CallHangUp::Reason::ICEFailed: + return "ICE failed"; + case CallHangUp::Reason::InviteTimeOut: + return "Invite time out"; + default: + return "User"; + } +} +} + void CallManager::hangUp(CallHangUp::Reason reason) { if (!callid_.empty()) { - nhlog::ui()->debug("WebRTC: call id: {} - hanging up", callid_); + nhlog::ui()->debug("WebRTC: call id: {} - hanging up ({})", callid_, + callHangUpReasonString(reason)); emit newMessage(roomid_, CallHangUp{callid_, 0, reason}); endCall(); } @@ -221,8 +250,8 @@ CallManager::answerInvite(const CallInvite &invite) return; } - setTurnServers(); session_.setStunServer(settings_->useStunServer() ? STUN_SERVER : ""); + session_.setTurnServers(turnURIs_); if (!session_.acceptOffer(invite.sdp)) { emit ChatPage::instance()->showNotification("Problem setting up call."); @@ -279,8 +308,9 @@ CallManager::handleEvent(const RoomEvent &callAnswerEvent) void CallManager::handleEvent(const RoomEvent &callHangUpEvent) { - nhlog::ui()->debug("WebRTC: call id: {} - incoming CallHangUp from {}", - callHangUpEvent.content.call_id, callHangUpEvent.sender); + nhlog::ui()->debug("WebRTC: call id: {} - incoming CallHangUp ({}) from {}", + callHangUpEvent.content.call_id, callHangUpReasonString(callHangUpEvent.content.reason), + callHangUpEvent.sender); if (callid_ == callHangUpEvent.content.call_id) { MainWindow::instance()->hideOverlay(); @@ -319,35 +349,6 @@ CallManager::retrieveTurnServer() }); } -void -CallManager::setTurnServers() -{ - // gstreamer expects: turn(s)://username:password@host:port?transport=udp(tcp) - // where username and password are percent-encoded - std::vector uris; - for (const auto &uri : turnServer_.uris) { - if (auto c = uri.find(':'); c == std::string::npos) { - nhlog::ui()->error("Invalid TURN server uri: {}", uri); - continue; - } - else { - std::string scheme = std::string(uri, 0, c); - if (scheme != "turn" && scheme != "turns") { - nhlog::ui()->error("Invalid TURN server uri: {}", uri); - continue; - } - - QString encodedUri = QString::fromStdString(scheme) + "://" + - QUrl::toPercentEncoding(QString::fromStdString(turnServer_.username)) + ":" + - QUrl::toPercentEncoding(QString::fromStdString(turnServer_.password)) + "@" + - QString::fromStdString(std::string(uri, ++c)); - uris.push_back(encodedUri.toStdString()); - } - } - if (!uris.empty()) - session_.setTurnServers(uris); -} - void CallManager::playRingtone(const QString &ringtone, bool repeat) { @@ -364,3 +365,34 @@ CallManager::stopRingtone() { player_.setPlaylist(nullptr); } + +namespace { +std::vector +getTurnURIs(const mtx::responses::TurnServer &turnServer) +{ + // gstreamer expects: turn(s)://username:password@host:port?transport=udp(tcp) + // where username and password are percent-encoded + std::vector ret; + for (const auto &uri : turnServer.uris) { + if (auto c = uri.find(':'); c == std::string::npos) { + nhlog::ui()->error("Invalid TURN server uri: {}", uri); + continue; + } + else { + std::string scheme = std::string(uri, 0, c); + if (scheme != "turn" && scheme != "turns") { + nhlog::ui()->error("Invalid TURN server uri: {}", uri); + continue; + } + + QString encodedUri = QString::fromStdString(scheme) + "://" + + QUrl::toPercentEncoding(QString::fromStdString(turnServer.username)) + ":" + + QUrl::toPercentEncoding(QString::fromStdString(turnServer.password)) + "@" + + QString::fromStdString(std::string(uri, ++c)); + ret.push_back(encodedUri.toStdString()); + } + } + return ret; +} +} + diff --git a/src/CallManager.h b/src/CallManager.h index 3debf2e8..6518fd13 100644 --- a/src/CallManager.h +++ b/src/CallManager.h @@ -11,7 +11,10 @@ #include "mtx/events/collections.hpp" #include "mtx/events/voip.hpp" -#include "mtx/responses/turn_server.hpp" + +namespace mtx::responses { +struct TurnServer; +} class UserSettings; class WebRTCSession; @@ -51,7 +54,7 @@ private: std::string callid_; const uint32_t timeoutms_ = 120000; std::vector remoteICECandidates_; - mtx::responses::TurnServer turnServer_; + std::vector turnURIs_; QTimer turnServerTimer_; QSharedPointer settings_; QMediaPlayer player_; @@ -65,7 +68,6 @@ private: void answerInvite(const mtx::events::msg::CallInvite&); void generateCallID(); void endCall(); - void setTurnServers(); void playRingtone(const QString &ringtone, bool repeat); void stopRingtone(); }; diff --git a/src/ChatPage.cpp b/src/ChatPage.cpp index 5b8ea475..b53a5761 100644 --- a/src/ChatPage.cpp +++ b/src/ChatPage.cpp @@ -137,15 +137,6 @@ ChatPage::ChatPage(QSharedPointer userSettings, QWidget *parent) activeCallBar_->hide(); connect( &callManager_, &CallManager::newCallParty, activeCallBar_, &ActiveCallBar::setCallParty); - connect(&WebRTCSession::instance(), - &WebRTCSession::stateChanged, - this, - [this](WebRTCSession::State state) { - if (state == WebRTCSession::State::DISCONNECTED) - activeCallBar_->hide(); - else - activeCallBar_->show(); - }); // Splitter splitter->addWidget(sideBar_); diff --git a/src/TextInputWidget.cpp b/src/TextInputWidget.cpp index d49fc746..9aadc101 100644 --- a/src/TextInputWidget.cpp +++ b/src/TextInputWidget.cpp @@ -666,7 +666,8 @@ void TextInputWidget::changeCallButtonState(WebRTCSession::State state) { QIcon icon; - if (state == WebRTCSession::State::DISCONNECTED) { + if (state == WebRTCSession::State::ICEFAILED || + state == WebRTCSession::State::DISCONNECTED) { callBtn_->setToolTip(tr("Place a call")); icon.addFile(":/icons/icons/ui/place-call.png"); } else { diff --git a/src/WebRTCSession.cpp b/src/WebRTCSession.cpp index ff9ec661..95a9041e 100644 --- a/src/WebRTCSession.cpp +++ b/src/WebRTCSession.cpp @@ -14,9 +14,9 @@ extern "C" { Q_DECLARE_METATYPE(WebRTCSession::State) namespace { -bool gisoffer; -std::string glocalsdp; -std::vector gcandidates; +bool isoffering_; +std::string localsdp_; +std::vector localcandidates_; gboolean newBusMessage(GstBus *bus G_GNUC_UNUSED, GstMessage *msg, gpointer user_data); GstWebRTCSessionDescription* parseSDP(const std::string &sdp, GstWebRTCSDPType type); @@ -24,6 +24,7 @@ void generateOffer(GstElement *webrtc); void setLocalDescription(GstPromise *promise, gpointer webrtc); void addLocalICECandidate(GstElement *webrtc G_GNUC_UNUSED, guint mlineIndex, gchar *candidate, gpointer G_GNUC_UNUSED); gboolean onICEGatheringCompletion(gpointer timerid); +void iceConnectionStateChanged(GstElement *webrtcbin, GParamSpec *pspec G_GNUC_UNUSED, gpointer user_data G_GNUC_UNUSED); void createAnswer(GstPromise *promise, gpointer webrtc); void addDecodeBin(GstElement *webrtc G_GNUC_UNUSED, GstPad *newpad, GstElement *pipe); void linkNewPad(GstElement *decodebin G_GNUC_UNUSED, GstPad *newpad, GstElement *pipe); @@ -92,9 +93,9 @@ WebRTCSession::init(std::string *errorMessage) bool WebRTCSession::createOffer() { - gisoffer = true; - glocalsdp.clear(); - gcandidates.clear(); + isoffering_ = true; + localsdp_.clear(); + localcandidates_.clear(); return startPipeline(111); // a dynamic opus payload type } @@ -105,9 +106,9 @@ WebRTCSession::acceptOffer(const std::string &sdp) if (state_ != State::DISCONNECTED) return false; - gisoffer = false; - glocalsdp.clear(); - gcandidates.clear(); + isoffering_ = false; + localsdp_.clear(); + localcandidates_.clear(); int opusPayloadType = getPayloadType(sdp, "opus"); if (opusPayloadType == -1) @@ -152,14 +153,20 @@ WebRTCSession::startPipeline(int opusPayloadType) gboolean udata; g_signal_emit_by_name(webrtc_, "add-turn-server", uri.c_str(), (gpointer)(&udata)); } + if (turnServers_.empty()) + nhlog::ui()->warn("WebRTC: no TURN server provided"); // generate the offer when the pipeline goes to PLAYING - if (gisoffer) + if (isoffering_) g_signal_connect(webrtc_, "on-negotiation-needed", G_CALLBACK(generateOffer), nullptr); // on-ice-candidate is emitted when a local ICE candidate has been gathered g_signal_connect(webrtc_, "on-ice-candidate", G_CALLBACK(addLocalICECandidate), nullptr); + // capture ICE failure + g_signal_connect(webrtc_, "notify::ice-connection-state", + G_CALLBACK(iceConnectionStateChanged), nullptr); + // incoming streams trigger pad-added gst_element_set_state(pipe_, GST_STATE_READY); g_signal_connect(webrtc_, "pad-added", G_CALLBACK(addDecodeBin), pipe_); @@ -229,8 +236,6 @@ WebRTCSession::acceptICECandidates(const std::vectordebug("WebRTC: remote candidate: (m-line:{}):{}", c.sdpMLineIndex, c.candidate); g_signal_emit_by_name(webrtc_, "add-ice-candidate", c.sdpMLineIndex, c.candidate.c_str()); } - if (state_ == State::OFFERSENT) - emit stateChanged(State::CONNECTING); } } @@ -357,11 +362,11 @@ setLocalDescription(GstPromise *promise, gpointer webrtc) g_signal_emit_by_name(webrtc, "set-local-description", gstsdp, nullptr); gchar *sdp = gst_sdp_message_as_text(gstsdp->sdp); - glocalsdp = std::string(sdp); + localsdp_ = std::string(sdp); g_free(sdp); gst_webrtc_session_description_free(gstsdp); - nhlog::ui()->debug("WebRTC: local description set ({}):\n{}", isAnswer ? "answer" : "offer", glocalsdp); + nhlog::ui()->debug("WebRTC: local description set ({}):\n{}", isAnswer ? "answer" : "offer", localsdp_); } void @@ -369,12 +374,12 @@ addLocalICECandidate(GstElement *webrtc G_GNUC_UNUSED, guint mlineIndex, gchar * { nhlog::ui()->debug("WebRTC: local candidate: (m-line:{}):{}", mlineIndex, candidate); - if (WebRTCSession::instance().state() == WebRTCSession::State::CONNECTED) { + if (WebRTCSession::instance().state() >= WebRTCSession::State::OFFERSENT) { emit WebRTCSession::instance().newICECandidate({"audio", (uint16_t)mlineIndex, candidate}); return; } - gcandidates.push_back({"audio", (uint16_t)mlineIndex, candidate}); + localcandidates_.push_back({"audio", (uint16_t)mlineIndex, candidate}); // GStreamer v1.16: webrtcbin's notify::ice-gathering-state triggers GST_WEBRTC_ICE_GATHERING_STATE_COMPLETE too early // fixed in v1.18 @@ -390,18 +395,36 @@ gboolean onICEGatheringCompletion(gpointer timerid) { *(guint*)(timerid) = 0; - if (gisoffer) { - emit WebRTCSession::instance().offerCreated(glocalsdp, gcandidates); + if (isoffering_) { + emit WebRTCSession::instance().offerCreated(localsdp_, localcandidates_); emit WebRTCSession::instance().stateChanged(WebRTCSession::State::OFFERSENT); } else { - emit WebRTCSession::instance().answerCreated(glocalsdp, gcandidates); - emit WebRTCSession::instance().stateChanged(WebRTCSession::State::CONNECTING); + emit WebRTCSession::instance().answerCreated(localsdp_, localcandidates_); + emit WebRTCSession::instance().stateChanged(WebRTCSession::State::ANSWERSENT); } - return FALSE; } +void +iceConnectionStateChanged(GstElement *webrtc, GParamSpec *pspec G_GNUC_UNUSED, gpointer user_data G_GNUC_UNUSED) +{ + GstWebRTCICEConnectionState newState; + g_object_get(webrtc, "ice-connection-state", &newState, nullptr); + switch (newState) { + case GST_WEBRTC_ICE_CONNECTION_STATE_CHECKING: + nhlog::ui()->debug("WebRTC: GstWebRTCICEConnectionState -> Checking"); + emit WebRTCSession::instance().stateChanged(WebRTCSession::State::CONNECTING); + break; + case GST_WEBRTC_ICE_CONNECTION_STATE_FAILED: + nhlog::ui()->error("WebRTC: GstWebRTCICEConnectionState -> Failed"); + emit WebRTCSession::instance().stateChanged(WebRTCSession::State::ICEFAILED); + break; + default: + break; + } +} + void createAnswer(GstPromise *promise, gpointer webrtc) { diff --git a/src/WebRTCSession.h b/src/WebRTCSession.h index f9882089..d79047a8 100644 --- a/src/WebRTCSession.h +++ b/src/WebRTCSession.h @@ -15,10 +15,12 @@ class WebRTCSession : public QObject public: enum class State { + ICEFAILED, DISCONNECTED, INITIATING, INITIATED, OFFERSENT, + ANSWERSENT, CONNECTING, CONNECTED }; @@ -30,13 +32,13 @@ public: } bool init(std::string *errorMessage = nullptr); + State state() const {return state_;} bool createOffer(); bool acceptOffer(const std::string &sdp); bool acceptAnswer(const std::string &sdp); void acceptICECandidates(const std::vector&); - State state() const {return state_;} bool toggleMuteAudioSrc(bool &isMuted); void end();