Handle ICE failure

src/ActiveCallBar.cpp

@@ -123,25 +123,32 @@ ActiveCallBar::update(WebRTCSession::State state)
 {
         switch (state) {
           case WebRTCSession::State::INITIATING:
+            show();
             stateLabel_->setText("Initiating call...");
             break;
           case WebRTCSession::State::INITIATED:
+            show();
             stateLabel_->setText("Call initiated...");
             break;
           case WebRTCSession::State::OFFERSENT:
+            show();
             stateLabel_->setText("Calling...");
             break;
           case WebRTCSession::State::CONNECTING:
+            show();
             stateLabel_->setText("Connecting...");
             break;
           case WebRTCSession::State::CONNECTED:
+            show();
             callStartTime_ = QDateTime::currentSecsSinceEpoch();
             timer_->start(1000);
             stateLabel_->setText("Voice call:");
             durationLabel_->setText("00:00");
             durationLabel_->show();
             break;
+          case WebRTCSession::State::ICEFAILED:
           case WebRTCSession::State::DISCONNECTED:
+            hide();
             timer_->stop();
             callPartyLabel_->setText(QString());
             stateLabel_->setText(QString());

src/CallManager.cpp

@@ -11,9 +11,10 @@
 #include "MatrixClient.h"
 #include "UserSettingsPage.h"
 #include "WebRTCSession.h"
-
 #include "dialogs/AcceptCall.h"
 
+#include "mtx/responses/turn_server.hpp"
+
 Q_DECLARE_METATYPE(std::vector<mtx::events::msg::CallCandidates::Candidate>)
 Q_DECLARE_METATYPE(mtx::events::msg::CallCandidates::Candidate)
 Q_DECLARE_METATYPE(mtx::responses::TurnServer)
@@ -24,6 +25,11 @@ using namespace mtx::events::msg;
 // https://github.com/vector-im/riot-web/issues/10173
 #define STUN_SERVER "stun://turn.matrix.org:3478"
 
+namespace {
+std::vector<std::string>
+getTurnURIs(const mtx::responses::TurnServer &turnServer);
+}
+
 CallManager::CallManager(QSharedPointer<UserSettings> userSettings)
   : QObject(),
     session_(WebRTCSession::instance()),
@@ -80,15 +86,23 @@ CallManager::CallManager(QSharedPointer<UserSettings> userSettings)
 
               // Request new credentials close to expiry
               // See https://tools.ietf.org/html/draft-uberti-behave-turn-rest-00
-              turnServer_ = res;
+              turnURIs_ = getTurnURIs(res);
               turnServerTimer_.setInterval(res.ttl * 1000 * 0.9);
       });
 
   connect(&session_, &WebRTCSession::stateChanged, this,
       [this](WebRTCSession::State state) {
-        if (state == WebRTCSession::State::DISCONNECTED)
+        if (state == WebRTCSession::State::DISCONNECTED) {
           playRingtone("qrc:/media/media/callend.ogg", false);
-        });
+        }
+        else if (state == WebRTCSession::State::ICEFAILED) {
+          QString error("Call connection failed.");
+          if (turnURIs_.empty())
+            error += " Your homeserver has no configured TURN server.";
+          emit ChatPage::instance()->showNotification(error);
+          hangUp(CallHangUp::Reason::ICEFailed);
+        }
+      });
 
   connect(&player_, &QMediaPlayer::mediaStatusChanged, this,
       [this](QMediaPlayer::MediaStatus status) {
@@ -116,8 +130,8 @@ CallManager::sendInvite(const QString &roomid)
     }
 
     roomid_ = roomid;
-    setTurnServers();
     session_.setStunServer(settings_->useStunServer() ? STUN_SERVER : "");
+    session_.setTurnServers(turnURIs_);
 
     generateCallID();
     nhlog::ui()->debug("WebRTC: call id: {} - creating invite", callid_);
@@ -132,11 +146,26 @@ CallManager::sendInvite(const QString &roomid)
     }
 }
 
+namespace {
+std::string callHangUpReasonString(CallHangUp::Reason reason)
+{
+  switch (reason) {
+    case CallHangUp::Reason::ICEFailed:
+      return "ICE failed";
+    case CallHangUp::Reason::InviteTimeOut:
+      return "Invite time out";
+    default:
+      return "User";
+  }
+}
+}
+
 void
 CallManager::hangUp(CallHangUp::Reason reason)
 {
   if (!callid_.empty()) {
-    nhlog::ui()->debug("WebRTC: call id: {} - hanging up", callid_);
+    nhlog::ui()->debug("WebRTC: call id: {} - hanging up ({})", callid_,
+        callHangUpReasonString(reason));
     emit newMessage(roomid_, CallHangUp{callid_, 0, reason});
     endCall();
   }
@@ -221,8 +250,8 @@ CallManager::answerInvite(const CallInvite &invite)
     return;
   }
 
-  setTurnServers();
   session_.setStunServer(settings_->useStunServer() ? STUN_SERVER : "");
+  session_.setTurnServers(turnURIs_);
 
   if (!session_.acceptOffer(invite.sdp)) {
     emit ChatPage::instance()->showNotification("Problem setting up call.");
@@ -279,8 +308,9 @@ CallManager::handleEvent(const RoomEvent<CallAnswer> &callAnswerEvent)
 void
 CallManager::handleEvent(const RoomEvent<CallHangUp> &callHangUpEvent)
 {
-  nhlog::ui()->debug("WebRTC: call id: {} - incoming CallHangUp from {}",
+  nhlog::ui()->debug("WebRTC: call id: {} - incoming CallHangUp ({}) from {}",
-      callHangUpEvent.content.call_id, callHangUpEvent.sender);
+      callHangUpEvent.content.call_id, callHangUpReasonString(callHangUpEvent.content.reason),
+         callHangUpEvent.sender);
 
   if (callid_ == callHangUpEvent.content.call_id) {
     MainWindow::instance()->hideOverlay();
@@ -319,35 +349,6 @@ CallManager::retrieveTurnServer()
   });
 }
 
-void
-CallManager::setTurnServers()
-{
-  // gstreamer expects: turn(s)://username:password@host:port?transport=udp(tcp)
-  // where username and password are percent-encoded
-  std::vector<std::string> uris;
-  for (const auto &uri : turnServer_.uris) {
-    if (auto c = uri.find(':'); c == std::string::npos) {
-      nhlog::ui()->error("Invalid TURN server uri: {}", uri);
-      continue;
-    }
-    else {
-      std::string scheme = std::string(uri, 0, c);
-      if (scheme != "turn" && scheme != "turns") {
-        nhlog::ui()->error("Invalid TURN server uri: {}", uri);
-        continue;
-      }
-
-      QString encodedUri = QString::fromStdString(scheme) + "://" + 
-                           QUrl::toPercentEncoding(QString::fromStdString(turnServer_.username)) + ":" +
-                           QUrl::toPercentEncoding(QString::fromStdString(turnServer_.password)) + "@" +
-                           QString::fromStdString(std::string(uri, ++c));
-      uris.push_back(encodedUri.toStdString());
-    }
-  }
-  if (!uris.empty())
-    session_.setTurnServers(uris);
-}
-
 void
 CallManager::playRingtone(const QString &ringtone, bool repeat)
 {
@@ -364,3 +365,34 @@ CallManager::stopRingtone()
 {
   player_.setPlaylist(nullptr);
 }
+
+namespace {
+std::vector<std::string>
+getTurnURIs(const mtx::responses::TurnServer &turnServer)
+{
+  // gstreamer expects: turn(s)://username:password@host:port?transport=udp(tcp)
+  // where username and password are percent-encoded
+  std::vector<std::string> ret;
+  for (const auto &uri : turnServer.uris) {
+    if (auto c = uri.find(':'); c == std::string::npos) {
+      nhlog::ui()->error("Invalid TURN server uri: {}", uri);
+      continue;
+    }
+    else {
+      std::string scheme = std::string(uri, 0, c);
+      if (scheme != "turn" && scheme != "turns") {
+        nhlog::ui()->error("Invalid TURN server uri: {}", uri);
+        continue;
+      }
+
+      QString encodedUri = QString::fromStdString(scheme) + "://" + 
+                           QUrl::toPercentEncoding(QString::fromStdString(turnServer.username)) + ":" +
+                           QUrl::toPercentEncoding(QString::fromStdString(turnServer.password)) + "@" +
+                           QString::fromStdString(std::string(uri, ++c));
+      ret.push_back(encodedUri.toStdString());
+    }
+  }
+  return ret;
+}
+}
+

src/CallManager.h

@@ -11,7 +11,10 @@
 
 #include "mtx/events/collections.hpp"
 #include "mtx/events/voip.hpp"
-#include "mtx/responses/turn_server.hpp"
+
+namespace mtx::responses {
+struct TurnServer;
+}
 
 class UserSettings;
 class WebRTCSession;
@@ -51,7 +54,7 @@ private:
         std::string callid_;
         const uint32_t timeoutms_ = 120000;
         std::vector<mtx::events::msg::CallCandidates::Candidate> remoteICECandidates_;
-        mtx::responses::TurnServer turnServer_;
+        std::vector<std::string> turnURIs_;
         QTimer turnServerTimer_;
         QSharedPointer<UserSettings> settings_;
         QMediaPlayer player_;
@@ -65,7 +68,6 @@ private:
         void answerInvite(const mtx::events::msg::CallInvite&);
         void generateCallID();
         void endCall();
-        void setTurnServers();
         void playRingtone(const QString &ringtone, bool repeat);
         void stopRingtone();
 };

src/ChatPage.cpp

@@ -137,15 +137,6 @@ ChatPage::ChatPage(QSharedPointer<UserSettings> userSettings, QWidget *parent)
         activeCallBar_->hide();
         connect(
           &callManager_, &CallManager::newCallParty, activeCallBar_, &ActiveCallBar::setCallParty);
-        connect(&WebRTCSession::instance(),
-                &WebRTCSession::stateChanged,
-                this,
-                [this](WebRTCSession::State state) {
-                        if (state == WebRTCSession::State::DISCONNECTED)
-                                activeCallBar_->hide();
-                        else
-                                activeCallBar_->show();
-                });
 
         // Splitter
         splitter->addWidget(sideBar_);

src/TextInputWidget.cpp

@@ -666,7 +666,8 @@ void
 TextInputWidget::changeCallButtonState(WebRTCSession::State state)
 {
         QIcon icon;
-        if (state == WebRTCSession::State::DISCONNECTED) {
+        if (state == WebRTCSession::State::ICEFAILED ||
+            state == WebRTCSession::State::DISCONNECTED) {
                 callBtn_->setToolTip(tr("Place a call"));
                 icon.addFile(":/icons/icons/ui/place-call.png");
         } else {

src/WebRTCSession.cpp

@@ -14,9 +14,9 @@ extern "C" {
 Q_DECLARE_METATYPE(WebRTCSession::State)
 
 namespace {
-bool gisoffer;
+bool isoffering_;
-std::string glocalsdp;
+std::string localsdp_;
-std::vector<mtx::events::msg::CallCandidates::Candidate> gcandidates;
+std::vector<mtx::events::msg::CallCandidates::Candidate> localcandidates_;
 
 gboolean newBusMessage(GstBus *bus G_GNUC_UNUSED, GstMessage *msg, gpointer user_data);
 GstWebRTCSessionDescription* parseSDP(const std::string &sdp, GstWebRTCSDPType type);
@@ -24,6 +24,7 @@ void generateOffer(GstElement *webrtc);
 void setLocalDescription(GstPromise *promise, gpointer webrtc);
 void addLocalICECandidate(GstElement *webrtc G_GNUC_UNUSED, guint mlineIndex, gchar *candidate, gpointer G_GNUC_UNUSED);
 gboolean onICEGatheringCompletion(gpointer timerid);
+void iceConnectionStateChanged(GstElement *webrtcbin, GParamSpec *pspec G_GNUC_UNUSED, gpointer user_data G_GNUC_UNUSED);
 void createAnswer(GstPromise *promise, gpointer webrtc);
 void addDecodeBin(GstElement *webrtc G_GNUC_UNUSED, GstPad *newpad, GstElement *pipe);
 void linkNewPad(GstElement *decodebin G_GNUC_UNUSED, GstPad *newpad, GstElement *pipe);
@@ -92,9 +93,9 @@ WebRTCSession::init(std::string *errorMessage)
 bool
 WebRTCSession::createOffer()
 {
-  gisoffer = true;
+  isoffering_ = true;
-  glocalsdp.clear();
+  localsdp_.clear();
-  gcandidates.clear();
+  localcandidates_.clear();
   return startPipeline(111); // a dynamic opus payload type
 }
 
@@ -105,9 +106,9 @@ WebRTCSession::acceptOffer(const std::string &sdp)
   if (state_ != State::DISCONNECTED)
     return false;
 
-  gisoffer = false;
+  isoffering_ = false;
-  glocalsdp.clear();
+  localsdp_.clear();
-  gcandidates.clear();
+  localcandidates_.clear();
 
   int opusPayloadType = getPayloadType(sdp, "opus"); 
   if (opusPayloadType == -1)
@@ -152,14 +153,20 @@ WebRTCSession::startPipeline(int opusPayloadType)
     gboolean udata;
     g_signal_emit_by_name(webrtc_, "add-turn-server", uri.c_str(), (gpointer)(&udata));
   }
+  if (turnServers_.empty())
+    nhlog::ui()->warn("WebRTC: no TURN server provided");
 
   // generate the offer when the pipeline goes to PLAYING
-  if (gisoffer)
+  if (isoffering_)
     g_signal_connect(webrtc_, "on-negotiation-needed", G_CALLBACK(generateOffer), nullptr);
 
   // on-ice-candidate is emitted when a local ICE candidate has been gathered
   g_signal_connect(webrtc_, "on-ice-candidate", G_CALLBACK(addLocalICECandidate), nullptr);
 
+  // capture ICE failure
+  g_signal_connect(webrtc_, "notify::ice-connection-state",
+    G_CALLBACK(iceConnectionStateChanged), nullptr);
+
   // incoming streams trigger pad-added
   gst_element_set_state(pipe_, GST_STATE_READY);
   g_signal_connect(webrtc_, "pad-added", G_CALLBACK(addDecodeBin), pipe_);
@@ -229,8 +236,6 @@ WebRTCSession::acceptICECandidates(const std::vector<mtx::events::msg::CallCandi
       nhlog::ui()->debug("WebRTC: remote candidate: (m-line:{}):{}", c.sdpMLineIndex, c.candidate);
       g_signal_emit_by_name(webrtc_, "add-ice-candidate", c.sdpMLineIndex, c.candidate.c_str());
     }
-    if (state_ == State::OFFERSENT)
-      emit stateChanged(State::CONNECTING);
   }
 }
 
@@ -357,11 +362,11 @@ setLocalDescription(GstPromise *promise, gpointer webrtc)
   g_signal_emit_by_name(webrtc, "set-local-description", gstsdp, nullptr);
 
   gchar *sdp = gst_sdp_message_as_text(gstsdp->sdp);
-  glocalsdp = std::string(sdp);
+  localsdp_ = std::string(sdp);
   g_free(sdp);
   gst_webrtc_session_description_free(gstsdp);
 
-  nhlog::ui()->debug("WebRTC: local description set ({}):\n{}", isAnswer ? "answer" : "offer", glocalsdp);
+  nhlog::ui()->debug("WebRTC: local description set ({}):\n{}", isAnswer ? "answer" : "offer", localsdp_);
 }
 
 void
@@ -369,12 +374,12 @@ addLocalICECandidate(GstElement *webrtc G_GNUC_UNUSED, guint mlineIndex, gchar *
 {
   nhlog::ui()->debug("WebRTC: local candidate: (m-line:{}):{}", mlineIndex, candidate);
 
-  if (WebRTCSession::instance().state() == WebRTCSession::State::CONNECTED) {
+  if (WebRTCSession::instance().state() >= WebRTCSession::State::OFFERSENT) {
     emit WebRTCSession::instance().newICECandidate({"audio", (uint16_t)mlineIndex, candidate});
     return;
   }
 
-  gcandidates.push_back({"audio", (uint16_t)mlineIndex, candidate});
+  localcandidates_.push_back({"audio", (uint16_t)mlineIndex, candidate});
 
   // GStreamer v1.16: webrtcbin's notify::ice-gathering-state triggers GST_WEBRTC_ICE_GATHERING_STATE_COMPLETE too early
   // fixed in v1.18
@@ -390,18 +395,36 @@ gboolean
 onICEGatheringCompletion(gpointer timerid)
 {
   *(guint*)(timerid) = 0;
-  if (gisoffer) {
+  if (isoffering_) {
-    emit WebRTCSession::instance().offerCreated(glocalsdp, gcandidates);
+    emit WebRTCSession::instance().offerCreated(localsdp_, localcandidates_);
     emit WebRTCSession::instance().stateChanged(WebRTCSession::State::OFFERSENT);
   }
   else {
-    emit WebRTCSession::instance().answerCreated(glocalsdp, gcandidates);
+    emit WebRTCSession::instance().answerCreated(localsdp_, localcandidates_);
-    emit WebRTCSession::instance().stateChanged(WebRTCSession::State::CONNECTING);
+    emit WebRTCSession::instance().stateChanged(WebRTCSession::State::ANSWERSENT);
   }
-
   return FALSE;
 }
 
+void
+iceConnectionStateChanged(GstElement *webrtc, GParamSpec *pspec G_GNUC_UNUSED, gpointer user_data G_GNUC_UNUSED)
+{
+  GstWebRTCICEConnectionState newState;
+  g_object_get(webrtc, "ice-connection-state", &newState, nullptr);
+  switch (newState) {
+    case GST_WEBRTC_ICE_CONNECTION_STATE_CHECKING:
+      nhlog::ui()->debug("WebRTC: GstWebRTCICEConnectionState -> Checking");
+      emit WebRTCSession::instance().stateChanged(WebRTCSession::State::CONNECTING);
+      break;
+    case GST_WEBRTC_ICE_CONNECTION_STATE_FAILED:
+      nhlog::ui()->error("WebRTC: GstWebRTCICEConnectionState -> Failed");
+      emit WebRTCSession::instance().stateChanged(WebRTCSession::State::ICEFAILED);
+      break;
+    default:
+      break;
+  }
+}
+
 void
 createAnswer(GstPromise *promise, gpointer webrtc)
 {

src/WebRTCSession.h

@@ -15,10 +15,12 @@ class WebRTCSession : public QObject
 
 public:
         enum class State {
+          ICEFAILED,
           DISCONNECTED,
           INITIATING,
           INITIATED,
           OFFERSENT,
+          ANSWERSENT,
           CONNECTING,
           CONNECTED
         };
@@ -30,13 +32,13 @@ public:
         }
 
         bool init(std::string *errorMessage = nullptr);
+        State state() const {return state_;} 
 
         bool createOffer();
         bool acceptOffer(const std::string &sdp);
         bool acceptAnswer(const std::string &sdp);
         void acceptICECandidates(const std::vector<mtx::events::msg::CallCandidates::Candidate>&);
 
-        State state() const {return state_;} 
         bool toggleMuteAudioSrc(bool &isMuted);
         void end();