d3v1l/matrixion
1
0
Fork 0
mirror of https://github.com/Nheko-Reborn/nheko.git synced 2024-11-22 11:00:48 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Handle ICE failure

Browse source
This commit is contained in:
trilene 2020-07-26 10:59:50 -04:00
parent 57d5a3d31f
commit 43ec0c0624
7 changed files with 131 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/ActiveCallBar.cpp
View file

@ -123,25 +123,32 @@ ActiveCallBar::update(WebRTCSession::State state)
{
switch (state) {
case WebRTCSession::State::INITIATING:
show();
stateLabel_->setText("Initiating call...");
break;
case WebRTCSession::State::INITIATED:
show();
stateLabel_->setText("Call initiated...");
break;
case WebRTCSession::State::OFFERSENT:
show();
stateLabel_->setText("Calling...");
break;
case WebRTCSession::State::CONNECTING:
show();
stateLabel_->setText("Connecting...");
break;
case WebRTCSession::State::CONNECTED:
show();
callStartTime_ = QDateTime::currentSecsSinceEpoch();
timer_->start(1000);
stateLabel_->setText("Voice call:");
durationLabel_->setText("00:00");
durationLabel_->show();
break;
case WebRTCSession::State::ICEFAILED:
case WebRTCSession::State::DISCONNECTED:
hide();
timer_->stop();
callPartyLabel_->setText(QString());
stateLabel_->setText(QString());

108
src/CallManager.cpp
View file

@ -11,9 +11,10 @@
#include "MatrixClient.h"
#include "UserSettingsPage.h"
#include "WebRTCSession.h"
#include "dialogs/AcceptCall.h"
#include "mtx/responses/turn_server.hpp"
Q_DECLARE_METATYPE(std::vector<mtx::events::msg::CallCandidates::Candidate>)
Q_DECLARE_METATYPE(mtx::events::msg::CallCandidates::Candidate)
Q_DECLARE_METATYPE(mtx::responses::TurnServer)
@ -24,6 +25,11 @@ using namespace mtx::events::msg;
// https://github.com/vector-im/riot-web/issues/10173
#define STUN_SERVER "stun://turn.matrix.org:3478"
namespace {
std::vector<std::string>
getTurnURIs(const mtx::responses::TurnServer &turnServer);
}
CallManager::CallManager(QSharedPointer<UserSettings> userSettings)
: QObject(),
session_(WebRTCSession::instance()),
@ -80,15 +86,23 @@ CallManager::CallManager(QSharedPointer<UserSettings> userSettings)
// Request new credentials close to expiry
// See https://tools.ietf.org/html/draft-uberti-behave-turn-rest-00
turnServer_ = res;
turnURIs_ = getTurnURIs(res);
turnServerTimer_.setInterval(res.ttl * 1000 * 0.9);
});
connect(&session_, &WebRTCSession::stateChanged, this,
[this](WebRTCSession::State state) {
if (state == WebRTCSession::State::DISCONNECTED)
if (state == WebRTCSession::State::DISCONNECTED) {
playRingtone("qrc:/media/media/callend.ogg", false);
});
}
else if (state == WebRTCSession::State::ICEFAILED) {
QString error("Call connection failed.");
if (turnURIs_.empty())
error += " Your homeserver has no configured TURN server.";
emit ChatPage::instance()->showNotification(error);
hangUp(CallHangUp::Reason::ICEFailed);
}
});
connect(&player_, &QMediaPlayer::mediaStatusChanged, this,
[this](QMediaPlayer::MediaStatus status) {
@ -116,8 +130,8 @@ CallManager::sendInvite(const QString &roomid)
}
roomid_ = roomid;
setTurnServers();
session_.setStunServer(settings_->useStunServer() ? STUN_SERVER : "");
session_.setTurnServers(turnURIs_);
generateCallID();
nhlog::ui()->debug("WebRTC: call id: {} - creating invite", callid_);
@ -132,11 +146,26 @@ CallManager::sendInvite(const QString &roomid)
}
}
namespace {
std::string callHangUpReasonString(CallHangUp::Reason reason)
{
switch (reason) {
case CallHangUp::Reason::ICEFailed:
return "ICE failed";
case CallHangUp::Reason::InviteTimeOut:
return "Invite time out";
default:
return "User";
}
}
}
void
CallManager::hangUp(CallHangUp::Reason reason)
{
if (!callid_.empty()) {
nhlog::ui()->debug("WebRTC: call id: {} - hanging up", callid_);
nhlog::ui()->debug("WebRTC: call id: {} - hanging up ({})", callid_,
callHangUpReasonString(reason));
emit newMessage(roomid_, CallHangUp{callid_, 0, reason});
endCall();
}
@ -221,8 +250,8 @@ CallManager::answerInvite(const CallInvite &invite)
return;
}
setTurnServers();
session_.setStunServer(settings_->useStunServer() ? STUN_SERVER : "");
session_.setTurnServers(turnURIs_);
if (!session_.acceptOffer(invite.sdp)) {
emit ChatPage::instance()->showNotification("Problem setting up call.");
@ -279,8 +308,9 @@ CallManager::handleEvent(const RoomEvent<CallAnswer> &callAnswerEvent)
void
CallManager::handleEvent(const RoomEvent<CallHangUp> &callHangUpEvent)
{
nhlog::ui()->debug("WebRTC: call id: {} - incoming CallHangUp from {}",
callHangUpEvent.content.call_id, callHangUpEvent.sender);
nhlog::ui()->debug("WebRTC: call id: {} - incoming CallHangUp ({}) from {}",
callHangUpEvent.content.call_id, callHangUpReasonString(callHangUpEvent.content.reason),
callHangUpEvent.sender);
if (callid_ == callHangUpEvent.content.call_id) {
MainWindow::instance()->hideOverlay();
@ -319,35 +349,6 @@ CallManager::retrieveTurnServer()
});
}
void
CallManager::setTurnServers()
{
// gstreamer expects: turn(s)://username:password@host:port?transport=udp(tcp)
// where username and password are percent-encoded
std::vector<std::string> uris;
for (const auto &uri : turnServer_.uris) {
if (auto c = uri.find(':'); c == std::string::npos) {
nhlog::ui()->error("Invalid TURN server uri: {}", uri);
continue;
}
else {
std::string scheme = std::string(uri, 0, c);
if (scheme != "turn" && scheme != "turns") {
nhlog::ui()->error("Invalid TURN server uri: {}", uri);
continue;
}
QString encodedUri = QString::fromStdString(scheme) + "://" +
QUrl::toPercentEncoding(QString::fromStdString(turnServer_.username)) + ":" +
QUrl::toPercentEncoding(QString::fromStdString(turnServer_.password)) + "@" +
QString::fromStdString(std::string(uri, ++c));
uris.push_back(encodedUri.toStdString());
}
}
if (!uris.empty())
session_.setTurnServers(uris);
}
void
CallManager::playRingtone(const QString &ringtone, bool repeat)
{
@ -364,3 +365,34 @@ CallManager::stopRingtone()
{
player_.setPlaylist(nullptr);
}
namespace {
std::vector<std::string>
getTurnURIs(const mtx::responses::TurnServer &turnServer)
{
// gstreamer expects: turn(s)://username:password@host:port?transport=udp(tcp)
// where username and password are percent-encoded
std::vector<std::string> ret;
for (const auto &uri : turnServer.uris) {
if (auto c = uri.find(':'); c == std::string::npos) {
nhlog::ui()->error("Invalid TURN server uri: {}", uri);
continue;
}
else {
std::string scheme = std::string(uri, 0, c);
if (scheme != "turn" && scheme != "turns") {
nhlog::ui()->error("Invalid TURN server uri: {}", uri);
continue;
}
QString encodedUri = QString::fromStdString(scheme) + "://" +
QUrl::toPercentEncoding(QString::fromStdString(turnServer.username)) + ":" +
QUrl::toPercentEncoding(QString::fromStdString(turnServer.password)) + "@" +
QString::fromStdString(std::string(uri, ++c));
ret.push_back(encodedUri.toStdString());
}
}
return ret;
}
}

8
src/CallManager.h
View file

@ -11,7 +11,10 @@
#include "mtx/events/collections.hpp"
#include "mtx/events/voip.hpp"
#include "mtx/responses/turn_server.hpp"
namespace mtx::responses {
struct TurnServer;
}
class UserSettings;
class WebRTCSession;
@ -51,7 +54,7 @@ private:
std::string callid_;
const uint32_t timeoutms_ = 120000;
std::vector<mtx::events::msg::CallCandidates::Candidate> remoteICECandidates_;
mtx::responses::TurnServer turnServer_;
std::vector<std::string> turnURIs_;
QTimer turnServerTimer_;
QSharedPointer<UserSettings> settings_;
QMediaPlayer player_;
@ -65,7 +68,6 @@ private:
void answerInvite(const mtx::events::msg::CallInvite&);
void generateCallID();
void endCall();
void setTurnServers();
void playRingtone(const QString &ringtone, bool repeat);
void stopRingtone();
};

9
src/ChatPage.cpp
View file

@ -137,15 +137,6 @@ ChatPage::ChatPage(QSharedPointer<UserSettings> userSettings, QWidget *parent)
activeCallBar_->hide();
connect(
&callManager_, &CallManager::newCallParty, activeCallBar_, &ActiveCallBar::setCallParty);
connect(&WebRTCSession::instance(),
&WebRTCSession::stateChanged,
this,
[this](WebRTCSession::State state) {
if (state == WebRTCSession::State::DISCONNECTED)
activeCallBar_->hide();
else
activeCallBar_->show();
});
// Splitter
splitter->addWidget(sideBar_);

3
src/TextInputWidget.cpp
View file

@ -666,7 +666,8 @@ void
TextInputWidget::changeCallButtonState(WebRTCSession::State state)
{
QIcon icon;
if (state == WebRTCSession::State::DISCONNECTED) {
if (state == WebRTCSession::State::ICEFAILED ||
state == WebRTCSession::State::DISCONNECTED) {
callBtn_->setToolTip(tr("Place a call"));
icon.addFile(":/icons/icons/ui/place-call.png");
} else {

65
src/WebRTCSession.cpp
View file

@ -14,9 +14,9 @@ extern "C" {
Q_DECLARE_METATYPE(WebRTCSession::State)
namespace {
bool gisoffer;
std::string glocalsdp;
std::vector<mtx::events::msg::CallCandidates::Candidate> gcandidates;
bool isoffering_;
std::string localsdp_;
std::vector<mtx::events::msg::CallCandidates::Candidate> localcandidates_;
gboolean newBusMessage(GstBus *bus G_GNUC_UNUSED, GstMessage *msg, gpointer user_data);
GstWebRTCSessionDescription* parseSDP(const std::string &sdp, GstWebRTCSDPType type);
@ -24,6 +24,7 @@ void generateOffer(GstElement *webrtc);
void setLocalDescription(GstPromise *promise, gpointer webrtc);
void addLocalICECandidate(GstElement *webrtc G_GNUC_UNUSED, guint mlineIndex, gchar *candidate, gpointer G_GNUC_UNUSED);
gboolean onICEGatheringCompletion(gpointer timerid);
void iceConnectionStateChanged(GstElement *webrtcbin, GParamSpec *pspec G_GNUC_UNUSED, gpointer user_data G_GNUC_UNUSED);
void createAnswer(GstPromise *promise, gpointer webrtc);
void addDecodeBin(GstElement *webrtc G_GNUC_UNUSED, GstPad *newpad, GstElement *pipe);
void linkNewPad(GstElement *decodebin G_GNUC_UNUSED, GstPad *newpad, GstElement *pipe);
@ -92,9 +93,9 @@ WebRTCSession::init(std::string *errorMessage)
bool
WebRTCSession::createOffer()
{
gisoffer = true;
glocalsdp.clear();
gcandidates.clear();
isoffering_ = true;
localsdp_.clear();
localcandidates_.clear();
return startPipeline(111); // a dynamic opus payload type
}
@ -105,9 +106,9 @@ WebRTCSession::acceptOffer(const std::string &sdp)
if (state_ != State::DISCONNECTED)
return false;
gisoffer = false;
glocalsdp.clear();
gcandidates.clear();
isoffering_ = false;
localsdp_.clear();
localcandidates_.clear();
int opusPayloadType = getPayloadType(sdp, "opus");
if (opusPayloadType == -1)
@ -152,14 +153,20 @@ WebRTCSession::startPipeline(int opusPayloadType)
gboolean udata;
g_signal_emit_by_name(webrtc_, "add-turn-server", uri.c_str(), (gpointer)(&udata));
}
if (turnServers_.empty())
nhlog::ui()->warn("WebRTC: no TURN server provided");
// generate the offer when the pipeline goes to PLAYING
if (gisoffer)
if (isoffering_)
g_signal_connect(webrtc_, "on-negotiation-needed", G_CALLBACK(generateOffer), nullptr);
// on-ice-candidate is emitted when a local ICE candidate has been gathered
g_signal_connect(webrtc_, "on-ice-candidate", G_CALLBACK(addLocalICECandidate), nullptr);
// capture ICE failure
g_signal_connect(webrtc_, "notify::ice-connection-state",
G_CALLBACK(iceConnectionStateChanged), nullptr);
// incoming streams trigger pad-added
gst_element_set_state(pipe_, GST_STATE_READY);
g_signal_connect(webrtc_, "pad-added", G_CALLBACK(addDecodeBin), pipe_);
@ -229,8 +236,6 @@ WebRTCSession::acceptICECandidates(const std::vector<mtx::events::msg::CallCandi
nhlog::ui()->debug("WebRTC: remote candidate: (m-line:{}):{}", c.sdpMLineIndex, c.candidate);
g_signal_emit_by_name(webrtc_, "add-ice-candidate", c.sdpMLineIndex, c.candidate.c_str());
}
if (state_ == State::OFFERSENT)
emit stateChanged(State::CONNECTING);
}
}
@ -357,11 +362,11 @@ setLocalDescription(GstPromise *promise, gpointer webrtc)
g_signal_emit_by_name(webrtc, "set-local-description", gstsdp, nullptr);
gchar *sdp = gst_sdp_message_as_text(gstsdp->sdp);
glocalsdp = std::string(sdp);
localsdp_ = std::string(sdp);
g_free(sdp);
gst_webrtc_session_description_free(gstsdp);
nhlog::ui()->debug("WebRTC: local description set ({}):\n{}", isAnswer ? "answer" : "offer", glocalsdp);
nhlog::ui()->debug("WebRTC: local description set ({}):\n{}", isAnswer ? "answer" : "offer", localsdp_);
}
void
@ -369,12 +374,12 @@ addLocalICECandidate(GstElement *webrtc G_GNUC_UNUSED, guint mlineIndex, gchar *
{
nhlog::ui()->debug("WebRTC: local candidate: (m-line:{}):{}", mlineIndex, candidate);
if (WebRTCSession::instance().state() == WebRTCSession::State::CONNECTED) {
if (WebRTCSession::instance().state() >= WebRTCSession::State::OFFERSENT) {
emit WebRTCSession::instance().newICECandidate({"audio", (uint16_t)mlineIndex, candidate});
return;
}
gcandidates.push_back({"audio", (uint16_t)mlineIndex, candidate});
localcandidates_.push_back({"audio", (uint16_t)mlineIndex, candidate});
// GStreamer v1.16: webrtcbin's notify::ice-gathering-state triggers GST_WEBRTC_ICE_GATHERING_STATE_COMPLETE too early
// fixed in v1.18
@ -390,18 +395,36 @@ gboolean
onICEGatheringCompletion(gpointer timerid)
{
*(guint*)(timerid) = 0;
if (gisoffer) {
emit WebRTCSession::instance().offerCreated(glocalsdp, gcandidates);
if (isoffering_) {
emit WebRTCSession::instance().offerCreated(localsdp_, localcandidates_);
emit WebRTCSession::instance().stateChanged(WebRTCSession::State::OFFERSENT);
}
else {
emit WebRTCSession::instance().answerCreated(glocalsdp, gcandidates);
emit WebRTCSession::instance().stateChanged(WebRTCSession::State::CONNECTING);
emit WebRTCSession::instance().answerCreated(localsdp_, localcandidates_);
emit WebRTCSession::instance().stateChanged(WebRTCSession::State::ANSWERSENT);
}
return FALSE;
}
void
iceConnectionStateChanged(GstElement *webrtc, GParamSpec *pspec G_GNUC_UNUSED, gpointer user_data G_GNUC_UNUSED)
{
GstWebRTCICEConnectionState newState;
g_object_get(webrtc, "ice-connection-state", &newState, nullptr);
switch (newState) {
case GST_WEBRTC_ICE_CONNECTION_STATE_CHECKING:
nhlog::ui()->debug("WebRTC: GstWebRTCICEConnectionState -> Checking");
emit WebRTCSession::instance().stateChanged(WebRTCSession::State::CONNECTING);
break;
case GST_WEBRTC_ICE_CONNECTION_STATE_FAILED:
nhlog::ui()->error("WebRTC: GstWebRTCICEConnectionState -> Failed");
emit WebRTCSession::instance().stateChanged(WebRTCSession::State::ICEFAILED);
break;
default:
break;
}
}
void
createAnswer(GstPromise *promise, gpointer webrtc)
{

4
src/WebRTCSession.h
View file

@ -15,10 +15,12 @@ class WebRTCSession : public QObject
public:
enum class State {
ICEFAILED,
DISCONNECTED,
INITIATING,
INITIATED,
OFFERSENT,
ANSWERSENT,
CONNECTING,
CONNECTED
};
@ -30,13 +32,13 @@ public:
}
bool init(std::string *errorMessage = nullptr);
State state() const {return state_;}
bool createOffer();
bool acceptOffer(const std::string &sdp);
bool acceptAnswer(const std::string &sdp);
void acceptICECandidates(const std::vector<mtx::events::msg::CallCandidates::Candidate>&);
State state() const {return state_;}
bool toggleMuteAudioSrc(bool &isMuted);
void end();