From c755d54243b7408bdf204bc32226c53ae5314e75 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sat, 10 Jul 2021 16:12:38 +0200 Subject: [PATCH] Remove db bottleneck by caching ro txn for 100 requests --- resources/qml/TimelineRow.qml | 2 +- src/Cache.cpp | 187 ++++++++++++++++++--------------- src/timeline/TimelineModel.cpp | 12 ++- src/timeline/TimelineModel.h | 5 +- 4 files changed, 116 insertions(+), 90 deletions(-) diff --git a/resources/qml/TimelineRow.qml b/resources/qml/TimelineRow.qml index 3fa1ad8e..e66dd2de 100644 --- a/resources/qml/TimelineRow.qml +++ b/resources/qml/TimelineRow.qml @@ -84,7 +84,7 @@ Item { } EncryptionIndicator { - visible: model.isRoomEncrypted + visible: room.isEncrypted encrypted: model.isEncrypted trust: model.trustlevel Layout.alignment: Qt.AlignRight | Qt.AlignTop diff --git a/src/Cache.cpp b/src/Cache.cpp index 8146e2cc..684883e4 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -92,6 +92,33 @@ namespace { std::unique_ptr instance_ = nullptr; } +struct RO_txn +{ + ~RO_txn() { txn.reset(); } + operator MDB_txn *() const noexcept { return txn.handle(); } + operator lmdb::txn &() noexcept { return txn; } + + lmdb::txn &txn; +}; + +RO_txn +ro_txn(lmdb::env &env) +{ + thread_local lmdb::txn txn = lmdb::txn::begin(env, nullptr, MDB_RDONLY); + thread_local int reuse_counter = 0; + + if (reuse_counter >= 100 || txn.env() != env.handle()) { + txn.abort(); + txn = lmdb::txn::begin(env, nullptr, MDB_RDONLY); + reuse_counter = 0; + } else if (reuse_counter > 0) { + txn.renew(); + } + reuse_counter++; + + return RO_txn{txn}; +} + template bool containsStateUpdates(const T &e) @@ -277,10 +304,9 @@ Cache::isRoomEncrypted(const std::string &room_id) { std::string_view unused; - auto txn = lmdb::txn::begin(env_); + auto txn = ro_txn(env_); auto db = lmdb::dbi::open(txn, ENCRYPTED_ROOMS_DB, MDB_CREATE); auto res = db.get(txn, room_id, unused); - txn.commit(); return res; } @@ -292,7 +318,7 @@ Cache::roomEncryptionSettings(const std::string &room_id) using namespace mtx::events::state; try { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); auto statesdb = getStatesDb(txn, room_id); std::string_view event; bool res = @@ -322,7 +348,7 @@ Cache::exportSessionKeys() ExportedSessionKeys keys; - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); auto cursor = lmdb::cursor::open(txn, inboundMegolmSessionDb_); std::string_view key, value; @@ -348,7 +374,6 @@ Cache::exportSessionKeys() } cursor.close(); - txn.commit(); return keys; } @@ -404,7 +429,7 @@ Cache::getInboundMegolmSession(const MegolmSessionIndex &index) using namespace mtx::crypto; try { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); std::string key = json(index).dump(); std::string_view value; @@ -425,7 +450,7 @@ Cache::inboundMegolmSessionExists(const MegolmSessionIndex &index) using namespace mtx::crypto; try { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); std::string key = json(index).dump(); std::string_view value; @@ -498,7 +523,7 @@ bool Cache::outboundMegolmSessionExists(const std::string &room_id) noexcept { try { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); std::string_view value; return outboundMegolmSessionDb_.get(txn, room_id, value); } catch (std::exception &e) { @@ -513,7 +538,7 @@ Cache::getOutboundMegolmSession(const std::string &room_id) try { using namespace mtx::crypto; - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); std::string_view value; outboundMegolmSessionDb_.get(txn, room_id, value); auto obj = json::parse(value); @@ -634,10 +659,9 @@ Cache::saveOlmAccount(const std::string &data) std::string Cache::restoreOlmAccount() { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); std::string_view pickled; syncStateDb_.get(txn, OLM_ACCOUNT_KEY, pickled); - txn.commit(); return std::string(pickled.data(), pickled.size()); } @@ -764,12 +788,11 @@ Cache::setNextBatchToken(lmdb::txn &txn, const QString &token) bool Cache::isInitialized() { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); std::string_view token; bool res = syncStateDb_.get(txn, NEXT_BATCH_KEY, token); - txn.commit(); return res; } @@ -780,12 +803,11 @@ Cache::nextBatchToken() if (!env_.handle()) throw lmdb::error("Env already closed", MDB_INVALID); - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); std::string_view token; bool result = syncStateDb_.get(txn, NEXT_BATCH_KEY, token); - txn.commit(); if (result) return std::string(token.data(), token.size()); @@ -829,17 +851,18 @@ Cache::deleteData() bool Cache::runMigrations() { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + std::string stored_version; + { + auto txn = ro_txn(env_); - std::string_view current_version; - bool res = syncStateDb_.get(txn, CACHE_FORMAT_VERSION_KEY, current_version); + std::string_view current_version; + bool res = syncStateDb_.get(txn, CACHE_FORMAT_VERSION_KEY, current_version); - txn.commit(); + if (!res) + return false; - if (!res) - return false; - - std::string stored_version(current_version.data(), current_version.size()); + stored_version = std::string(current_version); + } std::vector>> migrations{ {"2020.05.01", @@ -1019,13 +1042,11 @@ Cache::runMigrations() cache::CacheVersion Cache::formatVersion() { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); std::string_view current_version; bool res = syncStateDb_.get(txn, CACHE_FORMAT_VERSION_KEY, current_version); - txn.commit(); - if (!res) return cache::CacheVersion::Older; @@ -1058,15 +1079,13 @@ Cache::readReceipts(const QString &event_id, const QString &room_id) nlohmann::json json_key = receipt_key; try { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); auto key = json_key.dump(); std::string_view value; bool res = readReceiptsDb_.get(txn, key, value); - txn.commit(); - if (res) { auto json_response = json::parse(std::string_view(value.data(), value.size())); @@ -1151,21 +1170,22 @@ Cache::calculateRoomReadStatus() bool Cache::calculateRoomReadStatus(const std::string &room_id) { - auto txn = lmdb::txn::begin(env_); + std::string last_event_id_, fullyReadEventId_; + { + auto txn = ro_txn(env_); - // Get last event id on the room. - const auto last_event_id = getLastEventId(txn, room_id); - const auto localUser = utils::localUser().toStdString(); + // Get last event id on the room. + const auto last_event_id = getLastEventId(txn, room_id); + const auto localUser = utils::localUser().toStdString(); - std::string fullyReadEventId; - if (auto ev = getAccountData(txn, mtx::events::EventType::FullyRead, room_id)) { - if (auto fr = std::get_if< - mtx::events::AccountDataEvent>( - &ev.value())) { - fullyReadEventId = fr->content.event_id; + std::string fullyReadEventId; + if (auto ev = getAccountData(txn, mtx::events::EventType::FullyRead, room_id)) { + if (auto fr = std::get_if< + mtx::events::AccountDataEvent>( + &ev.value())) { + fullyReadEventId = fr->content.event_id; + } } - } - txn.commit(); if (last_event_id.empty() || fullyReadEventId.empty()) return true; @@ -1173,8 +1193,12 @@ Cache::calculateRoomReadStatus(const std::string &room_id) if (last_event_id == fullyReadEventId) return false; + last_event_id_ = std::string(last_event_id); + fullyReadEventId_ = std::string(fullyReadEventId); + } + // Retrieve all read receipts for that event. - return getEventIndex(room_id, last_event_id) > getEventIndex(room_id, fullyReadEventId); + return getEventIndex(room_id, last_event_id_) > getEventIndex(room_id, fullyReadEventId_); } void @@ -1490,7 +1514,7 @@ Cache::roomsWithStateUpdates(const mtx::responses::Sync &res) RoomInfo Cache::singleRoomInfo(const std::string &room_id) { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); auto statesdb = getStatesDb(txn, room_id); std::string_view data; @@ -1503,7 +1527,6 @@ Cache::singleRoomInfo(const std::string &room_id) tmp.join_rule = getRoomJoinRule(txn, statesdb); tmp.guest_access = getRoomGuestAccess(txn, statesdb); - txn.commit(); return tmp; } catch (const json::exception &e) { @@ -1514,7 +1537,6 @@ Cache::singleRoomInfo(const std::string &room_id) } } - txn.commit(); return RoomInfo(); } @@ -1574,7 +1596,7 @@ Cache::getRoomInfo(const std::vector &rooms) std::vector Cache::roomIds() { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); std::vector rooms; std::string_view room_id, unused; @@ -1584,7 +1606,6 @@ Cache::roomIds() rooms.push_back(QString::fromStdString(std::string(room_id))); roomsCursor.close(); - txn.commit(); return rooms; } @@ -1690,7 +1711,7 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, uint64_t std::optional Cache::getEvent(const std::string &room_id, const std::string &event_id) { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); auto eventsDb = getEventsDb(txn, room_id); std::string_view event{}; @@ -1744,7 +1765,7 @@ Cache::replaceEvent(const std::string &room_id, std::vector Cache::relatedEvents(const std::string &room_id, const std::string &event_id) { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); auto relationsDb = getRelationsDb(txn, room_id); std::vector related_ids; @@ -1775,7 +1796,7 @@ Cache::relatedEvents(const std::string &room_id, const std::string &event_id) size_t Cache::memberCount(const std::string &room_id) { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); return getMembersDb(txn, room_id).size(txn); } @@ -1784,7 +1805,7 @@ Cache::roomInfo(bool withInvites) { QMap result; - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); std::string_view room_id; std::string_view room_data; @@ -1809,8 +1830,6 @@ Cache::roomInfo(bool withInvites) invitesCursor.close(); } - txn.commit(); - return result; } @@ -1840,7 +1859,7 @@ Cache::getLastEventId(lmdb::txn &txn, const std::string &room_id) std::optional Cache::getTimelineRange(const std::string &room_id) { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); lmdb::dbi orderDb; try { orderDb = getOrderToMessageDb(txn, room_id); @@ -1874,7 +1893,7 @@ Cache::getTimelineIndex(const std::string &room_id, std::string_view event_id) if (event_id.empty() || room_id.empty()) return {}; - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); lmdb::dbi orderDb; try { @@ -1902,7 +1921,7 @@ Cache::getEventIndex(const std::string &room_id, std::string_view event_id) if (event_id.empty()) return {}; - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); lmdb::dbi orderDb; try { @@ -1930,7 +1949,7 @@ Cache::lastInvisibleEventAfter(const std::string &room_id, std::string_view even if (event_id.empty()) return {}; - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); lmdb::dbi orderDb; lmdb::dbi eventOrderDb; @@ -1974,7 +1993,7 @@ Cache::lastInvisibleEventAfter(const std::string &room_id, std::string_view even std::optional Cache::getArrivalIndex(const std::string &room_id, std::string_view event_id) { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); lmdb::dbi orderDb; try { @@ -1999,7 +2018,7 @@ Cache::getArrivalIndex(const std::string &room_id, std::string_view event_id) std::optional Cache::getTimelineEventId(const std::string &room_id, uint64_t index) { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); lmdb::dbi orderDb; try { orderDb = getOrderToMessageDb(txn, room_id); @@ -2025,7 +2044,7 @@ Cache::invites() { QHash result; - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); auto cursor = lmdb::cursor::open(txn, invitesDb_); std::string_view room_id, room_data; @@ -2054,7 +2073,7 @@ Cache::invite(std::string_view roomid) { std::optional result; - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); std::string_view room_data; @@ -2320,7 +2339,7 @@ Cache::getRoomAliases(const std::string &roomid) using namespace mtx::events; using namespace mtx::events::state; - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); auto statesdb = getStatesDb(txn, roomid); std::string_view event; @@ -2465,7 +2484,7 @@ Cache::getInviteRoomIsSpace(lmdb::txn &txn, lmdb::dbi &db) std::vector Cache::joinedRooms() { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); auto roomsCursor = lmdb::cursor::open(txn, roomsDb_); std::string_view id, data; @@ -2476,7 +2495,6 @@ Cache::joinedRooms() room_ids.emplace_back(id); roomsCursor.close(); - txn.commit(); return room_ids; } @@ -2488,7 +2506,7 @@ Cache::getMember(const std::string &room_id, const std::string &user_id) return std::nullopt; try { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); auto membersdb = getMembersDb(txn, room_id); @@ -2507,7 +2525,7 @@ Cache::getMember(const std::string &room_id, const std::string &user_id) std::vector Cache::getMembers(const std::string &room_id, std::size_t startIndex, std::size_t len) { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); auto db = getMembersDb(txn, room_id); auto cursor = lmdb::cursor::open(txn, db); @@ -2540,7 +2558,6 @@ Cache::getMembers(const std::string &room_id, std::size_t startIndex, std::size_ } cursor.close(); - txn.commit(); return members; } @@ -2548,14 +2565,21 @@ Cache::getMembers(const std::string &room_id, std::size_t startIndex, std::size_ bool Cache::isRoomMember(const std::string &user_id, const std::string &room_id) { - auto txn = lmdb::txn::begin(env_); - auto db = getMembersDb(txn, room_id); + try { + auto txn = ro_txn(env_); + auto db = getMembersDb(txn, room_id); - std::string_view value; - bool res = db.get(txn, user_id, value); - txn.commit(); + std::string_view value; + bool res = db.get(txn, user_id, value); - return res; + return res; + } catch (std::exception &e) { + nhlog::db()->warn("Failed to read member membership ({}) in room ({}): {}", + user_id, + room_id, + e.what()); + } + return false; } void @@ -3070,11 +3094,10 @@ Cache::removeReadNotification(const std::string &event_id) bool Cache::isNotificationSent(const std::string &event_id) { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); std::string_view value; bool res = notificationsDb_.get(txn, event_id, value); - txn.commit(); return res; } @@ -3230,7 +3253,7 @@ Cache::updateSpaces(lmdb::txn &txn, QMap> Cache::spaces() { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); QMap> ret; { @@ -3262,7 +3285,7 @@ Cache::spaces() std::vector Cache::getParentRoomIds(const std::string &room_id) { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); std::vector roomids; { @@ -3286,7 +3309,7 @@ Cache::getParentRoomIds(const std::string &room_id) std::vector Cache::getChildRoomIds(const std::string &room_id) { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); std::vector roomids; { @@ -3369,7 +3392,7 @@ Cache::hasEnoughPowerLevel(const std::vector &eventTypes std::vector Cache::roomMembers(const std::string &room_id) { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); std::vector members; std::string_view user_id, unused; @@ -3381,8 +3404,6 @@ Cache::roomMembers(const std::string &room_id) members.emplace_back(user_id); cursor.close(); - txn.commit(); - return members; } @@ -3392,7 +3413,7 @@ Cache::getMembersWithKeys(const std::string &room_id) std::string_view keys; try { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); std::map> members; auto db = getMembersDb(txn, room_id); @@ -3528,7 +3549,7 @@ Cache::userKeys(const std::string &user_id) std::string_view keys; try { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto txn = ro_txn(env_); auto db = getUserKeysDb(txn); auto res = db.get(txn, user_id, keys); diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp index caa40353..4cb97e07 100644 --- a/src/timeline/TimelineModel.cpp +++ b/src/timeline/TimelineModel.cpp @@ -323,6 +323,7 @@ TimelineModel::TimelineModel(TimelineViewManager *manager, QString room_id, QObj if (auto create = cache::client()->getStateEvent(room_id.toStdString())) this->isSpace_ = create->content.type == mtx::events::state::room_type::space; + this->isEncrypted_ = cache::isRoomEncrypted(room_id_.toStdString()); connect( this, @@ -435,7 +436,6 @@ TimelineModel::roleNames() const {IsEditable, "isEditable"}, {IsEncrypted, "isEncrypted"}, {Trustlevel, "trustlevel"}, - {IsRoomEncrypted, "isRoomEncrypted"}, {ReplyTo, "replyTo"}, {Reactions, "reactions"}, {RoomId, "roomId"}, @@ -622,9 +622,6 @@ TimelineModel::data(const mtx::events::collections::TimelineEvents &event, int r return crypto::Trust::Unverified; } - case IsRoomEncrypted: { - return cache::isRoomEncrypted(room_id_.toStdString()); - } case ReplyTo: return QVariant(QString::fromStdString(relations(event).reply_to().value_or(""))); case Reactions: { @@ -672,7 +669,6 @@ TimelineModel::data(const mtx::events::collections::TimelineEvents &event, int r m.insert(names[IsEdited], data(event, static_cast(IsEdited))); m.insert(names[IsEditable], data(event, static_cast(IsEditable))); m.insert(names[IsEncrypted], data(event, static_cast(IsEncrypted))); - m.insert(names[IsRoomEncrypted], data(event, static_cast(IsRoomEncrypted))); m.insert(names[ReplyTo], data(event, static_cast(ReplyTo))); m.insert(names[RoomName], data(event, static_cast(RoomName))); m.insert(names[RoomTopic], data(event, static_cast(RoomTopic))); @@ -785,6 +781,9 @@ TimelineModel::syncState(const mtx::responses::State &s) emit roomAvatarUrlChanged(); emit roomNameChanged(); emit roomMemberCountChanged(); + } else if (std::holds_alternative>(e)) { + this->isEncrypted_ = cache::isRoomEncrypted(room_id_.toStdString()); + emit encryptionChanged(); } } } @@ -842,6 +841,9 @@ TimelineModel::addEvents(const mtx::responses::Timeline &timeline) emit roomAvatarUrlChanged(); emit roomNameChanged(); emit roomMemberCountChanged(); + } else if (std::holds_alternative>(e)) { + this->isEncrypted_ = cache::isRoomEncrypted(room_id_.toStdString()); + emit encryptionChanged(); } } updateLastMessage(); diff --git a/src/timeline/TimelineModel.h b/src/timeline/TimelineModel.h index 3392d474..f093acb4 100644 --- a/src/timeline/TimelineModel.h +++ b/src/timeline/TimelineModel.h @@ -162,6 +162,7 @@ class TimelineModel : public QAbstractListModel Q_PROPERTY(QString roomAvatarUrl READ roomAvatarUrl NOTIFY roomAvatarUrlChanged) Q_PROPERTY(QString roomTopic READ roomTopic NOTIFY roomTopicChanged) Q_PROPERTY(int roomMemberCount READ roomMemberCount NOTIFY roomMemberCountChanged) + Q_PROPERTY(bool isEncrypted READ isEncrypted NOTIFY encryptionChanged) Q_PROPERTY(bool isSpace READ isSpace CONSTANT) Q_PROPERTY(InputBar *input READ input CONSTANT) Q_PROPERTY(Permissions *permissions READ permissions NOTIFY permissionsChanged) @@ -200,7 +201,6 @@ public: IsEditable, IsEncrypted, Trustlevel, - IsRoomEncrypted, ReplyTo, Reactions, RoomId, @@ -265,6 +265,7 @@ public: DescInfo lastMessage() const { return lastMessage_; } bool isSpace() const { return isSpace_; } + bool isEncrypted() const { return isEncrypted_; } int roomMemberCount() const; public slots: @@ -349,6 +350,7 @@ signals: void addPendingMessageToStore(mtx::events::collections::TimelineEvents event); void updateFlowEventId(std::string event_id); + void encryptionChanged(); void roomNameChanged(); void roomTopicChanged(); void roomAvatarUrlChanged(); @@ -394,6 +396,7 @@ private: bool decryptDescription = true; bool m_paginationInProgress = false; bool isSpace_ = false; + bool isEncrypted_ = false; }; template