WIP: Event Store split out

CMakeLists.txt

@@ -250,6 +250,7 @@ set(SRC_FILES
 
 
 	# Timeline
+	src/timeline/EventStore.cpp
 	src/timeline/ReactionsModel.cpp
 	src/timeline/TimelineViewManager.cpp
 	src/timeline/TimelineModel.cpp
@@ -453,6 +454,7 @@ qt5_wrap_cpp(MOC_HEADERS
 	src/emoji/Provider.h
 
 	# Timeline
+	src/timeline/EventStore.h
 	src/timeline/ReactionsModel.h
 	src/timeline/TimelineViewManager.h
 	src/timeline/TimelineModel.h

resources/qml/TimelineRow.qml

@@ -45,7 +45,7 @@ MouseArea {
 			// fancy reply, if this is a reply
 			Reply {
 				visible: model.replyTo
-				modelData: chat.model.getDump(model.replyTo)
+				modelData: chat.model.getDump(model.replyTo, model.id)
 				userColor: timelineManager.userColor(modelData.userId, colors.window)
 			}
 

resources/qml/TimelineView.qml

@@ -353,7 +353,7 @@ Page {
 						anchors.rightMargin: 20
 						anchors.bottom: parent.bottom
 
-						modelData: chat.model ? chat.model.getDump(chat.model.reply) : {}
+						modelData: chat.model ? chat.model.getDump(chat.model.reply, chat.model.id) : {}
 						userColor: timelineManager.userColor(modelData.userId, colors.window)
 					}
 

src/Cache.cpp

@@ -1272,7 +1272,10 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i
         int counter = 0;
 
         bool ret;
-        while ((ret = cursor.get(indexVal, event_id, forward ? MDB_NEXT : MDB_LAST)) &&
+        while ((ret = cursor.get(indexVal,
+                                 event_id,
+                                 counter == 0 ? (forward ? MDB_FIRST : MDB_LAST)
+                                              : (forward ? MDB_NEXT : MDB_PREV))) &&
                counter++ < BATCH_SIZE) {
                 lmdb::val event;
                 bool success = lmdb::dbi_get(txn, eventsDb, event_id, event);
@@ -1280,8 +1283,13 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i
                         continue;
 
                 mtx::events::collections::TimelineEvent te;
-                mtx::events::collections::from_json(
+                try {
-                  json::parse(std::string_view(event.data(), event.size())), te);
+                        mtx::events::collections::from_json(
+                          json::parse(std::string_view(event.data(), event.size())), te);
+                } catch (std::exception &e) {
+                        nhlog::db()->error("Failed to parse message from cache {}", e.what());
+                        continue;
+                }
 
                 messages.timeline.events.push_back(std::move(te.data));
         }
@@ -1306,8 +1314,13 @@ Cache::getEvent(const std::string &room_id, const std::string &event_id)
                 return {};
 
         mtx::events::collections::TimelineEvent te;
-        mtx::events::collections::from_json(
+        try {
-          json::parse(std::string_view(event.data(), event.size())), te);
+                mtx::events::collections::from_json(
+                  json::parse(std::string_view(event.data(), event.size())), te);
+        } catch (std::exception &e) {
+                nhlog::db()->error("Failed to parse message from cache {}", e.what());
+                return std::nullopt;
+        }
 
         return te;
 }
@@ -1364,6 +1377,61 @@ Cache::getLastEventId(lmdb::txn &txn, const std::string &room_id)
         return std::string(val.data(), val.size());
 }
 
+std::optional<Cache::TimelineRange>
+Cache::getTimelineRange(const std::string &room_id)
+{
+        auto txn     = lmdb::txn::begin(env_, nullptr, MDB_RDONLY);
+        auto orderDb = getOrderToMessageDb(txn, room_id);
+
+        lmdb::val indexVal, val;
+
+        auto cursor = lmdb::cursor::open(txn, orderDb);
+        if (!cursor.get(indexVal, val, MDB_LAST)) {
+                return {};
+        }
+
+        TimelineRange range{};
+        range.last = *indexVal.data<int64_t>();
+
+        if (!cursor.get(indexVal, val, MDB_FIRST)) {
+                return {};
+        }
+        range.first = *indexVal.data<int64_t>();
+
+        return range;
+}
+std::optional<int64_t>
+Cache::getTimelineIndex(const std::string &room_id, std::string_view event_id)
+{
+        auto txn     = lmdb::txn::begin(env_, nullptr, MDB_RDONLY);
+        auto orderDb = getMessageToOrderDb(txn, room_id);
+
+        lmdb::val indexVal{event_id.data(), event_id.size()}, val;
+
+        bool success = lmdb::dbi_get(txn, orderDb, indexVal, val);
+        if (!success) {
+                return {};
+        }
+
+        return *val.data<int64_t>();
+}
+
+std::optional<std::string>
+Cache::getTimelineEventId(const std::string &room_id, int64_t index)
+{
+        auto txn     = lmdb::txn::begin(env_, nullptr, MDB_RDONLY);
+        auto orderDb = getOrderToMessageDb(txn, room_id);
+
+        lmdb::val indexVal{&index, sizeof(index)}, val;
+
+        bool success = lmdb::dbi_get(txn, orderDb, indexVal, val);
+        if (!success) {
+                return {};
+        }
+
+        return std::string(val.data(), val.size());
+}
+
 DescInfo
 Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id)
 {
@@ -1379,8 +1447,10 @@ Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id)
         lmdb::val indexVal, event_id;
 
         auto cursor = lmdb::cursor::open(txn, orderDb);
-        cursor.get(indexVal, event_id, MDB_LAST);
+        bool first  = true;
-        while (cursor.get(indexVal, event_id, MDB_PREV)) {
+        while (cursor.get(indexVal, event_id, first ? MDB_LAST : MDB_PREV)) {
+                first = false;
+
                 lmdb::val event;
                 bool success = lmdb::dbi_get(txn, eventsDb, event_id, event);
                 if (!success)
@@ -2026,8 +2096,43 @@ Cache::saveTimelineMessages(lmdb::txn &txn,
                 auto event = mtx::accessors::serialize_event(e);
                 if (auto redaction =
                       std::get_if<mtx::events::RedactionEvent<mtx::events::msg::Redaction>>(&e)) {
-                        lmdb::dbi_put(
+                        if (redaction->redacts.empty())
-                          txn, eventsDb, lmdb::val(redaction->redacts), lmdb::val(event.dump()));
+                                continue;
+
+                        lmdb::val ev{};
+                        bool success =
+                          lmdb::dbi_get(txn, eventsDb, lmdb::val(redaction->redacts), ev);
+                        if (!success)
+                                continue;
+
+                        mtx::events::collections::TimelineEvent te;
+
+                        try {
+                                mtx::events::collections::from_json(
+                                  json::parse(std::string_view(ev.data(), ev.size())), te);
+                        } catch (std::exception &e) {
+                                nhlog::db()->error("Failed to parse message from cache {}",
+                                                   e.what());
+                                continue;
+                        }
+
+                        auto redactedEvent = std::visit(
+                          [](const auto &ev) -> mtx::events::RoomEvent<mtx::events::msg::Redacted> {
+                                  mtx::events::RoomEvent<mtx::events::msg::Redacted> replacement =
+                                    {};
+                                  replacement.event_id         = ev.event_id;
+                                  replacement.room_id          = ev.room_id;
+                                  replacement.sender           = ev.sender;
+                                  replacement.origin_server_ts = ev.origin_server_ts;
+                                  replacement.type             = ev.type;
+                                  return replacement;
+                          },
+                          te.data);
+
+                        lmdb::dbi_put(txn,
+                                      eventsDb,
+                                      lmdb::val(redaction->redacts),
+                                      lmdb::val(json(redactedEvent).dump()));
                 } else {
                         std::string event_id_val = event["event_id"].get<std::string>();
                         lmdb::val event_id       = event_id_val;
@@ -2237,8 +2342,10 @@ Cache::deleteOldMessages()
                 if (message_count < MAX_RESTORED_MESSAGES)
                         continue;
 
-                while (cursor.get(indexVal, val, MDB_NEXT) &&
+                bool start = true;
+                while (cursor.get(indexVal, val, start ? MDB_FIRST : MDB_NEXT) &&
                        message_count-- < MAX_RESTORED_MESSAGES) {
+                        start    = false;
                         auto obj = json::parse(std::string_view(val.data(), val.size()));
 
                         if (obj.count("event_id") != 0) {
@@ -2394,6 +2501,9 @@ Cache::removeAvatarUrl(const QString &room_id, const QString &user_id)
 mtx::presence::PresenceState
 Cache::presenceState(const std::string &user_id)
 {
+        if (user_id.empty())
+                return {};
+
         lmdb::val presenceVal;
 
         auto txn = lmdb::txn::begin(env_);
@@ -2416,6 +2526,9 @@ Cache::presenceState(const std::string &user_id)
 std::string
 Cache::statusMessage(const std::string &user_id)
 {
+        if (user_id.empty())
+                return {};
+
         lmdb::val presenceVal;
 
         auto txn = lmdb::txn::begin(env_);

src/Cache_p.h

@@ -170,6 +170,30 @@ public:
         //! Add all notifications containing a user mention to the db.
         void saveTimelineMentions(const mtx::responses::Notifications &res);
 
+        //! retrieve events in timeline and related functions
+        struct Messages
+        {
+                mtx::responses::Timeline timeline;
+                uint64_t next_index;
+                bool end_of_cache = false;
+        };
+        Messages getTimelineMessages(lmdb::txn &txn,
+                                     const std::string &room_id,
+                                     int64_t index = std::numeric_limits<int64_t>::max(),
+                                     bool forward  = false);
+
+        std::optional<mtx::events::collections::TimelineEvent> getEvent(
+          const std::string &room_id,
+          const std::string &event_id);
+        struct TimelineRange
+        {
+                int64_t first, last;
+        };
+        std::optional<TimelineRange> getTimelineRange(const std::string &room_id);
+        std::optional<int64_t> getTimelineIndex(const std::string &room_id,
+                                                std::string_view event_id);
+        std::optional<std::string> getTimelineEventId(const std::string &room_id, int64_t index);
+
         //! Remove old unused data.
         void deleteOldMessages();
         void deleteOldData() noexcept;
@@ -248,21 +272,6 @@ private:
                                   const std::string &room_id,
                                   const mtx::responses::Timeline &res);
 
-        struct Messages
-        {
-                mtx::responses::Timeline timeline;
-                uint64_t next_index;
-                bool end_of_cache = false;
-        };
-        Messages getTimelineMessages(lmdb::txn &txn,
-                                     const std::string &room_id,
-                                     int64_t index = std::numeric_limits<int64_t>::max(),
-                                     bool forward  = false);
-
-        std::optional<mtx::events::collections::TimelineEvent> getEvent(
-          const std::string &room_id,
-          const std::string &event_id);
-
         //! Remove a room from the cache.
         // void removeLeftRoom(lmdb::txn &txn, const std::string &room_id);
         template<class T>

src/timeline/EventStore.cpp

@@ -0,0 +1,259 @@
+#include "EventStore.h"
+
+#include <QThread>
+
+#include "Cache_p.h"
+#include "EventAccessors.h"
+#include "Logging.h"
+#include "Olm.h"
+
+QCache<EventStore::IdIndex, mtx::events::collections::TimelineEvents> EventStore::decryptedEvents_{
+  1000};
+QCache<EventStore::IdIndex, mtx::events::collections::TimelineEvents> EventStore::events_by_id_{
+  1000};
+QCache<EventStore::Index, mtx::events::collections::TimelineEvents> EventStore::events_{1000};
+
+EventStore::EventStore(std::string room_id, QObject *)
+  : room_id_(std::move(room_id))
+{
+        auto range = cache::client()->getTimelineRange(room_id_);
+
+        if (range) {
+                this->first = range->first;
+                this->last  = range->last;
+        }
+}
+
+void
+EventStore::handleSync(const mtx::responses::Timeline &events)
+{
+        if (this->thread() != QThread::currentThread())
+                nhlog::db()->warn("{} called from a different thread!", __func__);
+
+        auto range = cache::client()->getTimelineRange(room_id_);
+
+        if (range && range->last > this->last) {
+                emit beginInsertRows(toExternalIdx(this->last + 1), toExternalIdx(range->last));
+                this->last = range->last;
+                emit endInsertRows();
+        }
+
+        for (const auto &event : events.events) {
+                std::string relates_to;
+                if (auto redaction =
+                      std::get_if<mtx::events::RedactionEvent<mtx::events::msg::Redaction>>(
+                        &event)) {
+                        relates_to = redaction->redacts;
+                } else if (auto reaction =
+                             std::get_if<mtx::events::RoomEvent<mtx::events::msg::Reaction>>(
+                               &event)) {
+                        relates_to = reaction->content.relates_to.event_id;
+                } else {
+                        relates_to = mtx::accessors::in_reply_to_event(event);
+                }
+
+                if (!relates_to.empty()) {
+                        auto idx = cache::client()->getTimelineIndex(room_id_, relates_to);
+                        if (idx)
+                                emit dataChanged(toExternalIdx(*idx), toExternalIdx(*idx));
+                }
+        }
+}
+
+mtx::events::collections::TimelineEvents *
+EventStore::event(int idx, bool decrypt)
+{
+        if (this->thread() != QThread::currentThread())
+                nhlog::db()->warn("{} called from a different thread!", __func__);
+
+        Index index{room_id_, toInternalIdx(idx)};
+        if (index.idx > last || index.idx < first)
+                return nullptr;
+
+        auto event_ptr = events_.object(index);
+        if (!event_ptr) {
+                auto event_id = cache::client()->getTimelineEventId(room_id_, index.idx);
+                if (!event_id)
+                        return nullptr;
+
+                auto event = cache::client()->getEvent(room_id_, *event_id);
+                if (!event)
+                        return nullptr;
+                else
+                        event_ptr =
+                          new mtx::events::collections::TimelineEvents(std::move(event->data));
+                events_.insert(index, event_ptr);
+        }
+
+        if (decrypt)
+                if (auto encrypted =
+                      std::get_if<mtx::events::EncryptedEvent<mtx::events::msg::Encrypted>>(
+                        event_ptr))
+                        return decryptEvent({room_id_, encrypted->event_id}, *encrypted);
+
+        return event_ptr;
+}
+
+std::optional<int>
+EventStore::idToIndex(std::string_view id) const
+{
+        if (this->thread() != QThread::currentThread())
+                nhlog::db()->warn("{} called from a different thread!", __func__);
+
+        auto idx = cache::client()->getTimelineIndex(room_id_, id);
+        if (idx)
+                return toExternalIdx(*idx);
+        else
+                return std::nullopt;
+}
+std::optional<std::string>
+EventStore::indexToId(int idx) const
+{
+        if (this->thread() != QThread::currentThread())
+                nhlog::db()->warn("{} called from a different thread!", __func__);
+
+        return cache::client()->getTimelineEventId(room_id_, toInternalIdx(idx));
+}
+
+mtx::events::collections::TimelineEvents *
+EventStore::decryptEvent(const IdIndex &idx,
+                         const mtx::events::EncryptedEvent<mtx::events::msg::Encrypted> &e)
+{
+        if (auto cachedEvent = decryptedEvents_.object(idx))
+                return cachedEvent;
+
+        MegolmSessionIndex index;
+        index.room_id    = room_id_;
+        index.session_id = e.content.session_id;
+        index.sender_key = e.content.sender_key;
+
+        mtx::events::RoomEvent<mtx::events::msg::Notice> dummy;
+        dummy.origin_server_ts = e.origin_server_ts;
+        dummy.event_id         = e.event_id;
+        dummy.sender           = e.sender;
+        dummy.content.body =
+          tr("-- Encrypted Event (No keys found for decryption) --",
+             "Placeholder, when the message was not decrypted yet or can't be decrypted.")
+            .toStdString();
+
+        auto asCacheEntry = [&idx](mtx::events::collections::TimelineEvents &&event) {
+                auto event_ptr = new mtx::events::collections::TimelineEvents(std::move(event));
+                decryptedEvents_.insert(idx, event_ptr);
+                return event_ptr;
+        };
+
+        try {
+                if (!cache::client()->inboundMegolmSessionExists(index)) {
+                        nhlog::crypto()->info("Could not find inbound megolm session ({}, {}, {})",
+                                              index.room_id,
+                                              index.session_id,
+                                              e.sender);
+                        // TODO: request megolm session_id & session_key from the sender.
+                        return asCacheEntry(std::move(dummy));
+                }
+        } catch (const lmdb::error &e) {
+                nhlog::db()->critical("failed to check megolm session's existence: {}", e.what());
+                dummy.content.body = tr("-- Decryption Error (failed to communicate with DB) --",
+                                        "Placeholder, when the message can't be decrypted, because "
+                                        "the DB access failed when trying to lookup the session.")
+                                       .toStdString();
+                return asCacheEntry(std::move(dummy));
+        }
+
+        std::string msg_str;
+        try {
+                auto session = cache::client()->getInboundMegolmSession(index);
+                auto res     = olm::client()->decrypt_group_message(session, e.content.ciphertext);
+                msg_str      = std::string((char *)res.data.data(), res.data.size());
+        } catch (const lmdb::error &e) {
+                nhlog::db()->critical("failed to retrieve megolm session with index ({}, {}, {})",
+                                      index.room_id,
+                                      index.session_id,
+                                      index.sender_key,
+                                      e.what());
+                dummy.content.body =
+                  tr("-- Decryption Error (failed to retrieve megolm keys from db) --",
+                     "Placeholder, when the message can't be decrypted, because the DB access "
+                     "failed.")
+                    .toStdString();
+                return asCacheEntry(std::move(dummy));
+        } catch (const mtx::crypto::olm_exception &e) {
+                nhlog::crypto()->critical("failed to decrypt message with index ({}, {}, {}): {}",
+                                          index.room_id,
+                                          index.session_id,
+                                          index.sender_key,
+                                          e.what());
+                dummy.content.body =
+                  tr("-- Decryption Error (%1) --",
+                     "Placeholder, when the message can't be decrypted. In this case, the Olm "
+                     "decrytion returned an error, which is passed as %1.")
+                    .arg(e.what())
+                    .toStdString();
+                return asCacheEntry(std::move(dummy));
+        }
+
+        // Add missing fields for the event.
+        json body                = json::parse(msg_str);
+        body["event_id"]         = e.event_id;
+        body["sender"]           = e.sender;
+        body["origin_server_ts"] = e.origin_server_ts;
+        body["unsigned"]         = e.unsigned_data;
+
+        // relations are unencrypted in content...
+        if (json old_ev = e; old_ev["content"].count("m.relates_to") != 0)
+                body["content"]["m.relates_to"] = old_ev["content"]["m.relates_to"];
+
+        json event_array = json::array();
+        event_array.push_back(body);
+
+        std::vector<mtx::events::collections::TimelineEvents> temp_events;
+        mtx::responses::utils::parse_timeline_events(event_array, temp_events);
+
+        if (temp_events.size() == 1) {
+                auto encInfo = mtx::accessors::file(temp_events[0]);
+
+                if (encInfo)
+                        emit newEncryptedImage(encInfo.value());
+
+                return asCacheEntry(std::move(temp_events[0]));
+        }
+
+        dummy.content.body =
+          tr("-- Encrypted Event (Unknown event type) --",
+             "Placeholder, when the message was decrypted, but we couldn't parse it, because "
+             "Nheko/mtxclient don't support that event type yet.")
+            .toStdString();
+        return asCacheEntry(std::move(dummy));
+}
+
+mtx::events::collections::TimelineEvents *
+EventStore::event(std::string_view id, std::string_view related_to, bool decrypt)
+{
+        if (this->thread() != QThread::currentThread())
+                nhlog::db()->warn("{} called from a different thread!", __func__);
+
+        if (id.empty())
+                return nullptr;
+
+        IdIndex index{room_id_, std::string(id.data(), id.size())};
+
+        auto event_ptr = events_by_id_.object(index);
+        if (!event_ptr) {
+                auto event = cache::client()->getEvent(room_id_, index.id);
+                if (!event) {
+                        // fetch
+                        (void)related_to;
+                        return nullptr;
+                }
+                event_ptr = new mtx::events::collections::TimelineEvents(std::move(event->data));
+                events_by_id_.insert(index, event_ptr);
+        }
+
+        if (decrypt)
+                if (auto encrypted =
+                      std::get_if<mtx::events::EncryptedEvent<mtx::events::msg::Encrypted>>(
+                        event_ptr))
+                        return decryptEvent(index, *encrypted);
+
+        return event_ptr;
+}

src/timeline/EventStore.h

@@ -0,0 +1,98 @@
+#pragma once
+
+#include <limits>
+#include <string>
+
+#include <QCache>
+#include <QObject>
+#include <qhashfunctions.h>
+
+#include <mtx/events/collections.hpp>
+#include <mtx/responses/sync.hpp>
+
+class EventStore : public QObject
+{
+        Q_OBJECT
+
+public:
+        EventStore(std::string room_id, QObject *parent);
+
+        struct Index
+        {
+                std::string room;
+                int64_t idx;
+
+                friend uint qHash(const Index &i, uint seed = 0) noexcept
+                {
+                        QtPrivate::QHashCombine hash;
+                        seed = hash(seed, QByteArray::fromRawData(i.room.data(), i.room.size()));
+                        seed = hash(seed, i.idx);
+                        return seed;
+                }
+
+                friend bool operator==(const Index &a, const Index &b) noexcept
+                {
+                        return a.idx == b.idx && a.room == b.room;
+                }
+        };
+        struct IdIndex
+        {
+                std::string room, id;
+
+                friend uint qHash(const IdIndex &i, uint seed = 0) noexcept
+                {
+                        QtPrivate::QHashCombine hash;
+                        seed = hash(seed, QByteArray::fromRawData(i.room.data(), i.room.size()));
+                        seed = hash(seed, QByteArray::fromRawData(i.id.data(), i.id.size()));
+                        return seed;
+                }
+
+                friend bool operator==(const IdIndex &a, const IdIndex &b) noexcept
+                {
+                        return a.id == b.id && a.room == b.room;
+                }
+        };
+
+        void fetchMore();
+        void handleSync(const mtx::responses::Timeline &events);
+
+        // optionally returns the event or nullptr and fetches it, after which it emits a
+        // relatedFetched event
+        mtx::events::collections::TimelineEvents *event(std::string_view id,
+                                                        std::string_view related_to,
+                                                        bool decrypt = true);
+        // always returns a proper event as long as the idx is valid
+        mtx::events::collections::TimelineEvents *event(int idx, bool decrypt = true);
+
+        int size() const
+        {
+                return last != std::numeric_limits<int64_t>::max()
+                         ? static_cast<int>(last - first) + 1
+                         : 0;
+        }
+        int toExternalIdx(int64_t idx) const { return static_cast<int>(idx - first); }
+        int64_t toInternalIdx(int idx) const { return first + idx; }
+
+        std::optional<int> idToIndex(std::string_view id) const;
+        std::optional<std::string> indexToId(int idx) const;
+
+signals:
+        void beginInsertRows(int from, int to);
+        void endInsertRows();
+        void dataChanged(int from, int to);
+        void newEncryptedImage(mtx::crypto::EncryptedFile encryptionInfo);
+
+private:
+        mtx::events::collections::TimelineEvents *decryptEvent(
+          const IdIndex &idx,
+          const mtx::events::EncryptedEvent<mtx::events::msg::Encrypted> &e);
+
+        std::string room_id_;
+
+        int64_t first = std::numeric_limits<int64_t>::max(),
+                last  = std::numeric_limits<int64_t>::max();
+
+        static QCache<IdIndex, mtx::events::collections::TimelineEvents> decryptedEvents_;
+        static QCache<Index, mtx::events::collections::TimelineEvents> events_;
+        static QCache<IdIndex, mtx::events::collections::TimelineEvents> events_by_id_;
+};

src/timeline/TimelineModel.cpp

@@ -141,6 +141,7 @@ toRoomEventTypeString(const mtx::events::collections::TimelineEvents &event)
 
 TimelineModel::TimelineModel(TimelineViewManager *manager, QString room_id, QObject *parent)
   : QAbstractListModel(parent)
+  , events(room_id.toStdString(), this)
   , room_id_(room_id)
   , manager_(manager)
 {
@@ -165,41 +166,41 @@ TimelineModel::TimelineModel(TimelineViewManager *manager, QString room_id, QObj
           this,
           [this](QString txn_id, QString event_id) {
                   pending.removeOne(txn_id);
+                  (void)event_id;
+                  // auto ev = events.value(txn_id);
 
-                  auto ev = events.value(txn_id);
+                  // if (auto reaction =
+                  //      std::get_if<mtx::events::RoomEvent<mtx::events::msg::Reaction>>(&ev)) {
+                  //        QString reactedTo =
+                  //          QString::fromStdString(reaction->content.relates_to.event_id);
+                  //        auto &rModel = reactions[reactedTo];
+                  //        rModel.removeReaction(*reaction);
+                  //        auto rCopy     = *reaction;
+                  //        rCopy.event_id = event_id.toStdString();
+                  //        rModel.addReaction(room_id_.toStdString(), rCopy);
+                  //}
 
-                  if (auto reaction =
+                  // int idx = idToIndex(txn_id);
-                        std::get_if<mtx::events::RoomEvent<mtx::events::msg::Reaction>>(&ev)) {
+                  // if (idx < 0) {
-                          QString reactedTo =
+                  //        // transaction already received via sync
-                            QString::fromStdString(reaction->content.relates_to.event_id);
+                  //        return;
-                          auto &rModel = reactions[reactedTo];
+                  //}
-                          rModel.removeReaction(*reaction);
+                  // eventOrder[idx] = event_id;
-                          auto rCopy     = *reaction;
+                  // ev              = std::visit(
-                          rCopy.event_id = event_id.toStdString();
+                  //  [event_id](const auto &e) -> mtx::events::collections::TimelineEvents {
-                          rModel.addReaction(room_id_.toStdString(), rCopy);
+                  //          auto eventCopy     = e;
-                  }
+                  //          eventCopy.event_id = event_id.toStdString();
+                  //          return eventCopy;
+                  //  },
+                  //  ev);
 
-                  int idx = idToIndex(txn_id);
+                  // events.remove(txn_id);
-                  if (idx < 0) {
+                  // events.insert(event_id, ev);
-                          // transaction already received via sync
-                          return;
-                  }
-                  eventOrder[idx] = event_id;
-                  ev              = std::visit(
-                    [event_id](const auto &e) -> mtx::events::collections::TimelineEvents {
-                            auto eventCopy     = e;
-                            eventCopy.event_id = event_id.toStdString();
-                            return eventCopy;
-                    },
-                    ev);
 
-                  events.remove(txn_id);
+                  //// mark our messages as read
-                  events.insert(event_id, ev);
+                  // readEvent(event_id.toStdString());
 
-                  // mark our messages as read
+                  // emit dataChanged(index(idx, 0), index(idx, 0));
-                  readEvent(event_id.toStdString());
-
-                  emit dataChanged(index(idx, 0), index(idx, 0));
 
                   if (pending.size() > 0)
                           emit nextPendingMessage();
@@ -224,16 +225,24 @@ TimelineModel::TimelineModel(TimelineViewManager *manager, QString room_id, QObj
                 Qt::QueuedConnection);
 
         connect(
+          &events,
+          &EventStore::dataChanged,
           this,
-          &TimelineModel::eventFetched,
+          [this](int from, int to) {
-          this,
+                  emit dataChanged(index(events.size() - to, 0), index(events.size() - from, 0));
-          [this](QString requestingEvent, mtx::events::collections::TimelineEvents event) {
-                  events.insert(QString::fromStdString(mtx::accessors::event_id(event)), event);
-                  auto idx = idToIndex(requestingEvent);
-                  if (idx >= 0)
-                          emit dataChanged(index(idx, 0), index(idx, 0));
           },
           Qt::QueuedConnection);
+
+        connect(&events, &EventStore::beginInsertRows, this, [this](int from, int to) {
+                nhlog::ui()->info("begin insert from {} to {}",
+                                  events.size() - to + (to - from),
+                                  events.size() - from + (to - from));
+                beginInsertRows(QModelIndex(),
+                                events.size() - to + (to - from),
+                                events.size() - from + (to - from));
+        });
+        connect(&events, &EventStore::endInsertRows, this, [this]() { endInsertRows(); });
+        connect(&events, &EventStore::newEncryptedImage, this, &TimelineModel::newEncryptedImage);
 }
 
 QHash<int, QByteArray>
@@ -274,28 +283,22 @@ int
 TimelineModel::rowCount(const QModelIndex &parent) const
 {
         Q_UNUSED(parent);
-        return (int)this->eventOrder.size();
+        return this->events.size() + static_cast<int>(pending.size());
 }
 
 QVariantMap
-TimelineModel::getDump(QString eventId) const
+TimelineModel::getDump(QString eventId, QString relatedTo) const
 {
-        if (events.contains(eventId))
+        if (auto event = events.event(eventId.toStdString(), relatedTo.toStdString()))
-                return data(eventId, Dump).toMap();
+                return data(*event, Dump).toMap();
         return {};
 }
 
 QVariant
-TimelineModel::data(const QString &id, int role) const
+TimelineModel::data(const mtx::events::collections::TimelineEvents &event, int role) const
 {
         using namespace mtx::accessors;
-        namespace acc                                  = mtx::accessors;
+        namespace acc = mtx::accessors;
-        mtx::events::collections::TimelineEvents event = events.value(id);
-
-        if (auto e =
-              std::get_if<mtx::events::EncryptedEvent<mtx::events::msg::Encrypted>>(&event)) {
-                event = decryptEvent(*e).event;
-        }
 
         switch (role) {
         case UserId:
@@ -381,8 +384,9 @@ TimelineModel::data(const QString &id, int role) const
                 return QVariant(prop > 0 ? prop : 1.);
         }
         case Id:
-                return id;
+                return QVariant(QString::fromStdString(event_id(event)));
         case State: {
+                auto id             = QString::fromStdString(event_id(event));
                 auto containsOthers = [](const auto &vec) {
                         for (const auto &e : vec)
                                 if (e.second != http::client()->user_id().to_string())
@@ -401,19 +405,22 @@ TimelineModel::data(const QString &id, int role) const
                         return qml_mtx_events::Received;
         }
         case IsEncrypted: {
-                return std::holds_alternative<
+                //        return std::holds_alternative<
-                  mtx::events::EncryptedEvent<mtx::events::msg::Encrypted>>(events[id]);
+                //          mtx::events::EncryptedEvent<mtx::events::msg::Encrypted>>(events[id]);
+                return false;
         }
         case IsRoomEncrypted: {
                 return cache::isRoomEncrypted(room_id_.toStdString());
         }
         case ReplyTo:
                 return QVariant(QString::fromStdString(in_reply_to_event(event)));
-        case Reactions:
+        case Reactions: {
+                auto id = QString::fromStdString(event_id(event));
                 if (reactions.count(id))
                         return QVariant::fromValue((QObject *)&reactions.at(id));
                 else
                         return {};
+        }
         case RoomId:
                 return QVariant(room_id_);
         case RoomName:
@@ -425,30 +432,31 @@ TimelineModel::data(const QString &id, int role) const
                 auto names = roleNames();
 
                 // m.insert(names[Section], data(id, static_cast<int>(Section)));
-                m.insert(names[Type], data(id, static_cast<int>(Type)));
+                m.insert(names[Type], data(event, static_cast<int>(Type)));
-                m.insert(names[TypeString], data(id, static_cast<int>(TypeString)));
+                m.insert(names[TypeString], data(event, static_cast<int>(TypeString)));
-                m.insert(names[IsOnlyEmoji], data(id, static_cast<int>(IsOnlyEmoji)));
+                m.insert(names[IsOnlyEmoji], data(event, static_cast<int>(IsOnlyEmoji)));
-                m.insert(names[Body], data(id, static_cast<int>(Body)));
+                m.insert(names[Body], data(event, static_cast<int>(Body)));
-                m.insert(names[FormattedBody], data(id, static_cast<int>(FormattedBody)));
+                m.insert(names[FormattedBody], data(event, static_cast<int>(FormattedBody)));
-                m.insert(names[UserId], data(id, static_cast<int>(UserId)));
+                m.insert(names[UserId], data(event, static_cast<int>(UserId)));
-                m.insert(names[UserName], data(id, static_cast<int>(UserName)));
+                m.insert(names[UserName], data(event, static_cast<int>(UserName)));
-                m.insert(names[Timestamp], data(id, static_cast<int>(Timestamp)));
+                m.insert(names[Timestamp], data(event, static_cast<int>(Timestamp)));
-                m.insert(names[Url], data(id, static_cast<int>(Url)));
+                m.insert(names[Url], data(event, static_cast<int>(Url)));
-                m.insert(names[ThumbnailUrl], data(id, static_cast<int>(ThumbnailUrl)));
+                m.insert(names[ThumbnailUrl], data(event, static_cast<int>(ThumbnailUrl)));
-                m.insert(names[Blurhash], data(id, static_cast<int>(Blurhash)));
+                m.insert(names[Blurhash], data(event, static_cast<int>(Blurhash)));
-                m.insert(names[Filename], data(id, static_cast<int>(Filename)));
+                m.insert(names[Filename], data(event, static_cast<int>(Filename)));
-                m.insert(names[Filesize], data(id, static_cast<int>(Filesize)));
+                m.insert(names[Filesize], data(event, static_cast<int>(Filesize)));
-                m.insert(names[MimeType], data(id, static_cast<int>(MimeType)));
+                m.insert(names[MimeType], data(event, static_cast<int>(MimeType)));
-                m.insert(names[Height], data(id, static_cast<int>(Height)));
+                m.insert(names[Height], data(event, static_cast<int>(Height)));
-                m.insert(names[Width], data(id, static_cast<int>(Width)));
+                m.insert(names[Width], data(event, static_cast<int>(Width)));
-                m.insert(names[ProportionalHeight], data(id, static_cast<int>(ProportionalHeight)));
+                m.insert(names[ProportionalHeight],
-                m.insert(names[Id], data(id, static_cast<int>(Id)));
+                         data(event, static_cast<int>(ProportionalHeight)));
-                m.insert(names[State], data(id, static_cast<int>(State)));
+                m.insert(names[Id], data(event, static_cast<int>(Id)));
-                m.insert(names[IsEncrypted], data(id, static_cast<int>(IsEncrypted)));
+                m.insert(names[State], data(event, static_cast<int>(State)));
-                m.insert(names[IsRoomEncrypted], data(id, static_cast<int>(IsRoomEncrypted)));
+                m.insert(names[IsEncrypted], data(event, static_cast<int>(IsEncrypted)));
-                m.insert(names[ReplyTo], data(id, static_cast<int>(ReplyTo)));
+                m.insert(names[IsRoomEncrypted], data(event, static_cast<int>(IsRoomEncrypted)));
-                m.insert(names[RoomName], data(id, static_cast<int>(RoomName)));
+                m.insert(names[ReplyTo], data(event, static_cast<int>(ReplyTo)));
-                m.insert(names[RoomTopic], data(id, static_cast<int>(RoomTopic)));
+                m.insert(names[RoomName], data(event, static_cast<int>(RoomName)));
+                m.insert(names[RoomTopic], data(event, static_cast<int>(RoomTopic)));
 
                 return QVariant(m);
         }
@@ -462,29 +470,33 @@ TimelineModel::data(const QModelIndex &index, int role) const
 {
         using namespace mtx::accessors;
         namespace acc = mtx::accessors;
-        if (index.row() < 0 && index.row() >= (int)eventOrder.size())
+        if (index.row() < 0 && index.row() >= rowCount())
                 return QVariant();
 
-        QString id = eventOrder[index.row()];
+        auto event = events.event(rowCount() - index.row() - 1);
 
-        mtx::events::collections::TimelineEvents event = events.value(id);
+        if (!event)
+                return "";
 
         if (role == Section) {
-                QDateTime date = origin_server_ts(event);
+                QDateTime date = origin_server_ts(*event);
                 date.setTime(QTime());
 
-                std::string userId = acc::sender(event);
+                std::string userId = acc::sender(*event);
 
-                for (size_t r = index.row() + 1; r < eventOrder.size(); r++) {
+                for (int r = rowCount() - index.row(); r < events.size(); r++) {
-                        auto tempEv        = events.value(eventOrder[r]);
+                        auto tempEv = events.event(r);
-                        QDateTime prevDate = origin_server_ts(tempEv);
+                        if (!tempEv)
+                                break;
+
+                        QDateTime prevDate = origin_server_ts(*tempEv);
                         prevDate.setTime(QTime());
                         if (prevDate != date)
                                 return QString("%2 %1")
                                   .arg(date.toMSecsSinceEpoch())
                                   .arg(QString::fromStdString(userId));
 
-                        std::string prevUserId = acc::sender(tempEv);
+                        std::string prevUserId = acc::sender(*tempEv);
                         if (userId != prevUserId)
                                 break;
                 }
@@ -492,16 +504,16 @@ TimelineModel::data(const QModelIndex &index, int role) const
                 return QString("%1").arg(QString::fromStdString(userId));
         }
 
-        return data(id, role);
+        return data(*event, role);
 }
 
 bool
 TimelineModel::canFetchMore(const QModelIndex &) const
 {
-        if (eventOrder.empty())
+        if (!events.size())
                 return true;
         if (!std::holds_alternative<mtx::events::StateEvent<mtx::events::state::Create>>(
-              events[eventOrder.back()]))
+              *events.event(0)))
                 return true;
         else
 
@@ -562,13 +574,9 @@ TimelineModel::addEvents(const mtx::responses::Timeline &timeline)
         if (timeline.events.empty())
                 return;
 
-        std::vector<QString> ids = internalAddEvents(timeline.events);
+        internalAddEvents(timeline.events);
 
-        if (!ids.empty()) {
+        events.handleSync(timeline);
-                beginInsertRows(QModelIndex(), 0, static_cast<int>(ids.size() - 1));
-                this->eventOrder.insert(this->eventOrder.begin(), ids.rbegin(), ids.rend());
-                endInsertRows();
-        }
 
         if (!timeline.events.empty())
                 updateLastMessage();
@@ -613,21 +621,17 @@ isYourJoin(const mtx::events::Event<T> &)
 void
 TimelineModel::updateLastMessage()
 {
-        for (auto it = eventOrder.begin(); it != eventOrder.end(); ++it) {
+        for (auto it = events.size() - 1; it >= 0; --it) {
-                auto event = events.value(*it);
+                auto event = events.event(it, decryptDescription);
-                if (auto e = std::get_if<mtx::events::EncryptedEvent<mtx::events::msg::Encrypted>>(
+                if (!event)
-                      &event)) {
+                        continue;
-                        if (decryptDescription) {
-                                event = decryptEvent(*e).event;
-                        }
-                }
 
-                if (std::visit([](const auto &e) -> bool { return isYourJoin(e); }, event)) {
+                if (std::visit([](const auto &e) -> bool { return isYourJoin(e); }, *event)) {
-                        auto time   = mtx::accessors::origin_server_ts(event);
+                        auto time   = mtx::accessors::origin_server_ts(*event);
                         uint64_t ts = time.toMSecsSinceEpoch();
                         emit manager_->updateRoomsLastMessage(
                           room_id_,
-                          DescInfo{QString::fromStdString(mtx::accessors::event_id(event)),
+                          DescInfo{QString::fromStdString(mtx::accessors::event_id(*event)),
                                    QString::fromStdString(http::client()->user_id().to_string()),
                                    tr("You joined this room."),
                                    utils::descriptiveTime(time),
@@ -635,54 +639,34 @@ TimelineModel::updateLastMessage()
                                    time});
                         return;
                 }
-                if (!std::visit([](const auto &e) -> bool { return isMessage(e); }, event))
+                if (!std::visit([](const auto &e) -> bool { return isMessage(e); }, *event))
                         continue;
 
                 auto description = utils::getMessageDescription(
-                  event, QString::fromStdString(http::client()->user_id().to_string()), room_id_);
+                  *event, QString::fromStdString(http::client()->user_id().to_string()), room_id_);
                 emit manager_->updateRoomsLastMessage(room_id_, description);
                 return;
         }
 }
 
-std::vector<QString>
+void
 TimelineModel::internalAddEvents(
   const std::vector<mtx::events::collections::TimelineEvents> &timeline)
 {
-        std::vector<QString> ids;
         for (auto e : timeline) {
                 QString id = QString::fromStdString(mtx::accessors::event_id(e));
 
-                if (this->events.contains(id)) {
-                        this->events.insert(id, e);
-                        int idx = idToIndex(id);
-                        emit dataChanged(index(idx, 0), index(idx, 0));
-                        continue;
-                }
-
-                QString txid = QString::fromStdString(mtx::accessors::transaction_id(e));
-                if (this->pending.removeOne(txid)) {
-                        this->events.insert(id, e);
-                        this->events.remove(txid);
-                        int idx = idToIndex(txid);
-                        if (idx < 0) {
-                                nhlog::ui()->warn("Received index out of range");
-                                continue;
-                        }
-                        eventOrder[idx] = id;
-                        emit dataChanged(index(idx, 0), index(idx, 0));
-                        continue;
-                }
-
                 if (auto redaction =
                       std::get_if<mtx::events::RedactionEvent<mtx::events::msg::Redaction>>(&e)) {
                         QString redacts = QString::fromStdString(redaction->redacts);
-                        auto redacted   = std::find(eventOrder.begin(), eventOrder.end(), redacts);
 
-                        auto event = events.value(redacts);
+                        auto event = events.event(redaction->redacts, redaction->event_id);
+                        if (!event)
+                                continue;
+
                         if (auto reaction =
                               std::get_if<mtx::events::RoomEvent<mtx::events::msg::Reaction>>(
-                                &event)) {
+                                event)) {
                                 QString reactedTo =
                                   QString::fromStdString(reaction->content.relates_to.event_id);
                                 reactions[reactedTo].removeReaction(*reaction);
@@ -691,26 +675,6 @@ TimelineModel::internalAddEvents(
                                         emit dataChanged(index(idx, 0), index(idx, 0));
                         }
 
-                        if (redacted != eventOrder.end()) {
-                                auto redactedEvent = std::visit(
-                                  [](const auto &ev)
-                                    -> mtx::events::RoomEvent<mtx::events::msg::Redacted> {
-                                          mtx::events::RoomEvent<mtx::events::msg::Redacted>
-                                            replacement                = {};
-                                          replacement.event_id         = ev.event_id;
-                                          replacement.room_id          = ev.room_id;
-                                          replacement.sender           = ev.sender;
-                                          replacement.origin_server_ts = ev.origin_server_ts;
-                                          replacement.type             = ev.type;
-                                          return replacement;
-                                  },
-                                  e);
-                                events.insert(redacts, redactedEvent);
-
-                                int row = (int)std::distance(eventOrder.begin(), redacted);
-                                emit dataChanged(index(row, 0), index(row, 0));
-                        }
-
                         continue; // don't insert redaction into timeline
                 }
 
@@ -718,14 +682,13 @@ TimelineModel::internalAddEvents(
                       std::get_if<mtx::events::RoomEvent<mtx::events::msg::Reaction>>(&e)) {
                         QString reactedTo =
                           QString::fromStdString(reaction->content.relates_to.event_id);
-                        events.insert(id, e);
 
-                        // remove local echo
+                        // // remove local echo
-                        if (!txid.isEmpty()) {
+                        // if (!txid.isEmpty()) {
-                                auto rCopy     = *reaction;
+                        //         auto rCopy     = *reaction;
-                                rCopy.event_id = txid.toStdString();
+                        //         rCopy.event_id = txid.toStdString();
-                                reactions[reactedTo].removeReaction(rCopy);
+                        //         reactions[reactedTo].removeReaction(rCopy);
-                        }
+                        // }
 
                         reactions[reactedTo].addReaction(room_id_.toStdString(), *reaction);
                         int idx = idToIndex(reactedTo);
@@ -734,40 +697,27 @@ TimelineModel::internalAddEvents(
                         continue; // don't insert reaction into timeline
                 }
 
-                if (auto event =
+                // auto replyTo  = mtx::accessors::in_reply_to_event(e);
-                      std::get_if<mtx::events::EncryptedEvent<mtx::events::msg::Encrypted>>(&e)) {
+                // auto qReplyTo = QString::fromStdString(replyTo);
-                        auto e_      = decryptEvent(*event).event;
+                // if (!replyTo.empty() && !events.contains(qReplyTo)) {
-                        auto encInfo = mtx::accessors::file(e_);
+                //        http::client()->get_event(
-
+                //          this->room_id_.toStdString(),
-                        if (encInfo)
+                //          replyTo,
-                                emit newEncryptedImage(encInfo.value());
+                //          [this, id, replyTo](
-                }
+                //            const mtx::events::collections::TimelineEvents &timeline,
-
+                //            mtx::http::RequestErr err) {
-                this->events.insert(id, e);
+                //                  if (err) {
-                ids.push_back(id);
+                //                          nhlog::net()->error(
-
+                //                            "Failed to retrieve event with id {}, which was "
-                auto replyTo  = mtx::accessors::in_reply_to_event(e);
+                //                            "requested to show the replyTo for event {}",
-                auto qReplyTo = QString::fromStdString(replyTo);
+                //                            replyTo,
-                if (!replyTo.empty() && !events.contains(qReplyTo)) {
+                //                            id.toStdString());
-                        http::client()->get_event(
+                //                          return;
-                          this->room_id_.toStdString(),
+                //                  }
-                          replyTo,
+                //                  emit eventFetched(id, timeline);
-                          [this, id, replyTo](
+                //          });
-                            const mtx::events::collections::TimelineEvents &timeline,
+                //}
-                            mtx::http::RequestErr err) {
-                                  if (err) {
-                                          nhlog::net()->error(
-                                            "Failed to retrieve event with id {}, which was "
-                                            "requested to show the replyTo for event {}",
-                                            replyTo,
-                                            id.toStdString());
-                                          return;
-                                  }
-                                  emit eventFetched(id, timeline);
-                          });
-                }
         }
-        return ids;
 }
 
 void
@@ -798,22 +748,23 @@ TimelineModel::readEvent(const std::string &id)
 void
 TimelineModel::addBackwardsEvents(const mtx::responses::Messages &msgs)
 {
-        std::vector<QString> ids = internalAddEvents(msgs.chunk);
+        (void)msgs;
+        // std::vector<QString> ids = internalAddEvents(msgs.chunk);
 
-        if (!ids.empty()) {
+        // if (!ids.empty()) {
-                beginInsertRows(QModelIndex(),
+        //        beginInsertRows(QModelIndex(),
-                                static_cast<int>(this->eventOrder.size()),
+        //                        static_cast<int>(this->eventOrder.size()),
-                                static_cast<int>(this->eventOrder.size() + ids.size() - 1));
+        //                        static_cast<int>(this->eventOrder.size() + ids.size() - 1));
-                this->eventOrder.insert(this->eventOrder.end(), ids.begin(), ids.end());
+        //        this->eventOrder.insert(this->eventOrder.end(), ids.begin(), ids.end());
-                endInsertRows();
+        //        endInsertRows();
-        }
+        //}
 
-        prev_batch_token_ = QString::fromStdString(msgs.end);
+        // prev_batch_token_ = QString::fromStdString(msgs.end);
 
-        if (ids.empty() && !msgs.chunk.empty()) {
+        // if (ids.empty() && !msgs.chunk.empty()) {
-                // no visible events fetched, prevent loading from stopping
+        //        // no visible events fetched, prevent loading from stopping
-                fetchMore(QModelIndex());
+        //        fetchMore(QModelIndex());
-        }
+        //}
 }
 
 QString
@@ -852,7 +803,10 @@ TimelineModel::escapeEmoji(QString str) const
 void
 TimelineModel::viewRawMessage(QString id) const
 {
-        std::string ev = mtx::accessors::serialize_event(events.value(id)).dump(4);
+        auto e = events.event(id.toStdString(), "", false);
+        if (!e)
+                return;
+        std::string ev = mtx::accessors::serialize_event(*e).dump(4);
         auto dialog    = new dialogs::RawMessage(QString::fromStdString(ev));
         Q_UNUSED(dialog);
 }
@@ -860,13 +814,11 @@ TimelineModel::viewRawMessage(QString id) const
 void
 TimelineModel::viewDecryptedRawMessage(QString id) const
 {
-        auto event = events.value(id);
+        auto e = events.event(id.toStdString(), "");
-        if (auto e =
+        if (!e)
-              std::get_if<mtx::events::EncryptedEvent<mtx::events::msg::Encrypted>>(&event)) {
+                return;
-                event = decryptEvent(*e).event;
-        }
 
-        std::string ev = mtx::accessors::serialize_event(event).dump(4);
+        std::string ev = mtx::accessors::serialize_event(*e).dump(4);
         auto dialog    = new dialogs::RawMessage(QString::fromStdString(ev));
         Q_UNUSED(dialog);
 }
@@ -877,114 +829,6 @@ TimelineModel::openUserProfile(QString userid) const
         MainWindow::instance()->openUserProfile(userid, room_id_);
 }
 
-DecryptionResult
-TimelineModel::decryptEvent(const mtx::events::EncryptedEvent<mtx::events::msg::Encrypted> &e) const
-{
-        static QCache<std::string, DecryptionResult> decryptedEvents{300};
-
-        if (auto cachedEvent = decryptedEvents.object(e.event_id))
-                return *cachedEvent;
-
-        MegolmSessionIndex index;
-        index.room_id    = room_id_.toStdString();
-        index.session_id = e.content.session_id;
-        index.sender_key = e.content.sender_key;
-
-        mtx::events::RoomEvent<mtx::events::msg::Notice> dummy;
-        dummy.origin_server_ts = e.origin_server_ts;
-        dummy.event_id         = e.event_id;
-        dummy.sender           = e.sender;
-        dummy.content.body =
-          tr("-- Encrypted Event (No keys found for decryption) --",
-             "Placeholder, when the message was not decrypted yet or can't be decrypted.")
-            .toStdString();
-
-        try {
-                if (!cache::inboundMegolmSessionExists(index)) {
-                        nhlog::crypto()->info("Could not find inbound megolm session ({}, {}, {})",
-                                              index.room_id,
-                                              index.session_id,
-                                              e.sender);
-                        // TODO: request megolm session_id & session_key from the sender.
-                        decryptedEvents.insert(
-                          dummy.event_id, new DecryptionResult{dummy, false}, 1);
-                        return {dummy, false};
-                }
-        } catch (const lmdb::error &e) {
-                nhlog::db()->critical("failed to check megolm session's existence: {}", e.what());
-                dummy.content.body = tr("-- Decryption Error (failed to communicate with DB) --",
-                                        "Placeholder, when the message can't be decrypted, because "
-                                        "the DB access failed when trying to lookup the session.")
-                                       .toStdString();
-                decryptedEvents.insert(dummy.event_id, new DecryptionResult{dummy, false}, 1);
-                return {dummy, false};
-        }
-
-        std::string msg_str;
-        try {
-                auto session = cache::getInboundMegolmSession(index);
-                auto res     = olm::client()->decrypt_group_message(session, e.content.ciphertext);
-                msg_str      = std::string((char *)res.data.data(), res.data.size());
-        } catch (const lmdb::error &e) {
-                nhlog::db()->critical("failed to retrieve megolm session with index ({}, {}, {})",
-                                      index.room_id,
-                                      index.session_id,
-                                      index.sender_key,
-                                      e.what());
-                dummy.content.body =
-                  tr("-- Decryption Error (failed to retrieve megolm keys from db) --",
-                     "Placeholder, when the message can't be decrypted, because the DB access "
-                     "failed.")
-                    .toStdString();
-                decryptedEvents.insert(dummy.event_id, new DecryptionResult{dummy, false}, 1);
-                return {dummy, false};
-        } catch (const mtx::crypto::olm_exception &e) {
-                nhlog::crypto()->critical("failed to decrypt message with index ({}, {}, {}): {}",
-                                          index.room_id,
-                                          index.session_id,
-                                          index.sender_key,
-                                          e.what());
-                dummy.content.body =
-                  tr("-- Decryption Error (%1) --",
-                     "Placeholder, when the message can't be decrypted. In this case, the Olm "
-                     "decrytion returned an error, which is passed ad %1.")
-                    .arg(e.what())
-                    .toStdString();
-                decryptedEvents.insert(dummy.event_id, new DecryptionResult{dummy, false}, 1);
-                return {dummy, false};
-        }
-
-        // Add missing fields for the event.
-        json body                = json::parse(msg_str);
-        body["event_id"]         = e.event_id;
-        body["sender"]           = e.sender;
-        body["origin_server_ts"] = e.origin_server_ts;
-        body["unsigned"]         = e.unsigned_data;
-
-        // relations are unencrypted in content...
-        if (json old_ev = e; old_ev["content"].count("m.relates_to") != 0)
-                body["content"]["m.relates_to"] = old_ev["content"]["m.relates_to"];
-
-        json event_array = json::array();
-        event_array.push_back(body);
-
-        std::vector<mtx::events::collections::TimelineEvents> temp_events;
-        mtx::responses::utils::parse_timeline_events(event_array, temp_events);
-
-        if (temp_events.size() == 1) {
-                decryptedEvents.insert(e.event_id, new DecryptionResult{temp_events[0], true}, 1);
-                return {temp_events[0], true};
-        }
-
-        dummy.content.body =
-          tr("-- Encrypted Event (Unknown event type) --",
-             "Placeholder, when the message was decrypted, but we couldn't parse it, because "
-             "Nheko/mtxclient don't support that event type yet.")
-            .toStdString();
-        decryptedEvents.insert(dummy.event_id, new DecryptionResult{dummy, false}, 1);
-        return {dummy, false};
-}
-
 void
 TimelineModel::replyAction(QString id)
 {
@@ -995,23 +839,18 @@ TimelineModel::replyAction(QString id)
 RelatedInfo
 TimelineModel::relatedInfo(QString id)
 {
-        if (!events.contains(id))
+        auto event = events.event(id.toStdString(), "");
+        if (!event)
                 return {};
 
-        auto event = events.value(id);
-        if (auto e =
-              std::get_if<mtx::events::EncryptedEvent<mtx::events::msg::Encrypted>>(&event)) {
-                event = decryptEvent(*e).event;
-        }
-
         RelatedInfo related   = {};
-        related.quoted_user   = QString::fromStdString(mtx::accessors::sender(event));
+        related.quoted_user   = QString::fromStdString(mtx::accessors::sender(*event));
-        related.related_event = mtx::accessors::event_id(event);
+        related.related_event = mtx::accessors::event_id(*event);
-        related.type          = mtx::accessors::msg_type(event);
+        related.type          = mtx::accessors::msg_type(*event);
 
         // get body, strip reply fallback, then transform the event to text, if it is a media event
         // etc
-        related.quoted_body = QString::fromStdString(mtx::accessors::body(event));
+        related.quoted_body = QString::fromStdString(mtx::accessors::body(*event));
         QRegularExpression plainQuote("^>.*?$\n?", QRegularExpression::MultilineOption);
         while (related.quoted_body.startsWith(">"))
                 related.quoted_body.remove(plainQuote);
@@ -1020,7 +859,7 @@ TimelineModel::relatedInfo(QString id)
         related.quoted_body = utils::getQuoteBody(related);
 
         // get quoted body and strip reply fallback
-        related.quoted_formatted_body = mtx::accessors::formattedBodyWithFallback(event);
+        related.quoted_formatted_body = mtx::accessors::formattedBodyWithFallback(*event);
         related.quoted_formatted_body.remove(QRegularExpression(
           "<mx-reply>.*</mx-reply>", QRegularExpression::DotMatchesEverythingOption));
         related.room = room_id_;
@@ -1058,18 +897,19 @@ TimelineModel::idToIndex(QString id) const
 {
         if (id.isEmpty())
                 return -1;
-        for (int i = 0; i < (int)eventOrder.size(); i++)
+
-                if (id == eventOrder[i])
+        auto idx = events.idToIndex(id.toStdString());
-                        return i;
+        if (idx)
-        return -1;
+                return events.size() - *idx;
+        else
+                return -1;
 }
 
 QString
 TimelineModel::indexToId(int index) const
 {
-        if (index < 0 || index >= (int)eventOrder.size())
+        auto id = events.indexToId(events.size() - index);
-                return "";
+        return id ? QString::fromStdString(*id) : "";
-        return eventOrder[index];
 }
 
 // Note: this will only be called for our messages
@@ -1477,58 +1317,56 @@ struct SendMessageVisitor
 void
 TimelineModel::processOnePendingMessage()
 {
-        if (pending.isEmpty())
+        // if (pending.isEmpty())
-                return;
+        //        return;
 
-        QString txn_id_qstr = pending.first();
+        // QString txn_id_qstr = pending.first();
 
-        auto event = events.value(txn_id_qstr);
+        // auto event = events.value(txn_id_qstr);
-        std::visit(SendMessageVisitor{txn_id_qstr, this}, event);
+        // std::visit(SendMessageVisitor{txn_id_qstr, this}, event);
 }
 
 void
 TimelineModel::addPendingMessage(mtx::events::collections::TimelineEvents event)
 {
-        std::visit(
+        (void)event;
-          [](auto &msg) {
+        // std::visit(
-                  msg.type             = mtx::events::EventType::RoomMessage;
+        //  [](auto &msg) {
-                  msg.event_id         = http::client()->generate_txn_id();
+        //          msg.type             = mtx::events::EventType::RoomMessage;
-                  msg.sender           = http::client()->user_id().to_string();
+        //          msg.event_id         = http::client()->generate_txn_id();
-                  msg.origin_server_ts = QDateTime::currentMSecsSinceEpoch();
+        //          msg.sender           = http::client()->user_id().to_string();
-          },
+        //          msg.origin_server_ts = QDateTime::currentMSecsSinceEpoch();
-          event);
+        //  },
+        //  event);
 
-        internalAddEvents({event});
+        // internalAddEvents({event});
 
-        QString txn_id_qstr = QString::fromStdString(mtx::accessors::event_id(event));
+        // QString txn_id_qstr = QString::fromStdString(mtx::accessors::event_id(event));
-        pending.push_back(txn_id_qstr);
+        // pending.push_back(txn_id_qstr);
-        if (!std::get_if<mtx::events::RoomEvent<mtx::events::msg::Reaction>>(&event)) {
+        // if (!std::get_if<mtx::events::RoomEvent<mtx::events::msg::Reaction>>(&event)) {
-                beginInsertRows(QModelIndex(), 0, 0);
+        //        beginInsertRows(QModelIndex(), 0, 0);
-                this->eventOrder.insert(this->eventOrder.begin(), txn_id_qstr);
+        //        this->eventOrder.insert(this->eventOrder.begin(), txn_id_qstr);
-                endInsertRows();
+        //        endInsertRows();
-        }
+        //}
-        updateLastMessage();
+        // updateLastMessage();
 
-        emit nextPendingMessage();
+        // emit nextPendingMessage();
 }
 
 bool
 TimelineModel::saveMedia(QString eventId) const
 {
-        mtx::events::collections::TimelineEvents event = events.value(eventId);
+        mtx::events::collections::TimelineEvents *event = events.event(eventId.toStdString(), "");
+        if (!event)
+                return false;
 
-        if (auto e =
+        QString mxcUrl           = QString::fromStdString(mtx::accessors::url(*event));
-              std::get_if<mtx::events::EncryptedEvent<mtx::events::msg::Encrypted>>(&event)) {
+        QString originalFilename = QString::fromStdString(mtx::accessors::filename(*event));
-                event = decryptEvent(*e).event;
+        QString mimeType         = QString::fromStdString(mtx::accessors::mimetype(*event));
-        }
 
-        QString mxcUrl           = QString::fromStdString(mtx::accessors::url(event));
+        auto encryptionInfo = mtx::accessors::file(*event);
-        QString originalFilename = QString::fromStdString(mtx::accessors::filename(event));
-        QString mimeType         = QString::fromStdString(mtx::accessors::mimetype(event));
 
-        auto encryptionInfo = mtx::accessors::file(event);
+        qml_mtx_events::EventType eventType = toRoomEventType(*event);
-
-        qml_mtx_events::EventType eventType = toRoomEventType(event);
 
         QString dialogTitle;
         if (eventType == qml_mtx_events::EventType::ImageMessage) {
@@ -1593,18 +1431,15 @@ TimelineModel::saveMedia(QString eventId) const
 void
 TimelineModel::cacheMedia(QString eventId)
 {
-        mtx::events::collections::TimelineEvents event = events.value(eventId);
+        mtx::events::collections::TimelineEvents *event = events.event(eventId.toStdString(), "");
+        if (!event)
+                return;
 
-        if (auto e =
+        QString mxcUrl           = QString::fromStdString(mtx::accessors::url(*event));
-              std::get_if<mtx::events::EncryptedEvent<mtx::events::msg::Encrypted>>(&event)) {
+        QString originalFilename = QString::fromStdString(mtx::accessors::filename(*event));
-                event = decryptEvent(*e).event;
+        QString mimeType         = QString::fromStdString(mtx::accessors::mimetype(*event));
-        }
 
-        QString mxcUrl           = QString::fromStdString(mtx::accessors::url(event));
+        auto encryptionInfo = mtx::accessors::file(*event);
-        QString originalFilename = QString::fromStdString(mtx::accessors::filename(event));
-        QString mimeType         = QString::fromStdString(mtx::accessors::mimetype(event));
-
-        auto encryptionInfo = mtx::accessors::file(event);
 
         // If the message is a link to a non mxcUrl, don't download it
         if (!mxcUrl.startsWith("mxc://")) {
@@ -1725,11 +1560,11 @@ TimelineModel::formatTypingUsers(const std::vector<QString> &users, QColor bg)
 QString
 TimelineModel::formatJoinRuleEvent(QString id)
 {
-        if (!events.contains(id))
+        mtx::events::collections::TimelineEvents *e = events.event(id.toStdString(), "");
+        if (!e)
                 return "";
 
-        auto event =
+        auto event = std::get_if<mtx::events::StateEvent<mtx::events::state::JoinRules>>(e);
-          std::get_if<mtx::events::StateEvent<mtx::events::state::JoinRules>>(&events[id]);
         if (!event)
                 return "";
 
@@ -1750,11 +1585,11 @@ TimelineModel::formatJoinRuleEvent(QString id)
 QString
 TimelineModel::formatGuestAccessEvent(QString id)
 {
-        if (!events.contains(id))
+        mtx::events::collections::TimelineEvents *e = events.event(id.toStdString(), "");
+        if (!e)
                 return "";
 
-        auto event =
+        auto event = std::get_if<mtx::events::StateEvent<mtx::events::state::GuestAccess>>(e);
-          std::get_if<mtx::events::StateEvent<mtx::events::state::GuestAccess>>(&events[id]);
         if (!event)
                 return "";
 
@@ -1774,11 +1609,11 @@ TimelineModel::formatGuestAccessEvent(QString id)
 QString
 TimelineModel::formatHistoryVisibilityEvent(QString id)
 {
-        if (!events.contains(id))
+        mtx::events::collections::TimelineEvents *e = events.event(id.toStdString(), "");
+        if (!e)
                 return "";
 
-        auto event =
+        auto event = std::get_if<mtx::events::StateEvent<mtx::events::state::HistoryVisibility>>(e);
-          std::get_if<mtx::events::StateEvent<mtx::events::state::HistoryVisibility>>(&events[id]);
 
         if (!event)
                 return "";
@@ -1808,11 +1643,11 @@ TimelineModel::formatHistoryVisibilityEvent(QString id)
 QString
 TimelineModel::formatPowerLevelEvent(QString id)
 {
-        if (!events.contains(id))
+        mtx::events::collections::TimelineEvents *e = events.event(id.toStdString(), "");
+        if (!e)
                 return "";
 
-        auto event =
+        auto event = std::get_if<mtx::events::StateEvent<mtx::events::state::PowerLevels>>(e);
-          std::get_if<mtx::events::StateEvent<mtx::events::state::PowerLevels>>(&events[id]);
         if (!event)
                 return "";
 
@@ -1826,28 +1661,30 @@ TimelineModel::formatPowerLevelEvent(QString id)
 QString
 TimelineModel::formatMemberEvent(QString id)
 {
-        if (!events.contains(id))
+        mtx::events::collections::TimelineEvents *e = events.event(id.toStdString(), "");
+        if (!e)
                 return "";
 
-        auto event = std::get_if<mtx::events::StateEvent<mtx::events::state::Member>>(&events[id]);
+        auto event = std::get_if<mtx::events::StateEvent<mtx::events::state::Member>>(e);
         if (!event)
                 return "";
 
         mtx::events::StateEvent<mtx::events::state::Member> *prevEvent = nullptr;
-        QString prevEventId = QString::fromStdString(event->unsigned_data.replaces_state);
+        if (!event->unsigned_data.replaces_state.empty()) {
-        if (!prevEventId.isEmpty()) {
+                auto tempPrevEvent =
-                if (!events.contains(prevEventId)) {
+                  events.event(event->unsigned_data.replaces_state, event->event_id);
+                if (!tempPrevEvent) {
                         http::client()->get_event(
                           this->room_id_.toStdString(),
                           event->unsigned_data.replaces_state,
-                          [this, id, prevEventId](
+                          [this, id, prevEventId = event->unsigned_data.replaces_state](
                             const mtx::events::collections::TimelineEvents &timeline,
                             mtx::http::RequestErr err) {
                                   if (err) {
                                           nhlog::net()->error(
                                             "Failed to retrieve event with id {}, which was "
                                             "requested to show the membership for event {}",
-                                            prevEventId.toStdString(),
+                                            prevEventId,
                                             id.toStdString());
                                           return;
                                   }
@@ -1856,7 +1693,7 @@ TimelineModel::formatMemberEvent(QString id)
                 } else {
                         prevEvent =
                           std::get_if<mtx::events::StateEvent<mtx::events::state::Member>>(
-                            &events[prevEventId]);
+                            tempPrevEvent);
                 }
         }
 

src/timeline/TimelineModel.h

@@ -9,6 +9,7 @@
 #include <mtxclient/http/errors.hpp>
 
 #include "CacheCryptoStructs.h"
+#include "EventStore.h"
 #include "ReactionsModel.h"
 
 namespace mtx::http {
@@ -170,7 +171,7 @@ public:
         QHash<int, QByteArray> roleNames() const override;
         int rowCount(const QModelIndex &parent = QModelIndex()) const override;
         QVariant data(const QModelIndex &index, int role = Qt::DisplayRole) const override;
-        QVariant data(const QString &id, int role) const;
+        QVariant data(const mtx::events::collections::TimelineEvents &event, int role) const;
 
         bool canFetchMore(const QModelIndex &) const override;
         void fetchMore(const QModelIndex &) override;
@@ -207,7 +208,7 @@ public slots:
         void setCurrentIndex(int index);
         int currentIndex() const { return idToIndex(currentId); }
         void markEventsAsRead(const std::vector<QString> &event_ids);
-        QVariantMap getDump(QString eventId) const;
+        QVariantMap getDump(QString eventId, QString relatedTo) const;
         void updateTypingUsers(const std::vector<QString> &users)
         {
                 if (this->typingUsers_ != users) {
@@ -257,9 +258,7 @@ signals:
         void paginationInProgressChanged(const bool);
 
 private:
-        DecryptionResult decryptEvent(
+        void internalAddEvents(
-          const mtx::events::EncryptedEvent<mtx::events::msg::Encrypted> &e) const;
-        std::vector<QString> internalAddEvents(
           const std::vector<mtx::events::collections::TimelineEvents> &timeline);
         void sendEncryptedMessage(const std::string &txn_id, nlohmann::json content);
         void handleClaimedKeys(std::shared_ptr<StateKeeper> keeper,
@@ -272,12 +271,12 @@ private:
 
         void setPaginationInProgress(const bool paginationInProgress);
 
-        QHash<QString, mtx::events::collections::TimelineEvents> events;
         QSet<QString> read;
         QList<QString> pending;
-        std::vector<QString> eventOrder;
         std::map<QString, ReactionsModel> reactions;
 
+        mutable EventStore events;
+
         QString room_id_;
         QString prev_batch_token_;