Readd pagination and fix redactions

resources/qml/delegates/MessageDelegate.qml

@@ -66,6 +66,12 @@ Item {
 				text: qsTr("redacted")
 			}
 		}
+		DelegateChoice {
+			roleValue: MtxEvent.Redaction
+			Pill {
+				text: qsTr("redacted")
+			}
+		}
 		DelegateChoice {
 			roleValue: MtxEvent.Encryption
 			Pill {

src/Cache.cpp

@@ -1241,8 +1241,25 @@ Cache::getTimelineMentions()
         return notifs;
 }
 
+std::string
+Cache::previousBatchToken(const std::string &room_id)
+{
+        auto txn     = lmdb::txn::begin(env_, nullptr);
+        auto orderDb = getEventOrderDb(txn, room_id);
+
+        auto cursor = lmdb::cursor::open(txn, orderDb);
+        lmdb::val indexVal, val;
+        if (!cursor.get(indexVal, val, MDB_FIRST)) {
+                return "";
+        }
+
+        auto j = json::parse(std::string_view(val.data(), val.size()));
+
+        return j.value("prev_batch", "");
+}
+
 Cache::Messages
-Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t index, bool forward)
+Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, uint64_t index, bool forward)
 {
         // TODO(nico): Limit the messages returned by this maybe?
         auto orderDb  = getOrderToMessageDb(txn, room_id);
@@ -1253,16 +1270,16 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i
         lmdb::val indexVal, event_id;
 
         auto cursor = lmdb::cursor::open(txn, orderDb);
-        if (index == std::numeric_limits<int64_t>::max()) {
+        if (index == std::numeric_limits<uint64_t>::max()) {
                 if (cursor.get(indexVal, event_id, forward ? MDB_FIRST : MDB_LAST)) {
-                        index = *indexVal.data<int64_t>();
+                        index = *indexVal.data<uint64_t>();
                 } else {
                         messages.end_of_cache = true;
                         return messages;
                 }
         } else {
                 if (cursor.get(indexVal, event_id, MDB_SET)) {
-                        index = *indexVal.data<int64_t>();
+                        index = *indexVal.data<uint64_t>();
                 } else {
                         messages.end_of_cache = true;
                         return messages;
@@ -1296,7 +1313,7 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i
         cursor.close();
 
         // std::reverse(timeline.events.begin(), timeline.events.end());
-        messages.next_index   = *indexVal.data<int64_t>();
+        messages.next_index   = *indexVal.data<uint64_t>();
         messages.end_of_cache = !ret;
 
         return messages;
@@ -1402,16 +1419,16 @@ Cache::getTimelineRange(const std::string &room_id)
         }
 
         TimelineRange range{};
-        range.last = *indexVal.data<int64_t>();
+        range.last = *indexVal.data<uint64_t>();
 
         if (!cursor.get(indexVal, val, MDB_FIRST)) {
                 return {};
         }
-        range.first = *indexVal.data<int64_t>();
+        range.first = *indexVal.data<uint64_t>();
 
         return range;
 }
-std::optional<int64_t>
+std::optional<uint64_t>
 Cache::getTimelineIndex(const std::string &room_id, std::string_view event_id)
 {
         auto txn     = lmdb::txn::begin(env_, nullptr, MDB_RDONLY);
@@ -1424,11 +1441,11 @@ Cache::getTimelineIndex(const std::string &room_id, std::string_view event_id)
                 return {};
         }
 
-        return *val.data<int64_t>();
+        return *val.data<uint64_t>();
 }
 
 std::optional<std::string>
-Cache::getTimelineEventId(const std::string &room_id, int64_t index)
+Cache::getTimelineEventId(const std::string &room_id, uint64_t index)
 {
         auto txn     = lmdb::txn::begin(env_, nullptr, MDB_RDONLY);
         auto orderDb = getOrderToMessageDb(txn, room_id);
@@ -2074,6 +2091,9 @@ Cache::saveTimelineMessages(lmdb::txn &txn,
                             const std::string &room_id,
                             const mtx::responses::Timeline &res)
 {
+        if (res.events.empty())
+                return;
+
         auto eventsDb    = getEventsDb(txn, room_id);
         auto relationsDb = getRelationsDb(txn, room_id);
 
@@ -2090,16 +2110,16 @@ Cache::saveTimelineMessages(lmdb::txn &txn,
         using namespace mtx::events::state;
 
         lmdb::val indexVal, val;
-        int64_t index = 0;
+        uint64_t index = std::numeric_limits<uint64_t>::max() / 2;
         auto cursor    = lmdb::cursor::open(txn, orderDb);
         if (cursor.get(indexVal, val, MDB_LAST)) {
                 index = *indexVal.data<int64_t>();
         }
 
-        int64_t msgIndex = 0;
+        uint64_t msgIndex = std::numeric_limits<uint64_t>::max() / 2;
         auto msgCursor    = lmdb::cursor::open(txn, order2msgDb);
         if (msgCursor.get(indexVal, val, MDB_LAST)) {
-                msgIndex = *indexVal.data<int64_t>();
+                msgIndex = *indexVal.data<uint64_t>();
         }
 
         bool first = true;
@@ -2111,39 +2131,19 @@ Cache::saveTimelineMessages(lmdb::txn &txn,
                                 continue;
 
                         lmdb::val ev{};
-                        bool success =
+                        lmdb::dbi_put(
-                          lmdb::dbi_get(txn, eventsDb, lmdb::val(redaction->redacts), ev);
+                          txn, eventsDb, lmdb::val(redaction->redacts), lmdb::val(event.dump()));
-                        if (!success)
+                        lmdb::dbi_put(
-                                continue;
+                          txn, eventsDb, lmdb::val(redaction->event_id), lmdb::val(event.dump()));
 
-                        mtx::events::collections::TimelineEvent te;
+                        lmdb::val oldIndex{};
-
+                        if (lmdb::dbi_get(
-                        try {
+                              txn, msg2orderDb, lmdb::val(redaction->redacts), oldIndex)) {
-                                mtx::events::collections::from_json(
+                                lmdb::dbi_put(
-                                  json::parse(std::string_view(ev.data(), ev.size())), te);
+                                  txn, order2msgDb, oldIndex, lmdb::val(redaction->event_id));
-                        } catch (std::exception &e) {
+                                lmdb::dbi_put(
-                                nhlog::db()->error("Failed to parse message from cache {}",
+                                  txn, msg2orderDb, lmdb::val(redaction->event_id), oldIndex);
-                                                   e.what());
-                                continue;
                         }
-
-                        auto redactedEvent = std::visit(
-                          [](const auto &ev) -> mtx::events::RoomEvent<mtx::events::msg::Redacted> {
-                                  mtx::events::RoomEvent<mtx::events::msg::Redacted> replacement =
-                                    {};
-                                  replacement.event_id         = ev.event_id;
-                                  replacement.room_id          = ev.room_id;
-                                  replacement.sender           = ev.sender;
-                                  replacement.origin_server_ts = ev.origin_server_ts;
-                                  replacement.type             = ev.type;
-                                  return replacement;
-                          },
-                          te.data);
-
-                        lmdb::dbi_put(txn,
-                                      eventsDb,
-                                      lmdb::val(redaction->redacts),
-                                      lmdb::val(json(redactedEvent).dump()));
                 } else {
                         std::string event_id_val = event["event_id"].get<std::string>();
                         lmdb::val event_id       = event_id_val;
@@ -2193,6 +2193,83 @@ Cache::saveTimelineMessages(lmdb::txn &txn,
         }
 }
 
+uint64_t
+Cache::saveOldMessages(const std::string &room_id, const mtx::responses::Messages &res)
+{
+        auto txn         = lmdb::txn::begin(env_);
+        auto eventsDb    = getEventsDb(txn, room_id);
+        auto relationsDb = getRelationsDb(txn, room_id);
+
+        auto orderDb     = getEventOrderDb(txn, room_id);
+        auto msg2orderDb = getMessageToOrderDb(txn, room_id);
+        auto order2msgDb = getOrderToMessageDb(txn, room_id);
+
+        lmdb::val indexVal, val;
+        uint64_t index = std::numeric_limits<uint64_t>::max() / 2;
+        auto cursor    = lmdb::cursor::open(txn, orderDb);
+        if (cursor.get(indexVal, val, MDB_FIRST)) {
+                index = *indexVal.data<uint64_t>();
+        }
+
+        uint64_t msgIndex = std::numeric_limits<uint64_t>::max() / 2;
+        auto msgCursor    = lmdb::cursor::open(txn, order2msgDb);
+        if (msgCursor.get(indexVal, val, MDB_FIRST)) {
+                msgIndex = *indexVal.data<uint64_t>();
+        }
+
+        if (res.chunk.empty())
+                return index;
+
+        std::string event_id_val;
+        for (const auto &e : res.chunk) {
+                auto event         = mtx::accessors::serialize_event(e);
+                event_id_val       = event["event_id"].get<std::string>();
+                lmdb::val event_id = event_id_val;
+                lmdb::dbi_put(txn, eventsDb, event_id, lmdb::val(event.dump()));
+
+                --index;
+
+                json orderEntry        = json::object();
+                orderEntry["event_id"] = event_id_val;
+
+                nhlog::db()->debug("saving '{}'", orderEntry.dump());
+
+                lmdb::dbi_put(
+                  txn, orderDb, lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump()));
+
+                // TODO(Nico): Allow blacklisting more event types in UI
+                if (event["type"] != "m.reaction" && event["type"] != "m.dummy") {
+                        --msgIndex;
+                        lmdb::dbi_put(
+                          txn, order2msgDb, lmdb::val(&msgIndex, sizeof(msgIndex)), event_id);
+
+                        lmdb::dbi_put(
+                          txn, msg2orderDb, event_id, lmdb::val(&msgIndex, sizeof(msgIndex)));
+                }
+
+                if (event.contains("content") && event["content"].contains("m.relates_to")) {
+                        auto temp              = event["content"]["m.relates_to"];
+                        std::string relates_to = temp.contains("m.in_reply_to")
+                                                   ? temp["m.in_reply_to"]["event_id"]
+                                                   : temp["event_id"];
+
+                        if (!relates_to.empty())
+                                lmdb::dbi_put(txn, relationsDb, lmdb::val(relates_to), event_id);
+                }
+        }
+
+        json orderEntry          = json::object();
+        orderEntry["event_id"]   = event_id_val;
+        orderEntry["prev_batch"] = res.end;
+        lmdb::cursor_put(
+          cursor.handle(), lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump()));
+        nhlog::db()->debug("saving '{}'", orderEntry.dump());
+
+        txn.commit();
+
+        return msgIndex;
+}
+
 mtx::responses::Notifications
 Cache::getTimelineMentionsForRoom(lmdb::txn &txn, const std::string &room_id)
 {
@@ -2337,14 +2414,14 @@ Cache::deleteOldMessages()
                 auto eventsDb = getEventsDb(txn, room_id);
                 auto cursor   = lmdb::cursor::open(txn, orderDb);
 
-                int64_t first, last;
+                uint64_t first, last;
                 if (cursor.get(indexVal, val, MDB_LAST)) {
-                        last = *indexVal.data<int64_t>();
+                        last = *indexVal.data<uint64_t>();
                 } else {
                         continue;
                 }
                 if (cursor.get(indexVal, val, MDB_FIRST)) {
-                        first = *indexVal.data<int64_t>();
+                        first = *indexVal.data<uint64_t>();
                 } else {
                         continue;
                 }

src/Cache_p.h

@@ -179,7 +179,7 @@ public:
         };
         Messages getTimelineMessages(lmdb::txn &txn,
                                      const std::string &room_id,
-                                     int64_t index = std::numeric_limits<int64_t>::max(),
+                                     uint64_t index = std::numeric_limits<uint64_t>::max(),
                                      bool forward   = false);
 
         std::optional<mtx::events::collections::TimelineEvent> getEvent(
@@ -190,12 +190,15 @@ public:
                         const mtx::events::collections::TimelineEvent &event);
         struct TimelineRange
         {
-                int64_t first, last;
+                uint64_t first, last;
         };
         std::optional<TimelineRange> getTimelineRange(const std::string &room_id);
-        std::optional<int64_t> getTimelineIndex(const std::string &room_id,
+        std::optional<uint64_t> getTimelineIndex(const std::string &room_id,
                                                  std::string_view event_id);
-        std::optional<std::string> getTimelineEventId(const std::string &room_id, int64_t index);
+        std::optional<std::string> getTimelineEventId(const std::string &room_id, uint64_t index);
+
+        std::string previousBatchToken(const std::string &room_id);
+        uint64_t saveOldMessages(const std::string &room_id, const mtx::responses::Messages &res);
 
         //! Remove old unused data.
         void deleteOldMessages();

src/ChatPage.cpp

@@ -795,7 +795,6 @@ ChatPage::loadStateFromCache()
 
         nhlog::db()->info("restoring state from cache");
 
-        QtConcurrent::run([this]() {
         try {
                 cache::restoreSessions();
                 olm::client()->load(cache::restoreOlmAccount(), STORAGE_SECRET_KEY);
@@ -811,13 +810,11 @@ ChatPage::loadStateFromCache()
 
         } catch (const mtx::crypto::olm_exception &e) {
                 nhlog::crypto()->critical("failed to restore olm account: {}", e.what());
-                        emit dropToLoginPageCb(
+                emit dropToLoginPageCb(tr("Failed to restore OLM account. Please login again."));
-                          tr("Failed to restore OLM account. Please login again."));
                 return;
         } catch (const lmdb::error &e) {
                 nhlog::db()->critical("failed to restore cache: {}", e.what());
-                        emit dropToLoginPageCb(
+                emit dropToLoginPageCb(tr("Failed to restore save data. Please login again."));
-                          tr("Failed to restore save data. Please login again."));
                 return;
         } catch (const json::exception &e) {
                 nhlog::db()->critical("failed to parse cache data: {}", e.what());
@@ -831,7 +828,6 @@ ChatPage::loadStateFromCache()
 
         // Start receiving events.
         emit trySyncCb();
-        });
 }
 
 void

src/timeline/EventStore.cpp

@@ -34,12 +34,31 @@ EventStore::EventStore(std::string room_id, QObject *)
                   cache::client()->storeEvent(room_id_, id, {timeline});
 
                   if (!relatedTo.empty()) {
-                          auto idx = idToIndex(id);
+                          auto idx = idToIndex(relatedTo);
                           if (idx)
                                   emit dataChanged(*idx, *idx);
                   }
           },
           Qt::QueuedConnection);
+
+        connect(
+          this,
+          &EventStore::oldMessagesRetrieved,
+          this,
+          [this](const mtx::responses::Messages &res) {
+                  //
+                  uint64_t newFirst = cache::client()->saveOldMessages(room_id_, res);
+                  if (newFirst == first)
+                          fetchMore();
+                  else {
+                          emit beginInsertRows(toExternalIdx(newFirst),
+                                               toExternalIdx(this->first - 1));
+                          this->first = newFirst;
+                          emit endInsertRows();
+                          emit fetchedMore();
+                  }
+          },
+          Qt::QueuedConnection);
 }
 
 void
@@ -49,8 +68,16 @@ EventStore::handleSync(const mtx::responses::Timeline &events)
                 nhlog::db()->warn("{} called from a different thread!", __func__);
 
         auto range = cache::client()->getTimelineRange(room_id_);
+        if (!range)
+                return;
 
-        if (range && range->last > this->last) {
+        if (events.limited) {
+                emit beginResetModel();
+                this->last  = range->last;
+                this->first = range->first;
+                emit endResetModel();
+
+        } else if (range->last > this->last) {
                 emit beginInsertRows(toExternalIdx(this->last + 1), toExternalIdx(range->last));
                 this->last = range->last;
                 emit endInsertRows();
@@ -290,3 +317,27 @@ EventStore::event(std::string_view id, std::string_view related_to, bool decrypt
 
         return event_ptr;
 }
+
+void
+EventStore::fetchMore()
+{
+        mtx::http::MessagesOpts opts;
+        opts.room_id = room_id_;
+        opts.from    = cache::client()->previousBatchToken(room_id_);
+
+        nhlog::ui()->debug("Paginating room {}, token {}", opts.room_id, opts.from);
+
+        http::client()->messages(
+          opts, [this, opts](const mtx::responses::Messages &res, mtx::http::RequestErr err) {
+                  if (err) {
+                          nhlog::net()->error("failed to call /messages ({}): {} - {} - {}",
+                                              opts.room_id,
+                                              mtx::errors::to_string(err->matrix_error.errcode),
+                                              err->matrix_error.error,
+                                              err->parse_error);
+                          return;
+                  }
+
+                  emit oldMessagesRetrieved(std::move(res));
+          });
+}

src/timeline/EventStore.h

@@ -8,6 +8,7 @@
 #include <qhashfunctions.h>
 
 #include <mtx/events/collections.hpp>
+#include <mtx/responses/messages.hpp>
 #include <mtx/responses/sync.hpp>
 
 class EventStore : public QObject
@@ -20,7 +21,7 @@ public:
         struct Index
         {
                 std::string room;
-                int64_t idx;
+                uint64_t idx;
 
                 friend uint qHash(const Index &i, uint seed = 0) noexcept
                 {
@@ -66,12 +67,12 @@ public:
 
         int size() const
         {
-                return last != std::numeric_limits<int64_t>::max()
+                return last != std::numeric_limits<uint64_t>::max()
                          ? static_cast<int>(last - first) + 1
                          : 0;
         }
-        int toExternalIdx(int64_t idx) const { return static_cast<int>(idx - first); }
+        int toExternalIdx(uint64_t idx) const { return static_cast<int>(idx - first); }
-        int64_t toInternalIdx(int idx) const { return first + idx; }
+        uint64_t toInternalIdx(int idx) const { return first + idx; }
 
         std::optional<int> idToIndex(std::string_view id) const;
         std::optional<std::string> indexToId(int idx) const;
@@ -79,11 +80,15 @@ public:
 signals:
         void beginInsertRows(int from, int to);
         void endInsertRows();
+        void beginResetModel();
+        void endResetModel();
         void dataChanged(int from, int to);
         void newEncryptedImage(mtx::crypto::EncryptedFile encryptionInfo);
         void eventFetched(std::string id,
                           std::string relatedTo,
                           mtx::events::collections::TimelineEvents timeline);
+        void oldMessagesRetrieved(const mtx::responses::Messages &);
+        void fetchedMore();
 
 private:
         mtx::events::collections::TimelineEvents *decryptEvent(
@@ -92,8 +97,8 @@ private:
 
         std::string room_id_;
 
-        int64_t first = std::numeric_limits<int64_t>::max(),
+        uint64_t first = std::numeric_limits<uint64_t>::max(),
-                last  = std::numeric_limits<int64_t>::max();
+                 last  = std::numeric_limits<uint64_t>::max();
 
         static QCache<IdIndex, mtx::events::collections::TimelineEvents> decryptedEvents_;
         static QCache<Index, mtx::events::collections::TimelineEvents> events_;

src/timeline/TimelineModel.cpp

@@ -229,20 +229,33 @@ TimelineModel::TimelineModel(TimelineViewManager *manager, QString room_id, QObj
           &EventStore::dataChanged,
           this,
           [this](int from, int to) {
-                  emit dataChanged(index(events.size() - to, 0), index(events.size() - from, 0));
+                  nhlog::ui()->debug(
+                    "data changed {} to {}", events.size() - to - 1, events.size() - from - 1);
+                  emit dataChanged(index(events.size() - to - 1, 0),
+                                   index(events.size() - from - 1, 0));
           },
           Qt::QueuedConnection);
 
         connect(&events, &EventStore::beginInsertRows, this, [this](int from, int to) {
-                nhlog::ui()->info("begin insert from {} to {}",
+                int first = events.size() - to;
-                                  events.size() - to + (to - from),
+                int last  = events.size() - from;
-                                  events.size() - from + (to - from));
+                if (from >= events.size()) {
-                beginInsertRows(QModelIndex(),
+                        int batch_size = to - from;
-                                events.size() - to + (to - from),
+                        first += batch_size;
-                                events.size() - from + (to - from));
+                        last += batch_size;
+                } else {
+                        first -= 1;
+                        last -= 1;
+                }
+                nhlog::ui()->debug("begin insert from {} to {}", first, last);
+                beginInsertRows(QModelIndex(), first, last);
         });
         connect(&events, &EventStore::endInsertRows, this, [this]() { endInsertRows(); });
+        connect(&events, &EventStore::beginResetModel, this, [this]() { beginResetModel(); });
+        connect(&events, &EventStore::endResetModel, this, [this]() { endResetModel(); });
         connect(&events, &EventStore::newEncryptedImage, this, &TimelineModel::newEncryptedImage);
+        connect(
+          &events, &EventStore::fetchedMore, this, [this]() { setPaginationInProgress(false); });
 }
 
 QHash<int, QByteArray>
@@ -512,8 +525,9 @@ TimelineModel::canFetchMore(const QModelIndex &) const
 {
         if (!events.size())
                 return true;
-        if (!std::holds_alternative<mtx::events::StateEvent<mtx::events::state::Create>>(
+        if (auto first = events.event(0);
-              *events.event(0)))
+            first &&
+            !std::holds_alternative<mtx::events::StateEvent<mtx::events::state::Create>>(*first))
                 return true;
         else
 
@@ -540,27 +554,8 @@ TimelineModel::fetchMore(const QModelIndex &)
         }
 
         setPaginationInProgress(true);
-        mtx::http::MessagesOpts opts;
-        opts.room_id = room_id_.toStdString();
-        opts.from    = prev_batch_token_.toStdString();
 
-        nhlog::ui()->debug("Paginating room {}", opts.room_id);
+        events.fetchMore();
-
-        http::client()->messages(
-          opts, [this, opts](const mtx::responses::Messages &res, mtx::http::RequestErr err) {
-                  if (err) {
-                          nhlog::net()->error("failed to call /messages ({}): {} - {} - {}",
-                                              opts.room_id,
-                                              mtx::errors::to_string(err->matrix_error.errcode),
-                                              err->matrix_error.error,
-                                              err->parse_error);
-                          setPaginationInProgress(false);
-                          return;
-                  }
-
-                  emit oldMessagesRetrieved(std::move(res));
-                  setPaginationInProgress(false);
-          });
 }
 
 void