Store timestamp with olm sessions

src/Cache.cpp

@@ -40,7 +40,7 @@
 
 //! Should be changed when a breaking change occurs in the cache format.
 //! This will reset client's data.
-static const std::string CURRENT_CACHE_FORMAT_VERSION("2020.07.05");
+static const std::string CURRENT_CACHE_FORMAT_VERSION("2020.10.20");
 static const std::string SECRET("secret");
 
 static lmdb::val NEXT_BATCH_KEY("next_batch");
@@ -437,7 +437,9 @@ Cache::getOutboundMegolmSession(const std::string &room_id)
 //
 
 void
-Cache::saveOlmSession(const std::string &curve25519, mtx::crypto::OlmSessionPtr session)
+Cache::saveOlmSession(const std::string &curve25519,
+                      mtx::crypto::OlmSessionPtr session,
+                      uint64_t timestamp)
 {
         using namespace mtx::crypto;
 
@@ -447,7 +449,11 @@ Cache::saveOlmSession(const std::string &curve25519, mtx::crypto::OlmSessionPtr
         const auto pickled    = pickle<SessionObject>(session.get(), SECRET);
         const auto session_id = mtx::crypto::session_id(session.get());
 
-        lmdb::dbi_put(txn, db, lmdb::val(session_id), lmdb::val(pickled));
+        StoredOlmSession stored_session;
+        stored_session.pickled_session = pickled;
+        stored_session.last_message_ts = timestamp;
+
+        lmdb::dbi_put(txn, db, lmdb::val(session_id), lmdb::val(json(stored_session).dump()));
 
         txn.commit();
 }
@@ -466,13 +472,44 @@ Cache::getOlmSession(const std::string &curve25519, const std::string &session_i
         txn.commit();
 
         if (found) {
-                auto data = std::string(pickled.data(), pickled.size());
+                std::string_view raw(pickled.data(), pickled.size());
-                return unpickle<SessionObject>(data, SECRET);
+                auto data = json::parse(raw).get<StoredOlmSession>();
+                return unpickle<SessionObject>(data.pickled_session, SECRET);
         }
 
         return std::nullopt;
 }
 
+std::optional<mtx::crypto::OlmSessionPtr>
+Cache::getLatestOlmSession(const std::string &curve25519)
+{
+        using namespace mtx::crypto;
+
+        auto txn = lmdb::txn::begin(env_);
+        auto db  = getOlmSessionsDb(txn, curve25519);
+
+        std::string session_id, pickled_session;
+        std::vector<std::string> res;
+
+        std::optional<StoredOlmSession> currentNewest;
+
+        auto cursor = lmdb::cursor::open(txn, db);
+        while (cursor.get(session_id, pickled_session, MDB_NEXT)) {
+                auto data =
+                  json::parse(std::string_view(pickled_session.data(), pickled_session.size()))
+                    .get<StoredOlmSession>();
+                if (!currentNewest || currentNewest->last_message_ts < data.last_message_ts)
+                        currentNewest = data;
+        }
+        cursor.close();
+
+        txn.commit();
+
+        return currentNewest
+                 ? std::optional(unpickle<SessionObject>(currentNewest->pickled_session, SECRET))
+                 : std::nullopt;
+}
+
 std::vector<std::string>
 Cache::getOlmSessions(const std::string &curve25519)
 {
@@ -828,6 +865,80 @@ Cache::runMigrations()
                    nhlog::db()->info("Successfully deleted pending receipts database.");
                    return true;
            }},
+          {"2020.10.20",
+           [this]() {
+                   try {
+                           using namespace mtx::crypto;
+
+                           auto txn = lmdb::txn::begin(env_);
+
+                           auto mainDb = lmdb::dbi::open(txn, nullptr);
+
+                           std::string dbName, ignored;
+                           auto olmDbCursor = lmdb::cursor::open(txn, mainDb);
+                           while (olmDbCursor.get(dbName, ignored, MDB_NEXT)) {
+                                   // skip every db but olm session dbs
+                                   nhlog::db()->debug("Db {}", dbName);
+                                   if (dbName.find("olm_sessions/") != 0)
+                                           continue;
+
+                                   nhlog::db()->debug("Migrating {}", dbName);
+
+                                   auto olmDb = lmdb::dbi::open(txn, dbName.c_str());
+
+                                   std::string session_id, session_value;
+
+                                   std::vector<std::pair<std::string, StoredOlmSession>> sessions;
+
+                                   auto cursor = lmdb::cursor::open(txn, olmDb);
+                                   while (cursor.get(session_id, session_value, MDB_NEXT)) {
+                                           nhlog::db()->debug("session_id {}, session_value {}",
+                                                              session_id,
+                                                              session_value);
+                                           StoredOlmSession session;
+                                           bool invalid = false;
+                                           for (auto c : session_value)
+                                                   if (!isprint(c)) {
+                                                           invalid = true;
+                                                           break;
+                                                   }
+                                           if (invalid)
+                                                   continue;
+
+                                           nhlog::db()->debug("Not skipped");
+
+                                           session.pickled_session = session_value;
+                                           sessions.emplace_back(session_id, session);
+                                   }
+                                   cursor.close();
+
+                                   olmDb.drop(txn, true);
+
+                                   auto newDbName = dbName;
+                                   newDbName.erase(0, sizeof("olm_sessions") - 1);
+                                   newDbName = "olm_sessions.v2" + newDbName;
+
+                                   auto newDb = lmdb::dbi::open(txn, newDbName.c_str(), MDB_CREATE);
+
+                                   for (const auto &[key, value] : sessions) {
+                                           nhlog::db()->debug("{}\n{}", key, json(value).dump());
+                                           lmdb::dbi_put(txn,
+                                                         newDb,
+                                                         lmdb::val(key),
+                                                         lmdb::val(json(value).dump()));
+                                   }
+                           }
+                           olmDbCursor.close();
+
+                           txn.commit();
+                   } catch (const lmdb::error &) {
+                           nhlog::db()->critical("Failed to migrate olm sessions,");
+                           return false;
+                   }
+
+                   nhlog::db()->info("Successfully migrated olm sessions.");
+                   return true;
+           }},
         };
 
         nhlog::db()->info("Running migrations, this may take a while!");
@@ -3629,6 +3740,19 @@ from_json(const nlohmann::json &obj, MegolmSessionIndex &msg)
         msg.sender_key = obj.at("sender_key");
 }
 
+void
+to_json(nlohmann::json &obj, const StoredOlmSession &msg)
+{
+        obj["ts"] = msg.last_message_ts;
+        obj["s"]  = msg.pickled_session;
+}
+void
+from_json(const nlohmann::json &obj, StoredOlmSession &msg)
+{
+        msg.last_message_ts = obj.at("ts").get<uint64_t>();
+        msg.pickled_session = obj.at("s").get<std::string>();
+}
+
 namespace cache {
 void
 init(const QString &user_id)
@@ -4114,9 +4238,11 @@ inboundMegolmSessionExists(const MegolmSessionIndex &index)
 // Olm Sessions
 //
 void
-saveOlmSession(const std::string &curve25519, mtx::crypto::OlmSessionPtr session)
+saveOlmSession(const std::string &curve25519,
+               mtx::crypto::OlmSessionPtr session,
+               uint64_t timestamp)
 {
-        instance_->saveOlmSession(curve25519, std::move(session));
+        instance_->saveOlmSession(curve25519, std::move(session), timestamp);
 }
 std::vector<std::string>
 getOlmSessions(const std::string &curve25519)
@@ -4128,6 +4254,11 @@ getOlmSession(const std::string &curve25519, const std::string &session_id)
 {
         return instance_->getOlmSession(curve25519, session_id);
 }
+std::optional<mtx::crypto::OlmSessionPtr>
+getLatestOlmSession(const std::string &curve25519)
+{
+        return instance_->getLatestOlmSession(curve25519);
+}
 
 void
 saveOlmAccount(const std::string &pickled)

src/Cache.h

@@ -292,11 +292,15 @@ inboundMegolmSessionExists(const MegolmSessionIndex &index);
 // Olm Sessions
 //
 void
-saveOlmSession(const std::string &curve25519, mtx::crypto::OlmSessionPtr session);
+saveOlmSession(const std::string &curve25519,
+               mtx::crypto::OlmSessionPtr session,
+               uint64_t timestamp);
 std::vector<std::string>
 getOlmSessions(const std::string &curve25519);
 std::optional<mtx::crypto::OlmSessionPtr>
 getOlmSession(const std::string &curve25519, const std::string &session_id);
+std::optional<mtx::crypto::OlmSessionPtr>
+getLatestOlmSession(const std::string &curve25519);
 
 void
 saveOlmAccount(const std::string &pickled);

src/CacheCryptoStructs.h

@@ -66,6 +66,16 @@ struct OlmSessionStorage
         std::mutex group_inbound_mtx;
 };
 
+struct StoredOlmSession
+{
+        std::uint64_t last_message_ts = 0;
+        std::string pickled_session;
+};
+void
+to_json(nlohmann::json &obj, const StoredOlmSession &msg);
+void
+from_json(const nlohmann::json &obj, StoredOlmSession &msg);
+
 //! Verification status of a single user
 struct VerificationStatus
 {

src/Cache_p.h

@@ -266,10 +266,14 @@ public:
         //
         // Olm Sessions
         //
-        void saveOlmSession(const std::string &curve25519, mtx::crypto::OlmSessionPtr session);
+        void saveOlmSession(const std::string &curve25519,
+                            mtx::crypto::OlmSessionPtr session,
+                            uint64_t timestamp);
         std::vector<std::string> getOlmSessions(const std::string &curve25519);
         std::optional<mtx::crypto::OlmSessionPtr> getOlmSession(const std::string &curve25519,
                                                                 const std::string &session_id);
+        std::optional<mtx::crypto::OlmSessionPtr> getLatestOlmSession(
+          const std::string &curve25519);
 
         void saveOlmAccount(const std::string &pickled);
         std::string restoreOlmAccount();
@@ -565,7 +569,7 @@ private:
         lmdb::dbi getOlmSessionsDb(lmdb::txn &txn, const std::string &curve25519_key)
         {
                 return lmdb::dbi::open(
-                  txn, std::string("olm_sessions/" + curve25519_key).c_str(), MDB_CREATE);
+                  txn, std::string("olm_sessions.v2/" + curve25519_key).c_str(), MDB_CREATE);
         }
 
         QString getDisplayName(const mtx::events::StateEvent<mtx::events::state::Member> &event)

src/Olm.cpp

@@ -47,7 +47,7 @@ handle_to_device_messages(const std::vector<mtx::events::collections::DeviceEven
 
                 if (msg_type == to_string(mtx::events::EventType::RoomEncrypted)) {
                         try {
-                                OlmMessage olm_msg = j_msg;
+                                olm::OlmMessage olm_msg = j_msg;
                                 handle_olm_message(std::move(olm_msg));
                         } catch (const nlohmann::json::exception &e) {
                                 nhlog::crypto()->warn(
@@ -56,10 +56,6 @@ handle_to_device_messages(const std::vector<mtx::events::collections::DeviceEven
                                 nhlog::crypto()->warn("validation error for olm message: {} {}",
                                                       e.what(),
                                                       j_msg.dump(2));
-
-                                nhlog::crypto()->warn("validation error for olm message: {} {}",
-                                                      e.what(),
-                                                      j_msg.dump(2));
                         }
 
                 } else if (msg_type == to_string(mtx::events::EventType::RoomKeyRequest)) {
@@ -250,7 +246,8 @@ handle_pre_key_olm_message(const std::string &sender,
         nhlog::crypto()->debug("decrypted message: \n {}", plaintext.dump(2));
 
         try {
-                cache::saveOlmSession(sender_key, std::move(inbound_session));
+                cache::saveOlmSession(
+                  sender_key, std::move(inbound_session), QDateTime::currentMSecsSinceEpoch());
         } catch (const lmdb::error &e) {
                 nhlog::db()->warn(
                   "failed to save inbound olm session from {}: {}", sender, e.what());
@@ -318,7 +315,8 @@ try_olm_decryption(const std::string &sender_key, const mtx::events::msg::OlmCip
 
                 try {
                         text = olm::client()->decrypt_message(session->get(), msg.type, msg.body);
-                        cache::saveOlmSession(id, std::move(session.value()));
+                        cache::saveOlmSession(
+                          id, std::move(session.value()), QDateTime::currentMSecsSinceEpoch());
                 } catch (const mtx::crypto::olm_exception &e) {
                         nhlog::crypto()->debug("failed to decrypt olm message ({}, {}) with {}: {}",
                                                msg.type,
@@ -672,7 +670,9 @@ send_megolm_key_to_device(const std::string &user_id,
                                     device_msg       = olm::client()->create_olm_encrypted_content(
                                       olm_session.get(), json(room_key).dump(), pks.curve25519);
 
-                                    cache::saveOlmSession(pks.curve25519, std::move(olm_session));
+                                    cache::saveOlmSession(pks.curve25519,
+                                                          std::move(olm_session),
+                                                          QDateTime::currentMSecsSinceEpoch());
                             } catch (const json::exception &e) {
                                     nhlog::crypto()->warn("creating outbound session: {}",
                                                           e.what());
@@ -749,4 +749,14 @@ decryptEvent(const MegolmSessionIndex &index,
 
         return {std::nullopt, std::nullopt, std::move(te.data)};
 }
+
+//! Send encrypted to device messages, targets is a map from userid to device ids or "*"
+void
+send_encrypted_to_device_messages(const std::map<std::string, std::vector<std::string>> targets,
+                                  const mtx::events::collections::DeviceEvents &event)
+{
+        (void)targets;
+        (void)event;
+}
+
 } // namespace olm

src/Olm.h

@@ -110,4 +110,9 @@ send_megolm_key_to_device(const std::string &user_id,
                           const std::string &device_id,
                           const mtx::events::msg::ForwardedRoomKey &payload);
 
+//! Send encrypted to device messages, targets is a map from userid to device ids or "*"
+void
+send_encrypted_to_device_messages(const std::map<std::string, std::vector<std::string>> targets,
+                                  const mtx::events::collections::DeviceEvents &event);
+
 } // namespace olm

src/timeline/EventStore.cpp

@@ -54,6 +54,9 @@ EventStore::EventStore(std::string room_id, QObject *)
           &EventStore::oldMessagesRetrieved,
           this,
           [this](const mtx::responses::Messages &res) {
+                  if (cache::client()->previousBatchToken(room_id_) == res.end)
+                          noMoreMessages = true;
+
                   uint64_t newFirst = cache::client()->saveOldMessages(room_id_, res);
                   if (newFirst == first)
                           fetchMore();
@@ -687,6 +690,9 @@ EventStore::get(std::string_view id, std::string_view related_to, bool decrypt)
 void
 EventStore::fetchMore()
 {
+        if (noMoreMessages)
+                return;
+
         mtx::http::MessagesOpts opts;
         opts.room_id = room_id_;
         opts.from    = cache::client()->previousBatchToken(room_id_);

src/timeline/EventStore.h

@@ -123,4 +123,5 @@ private:
 
         std::string current_txn;
         int current_txn_error_count = 0;
+        bool noMoreMessages         = false;
 };

src/timeline/TimelineModel.cpp

@@ -1138,7 +1138,8 @@ TimelineModel::handleClaimedKeys(
                           pks.at(user_id).at(device_id).curve25519);
 
                         try {
-                                cache::saveOlmSession(id_key, std::move(s));
+                                cache::saveOlmSession(
+                                  id_key, std::move(s), QDateTime::currentMSecsSinceEpoch());
                         } catch (const lmdb::error &e) {
                                 nhlog::db()->critical("failed to save outbound olm session: {}",
                                                       e.what());