Merge branch 'soru/performance' into 'master'

Greatly imporve initial loading performance

See merge request famedly/famedlysdk!377

lib/src/client.dart

@@ -58,6 +58,8 @@ class Client {
 
   Set<KeyVerificationMethod> verificationMethods;
 
+  Set<String> importantStateEvents;
+
   /// Create a client
   /// clientName = unique identifier of this client
   /// debug: Print debug output?
@@ -66,13 +68,37 @@ class Client {
   /// verificationMethods: A set of all the verification methods this client can handle. Includes:
   ///    KeyVerificationMethod.numbers: Compare numbers. Most basic, should be supported
   ///    KeyVerificationMethod.emoji: Compare emojis
+  /// importantStateEvents: A set of all the important state events to load when the client connects.
+  ///    To speed up performance only a set of state events is loaded on startup, those that are
+  ///    needed to display a room list. All the remaining state events are automatically post-loaded
+  ///    when opening the timeline of a room or manually by calling `room.postLoad()`.
+  ///    This set will always include the following state events:
+  ///     - m.room.name
+  ///     - m.room.avatar
+  ///     - m.room.message
+  ///     - m.room.encrypted
+  ///     - m.room.encryption
+  ///     - m.room.canonical_alias
+  ///     - m.room.tombstone
+  ///     - *some* m.room.member events, where needed
   Client(this.clientName,
       {this.debug = false,
       this.database,
       this.enableE2eeRecovery = false,
       this.verificationMethods,
-      http.Client httpClient}) {
+      http.Client httpClient,
+      this.importantStateEvents}) {
     verificationMethods ??= <KeyVerificationMethod>{};
+    importantStateEvents ??= <String>{};
+    importantStateEvents.addAll([
+      EventTypes.RoomName,
+      EventTypes.RoomAvatar,
+      EventTypes.Message,
+      EventTypes.Encrypted,
+      EventTypes.Encryption,
+      EventTypes.RoomCanonicalAlias,
+      EventTypes.RoomTombstone,
+    ]);
     api = MatrixApi(debug: debug, httpClient: httpClient);
     onLoginStateChanged.stream.listen((loginState) {
       if (debug) {
@@ -644,7 +670,7 @@ class Client {
       _rooms = await database.getRoomList(this, onlyLeft: false);
       _sortRooms();
       accountData = await database.getAccountData(id);
-      presences = await database.getPresences(id);
+      presences.clear();
     }
 
     onLoginStateChanged.add(LoginState.logged);
@@ -744,14 +770,6 @@ class Client {
     }
     if (sync.presence != null) {
       for (final newPresence in sync.presence) {
-        if (database != null) {
-          await database.storePresence(
-            id,
-            newPresence.type,
-            newPresence.senderId,
-            jsonEncode(newPresence.toJson()),
-          );
-        }
         presences[newPresence.senderId] = newPresence;
         onPresence.add(newPresence);
       }
@@ -969,6 +987,16 @@ class Client {
       if (event['type'] == EventTypes.Encrypted && encryptionEnabled) {
         update = await update.decrypt(room);
       }
+      if (event['type'] == EventTypes.Message &&
+          !room.isDirectChat &&
+          database != null &&
+          room.getState(EventTypes.RoomMember, event['sender']) == null) {
+        // In order to correctly render room list previews we need to fetch the member from the database
+        final user = await database.getUser(id, event['sender'], room);
+        if (user != null) {
+          room.setState(user);
+        }
+      }
       if (type != 'ephemeral' && database != null) {
         await database.storeEventUpdate(id, update);
       }

lib/src/database/database.dart

@@ -157,9 +157,12 @@ class Database extends _$Database {
               ? t.membership.equals('leave')
               : t.membership.equals('leave').not()))
         .get();
-    final resStates = await getAllRoomStates(client.id).get();
+    final resStates = await getImportantRoomStates(
+            client.id, client.importantStateEvents.toList())
+        .get();
     final resAccountData = await getAllRoomAccountData(client.id).get();
     final roomList = <sdk.Room>[];
+    final allMembersToPostload = <String, Set<String>>{};
     for (final r in res) {
       final room = await sdk.Room.getRoomFromTableRow(
         r,
@@ -168,6 +171,81 @@ class Database extends _$Database {
         roomAccountData: resAccountData.where((rs) => rs.roomId == r.roomId),
       );
       roomList.add(room);
+      // let's see if we need any m.room.member events
+      final membersToPostload = <String>{};
+      // the lastEvent message preview might have an author we need to fetch, if it is a group chat
+      if (room.getState(EventTypes.Message) != null && !room.isDirectChat) {
+        membersToPostload.add(room.getState(EventTypes.Message).senderId);
+      }
+      // if the room has no name and no canonical alias, its name is calculated
+      // based on the heroes of the room
+      if (room.getState(EventTypes.RoomName) == null &&
+          room.getState(EventTypes.RoomCanonicalAlias) == null &&
+          room.mHeroes != null) {
+        // we don't have a name and no canonical alias, so we'll need to
+        // post-load the heroes
+        membersToPostload.addAll(room.mHeroes.where((h) => h.isNotEmpty));
+      }
+      // okay, only load from the database if we actually have stuff to load
+      if (membersToPostload.isNotEmpty) {
+        // save it for loading later
+        allMembersToPostload[room.id] = membersToPostload;
+      }
+    }
+    // now we postload all members, if thre are any
+    if (allMembersToPostload.isNotEmpty) {
+      // we will generate a query to fetch as many events as possible at once, as that
+      // significantly improves performance. However, to prevent too large queries from being constructed,
+      // we limit to only fetching 500 rooms at once.
+      // This value might be fine-tune-able to be larger (and thus increase performance more for very large accounts),
+      // however this very conservative value should be on the safe side.
+      const MAX_ROOMS_PER_QUERY = 500;
+      // as we iterate over our entries in separate chunks one-by-one we use an iterator
+      // which persists accross the chunks, and thus we just re-sume iteration at the place
+      // we prreviously left off.
+      final entriesIterator = allMembersToPostload.entries.iterator;
+      // now we iterate over all our 500-room-chunks...
+      for (var i = 0;
+          i < allMembersToPostload.keys.length;
+          i += MAX_ROOMS_PER_QUERY) {
+        // query the current chunk and build the query
+        final membersRes = await (select(roomStates)
+              ..where((s) {
+                // all chunks have to have the reight client id and must be of type `m.room.member`
+                final basequery = s.clientId.equals(client.id) &
+                    s.type.equals('m.room.member');
+                // this is where the magic happens. Here we build a query with the form
+                // OR room_id = '!roomId1' AND state_key IN ('@member') OR room_id = '!roomId2' AND state_key IN ('@member')
+                // subqueries holds our query fragment
+                Expression<bool> subqueries;
+                // here we iterate over our chunk....we musn't forget to progress our iterator!
+                // we must check for if our chunk is done *before* progressing the
+                // iterator, else we might progress it twice around chunk edges, missing on rooms
+                for (var j = 0;
+                    j < MAX_ROOMS_PER_QUERY && entriesIterator.moveNext();
+                    j++) {
+                  final entry = entriesIterator.current;
+                  // builds room_id = '!roomId1' AND state_key IN ('@member')
+                  final q =
+                      s.roomId.equals(entry.key) & s.stateKey.isIn(entry.value);
+                  // adds it either as the start of subqueries or as a new OR condition to it
+                  if (subqueries == null) {
+                    subqueries = q;
+                  } else {
+                    subqueries = subqueries | q;
+                  }
+                }
+                // combinde the basequery with the subquery together, giving our final query
+                return basequery & subqueries;
+              }))
+            .get();
+        // now that we got all the entries from the database, set them as room states
+        for (final dbMember in membersRes) {
+          final room = roomList.firstWhere((r) => r.id == dbMember.roomId);
+          final event = sdk.Event.fromDb(dbMember, room);
+          room.setState(event);
+        }
+      }
     }
     return roomList;
   }
@@ -192,22 +270,6 @@ class Database extends _$Database {
     return newAccountData;
   }
 
-  Future<Map<String, api.Presence>> getPresences(int clientId) async {
-    final newPresences = <String, api.Presence>{};
-    final rawPresences = await getAllPresences(clientId).get();
-    for (final d in rawPresences) {
-      // TODO: Why is this not working?
-      try {
-        final content = sdk.Event.getMapFromPayload(d.content);
-        var presence = api.Presence.fromJson(content);
-        presence.senderId = d.sender;
-        presence.type = d.type;
-        newPresences[d.sender] = api.Presence.fromJson(content);
-      } catch (_) {}
-    }
-    return newPresences;
-  }
-
   /// Stores a RoomUpdate object in the database. Must be called inside of
   /// [transaction].
   final Set<String> _ensuredRooms = {};

lib/src/database/database.g.dart

@@ -5999,35 +5999,6 @@ abstract class _$Database extends GeneratedDatabase {
     );
   }
 
-  DbPresence _rowToDbPresence(QueryRow row) {
-    return DbPresence(
-      clientId: row.readInt('client_id'),
-      type: row.readString('type'),
-      sender: row.readString('sender'),
-      content: row.readString('content'),
-    );
-  }
-
-  Selectable<DbPresence> getAllPresences(int client_id) {
-    return customSelect('SELECT * FROM presences WHERE client_id = :client_id',
-        variables: [Variable.withInt(client_id)],
-        readsFrom: {presences}).map(_rowToDbPresence);
-  }
-
-  Future<int> storePresence(
-      int client_id, String type, String sender, String content) {
-    return customInsert(
-      'INSERT OR REPLACE INTO presences (client_id, type, sender, content) VALUES (:client_id, :type, :sender, :content)',
-      variables: [
-        Variable.withInt(client_id),
-        Variable.withString(type),
-        Variable.withString(sender),
-        Variable.withString(content)
-      ],
-      updates: {presences},
-    );
-  }
-
   Future<int> updateEvent(String unsigned, String content, String prev_content,
       int client_id, String event_id, String room_id) {
     return customUpdate(
@@ -6077,6 +6048,22 @@ abstract class _$Database extends GeneratedDatabase {
     );
   }
 
+  Selectable<DbRoomState> getImportantRoomStates(
+      int client_id, List<String> events) {
+    var $arrayStartIndex = 2;
+    final expandedevents = $expandVar($arrayStartIndex, events.length);
+    $arrayStartIndex += events.length;
+    return customSelect(
+        'SELECT * FROM room_states WHERE client_id = :client_id AND type IN ($expandedevents)',
+        variables: [
+          Variable.withInt(client_id),
+          for (var $ in events) Variable.withString($)
+        ],
+        readsFrom: {
+          roomStates
+        }).map(_rowToDbRoomState);
+  }
+
   Selectable<DbRoomState> getAllRoomStates(int client_id) {
     return customSelect(
         'SELECT * FROM room_states WHERE client_id = :client_id',
@@ -6084,6 +6071,23 @@ abstract class _$Database extends GeneratedDatabase {
         readsFrom: {roomStates}).map(_rowToDbRoomState);
   }
 
+  Selectable<DbRoomState> getUnimportantRoomStatesForRoom(
+      int client_id, String room_id, List<String> events) {
+    var $arrayStartIndex = 3;
+    final expandedevents = $expandVar($arrayStartIndex, events.length);
+    $arrayStartIndex += events.length;
+    return customSelect(
+        'SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type NOT IN ($expandedevents)',
+        variables: [
+          Variable.withInt(client_id),
+          Variable.withString(room_id),
+          for (var $ in events) Variable.withString($)
+        ],
+        readsFrom: {
+          roomStates
+        }).map(_rowToDbRoomState);
+  }
+
   Future<int> storeEvent(
       int client_id,
       String event_id,
@@ -6192,6 +6196,23 @@ abstract class _$Database extends GeneratedDatabase {
         }).map(_rowToDbRoomState);
   }
 
+  Selectable<DbRoomState> dbGetUsers(
+      int client_id, List<String> mxids, String room_id) {
+    var $arrayStartIndex = 2;
+    final expandedmxids = $expandVar($arrayStartIndex, mxids.length);
+    $arrayStartIndex += mxids.length;
+    return customSelect(
+        'SELECT * FROM room_states WHERE client_id = :client_id AND type = \'m.room.member\' AND state_key IN ($expandedmxids) AND room_id = :room_id',
+        variables: [
+          Variable.withInt(client_id),
+          for (var $ in mxids) Variable.withString($),
+          Variable.withString(room_id)
+        ],
+        readsFrom: {
+          roomStates
+        }).map(_rowToDbRoomState);
+  }
+
   DbEvent _rowToDbEvent(QueryRow row) {
     return DbEvent(
       clientId: row.readInt('client_id'),

lib/src/database/database.moor

@@ -206,11 +206,11 @@ setRoomPrevBatch: UPDATE rooms SET prev_batch = :prev_batch WHERE client_id = :c
 updateRoomSortOrder: UPDATE rooms SET oldest_sort_order = :oldest_sort_order, newest_sort_order = :newest_sort_order WHERE client_id = :client_id AND room_id = :room_id;
 getAllAccountData: SELECT * FROM account_data WHERE client_id = :client_id;
 storeAccountData: INSERT OR REPLACE INTO account_data (client_id, type, content) VALUES (:client_id, :type, :content);
-getAllPresences: SELECT * FROM presences WHERE client_id = :client_id;
-storePresence: INSERT OR REPLACE INTO presences (client_id, type, sender, content) VALUES (:client_id, :type, :sender, :content);
 updateEvent: UPDATE events SET unsigned = :unsigned, content = :content, prev_content = :prev_content WHERE client_id = :client_id AND event_id = :event_id AND room_id = :room_id;
 updateEventStatus: UPDATE events SET status = :status, event_id = :new_event_id WHERE client_id = :client_id AND event_id = :old_event_id AND room_id = :room_id;
+getImportantRoomStates: SELECT * FROM room_states WHERE client_id = :client_id AND type IN :events;
 getAllRoomStates: SELECT * FROM room_states WHERE client_id = :client_id;
+getUnimportantRoomStatesForRoom: SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type NOT IN :events;
 storeEvent: INSERT OR REPLACE INTO events (client_id, event_id, room_id, sort_order, origin_server_ts, sender, type, unsigned, content, prev_content, state_key, status) VALUES (:client_id, :event_id, :room_id, :sort_order, :origin_server_ts, :sender, :type, :unsigned, :content, :prev_content, :state_key, :status);
 storeRoomState: INSERT OR REPLACE INTO room_states (client_id, event_id, room_id, sort_order, origin_server_ts, sender, type, unsigned, content, prev_content, state_key) VALUES (:client_id, :event_id, :room_id, :sort_order, :origin_server_ts, :sender, :type, :unsigned, :content, :prev_content, :state_key);
 getAllRoomAccountData: SELECT * FROM room_account_data WHERE client_id = :client_id;

lib/src/room.dart

@@ -101,6 +101,25 @@ class Room {
         _oldestSortOrder, _newestSortOrder, client.id, id);
   }
 
+  /// Flag if the room is partial, meaning not all state events have been loaded yet
+  bool partial = true;
+
+  /// Load all the missing state events for the room from the database. If the room has already been loaded, this does nothing.
+  Future<void> postLoad() async {
+    if (!partial || client.database == null) {
+      return;
+    }
+    final allStates = await client.database
+        .getUnimportantRoomStatesForRoom(
+            client.id, id, client.importantStateEvents.toList())
+        .get();
+    for (final state in allStates) {
+      final newState = Event.fromDb(state, this);
+      setState(newState);
+    }
+    partial = false;
+  }
+
   /// Returns the [Event] for the given [typeKey] and optional [stateKey].
   /// If no [stateKey] is provided, it defaults to an empty string.
   Event getState(String typeKey, [String stateKey = '']) =>
@@ -934,6 +953,7 @@ class Room {
   Future<Timeline> getTimeline(
       {onTimelineUpdateCallback onUpdate,
       onTimelineInsertCallback onInsert}) async {
+    await postLoad();
     var events;
     if (client.database != null) {
       events = await client.database.getEventList(client.id, this);
@@ -1033,6 +1053,15 @@ class Room {
     if (getState(EventTypes.RoomMember, mxID) != null) {
       return getState(EventTypes.RoomMember, mxID).asUser;
     }
+    if (client.database != null) {
+      // it may be in the database
+      final user = await client.database.getUser(client.id, mxID, this);
+      if (user != null) {
+        setState(user);
+        if (onUpdate != null) onUpdate.add(id);
+        return user;
+      }
+    }
     if (mxID == null || !_requestingMatrixIds.add(mxID)) return null;
     Map<String, dynamic> resp;
     try {