re-work state lazy loading after discussion

lib/src/client.dart

@@ -58,6 +58,8 @@ class Client {
 
   Set<KeyVerificationMethod> verificationMethods;
 
+  Set<String> importantStateEvents;
+
   /// Create a client
   /// clientName = unique identifier of this client
   /// debug: Print debug output?
@@ -66,13 +68,37 @@ class Client {
   /// verificationMethods: A set of all the verification methods this client can handle. Includes:
   ///    KeyVerificationMethod.numbers: Compare numbers. Most basic, should be supported
   ///    KeyVerificationMethod.emoji: Compare emojis
+  /// importantStateEvents: A set of all the important state events to load when the client connects.
+  ///    To speed up performance only a set of state events is loaded on startup, those that are
+  ///    needed to display a room list. All the remaining state events are automatically post-loaded
+  ///    when opening the timeline of a room or manually by calling `room.postLoad()`.
+  ///    This set will always include the following state events:
+  ///     - m.room.name
+  ///     - m.room.avatar
+  ///     - m.room.message
+  ///     - m.room.encrypted
+  ///     - m.room.encryption
+  ///     - m.room.canonical_alias
+  ///     - m.room.tombstone
+  ///     - *some* m.room.member events, where needed
   Client(this.clientName,
       {this.debug = false,
       this.database,
       this.enableE2eeRecovery = false,
       this.verificationMethods,
-      http.Client httpClient}) {
+      http.Client httpClient,
+      this.importantStateEvents}) {
     verificationMethods ??= <KeyVerificationMethod>{};
+    importantStateEvents ??= <String>{};
+    importantStateEvents.addAll([
+      'm.room.name',
+      'm.room.avatar',
+      'm.room.message',
+      'm.room.encrypted',
+      'm.room.encryption',
+      'm.room.canonical_alias',
+      'm.room.tombstone',
+    ]);
     api = MatrixApi(debug: debug, httpClient: httpClient);
     onLoginStateChanged.stream.listen((loginState) {
       if (debug) {
@@ -533,10 +559,6 @@ class Client {
   final StreamController<KeyVerification> onKeyVerificationRequest =
       StreamController.broadcast();
 
-  /// When a new update entered, be it a sync or an avatar post-loaded
-  /// payload will always be true
-  final StreamController<bool> onUpdate = StreamController.broadcast();
-
   /// Matrix synchronisation is done with https long polling. This needs a
   /// timeout which is usually 30 seconds.
   int syncTimeoutSec = 30;
@@ -646,9 +668,6 @@ class Client {
       }
       _userDeviceKeys = await database.getUserDeviceKeys(this);
       _rooms = await database.getRoomList(this, onlyLeft: false);
-      for (final r in rooms) {
-        r.onUpdate.stream.listen((v) => _addUpdate());
-      }
       _sortRooms();
       accountData = await database.getAccountData(id);
       presences.clear();
@@ -768,7 +787,6 @@ class Client {
       encryption.handleDeviceOneTimeKeysCount(sync.deviceOneTimeKeysCount);
     }
     onSync.add(sync);
-    _addUpdate();
   }
 
   Future<void> _handleDeviceListsEvents(DeviceListsUpdate deviceLists) async {
@@ -1009,7 +1027,6 @@ class Client {
         roomAccountData: {},
         client: this,
       );
-      newRoom.onUpdate.stream.listen((v) => _addUpdate());
       rooms.insert(position, newRoom);
     }
     // If the membership is "leave" then remove the item and stop here
@@ -1420,16 +1437,6 @@ class Client {
     }
   }
 
-  Timer _updateTimer;
-
-  void _addUpdate() {
-    // we only want max. one update per 50ms
-    _updateTimer ??= Timer(Duration(milliseconds: 50), () {
-      onUpdate.add(true);
-      _updateTimer = null;
-    });
-  }
-
   bool _disposed = false;
 
   /// Stops the synchronization and closes the database. After this

lib/src/database/database.dart

@@ -157,9 +157,12 @@ class Database extends _$Database {
               ? t.membership.equals('leave')
               : t.membership.equals('leave').not()))
         .get();
-    final resStates = await getImportantRoomStates(client.id).get();
+    final resStates = await getImportantRoomStates(
+            client.id, client.importantStateEvents.toList())
+        .get();
     final resAccountData = await getAllRoomAccountData(client.id).get();
     final roomList = <sdk.Room>[];
+    final allMembersToPostload = <String, Set<String>>{};
     for (final r in res) {
       final room = await sdk.Room.getRoomFromTableRow(
         r,
@@ -168,6 +171,81 @@ class Database extends _$Database {
         roomAccountData: resAccountData.where((rs) => rs.roomId == r.roomId),
       );
       roomList.add(room);
+      // let's see if we need any m.room.member events
+      final membersToPostload = <String>{};
+      // the lastEvent message preview might have an author we need to fetch, if it is a group chat
+      if (room.getState(EventTypes.Message) != null && !room.isDirectChat) {
+        membersToPostload.add(room.getState(EventTypes.Message).senderId);
+      }
+      // if the room has no name and no canonical alias, its name is calculated
+      // based on the heroes of the room
+      if (room.getState(EventTypes.RoomName) == null &&
+          room.getState(EventTypes.RoomCanonicalAlias) == null &&
+          room.mHeroes != null) {
+        // we don't have a name and no canonical alias, so we'll need to
+        // post-load the heroes
+        membersToPostload.addAll(room.mHeroes.where((h) => h.isNotEmpty));
+      }
+      // okay, only load from the database if we actually have stuff to load
+      if (membersToPostload.isNotEmpty) {
+        // save it for loading later
+        allMembersToPostload[room.id] = membersToPostload;
+      }
+    }
+    // now we postload all members, if thre are any
+    if (allMembersToPostload.isNotEmpty) {
+      // we will generate a query to fetch as many events as possible at once, as that
+      // significantly improves performance. However, to prevent too large queries from being constructed,
+      // we limit to only fetching 500 rooms at once.
+      // This value might be fine-tune-able to be larger (and thus increase performance more for very large accounts),
+      // however this very conservative value should be on the safe side.
+      final MAX_ROOMS_PER_QUERY = 500;
+      // as we iterate over our entries in separate chunks one-by-one we use an iterator
+      // which persists accross the chunks, and thus we just re-sume iteration at the place
+      // we prreviously left off.
+      final entriesIterator = allMembersToPostload.entries.iterator;
+      // now we iterate over all our 500-room-chunks...
+      for (var i = 0;
+          i < allMembersToPostload.keys.length;
+          i += MAX_ROOMS_PER_QUERY) {
+        // query the current chunk and build the query
+        final membersRes = await (select(roomStates)
+              ..where((s) {
+                // all chunks have to have the reight client id and must be of type `m.room.member`
+                final basequery = s.clientId.equals(client.id) &
+                    s.type.equals('m.room.member');
+                // this is where the magic happens. Here we build a query with the form
+                // OR room_id = '!roomId1' AND state_key IN ('@member') OR room_id = '!roomId2' AND state_key IN ('@member')
+                // subqueries holds our query fragment
+                Expression<bool> subqueries;
+                // here we iterate over our chunk....we musn't forget to progress our iterator!
+                // we must check for if our chunk is done *before* progressing the
+                // iterator, else we might progress it twice around chunk edges, missing on rooms
+                for (var j = 0;
+                    j < MAX_ROOMS_PER_QUERY && entriesIterator.moveNext();
+                    j++) {
+                  final entry = entriesIterator.current;
+                  // builds room_id = '!roomId1' AND state_key IN ('@member')
+                  final q =
+                      s.roomId.equals(entry.key) & s.stateKey.isIn(entry.value);
+                  // adds it either as the start of subqueries or as a new OR condition to it
+                  if (subqueries == null) {
+                    subqueries = q;
+                  } else {
+                    subqueries = subqueries | q;
+                  }
+                }
+                // combinde the basequery with the subquery together, giving our final query
+                return basequery & subqueries;
+              }))
+            .get();
+        // now that we got all the entries from the database, set them as room states
+        for (final dbMember in membersRes) {
+          final room = roomList.firstWhere((r) => r.id == dbMember.roomId);
+          final event = sdk.Event.fromDb(dbMember, room);
+          room.setState(event);
+        }
+      }
     }
     return roomList;
   }

lib/src/database/database.g.dart

@@ -6048,11 +6048,20 @@ abstract class _$Database extends GeneratedDatabase {
     );
   }
 
-  Selectable<DbRoomState> getImportantRoomStates(int client_id) {
+  Selectable<DbRoomState> getImportantRoomStates(
+      int client_id, List<String> events) {
+    var $arrayStartIndex = 2;
+    final expandedevents = $expandVar($arrayStartIndex, events.length);
+    $arrayStartIndex += events.length;
     return customSelect(
-        'SELECT * FROM room_states WHERE client_id = :client_id AND type <> \'m.room.member\'',
+        'SELECT * FROM room_states WHERE client_id = :client_id AND type IN ($expandedevents)',
-        variables: [Variable.withInt(client_id)],
+        variables: [
-        readsFrom: {roomStates}).map(_rowToDbRoomState);
+          Variable.withInt(client_id),
+          for (var $ in events) Variable.withString($)
+        ],
+        readsFrom: {
+          roomStates
+        }).map(_rowToDbRoomState);
   }
 
   Selectable<DbRoomState> getAllRoomStates(int client_id) {
@@ -6063,11 +6072,20 @@ abstract class _$Database extends GeneratedDatabase {
   }
 
   Selectable<DbRoomState> getUnimportantRoomStatesForRoom(
-      int client_id, String room_id) {
+      int client_id, String room_id, List<String> events) {
+    var $arrayStartIndex = 3;
+    final expandedevents = $expandVar($arrayStartIndex, events.length);
+    $arrayStartIndex += events.length;
     return customSelect(
-        'SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type = \'m.room.member\'',
+        'SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type NOT IN ($expandedevents)',
-        variables: [Variable.withInt(client_id), Variable.withString(room_id)],
+        variables: [
-        readsFrom: {roomStates}).map(_rowToDbRoomState);
+          Variable.withInt(client_id),
+          Variable.withString(room_id),
+          for (var $ in events) Variable.withString($)
+        ],
+        readsFrom: {
+          roomStates
+        }).map(_rowToDbRoomState);
   }
 
   Future<int> storeEvent(
@@ -6178,6 +6196,23 @@ abstract class _$Database extends GeneratedDatabase {
         }).map(_rowToDbRoomState);
   }
 
+  Selectable<DbRoomState> dbGetUsers(
+      int client_id, List<String> mxids, String room_id) {
+    var $arrayStartIndex = 2;
+    final expandedmxids = $expandVar($arrayStartIndex, mxids.length);
+    $arrayStartIndex += mxids.length;
+    return customSelect(
+        'SELECT * FROM room_states WHERE client_id = :client_id AND type = \'m.room.member\' AND state_key IN ($expandedmxids) AND room_id = :room_id',
+        variables: [
+          Variable.withInt(client_id),
+          for (var $ in mxids) Variable.withString($),
+          Variable.withString(room_id)
+        ],
+        readsFrom: {
+          roomStates
+        }).map(_rowToDbRoomState);
+  }
+
   DbEvent _rowToDbEvent(QueryRow row) {
     return DbEvent(
       clientId: row.readInt('client_id'),

lib/src/database/database.moor

@@ -208,10 +208,9 @@ getAllAccountData: SELECT * FROM account_data WHERE client_id = :client_id;
 storeAccountData: INSERT OR REPLACE INTO account_data (client_id, type, content) VALUES (:client_id, :type, :content);
 updateEvent: UPDATE events SET unsigned = :unsigned, content = :content, prev_content = :prev_content WHERE client_id = :client_id AND event_id = :event_id AND room_id = :room_id;
 updateEventStatus: UPDATE events SET status = :status, event_id = :new_event_id WHERE client_id = :client_id AND event_id = :old_event_id AND room_id = :room_id;
---getImportantRoomStates: SELECT * FROM room_states WHERE client_id = :client_id AND type IN ('m.room.name', 'm.room.avatar', 'm.room.message', 'm.room.encrypted', 'm.room.encryption', 'im.ponies.room_emotes');
+getImportantRoomStates: SELECT * FROM room_states WHERE client_id = :client_id AND type IN :events;
-getImportantRoomStates: SELECT * FROM room_states WHERE client_id = :client_id AND type <> 'm.room.member';
 getAllRoomStates: SELECT * FROM room_states WHERE client_id = :client_id;
-getUnimportantRoomStatesForRoom: SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type = 'm.room.member';
+getUnimportantRoomStatesForRoom: SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type NOT IN :events;
 storeEvent: INSERT OR REPLACE INTO events (client_id, event_id, room_id, sort_order, origin_server_ts, sender, type, unsigned, content, prev_content, state_key, status) VALUES (:client_id, :event_id, :room_id, :sort_order, :origin_server_ts, :sender, :type, :unsigned, :content, :prev_content, :state_key, :status);
 storeRoomState: INSERT OR REPLACE INTO room_states (client_id, event_id, room_id, sort_order, origin_server_ts, sender, type, unsigned, content, prev_content, state_key) VALUES (:client_id, :event_id, :room_id, :sort_order, :origin_server_ts, :sender, :type, :unsigned, :content, :prev_content, :state_key);
 getAllRoomAccountData: SELECT * FROM room_account_data WHERE client_id = :client_id;

lib/src/room.dart

@@ -101,13 +101,17 @@ class Room {
         _oldestSortOrder, _newestSortOrder, client.id, id);
   }
 
+  /// Flag if the room is partial, meaning not all state events have been loaded yet
   bool partial = true;
+
+  /// Load all the missing state events for the room from the database. If the room has already been loaded, this does nothing.
   Future<void> postLoad() async {
     if (!partial || client.database == null) {
       return;
     }
     final allStates = await client.database
-        .getUnimportantRoomStatesForRoom(client.id, id)
+        .getUnimportantRoomStatesForRoom(
+            client.id, id, client.importantStateEvents.toList())
         .get();
     for (final state in allStates) {
       final newState = Event.fromDb(state, this);