From 8f122195c5edd17e50e762c93a9609321197744d Mon Sep 17 00:00:00 2001 From: Sorunome Date: Wed, 1 Jul 2020 11:09:31 +0200 Subject: [PATCH] re-work state lazy loading after discussion --- lib/src/client.dart | 47 +++++++++++-------- lib/src/database/database.dart | 80 +++++++++++++++++++++++++++++++- lib/src/database/database.g.dart | 51 ++++++++++++++++---- lib/src/database/database.moor | 5 +- lib/src/room.dart | 6 ++- 5 files changed, 156 insertions(+), 33 deletions(-) diff --git a/lib/src/client.dart b/lib/src/client.dart index 53d1186..5efec41 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -58,6 +58,8 @@ class Client { Set verificationMethods; + Set importantStateEvents; + /// Create a client /// clientName = unique identifier of this client /// debug: Print debug output? @@ -66,13 +68,37 @@ class Client { /// verificationMethods: A set of all the verification methods this client can handle. Includes: /// KeyVerificationMethod.numbers: Compare numbers. Most basic, should be supported /// KeyVerificationMethod.emoji: Compare emojis + /// importantStateEvents: A set of all the important state events to load when the client connects. + /// To speed up performance only a set of state events is loaded on startup, those that are + /// needed to display a room list. All the remaining state events are automatically post-loaded + /// when opening the timeline of a room or manually by calling `room.postLoad()`. + /// This set will always include the following state events: + /// - m.room.name + /// - m.room.avatar + /// - m.room.message + /// - m.room.encrypted + /// - m.room.encryption + /// - m.room.canonical_alias + /// - m.room.tombstone + /// - *some* m.room.member events, where needed Client(this.clientName, {this.debug = false, this.database, this.enableE2eeRecovery = false, this.verificationMethods, - http.Client httpClient}) { + http.Client httpClient, + this.importantStateEvents}) { verificationMethods ??= {}; + importantStateEvents ??= {}; + importantStateEvents.addAll([ + 'm.room.name', + 'm.room.avatar', + 'm.room.message', + 'm.room.encrypted', + 'm.room.encryption', + 'm.room.canonical_alias', + 'm.room.tombstone', + ]); api = MatrixApi(debug: debug, httpClient: httpClient); onLoginStateChanged.stream.listen((loginState) { if (debug) { @@ -533,10 +559,6 @@ class Client { final StreamController onKeyVerificationRequest = StreamController.broadcast(); - /// When a new update entered, be it a sync or an avatar post-loaded - /// payload will always be true - final StreamController onUpdate = StreamController.broadcast(); - /// Matrix synchronisation is done with https long polling. This needs a /// timeout which is usually 30 seconds. int syncTimeoutSec = 30; @@ -646,9 +668,6 @@ class Client { } _userDeviceKeys = await database.getUserDeviceKeys(this); _rooms = await database.getRoomList(this, onlyLeft: false); - for (final r in rooms) { - r.onUpdate.stream.listen((v) => _addUpdate()); - } _sortRooms(); accountData = await database.getAccountData(id); presences.clear(); @@ -768,7 +787,6 @@ class Client { encryption.handleDeviceOneTimeKeysCount(sync.deviceOneTimeKeysCount); } onSync.add(sync); - _addUpdate(); } Future _handleDeviceListsEvents(DeviceListsUpdate deviceLists) async { @@ -1009,7 +1027,6 @@ class Client { roomAccountData: {}, client: this, ); - newRoom.onUpdate.stream.listen((v) => _addUpdate()); rooms.insert(position, newRoom); } // If the membership is "leave" then remove the item and stop here @@ -1420,16 +1437,6 @@ class Client { } } - Timer _updateTimer; - - void _addUpdate() { - // we only want max. one update per 50ms - _updateTimer ??= Timer(Duration(milliseconds: 50), () { - onUpdate.add(true); - _updateTimer = null; - }); - } - bool _disposed = false; /// Stops the synchronization and closes the database. After this diff --git a/lib/src/database/database.dart b/lib/src/database/database.dart index dc6cdd6..69052c6 100644 --- a/lib/src/database/database.dart +++ b/lib/src/database/database.dart @@ -157,9 +157,12 @@ class Database extends _$Database { ? t.membership.equals('leave') : t.membership.equals('leave').not())) .get(); - final resStates = await getImportantRoomStates(client.id).get(); + final resStates = await getImportantRoomStates( + client.id, client.importantStateEvents.toList()) + .get(); final resAccountData = await getAllRoomAccountData(client.id).get(); final roomList = []; + final allMembersToPostload = >{}; for (final r in res) { final room = await sdk.Room.getRoomFromTableRow( r, @@ -168,6 +171,81 @@ class Database extends _$Database { roomAccountData: resAccountData.where((rs) => rs.roomId == r.roomId), ); roomList.add(room); + // let's see if we need any m.room.member events + final membersToPostload = {}; + // the lastEvent message preview might have an author we need to fetch, if it is a group chat + if (room.getState(EventTypes.Message) != null && !room.isDirectChat) { + membersToPostload.add(room.getState(EventTypes.Message).senderId); + } + // if the room has no name and no canonical alias, its name is calculated + // based on the heroes of the room + if (room.getState(EventTypes.RoomName) == null && + room.getState(EventTypes.RoomCanonicalAlias) == null && + room.mHeroes != null) { + // we don't have a name and no canonical alias, so we'll need to + // post-load the heroes + membersToPostload.addAll(room.mHeroes.where((h) => h.isNotEmpty)); + } + // okay, only load from the database if we actually have stuff to load + if (membersToPostload.isNotEmpty) { + // save it for loading later + allMembersToPostload[room.id] = membersToPostload; + } + } + // now we postload all members, if thre are any + if (allMembersToPostload.isNotEmpty) { + // we will generate a query to fetch as many events as possible at once, as that + // significantly improves performance. However, to prevent too large queries from being constructed, + // we limit to only fetching 500 rooms at once. + // This value might be fine-tune-able to be larger (and thus increase performance more for very large accounts), + // however this very conservative value should be on the safe side. + final MAX_ROOMS_PER_QUERY = 500; + // as we iterate over our entries in separate chunks one-by-one we use an iterator + // which persists accross the chunks, and thus we just re-sume iteration at the place + // we prreviously left off. + final entriesIterator = allMembersToPostload.entries.iterator; + // now we iterate over all our 500-room-chunks... + for (var i = 0; + i < allMembersToPostload.keys.length; + i += MAX_ROOMS_PER_QUERY) { + // query the current chunk and build the query + final membersRes = await (select(roomStates) + ..where((s) { + // all chunks have to have the reight client id and must be of type `m.room.member` + final basequery = s.clientId.equals(client.id) & + s.type.equals('m.room.member'); + // this is where the magic happens. Here we build a query with the form + // OR room_id = '!roomId1' AND state_key IN ('@member') OR room_id = '!roomId2' AND state_key IN ('@member') + // subqueries holds our query fragment + Expression subqueries; + // here we iterate over our chunk....we musn't forget to progress our iterator! + // we must check for if our chunk is done *before* progressing the + // iterator, else we might progress it twice around chunk edges, missing on rooms + for (var j = 0; + j < MAX_ROOMS_PER_QUERY && entriesIterator.moveNext(); + j++) { + final entry = entriesIterator.current; + // builds room_id = '!roomId1' AND state_key IN ('@member') + final q = + s.roomId.equals(entry.key) & s.stateKey.isIn(entry.value); + // adds it either as the start of subqueries or as a new OR condition to it + if (subqueries == null) { + subqueries = q; + } else { + subqueries = subqueries | q; + } + } + // combinde the basequery with the subquery together, giving our final query + return basequery & subqueries; + })) + .get(); + // now that we got all the entries from the database, set them as room states + for (final dbMember in membersRes) { + final room = roomList.firstWhere((r) => r.id == dbMember.roomId); + final event = sdk.Event.fromDb(dbMember, room); + room.setState(event); + } + } } return roomList; } diff --git a/lib/src/database/database.g.dart b/lib/src/database/database.g.dart index 9e11f41..e17101d 100644 --- a/lib/src/database/database.g.dart +++ b/lib/src/database/database.g.dart @@ -6048,11 +6048,20 @@ abstract class _$Database extends GeneratedDatabase { ); } - Selectable getImportantRoomStates(int client_id) { + Selectable getImportantRoomStates( + int client_id, List events) { + var $arrayStartIndex = 2; + final expandedevents = $expandVar($arrayStartIndex, events.length); + $arrayStartIndex += events.length; return customSelect( - 'SELECT * FROM room_states WHERE client_id = :client_id AND type <> \'m.room.member\'', - variables: [Variable.withInt(client_id)], - readsFrom: {roomStates}).map(_rowToDbRoomState); + 'SELECT * FROM room_states WHERE client_id = :client_id AND type IN ($expandedevents)', + variables: [ + Variable.withInt(client_id), + for (var $ in events) Variable.withString($) + ], + readsFrom: { + roomStates + }).map(_rowToDbRoomState); } Selectable getAllRoomStates(int client_id) { @@ -6063,11 +6072,20 @@ abstract class _$Database extends GeneratedDatabase { } Selectable getUnimportantRoomStatesForRoom( - int client_id, String room_id) { + int client_id, String room_id, List events) { + var $arrayStartIndex = 3; + final expandedevents = $expandVar($arrayStartIndex, events.length); + $arrayStartIndex += events.length; return customSelect( - 'SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type = \'m.room.member\'', - variables: [Variable.withInt(client_id), Variable.withString(room_id)], - readsFrom: {roomStates}).map(_rowToDbRoomState); + 'SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type NOT IN ($expandedevents)', + variables: [ + Variable.withInt(client_id), + Variable.withString(room_id), + for (var $ in events) Variable.withString($) + ], + readsFrom: { + roomStates + }).map(_rowToDbRoomState); } Future storeEvent( @@ -6178,6 +6196,23 @@ abstract class _$Database extends GeneratedDatabase { }).map(_rowToDbRoomState); } + Selectable dbGetUsers( + int client_id, List mxids, String room_id) { + var $arrayStartIndex = 2; + final expandedmxids = $expandVar($arrayStartIndex, mxids.length); + $arrayStartIndex += mxids.length; + return customSelect( + 'SELECT * FROM room_states WHERE client_id = :client_id AND type = \'m.room.member\' AND state_key IN ($expandedmxids) AND room_id = :room_id', + variables: [ + Variable.withInt(client_id), + for (var $ in mxids) Variable.withString($), + Variable.withString(room_id) + ], + readsFrom: { + roomStates + }).map(_rowToDbRoomState); + } + DbEvent _rowToDbEvent(QueryRow row) { return DbEvent( clientId: row.readInt('client_id'), diff --git a/lib/src/database/database.moor b/lib/src/database/database.moor index 1dbdef1..716bdc6 100644 --- a/lib/src/database/database.moor +++ b/lib/src/database/database.moor @@ -208,10 +208,9 @@ getAllAccountData: SELECT * FROM account_data WHERE client_id = :client_id; storeAccountData: INSERT OR REPLACE INTO account_data (client_id, type, content) VALUES (:client_id, :type, :content); updateEvent: UPDATE events SET unsigned = :unsigned, content = :content, prev_content = :prev_content WHERE client_id = :client_id AND event_id = :event_id AND room_id = :room_id; updateEventStatus: UPDATE events SET status = :status, event_id = :new_event_id WHERE client_id = :client_id AND event_id = :old_event_id AND room_id = :room_id; ---getImportantRoomStates: SELECT * FROM room_states WHERE client_id = :client_id AND type IN ('m.room.name', 'm.room.avatar', 'm.room.message', 'm.room.encrypted', 'm.room.encryption', 'im.ponies.room_emotes'); -getImportantRoomStates: SELECT * FROM room_states WHERE client_id = :client_id AND type <> 'm.room.member'; +getImportantRoomStates: SELECT * FROM room_states WHERE client_id = :client_id AND type IN :events; getAllRoomStates: SELECT * FROM room_states WHERE client_id = :client_id; -getUnimportantRoomStatesForRoom: SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type = 'm.room.member'; +getUnimportantRoomStatesForRoom: SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type NOT IN :events; storeEvent: INSERT OR REPLACE INTO events (client_id, event_id, room_id, sort_order, origin_server_ts, sender, type, unsigned, content, prev_content, state_key, status) VALUES (:client_id, :event_id, :room_id, :sort_order, :origin_server_ts, :sender, :type, :unsigned, :content, :prev_content, :state_key, :status); storeRoomState: INSERT OR REPLACE INTO room_states (client_id, event_id, room_id, sort_order, origin_server_ts, sender, type, unsigned, content, prev_content, state_key) VALUES (:client_id, :event_id, :room_id, :sort_order, :origin_server_ts, :sender, :type, :unsigned, :content, :prev_content, :state_key); getAllRoomAccountData: SELECT * FROM room_account_data WHERE client_id = :client_id; diff --git a/lib/src/room.dart b/lib/src/room.dart index 55992dc..40aa431 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -101,13 +101,17 @@ class Room { _oldestSortOrder, _newestSortOrder, client.id, id); } + /// Flag if the room is partial, meaning not all state events have been loaded yet bool partial = true; + + /// Load all the missing state events for the room from the database. If the room has already been loaded, this does nothing. Future postLoad() async { if (!partial || client.database == null) { return; } final allStates = await client.database - .getUnimportantRoomStatesForRoom(client.id, id) + .getUnimportantRoomStatesForRoom( + client.id, id, client.importantStateEvents.toList()) .get(); for (final state in allStates) { final newState = Event.fromDb(state, this);