From a1f8120c59fcb961e62287a181142f464d10adb1 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Tue, 30 Jun 2020 12:17:56 +0200 Subject: [PATCH 1/5] Greatly imporve initial loading performance --- lib/src/client.dart | 21 +++++++++++++++++++++ lib/src/database/database.dart | 2 +- lib/src/database/database.g.dart | 15 +++++++++++++++ lib/src/database/database.moor | 2 ++ lib/src/room.dart | 23 +++++++++++++++++++++++ 5 files changed, 62 insertions(+), 1 deletion(-) diff --git a/lib/src/client.dart b/lib/src/client.dart index 6208b28..7448ade 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -533,6 +533,10 @@ class Client { final StreamController onKeyVerificationRequest = StreamController.broadcast(); + /// When a new update entered, be it a sync or an avatar post-loaded + /// payload will always be true + final StreamController onUpdate = StreamController.broadcast(); + /// Matrix synchronisation is done with https long polling. This needs a /// timeout which is usually 30 seconds. int syncTimeoutSec = 30; @@ -642,6 +646,9 @@ class Client { } _userDeviceKeys = await database.getUserDeviceKeys(this); _rooms = await database.getRoomList(this, onlyLeft: false); + for (final r in rooms) { + r.onUpdate.stream.listen((v) => _addUpdate()); + } _sortRooms(); accountData = await database.getAccountData(id); presences = await database.getPresences(id); @@ -769,6 +776,7 @@ class Client { encryption.handleDeviceOneTimeKeysCount(sync.deviceOneTimeKeysCount); } onSync.add(sync); + _addUpdate(); } Future _handleDeviceListsEvents(DeviceListsUpdate deviceLists) async { @@ -1009,6 +1017,7 @@ class Client { roomAccountData: {}, client: this, ); + newRoom.onUpdate.stream.listen((v) => _addUpdate()); rooms.insert(position, newRoom); } // If the membership is "leave" then remove the item and stop here @@ -1419,6 +1428,18 @@ class Client { } } + Timer _updateTimer; + + void _addUpdate() { + // we only want max. one update per 50ms + if (_updateTimer == null) { + _updateTimer = Timer(Duration(milliseconds: 50), () { + onUpdate.add(true); + _updateTimer = null; + }); + } + } + bool _disposed = false; /// Stops the synchronization and closes the database. After this diff --git a/lib/src/database/database.dart b/lib/src/database/database.dart index 39a3783..7240a35 100644 --- a/lib/src/database/database.dart +++ b/lib/src/database/database.dart @@ -157,7 +157,7 @@ class Database extends _$Database { ? t.membership.equals('leave') : t.membership.equals('leave').not())) .get(); - final resStates = await getAllRoomStates(client.id).get(); + final resStates = await getImportantRoomStates(client.id).get(); final resAccountData = await getAllRoomAccountData(client.id).get(); final roomList = []; for (final r in res) { diff --git a/lib/src/database/database.g.dart b/lib/src/database/database.g.dart index d51eec5..8971453 100644 --- a/lib/src/database/database.g.dart +++ b/lib/src/database/database.g.dart @@ -6077,6 +6077,13 @@ abstract class _$Database extends GeneratedDatabase { ); } + Selectable getImportantRoomStates(int client_id) { + return customSelect( + 'SELECT * FROM room_states WHERE client_id = :client_id AND type IN (\'m.room.name\', \'m.room.avatar\', \'m.room.message\', \'m.room.encrypted\', \'m.room.encryption\')', + variables: [Variable.withInt(client_id)], + readsFrom: {roomStates}).map(_rowToDbRoomState); + } + Selectable getAllRoomStates(int client_id) { return customSelect( 'SELECT * FROM room_states WHERE client_id = :client_id', @@ -6084,6 +6091,14 @@ abstract class _$Database extends GeneratedDatabase { readsFrom: {roomStates}).map(_rowToDbRoomState); } + Selectable getAllRoomStatesForRoom( + int client_id, String room_id) { + return customSelect( + 'SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id', + variables: [Variable.withInt(client_id), Variable.withString(room_id)], + readsFrom: {roomStates}).map(_rowToDbRoomState); + } + Future storeEvent( int client_id, String event_id, diff --git a/lib/src/database/database.moor b/lib/src/database/database.moor index c2e5ff0..b5459bf 100644 --- a/lib/src/database/database.moor +++ b/lib/src/database/database.moor @@ -210,7 +210,9 @@ getAllPresences: SELECT * FROM presences WHERE client_id = :client_id; storePresence: INSERT OR REPLACE INTO presences (client_id, type, sender, content) VALUES (:client_id, :type, :sender, :content); updateEvent: UPDATE events SET unsigned = :unsigned, content = :content, prev_content = :prev_content WHERE client_id = :client_id AND event_id = :event_id AND room_id = :room_id; updateEventStatus: UPDATE events SET status = :status, event_id = :new_event_id WHERE client_id = :client_id AND event_id = :old_event_id AND room_id = :room_id; +getImportantRoomStates: SELECT * FROM room_states WHERE client_id = :client_id AND type IN ('m.room.name', 'm.room.avatar', 'm.room.message', 'm.room.encrypted', 'm.room.encryption'); getAllRoomStates: SELECT * FROM room_states WHERE client_id = :client_id; +getAllRoomStatesForRoom: SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id; storeEvent: INSERT OR REPLACE INTO events (client_id, event_id, room_id, sort_order, origin_server_ts, sender, type, unsigned, content, prev_content, state_key, status) VALUES (:client_id, :event_id, :room_id, :sort_order, :origin_server_ts, :sender, :type, :unsigned, :content, :prev_content, :state_key, :status); storeRoomState: INSERT OR REPLACE INTO room_states (client_id, event_id, room_id, sort_order, origin_server_ts, sender, type, unsigned, content, prev_content, state_key) VALUES (:client_id, :event_id, :room_id, :sort_order, :origin_server_ts, :sender, :type, :unsigned, :content, :prev_content, :state_key); getAllRoomAccountData: SELECT * FROM room_account_data WHERE client_id = :client_id; diff --git a/lib/src/room.dart b/lib/src/room.dart index 6f93ea3..2f92ade 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -101,6 +101,19 @@ class Room { _oldestSortOrder, _newestSortOrder, client.id, id); } + bool partial = true; + Future postLoad() async { + if (!partial || client.database == null) { + return; + } + final allStates = await client.database.getAllRoomStatesForRoom(client.id, id).get(); + for (final state in allStates) { + final newState = Event.fromDb(state, this); + setState(newState); + } + partial = false; + } + /// Returns the [Event] for the given [typeKey] and optional [stateKey]. /// If no [stateKey] is provided, it defaults to an empty string. Event getState(String typeKey, [String stateKey = '']) => @@ -934,6 +947,7 @@ class Room { Future getTimeline( {onTimelineUpdateCallback onUpdate, onTimelineInsertCallback onInsert}) async { + await postLoad(); var events; if (client.database != null) { events = await client.database.getEventList(client.id, this); @@ -1033,6 +1047,15 @@ class Room { if (getState(EventTypes.RoomMember, mxID) != null) { return getState(EventTypes.RoomMember, mxID).asUser; } + if (client.database != null) { + // it may be in the database + final user = await client.database.getUser(client.id, mxID, this); + if (user != null) { + setState(user); + if (onUpdate != null) onUpdate.add(id); + return user; + } + } if (mxID == null || !_requestingMatrixIds.add(mxID)) return null; Map resp; try { From 2e3d8205b1faa7907179301c8fab7c91667225b0 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Tue, 30 Jun 2020 12:21:03 +0200 Subject: [PATCH 2/5] analyze and format --- lib/src/client.dart | 10 ++++------ lib/src/room.dart | 3 ++- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/src/client.dart b/lib/src/client.dart index 7448ade..d431b33 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -1432,12 +1432,10 @@ class Client { void _addUpdate() { // we only want max. one update per 50ms - if (_updateTimer == null) { - _updateTimer = Timer(Duration(milliseconds: 50), () { - onUpdate.add(true); - _updateTimer = null; - }); - } + _updateTimer ??= Timer(Duration(milliseconds: 50), () { + onUpdate.add(true); + _updateTimer = null; + }); } bool _disposed = false; diff --git a/lib/src/room.dart b/lib/src/room.dart index 2f92ade..10ec758 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -106,7 +106,8 @@ class Room { if (!partial || client.database == null) { return; } - final allStates = await client.database.getAllRoomStatesForRoom(client.id, id).get(); + final allStates = + await client.database.getAllRoomStatesForRoom(client.id, id).get(); for (final state in allStates) { final newState = Event.fromDb(state, this); setState(newState); From b7b369923faee1c235b2d6c076e365878dfea04f Mon Sep 17 00:00:00 2001 From: Sorunome Date: Tue, 30 Jun 2020 13:41:52 +0200 Subject: [PATCH 3/5] only lazy-load m.room.member, not store presence --- lib/src/client.dart | 10 +-------- lib/src/database/database.dart | 16 --------------- lib/src/database/database.g.dart | 35 +++----------------------------- lib/src/database/database.moor | 7 +++---- lib/src/room.dart | 5 +++-- 5 files changed, 10 insertions(+), 63 deletions(-) diff --git a/lib/src/client.dart b/lib/src/client.dart index d431b33..53d1186 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -651,7 +651,7 @@ class Client { } _sortRooms(); accountData = await database.getAccountData(id); - presences = await database.getPresences(id); + presences.clear(); } onLoginStateChanged.add(LoginState.logged); @@ -744,14 +744,6 @@ class Client { } if (sync.presence != null) { for (final newPresence in sync.presence) { - if (database != null) { - await database.storePresence( - id, - newPresence.type, - newPresence.senderId, - jsonEncode(newPresence.toJson()), - ); - } presences[newPresence.senderId] = newPresence; onPresence.add(newPresence); } diff --git a/lib/src/database/database.dart b/lib/src/database/database.dart index 7240a35..dc6cdd6 100644 --- a/lib/src/database/database.dart +++ b/lib/src/database/database.dart @@ -192,22 +192,6 @@ class Database extends _$Database { return newAccountData; } - Future> getPresences(int clientId) async { - final newPresences = {}; - final rawPresences = await getAllPresences(clientId).get(); - for (final d in rawPresences) { - // TODO: Why is this not working? - try { - final content = sdk.Event.getMapFromPayload(d.content); - var presence = api.Presence.fromJson(content); - presence.senderId = d.sender; - presence.type = d.type; - newPresences[d.sender] = api.Presence.fromJson(content); - } catch (_) {} - } - return newPresences; - } - /// Stores a RoomUpdate object in the database. Must be called inside of /// [transaction]. final Set _ensuredRooms = {}; diff --git a/lib/src/database/database.g.dart b/lib/src/database/database.g.dart index 8971453..9e11f41 100644 --- a/lib/src/database/database.g.dart +++ b/lib/src/database/database.g.dart @@ -5999,35 +5999,6 @@ abstract class _$Database extends GeneratedDatabase { ); } - DbPresence _rowToDbPresence(QueryRow row) { - return DbPresence( - clientId: row.readInt('client_id'), - type: row.readString('type'), - sender: row.readString('sender'), - content: row.readString('content'), - ); - } - - Selectable getAllPresences(int client_id) { - return customSelect('SELECT * FROM presences WHERE client_id = :client_id', - variables: [Variable.withInt(client_id)], - readsFrom: {presences}).map(_rowToDbPresence); - } - - Future storePresence( - int client_id, String type, String sender, String content) { - return customInsert( - 'INSERT OR REPLACE INTO presences (client_id, type, sender, content) VALUES (:client_id, :type, :sender, :content)', - variables: [ - Variable.withInt(client_id), - Variable.withString(type), - Variable.withString(sender), - Variable.withString(content) - ], - updates: {presences}, - ); - } - Future updateEvent(String unsigned, String content, String prev_content, int client_id, String event_id, String room_id) { return customUpdate( @@ -6079,7 +6050,7 @@ abstract class _$Database extends GeneratedDatabase { Selectable getImportantRoomStates(int client_id) { return customSelect( - 'SELECT * FROM room_states WHERE client_id = :client_id AND type IN (\'m.room.name\', \'m.room.avatar\', \'m.room.message\', \'m.room.encrypted\', \'m.room.encryption\')', + 'SELECT * FROM room_states WHERE client_id = :client_id AND type <> \'m.room.member\'', variables: [Variable.withInt(client_id)], readsFrom: {roomStates}).map(_rowToDbRoomState); } @@ -6091,10 +6062,10 @@ abstract class _$Database extends GeneratedDatabase { readsFrom: {roomStates}).map(_rowToDbRoomState); } - Selectable getAllRoomStatesForRoom( + Selectable getUnimportantRoomStatesForRoom( int client_id, String room_id) { return customSelect( - 'SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id', + 'SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type = \'m.room.member\'', variables: [Variable.withInt(client_id), Variable.withString(room_id)], readsFrom: {roomStates}).map(_rowToDbRoomState); } diff --git a/lib/src/database/database.moor b/lib/src/database/database.moor index b5459bf..1dbdef1 100644 --- a/lib/src/database/database.moor +++ b/lib/src/database/database.moor @@ -206,13 +206,12 @@ setRoomPrevBatch: UPDATE rooms SET prev_batch = :prev_batch WHERE client_id = :c updateRoomSortOrder: UPDATE rooms SET oldest_sort_order = :oldest_sort_order, newest_sort_order = :newest_sort_order WHERE client_id = :client_id AND room_id = :room_id; getAllAccountData: SELECT * FROM account_data WHERE client_id = :client_id; storeAccountData: INSERT OR REPLACE INTO account_data (client_id, type, content) VALUES (:client_id, :type, :content); -getAllPresences: SELECT * FROM presences WHERE client_id = :client_id; -storePresence: INSERT OR REPLACE INTO presences (client_id, type, sender, content) VALUES (:client_id, :type, :sender, :content); updateEvent: UPDATE events SET unsigned = :unsigned, content = :content, prev_content = :prev_content WHERE client_id = :client_id AND event_id = :event_id AND room_id = :room_id; updateEventStatus: UPDATE events SET status = :status, event_id = :new_event_id WHERE client_id = :client_id AND event_id = :old_event_id AND room_id = :room_id; -getImportantRoomStates: SELECT * FROM room_states WHERE client_id = :client_id AND type IN ('m.room.name', 'm.room.avatar', 'm.room.message', 'm.room.encrypted', 'm.room.encryption'); +--getImportantRoomStates: SELECT * FROM room_states WHERE client_id = :client_id AND type IN ('m.room.name', 'm.room.avatar', 'm.room.message', 'm.room.encrypted', 'm.room.encryption', 'im.ponies.room_emotes'); +getImportantRoomStates: SELECT * FROM room_states WHERE client_id = :client_id AND type <> 'm.room.member'; getAllRoomStates: SELECT * FROM room_states WHERE client_id = :client_id; -getAllRoomStatesForRoom: SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id; +getUnimportantRoomStatesForRoom: SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type = 'm.room.member'; storeEvent: INSERT OR REPLACE INTO events (client_id, event_id, room_id, sort_order, origin_server_ts, sender, type, unsigned, content, prev_content, state_key, status) VALUES (:client_id, :event_id, :room_id, :sort_order, :origin_server_ts, :sender, :type, :unsigned, :content, :prev_content, :state_key, :status); storeRoomState: INSERT OR REPLACE INTO room_states (client_id, event_id, room_id, sort_order, origin_server_ts, sender, type, unsigned, content, prev_content, state_key) VALUES (:client_id, :event_id, :room_id, :sort_order, :origin_server_ts, :sender, :type, :unsigned, :content, :prev_content, :state_key); getAllRoomAccountData: SELECT * FROM room_account_data WHERE client_id = :client_id; diff --git a/lib/src/room.dart b/lib/src/room.dart index 10ec758..55992dc 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -106,8 +106,9 @@ class Room { if (!partial || client.database == null) { return; } - final allStates = - await client.database.getAllRoomStatesForRoom(client.id, id).get(); + final allStates = await client.database + .getUnimportantRoomStatesForRoom(client.id, id) + .get(); for (final state in allStates) { final newState = Event.fromDb(state, this); setState(newState); From 8f122195c5edd17e50e762c93a9609321197744d Mon Sep 17 00:00:00 2001 From: Sorunome Date: Wed, 1 Jul 2020 11:09:31 +0200 Subject: [PATCH 4/5] re-work state lazy loading after discussion --- lib/src/client.dart | 47 +++++++++++-------- lib/src/database/database.dart | 80 +++++++++++++++++++++++++++++++- lib/src/database/database.g.dart | 51 ++++++++++++++++---- lib/src/database/database.moor | 5 +- lib/src/room.dart | 6 ++- 5 files changed, 156 insertions(+), 33 deletions(-) diff --git a/lib/src/client.dart b/lib/src/client.dart index 53d1186..5efec41 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -58,6 +58,8 @@ class Client { Set verificationMethods; + Set importantStateEvents; + /// Create a client /// clientName = unique identifier of this client /// debug: Print debug output? @@ -66,13 +68,37 @@ class Client { /// verificationMethods: A set of all the verification methods this client can handle. Includes: /// KeyVerificationMethod.numbers: Compare numbers. Most basic, should be supported /// KeyVerificationMethod.emoji: Compare emojis + /// importantStateEvents: A set of all the important state events to load when the client connects. + /// To speed up performance only a set of state events is loaded on startup, those that are + /// needed to display a room list. All the remaining state events are automatically post-loaded + /// when opening the timeline of a room or manually by calling `room.postLoad()`. + /// This set will always include the following state events: + /// - m.room.name + /// - m.room.avatar + /// - m.room.message + /// - m.room.encrypted + /// - m.room.encryption + /// - m.room.canonical_alias + /// - m.room.tombstone + /// - *some* m.room.member events, where needed Client(this.clientName, {this.debug = false, this.database, this.enableE2eeRecovery = false, this.verificationMethods, - http.Client httpClient}) { + http.Client httpClient, + this.importantStateEvents}) { verificationMethods ??= {}; + importantStateEvents ??= {}; + importantStateEvents.addAll([ + 'm.room.name', + 'm.room.avatar', + 'm.room.message', + 'm.room.encrypted', + 'm.room.encryption', + 'm.room.canonical_alias', + 'm.room.tombstone', + ]); api = MatrixApi(debug: debug, httpClient: httpClient); onLoginStateChanged.stream.listen((loginState) { if (debug) { @@ -533,10 +559,6 @@ class Client { final StreamController onKeyVerificationRequest = StreamController.broadcast(); - /// When a new update entered, be it a sync or an avatar post-loaded - /// payload will always be true - final StreamController onUpdate = StreamController.broadcast(); - /// Matrix synchronisation is done with https long polling. This needs a /// timeout which is usually 30 seconds. int syncTimeoutSec = 30; @@ -646,9 +668,6 @@ class Client { } _userDeviceKeys = await database.getUserDeviceKeys(this); _rooms = await database.getRoomList(this, onlyLeft: false); - for (final r in rooms) { - r.onUpdate.stream.listen((v) => _addUpdate()); - } _sortRooms(); accountData = await database.getAccountData(id); presences.clear(); @@ -768,7 +787,6 @@ class Client { encryption.handleDeviceOneTimeKeysCount(sync.deviceOneTimeKeysCount); } onSync.add(sync); - _addUpdate(); } Future _handleDeviceListsEvents(DeviceListsUpdate deviceLists) async { @@ -1009,7 +1027,6 @@ class Client { roomAccountData: {}, client: this, ); - newRoom.onUpdate.stream.listen((v) => _addUpdate()); rooms.insert(position, newRoom); } // If the membership is "leave" then remove the item and stop here @@ -1420,16 +1437,6 @@ class Client { } } - Timer _updateTimer; - - void _addUpdate() { - // we only want max. one update per 50ms - _updateTimer ??= Timer(Duration(milliseconds: 50), () { - onUpdate.add(true); - _updateTimer = null; - }); - } - bool _disposed = false; /// Stops the synchronization and closes the database. After this diff --git a/lib/src/database/database.dart b/lib/src/database/database.dart index dc6cdd6..69052c6 100644 --- a/lib/src/database/database.dart +++ b/lib/src/database/database.dart @@ -157,9 +157,12 @@ class Database extends _$Database { ? t.membership.equals('leave') : t.membership.equals('leave').not())) .get(); - final resStates = await getImportantRoomStates(client.id).get(); + final resStates = await getImportantRoomStates( + client.id, client.importantStateEvents.toList()) + .get(); final resAccountData = await getAllRoomAccountData(client.id).get(); final roomList = []; + final allMembersToPostload = >{}; for (final r in res) { final room = await sdk.Room.getRoomFromTableRow( r, @@ -168,6 +171,81 @@ class Database extends _$Database { roomAccountData: resAccountData.where((rs) => rs.roomId == r.roomId), ); roomList.add(room); + // let's see if we need any m.room.member events + final membersToPostload = {}; + // the lastEvent message preview might have an author we need to fetch, if it is a group chat + if (room.getState(EventTypes.Message) != null && !room.isDirectChat) { + membersToPostload.add(room.getState(EventTypes.Message).senderId); + } + // if the room has no name and no canonical alias, its name is calculated + // based on the heroes of the room + if (room.getState(EventTypes.RoomName) == null && + room.getState(EventTypes.RoomCanonicalAlias) == null && + room.mHeroes != null) { + // we don't have a name and no canonical alias, so we'll need to + // post-load the heroes + membersToPostload.addAll(room.mHeroes.where((h) => h.isNotEmpty)); + } + // okay, only load from the database if we actually have stuff to load + if (membersToPostload.isNotEmpty) { + // save it for loading later + allMembersToPostload[room.id] = membersToPostload; + } + } + // now we postload all members, if thre are any + if (allMembersToPostload.isNotEmpty) { + // we will generate a query to fetch as many events as possible at once, as that + // significantly improves performance. However, to prevent too large queries from being constructed, + // we limit to only fetching 500 rooms at once. + // This value might be fine-tune-able to be larger (and thus increase performance more for very large accounts), + // however this very conservative value should be on the safe side. + final MAX_ROOMS_PER_QUERY = 500; + // as we iterate over our entries in separate chunks one-by-one we use an iterator + // which persists accross the chunks, and thus we just re-sume iteration at the place + // we prreviously left off. + final entriesIterator = allMembersToPostload.entries.iterator; + // now we iterate over all our 500-room-chunks... + for (var i = 0; + i < allMembersToPostload.keys.length; + i += MAX_ROOMS_PER_QUERY) { + // query the current chunk and build the query + final membersRes = await (select(roomStates) + ..where((s) { + // all chunks have to have the reight client id and must be of type `m.room.member` + final basequery = s.clientId.equals(client.id) & + s.type.equals('m.room.member'); + // this is where the magic happens. Here we build a query with the form + // OR room_id = '!roomId1' AND state_key IN ('@member') OR room_id = '!roomId2' AND state_key IN ('@member') + // subqueries holds our query fragment + Expression subqueries; + // here we iterate over our chunk....we musn't forget to progress our iterator! + // we must check for if our chunk is done *before* progressing the + // iterator, else we might progress it twice around chunk edges, missing on rooms + for (var j = 0; + j < MAX_ROOMS_PER_QUERY && entriesIterator.moveNext(); + j++) { + final entry = entriesIterator.current; + // builds room_id = '!roomId1' AND state_key IN ('@member') + final q = + s.roomId.equals(entry.key) & s.stateKey.isIn(entry.value); + // adds it either as the start of subqueries or as a new OR condition to it + if (subqueries == null) { + subqueries = q; + } else { + subqueries = subqueries | q; + } + } + // combinde the basequery with the subquery together, giving our final query + return basequery & subqueries; + })) + .get(); + // now that we got all the entries from the database, set them as room states + for (final dbMember in membersRes) { + final room = roomList.firstWhere((r) => r.id == dbMember.roomId); + final event = sdk.Event.fromDb(dbMember, room); + room.setState(event); + } + } } return roomList; } diff --git a/lib/src/database/database.g.dart b/lib/src/database/database.g.dart index 9e11f41..e17101d 100644 --- a/lib/src/database/database.g.dart +++ b/lib/src/database/database.g.dart @@ -6048,11 +6048,20 @@ abstract class _$Database extends GeneratedDatabase { ); } - Selectable getImportantRoomStates(int client_id) { + Selectable getImportantRoomStates( + int client_id, List events) { + var $arrayStartIndex = 2; + final expandedevents = $expandVar($arrayStartIndex, events.length); + $arrayStartIndex += events.length; return customSelect( - 'SELECT * FROM room_states WHERE client_id = :client_id AND type <> \'m.room.member\'', - variables: [Variable.withInt(client_id)], - readsFrom: {roomStates}).map(_rowToDbRoomState); + 'SELECT * FROM room_states WHERE client_id = :client_id AND type IN ($expandedevents)', + variables: [ + Variable.withInt(client_id), + for (var $ in events) Variable.withString($) + ], + readsFrom: { + roomStates + }).map(_rowToDbRoomState); } Selectable getAllRoomStates(int client_id) { @@ -6063,11 +6072,20 @@ abstract class _$Database extends GeneratedDatabase { } Selectable getUnimportantRoomStatesForRoom( - int client_id, String room_id) { + int client_id, String room_id, List events) { + var $arrayStartIndex = 3; + final expandedevents = $expandVar($arrayStartIndex, events.length); + $arrayStartIndex += events.length; return customSelect( - 'SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type = \'m.room.member\'', - variables: [Variable.withInt(client_id), Variable.withString(room_id)], - readsFrom: {roomStates}).map(_rowToDbRoomState); + 'SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type NOT IN ($expandedevents)', + variables: [ + Variable.withInt(client_id), + Variable.withString(room_id), + for (var $ in events) Variable.withString($) + ], + readsFrom: { + roomStates + }).map(_rowToDbRoomState); } Future storeEvent( @@ -6178,6 +6196,23 @@ abstract class _$Database extends GeneratedDatabase { }).map(_rowToDbRoomState); } + Selectable dbGetUsers( + int client_id, List mxids, String room_id) { + var $arrayStartIndex = 2; + final expandedmxids = $expandVar($arrayStartIndex, mxids.length); + $arrayStartIndex += mxids.length; + return customSelect( + 'SELECT * FROM room_states WHERE client_id = :client_id AND type = \'m.room.member\' AND state_key IN ($expandedmxids) AND room_id = :room_id', + variables: [ + Variable.withInt(client_id), + for (var $ in mxids) Variable.withString($), + Variable.withString(room_id) + ], + readsFrom: { + roomStates + }).map(_rowToDbRoomState); + } + DbEvent _rowToDbEvent(QueryRow row) { return DbEvent( clientId: row.readInt('client_id'), diff --git a/lib/src/database/database.moor b/lib/src/database/database.moor index 1dbdef1..716bdc6 100644 --- a/lib/src/database/database.moor +++ b/lib/src/database/database.moor @@ -208,10 +208,9 @@ getAllAccountData: SELECT * FROM account_data WHERE client_id = :client_id; storeAccountData: INSERT OR REPLACE INTO account_data (client_id, type, content) VALUES (:client_id, :type, :content); updateEvent: UPDATE events SET unsigned = :unsigned, content = :content, prev_content = :prev_content WHERE client_id = :client_id AND event_id = :event_id AND room_id = :room_id; updateEventStatus: UPDATE events SET status = :status, event_id = :new_event_id WHERE client_id = :client_id AND event_id = :old_event_id AND room_id = :room_id; ---getImportantRoomStates: SELECT * FROM room_states WHERE client_id = :client_id AND type IN ('m.room.name', 'm.room.avatar', 'm.room.message', 'm.room.encrypted', 'm.room.encryption', 'im.ponies.room_emotes'); -getImportantRoomStates: SELECT * FROM room_states WHERE client_id = :client_id AND type <> 'm.room.member'; +getImportantRoomStates: SELECT * FROM room_states WHERE client_id = :client_id AND type IN :events; getAllRoomStates: SELECT * FROM room_states WHERE client_id = :client_id; -getUnimportantRoomStatesForRoom: SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type = 'm.room.member'; +getUnimportantRoomStatesForRoom: SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type NOT IN :events; storeEvent: INSERT OR REPLACE INTO events (client_id, event_id, room_id, sort_order, origin_server_ts, sender, type, unsigned, content, prev_content, state_key, status) VALUES (:client_id, :event_id, :room_id, :sort_order, :origin_server_ts, :sender, :type, :unsigned, :content, :prev_content, :state_key, :status); storeRoomState: INSERT OR REPLACE INTO room_states (client_id, event_id, room_id, sort_order, origin_server_ts, sender, type, unsigned, content, prev_content, state_key) VALUES (:client_id, :event_id, :room_id, :sort_order, :origin_server_ts, :sender, :type, :unsigned, :content, :prev_content, :state_key); getAllRoomAccountData: SELECT * FROM room_account_data WHERE client_id = :client_id; diff --git a/lib/src/room.dart b/lib/src/room.dart index 55992dc..40aa431 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -101,13 +101,17 @@ class Room { _oldestSortOrder, _newestSortOrder, client.id, id); } + /// Flag if the room is partial, meaning not all state events have been loaded yet bool partial = true; + + /// Load all the missing state events for the room from the database. If the room has already been loaded, this does nothing. Future postLoad() async { if (!partial || client.database == null) { return; } final allStates = await client.database - .getUnimportantRoomStatesForRoom(client.id, id) + .getUnimportantRoomStatesForRoom( + client.id, id, client.importantStateEvents.toList()) .get(); for (final state in allStates) { final newState = Event.fromDb(state, this); From 7351319f28ec4096e39e2f39425ee7fea3ad6520 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Thu, 2 Jul 2020 10:32:11 +0200 Subject: [PATCH 5/5] address things --- lib/src/client.dart | 24 +++++++++++++++++------- lib/src/database/database.dart | 2 +- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/lib/src/client.dart b/lib/src/client.dart index 5efec41..8e7a713 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -91,13 +91,13 @@ class Client { verificationMethods ??= {}; importantStateEvents ??= {}; importantStateEvents.addAll([ - 'm.room.name', - 'm.room.avatar', - 'm.room.message', - 'm.room.encrypted', - 'm.room.encryption', - 'm.room.canonical_alias', - 'm.room.tombstone', + EventTypes.RoomName, + EventTypes.RoomAvatar, + EventTypes.Message, + EventTypes.Encrypted, + EventTypes.Encryption, + EventTypes.RoomCanonicalAlias, + EventTypes.RoomTombstone, ]); api = MatrixApi(debug: debug, httpClient: httpClient); onLoginStateChanged.stream.listen((loginState) { @@ -980,6 +980,16 @@ class Client { if (event['type'] == EventTypes.Encrypted && encryptionEnabled) { update = await update.decrypt(room); } + if (event['type'] == EventTypes.Message && + !room.isDirectChat && + database != null && + room.getState(EventTypes.RoomMember, event['sender']) == null) { + // In order to correctly render room list previews we need to fetch the member from the database + final user = await database.getUser(id, event['sender'], room); + if (user != null) { + room.setState(user); + } + } if (type != 'ephemeral' && database != null) { await database.storeEventUpdate(id, update); } diff --git a/lib/src/database/database.dart b/lib/src/database/database.dart index 69052c6..c329d0b 100644 --- a/lib/src/database/database.dart +++ b/lib/src/database/database.dart @@ -199,7 +199,7 @@ class Database extends _$Database { // we limit to only fetching 500 rooms at once. // This value might be fine-tune-able to be larger (and thus increase performance more for very large accounts), // however this very conservative value should be on the safe side. - final MAX_ROOMS_PER_QUERY = 500; + const MAX_ROOMS_PER_QUERY = 500; // as we iterate over our entries in separate chunks one-by-one we use an iterator // which persists accross the chunks, and thus we just re-sume iteration at the place // we prreviously left off.