inex/famedlysdk
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

re-work state lazy loading after discussion

Browse source
This commit is contained in:
Sorunome 2020-07-01 11:09:31 +02:00
parent b7b369923f
commit 8f122195c5
No known key found for this signature in database
GPG key ID: B19471D07FC9BE9C
5 changed files with 156 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
lib/src/client.dart
View file

@ -58,6 +58,8 @@ class Client {
Set<KeyVerificationMethod> verificationMethods;
Set<String> importantStateEvents;
/// Create a client
/// clientName = unique identifier of this client
/// debug: Print debug output?
@ -66,13 +68,37 @@ class Client {
/// verificationMethods: A set of all the verification methods this client can handle. Includes:
/// KeyVerificationMethod.numbers: Compare numbers. Most basic, should be supported
/// KeyVerificationMethod.emoji: Compare emojis
/// importantStateEvents: A set of all the important state events to load when the client connects.
/// To speed up performance only a set of state events is loaded on startup, those that are
/// needed to display a room list. All the remaining state events are automatically post-loaded
/// when opening the timeline of a room or manually by calling `room.postLoad()`.
/// This set will always include the following state events:
/// - m.room.name
/// - m.room.avatar
/// - m.room.message
/// - m.room.encrypted
/// - m.room.encryption
/// - m.room.canonical_alias
/// - m.room.tombstone
/// - *some* m.room.member events, where needed
Client(this.clientName,
{this.debug = false,
this.database,
this.enableE2eeRecovery = false,
this.verificationMethods,
http.Client httpClient}) {
http.Client httpClient,
this.importantStateEvents}) {
verificationMethods ??= <KeyVerificationMethod>{};
importantStateEvents ??= <String>{};
importantStateEvents.addAll([
'm.room.name',
'm.room.avatar',
'm.room.message',
'm.room.encrypted',
'm.room.encryption',
'm.room.canonical_alias',
'm.room.tombstone',
]);
api = MatrixApi(debug: debug, httpClient: httpClient);
onLoginStateChanged.stream.listen((loginState) {
if (debug) {
@ -533,10 +559,6 @@ class Client {
final StreamController<KeyVerification> onKeyVerificationRequest =
StreamController.broadcast();
/// When a new update entered, be it a sync or an avatar post-loaded
/// payload will always be true
final StreamController<bool> onUpdate = StreamController.broadcast();
/// Matrix synchronisation is done with https long polling. This needs a
/// timeout which is usually 30 seconds.
int syncTimeoutSec = 30;
@ -646,9 +668,6 @@ class Client {
}
_userDeviceKeys = await database.getUserDeviceKeys(this);
_rooms = await database.getRoomList(this, onlyLeft: false);
for (final r in rooms) {
r.onUpdate.stream.listen((v) => _addUpdate());
}
_sortRooms();
accountData = await database.getAccountData(id);
presences.clear();
@ -768,7 +787,6 @@ class Client {
encryption.handleDeviceOneTimeKeysCount(sync.deviceOneTimeKeysCount);
}
onSync.add(sync);
_addUpdate();
}
Future<void> _handleDeviceListsEvents(DeviceListsUpdate deviceLists) async {
@ -1009,7 +1027,6 @@ class Client {
roomAccountData: {},
client: this,
);
newRoom.onUpdate.stream.listen((v) => _addUpdate());
rooms.insert(position, newRoom);
}
// If the membership is "leave" then remove the item and stop here
@ -1420,16 +1437,6 @@ class Client {
}
}
Timer _updateTimer;
void _addUpdate() {
// we only want max. one update per 50ms
_updateTimer ??= Timer(Duration(milliseconds: 50), () {
onUpdate.add(true);
_updateTimer = null;
});
}
bool _disposed = false;
/// Stops the synchronization and closes the database. After this

80
lib/src/database/database.dart
View file

@ -157,9 +157,12 @@ class Database extends _$Database {
? t.membership.equals('leave')
: t.membership.equals('leave').not()))
.get();
final resStates = await getImportantRoomStates(client.id).get();
final resStates = await getImportantRoomStates(
client.id, client.importantStateEvents.toList())
.get();
final resAccountData = await getAllRoomAccountData(client.id).get();
final roomList = <sdk.Room>[];
final allMembersToPostload = <String, Set<String>>{};
for (final r in res) {
final room = await sdk.Room.getRoomFromTableRow(
r,
@ -168,6 +171,81 @@ class Database extends _$Database {
roomAccountData: resAccountData.where((rs) => rs.roomId == r.roomId),
);
roomList.add(room);
// let's see if we need any m.room.member events
final membersToPostload = <String>{};
// the lastEvent message preview might have an author we need to fetch, if it is a group chat
if (room.getState(EventTypes.Message) != null && !room.isDirectChat) {
membersToPostload.add(room.getState(EventTypes.Message).senderId);
}
// if the room has no name and no canonical alias, its name is calculated
// based on the heroes of the room
if (room.getState(EventTypes.RoomName) == null &&
room.getState(EventTypes.RoomCanonicalAlias) == null &&
room.mHeroes != null) {
// we don't have a name and no canonical alias, so we'll need to
// post-load the heroes
membersToPostload.addAll(room.mHeroes.where((h) => h.isNotEmpty));
}
// okay, only load from the database if we actually have stuff to load
if (membersToPostload.isNotEmpty) {
// save it for loading later
allMembersToPostload[room.id] = membersToPostload;
}
}
// now we postload all members, if thre are any
if (allMembersToPostload.isNotEmpty) {
// we will generate a query to fetch as many events as possible at once, as that
// significantly improves performance. However, to prevent too large queries from being constructed,
// we limit to only fetching 500 rooms at once.
// This value might be fine-tune-able to be larger (and thus increase performance more for very large accounts),
// however this very conservative value should be on the safe side.
final MAX_ROOMS_PER_QUERY = 500;
// as we iterate over our entries in separate chunks one-by-one we use an iterator
// which persists accross the chunks, and thus we just re-sume iteration at the place
// we prreviously left off.
final entriesIterator = allMembersToPostload.entries.iterator;
// now we iterate over all our 500-room-chunks...
for (var i = 0;
i < allMembersToPostload.keys.length;
i += MAX_ROOMS_PER_QUERY) {
// query the current chunk and build the query
final membersRes = await (select(roomStates)
..where((s) {
// all chunks have to have the reight client id and must be of type `m.room.member`
final basequery = s.clientId.equals(client.id) &
s.type.equals('m.room.member');
// this is where the magic happens. Here we build a query with the form
// OR room_id = '!roomId1' AND state_key IN ('@member') OR room_id = '!roomId2' AND state_key IN ('@member')
// subqueries holds our query fragment
Expression<bool> subqueries;
// here we iterate over our chunk....we musn't forget to progress our iterator!
// we must check for if our chunk is done *before* progressing the
// iterator, else we might progress it twice around chunk edges, missing on rooms
for (var j = 0;
j < MAX_ROOMS_PER_QUERY && entriesIterator.moveNext();
j++) {
final entry = entriesIterator.current;
// builds room_id = '!roomId1' AND state_key IN ('@member')
final q =
s.roomId.equals(entry.key) & s.stateKey.isIn(entry.value);
// adds it either as the start of subqueries or as a new OR condition to it
if (subqueries == null) {
subqueries = q;
} else {
subqueries = subqueries | q;
}
}
// combinde the basequery with the subquery together, giving our final query
return basequery & subqueries;
}))
.get();
// now that we got all the entries from the database, set them as room states
for (final dbMember in membersRes) {
final room = roomList.firstWhere((r) => r.id == dbMember.roomId);
final event = sdk.Event.fromDb(dbMember, room);
room.setState(event);
}
}
}
return roomList;
}

51
lib/src/database/database.g.dart
View file

@ -6048,11 +6048,20 @@ abstract class _$Database extends GeneratedDatabase {
);
}
Selectable<DbRoomState> getImportantRoomStates(int client_id) {
Selectable<DbRoomState> getImportantRoomStates(
int client_id, List<String> events) {
var $arrayStartIndex = 2;
final expandedevents = $expandVar($arrayStartIndex, events.length);
$arrayStartIndex += events.length;
return customSelect(
'SELECT * FROM room_states WHERE client_id = :client_id AND type <> \'m.room.member\'',
variables: [Variable.withInt(client_id)],
readsFrom: {roomStates}).map(_rowToDbRoomState);
'SELECT * FROM room_states WHERE client_id = :client_id AND type IN ($expandedevents)',
variables: [
Variable.withInt(client_id),
for (var $ in events) Variable.withString($)
],
readsFrom: {
roomStates
}).map(_rowToDbRoomState);
}
Selectable<DbRoomState> getAllRoomStates(int client_id) {
@ -6063,11 +6072,20 @@ abstract class _$Database extends GeneratedDatabase {
}
Selectable<DbRoomState> getUnimportantRoomStatesForRoom(
int client_id, String room_id) {
int client_id, String room_id, List<String> events) {
var $arrayStartIndex = 3;
final expandedevents = $expandVar($arrayStartIndex, events.length);
$arrayStartIndex += events.length;
return customSelect(
'SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type = \'m.room.member\'',
variables: [Variable.withInt(client_id), Variable.withString(room_id)],
readsFrom: {roomStates}).map(_rowToDbRoomState);
'SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type NOT IN ($expandedevents)',
variables: [
Variable.withInt(client_id),
Variable.withString(room_id),
for (var $ in events) Variable.withString($)
],
readsFrom: {
roomStates
}).map(_rowToDbRoomState);
}
Future<int> storeEvent(
@ -6178,6 +6196,23 @@ abstract class _$Database extends GeneratedDatabase {
}).map(_rowToDbRoomState);
}
Selectable<DbRoomState> dbGetUsers(
int client_id, List<String> mxids, String room_id) {
var $arrayStartIndex = 2;
final expandedmxids = $expandVar($arrayStartIndex, mxids.length);
$arrayStartIndex += mxids.length;
return customSelect(
'SELECT * FROM room_states WHERE client_id = :client_id AND type = \'m.room.member\' AND state_key IN ($expandedmxids) AND room_id = :room_id',
variables: [
Variable.withInt(client_id),
for (var $ in mxids) Variable.withString($),
Variable.withString(room_id)
],
readsFrom: {
roomStates
}).map(_rowToDbRoomState);
}
DbEvent _rowToDbEvent(QueryRow row) {
return DbEvent(
clientId: row.readInt('client_id'),

5
lib/src/database/database.moor
View file

@ -208,10 +208,9 @@ getAllAccountData: SELECT * FROM account_data WHERE client_id = :client_id;
storeAccountData: INSERT OR REPLACE INTO account_data (client_id, type, content) VALUES (:client_id, :type, :content);
updateEvent: UPDATE events SET unsigned = :unsigned, content = :content, prev_content = :prev_content WHERE client_id = :client_id AND event_id = :event_id AND room_id = :room_id;
updateEventStatus: UPDATE events SET status = :status, event_id = :new_event_id WHERE client_id = :client_id AND event_id = :old_event_id AND room_id = :room_id;
--getImportantRoomStates: SELECT * FROM room_states WHERE client_id = :client_id AND type IN ('m.room.name', 'm.room.avatar', 'm.room.message', 'm.room.encrypted', 'm.room.encryption', 'im.ponies.room_emotes');
getImportantRoomStates: SELECT * FROM room_states WHERE client_id = :client_id AND type <> 'm.room.member';
getImportantRoomStates: SELECT * FROM room_states WHERE client_id = :client_id AND type IN :events;
getAllRoomStates: SELECT * FROM room_states WHERE client_id = :client_id;
getUnimportantRoomStatesForRoom: SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type = 'm.room.member';
getUnimportantRoomStatesForRoom: SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type NOT IN :events;
storeEvent: INSERT OR REPLACE INTO events (client_id, event_id, room_id, sort_order, origin_server_ts, sender, type, unsigned, content, prev_content, state_key, status) VALUES (:client_id, :event_id, :room_id, :sort_order, :origin_server_ts, :sender, :type, :unsigned, :content, :prev_content, :state_key, :status);
storeRoomState: INSERT OR REPLACE INTO room_states (client_id, event_id, room_id, sort_order, origin_server_ts, sender, type, unsigned, content, prev_content, state_key) VALUES (:client_id, :event_id, :room_id, :sort_order, :origin_server_ts, :sender, :type, :unsigned, :content, :prev_content, :state_key);
getAllRoomAccountData: SELECT * FROM room_account_data WHERE client_id = :client_id;

6
lib/src/room.dart
View file

@ -101,13 +101,17 @@ class Room {
_oldestSortOrder, _newestSortOrder, client.id, id);
}
/// Flag if the room is partial, meaning not all state events have been loaded yet
bool partial = true;
/// Load all the missing state events for the room from the database. If the room has already been loaded, this does nothing.
Future<void> postLoad() async {
if (!partial || client.database == null) {
return;
}
final allStates = await client.database
.getUnimportantRoomStatesForRoom(client.id, id)
.getUnimportantRoomStatesForRoom(
client.id, id, client.importantStateEvents.toList())
.get();
for (final state in allStates) {
final newState = Event.fromDb(state, this);