inex/famedlysdk
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge branch 'soru/performance' into 'master'

Browse source 
Greatly imporve initial loading performance

See merge request famedly/famedlysdk!377
This commit is contained in:
Sorunome 2020-07-02 08:55:31 +00:00
parent ad098aa246 7351319f28
commit ac720df3d2
5 changed files with 198 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
lib/src/client.dart
View file

@ -58,6 +58,8 @@ class Client {
Set<KeyVerificationMethod> verificationMethods;
Set<String> importantStateEvents;
/// Create a client
/// clientName = unique identifier of this client
/// debug: Print debug output?
@ -66,13 +68,37 @@ class Client {
/// verificationMethods: A set of all the verification methods this client can handle. Includes:
/// KeyVerificationMethod.numbers: Compare numbers. Most basic, should be supported
/// KeyVerificationMethod.emoji: Compare emojis
/// importantStateEvents: A set of all the important state events to load when the client connects.
/// To speed up performance only a set of state events is loaded on startup, those that are
/// needed to display a room list. All the remaining state events are automatically post-loaded
/// when opening the timeline of a room or manually by calling `room.postLoad()`.
/// This set will always include the following state events:
/// - m.room.name
/// - m.room.avatar
/// - m.room.message
/// - m.room.encrypted
/// - m.room.encryption
/// - m.room.canonical_alias
/// - m.room.tombstone
/// - *some* m.room.member events, where needed
Client(this.clientName,
{this.debug = false,
this.database,
this.enableE2eeRecovery = false,
this.verificationMethods,
http.Client httpClient}) {
http.Client httpClient,
this.importantStateEvents}) {
verificationMethods ??= <KeyVerificationMethod>{};
importantStateEvents ??= <String>{};
importantStateEvents.addAll([
EventTypes.RoomName,
EventTypes.RoomAvatar,
EventTypes.Message,
EventTypes.Encrypted,
EventTypes.Encryption,
EventTypes.RoomCanonicalAlias,
EventTypes.RoomTombstone,
]);
api = MatrixApi(debug: debug, httpClient: httpClient);
onLoginStateChanged.stream.listen((loginState) {
if (debug) {
@ -644,7 +670,7 @@ class Client {
_rooms = await database.getRoomList(this, onlyLeft: false);
_sortRooms();
accountData = await database.getAccountData(id);
presences = await database.getPresences(id);
presences.clear();
}
onLoginStateChanged.add(LoginState.logged);
@ -744,14 +770,6 @@ class Client {
}
if (sync.presence != null) {
for (final newPresence in sync.presence) {
if (database != null) {
await database.storePresence(
id,
newPresence.type,
newPresence.senderId,
jsonEncode(newPresence.toJson()),
);
}
presences[newPresence.senderId] = newPresence;
onPresence.add(newPresence);
}
@ -969,6 +987,16 @@ class Client {
if (event['type'] == EventTypes.Encrypted && encryptionEnabled) {
update = await update.decrypt(room);
}
if (event['type'] == EventTypes.Message &&
!room.isDirectChat &&
database != null &&
room.getState(EventTypes.RoomMember, event['sender']) == null) {
// In order to correctly render room list previews we need to fetch the member from the database
final user = await database.getUser(id, event['sender'], room);
if (user != null) {
room.setState(user);
}
}
if (type != 'ephemeral' && database != null) {
await database.storeEventUpdate(id, update);
}

96
lib/src/database/database.dart
View file

@ -157,9 +157,12 @@ class Database extends _$Database {
? t.membership.equals('leave')
: t.membership.equals('leave').not()))
.get();
final resStates = await getAllRoomStates(client.id).get();
final resStates = await getImportantRoomStates(
client.id, client.importantStateEvents.toList())
.get();
final resAccountData = await getAllRoomAccountData(client.id).get();
final roomList = <sdk.Room>[];
final allMembersToPostload = <String, Set<String>>{};
for (final r in res) {
final room = await sdk.Room.getRoomFromTableRow(
r,
@ -168,6 +171,81 @@ class Database extends _$Database {
roomAccountData: resAccountData.where((rs) => rs.roomId == r.roomId),
);
roomList.add(room);
// let's see if we need any m.room.member events
final membersToPostload = <String>{};
// the lastEvent message preview might have an author we need to fetch, if it is a group chat
if (room.getState(EventTypes.Message) != null && !room.isDirectChat) {
membersToPostload.add(room.getState(EventTypes.Message).senderId);
}
// if the room has no name and no canonical alias, its name is calculated
// based on the heroes of the room
if (room.getState(EventTypes.RoomName) == null &&
room.getState(EventTypes.RoomCanonicalAlias) == null &&
room.mHeroes != null) {
// we don't have a name and no canonical alias, so we'll need to
// post-load the heroes
membersToPostload.addAll(room.mHeroes.where((h) => h.isNotEmpty));
}
// okay, only load from the database if we actually have stuff to load
if (membersToPostload.isNotEmpty) {
// save it for loading later
allMembersToPostload[room.id] = membersToPostload;
}
}
// now we postload all members, if thre are any
if (allMembersToPostload.isNotEmpty) {
// we will generate a query to fetch as many events as possible at once, as that
// significantly improves performance. However, to prevent too large queries from being constructed,
// we limit to only fetching 500 rooms at once.
// This value might be fine-tune-able to be larger (and thus increase performance more for very large accounts),
// however this very conservative value should be on the safe side.
const MAX_ROOMS_PER_QUERY = 500;
// as we iterate over our entries in separate chunks one-by-one we use an iterator
// which persists accross the chunks, and thus we just re-sume iteration at the place
// we prreviously left off.
final entriesIterator = allMembersToPostload.entries.iterator;
// now we iterate over all our 500-room-chunks...
for (var i = 0;
i < allMembersToPostload.keys.length;
i += MAX_ROOMS_PER_QUERY) {
// query the current chunk and build the query
final membersRes = await (select(roomStates)
..where((s) {
// all chunks have to have the reight client id and must be of type `m.room.member`
final basequery = s.clientId.equals(client.id) &
s.type.equals('m.room.member');
// this is where the magic happens. Here we build a query with the form
// OR room_id = '!roomId1' AND state_key IN ('@member') OR room_id = '!roomId2' AND state_key IN ('@member')
// subqueries holds our query fragment
Expression<bool> subqueries;
// here we iterate over our chunk....we musn't forget to progress our iterator!
// we must check for if our chunk is done *before* progressing the
// iterator, else we might progress it twice around chunk edges, missing on rooms
for (var j = 0;
j < MAX_ROOMS_PER_QUERY && entriesIterator.moveNext();
j++) {
final entry = entriesIterator.current;
// builds room_id = '!roomId1' AND state_key IN ('@member')
final q =
s.roomId.equals(entry.key) & s.stateKey.isIn(entry.value);
// adds it either as the start of subqueries or as a new OR condition to it
if (subqueries == null) {
subqueries = q;
} else {
subqueries = subqueries | q;
}
}
// combinde the basequery with the subquery together, giving our final query
return basequery & subqueries;
}))
.get();
// now that we got all the entries from the database, set them as room states
for (final dbMember in membersRes) {
final room = roomList.firstWhere((r) => r.id == dbMember.roomId);
final event = sdk.Event.fromDb(dbMember, room);
room.setState(event);
}
}
}
return roomList;
}
@ -192,22 +270,6 @@ class Database extends _$Database {
return newAccountData;
}
Future<Map<String, api.Presence>> getPresences(int clientId) async {
final newPresences = <String, api.Presence>{};
final rawPresences = await getAllPresences(clientId).get();
for (final d in rawPresences) {
// TODO: Why is this not working?
try {
final content = sdk.Event.getMapFromPayload(d.content);
var presence = api.Presence.fromJson(content);
presence.senderId = d.sender;
presence.type = d.type;
newPresences[d.sender] = api.Presence.fromJson(content);
} catch (_) {}
}
return newPresences;
}
/// Stores a RoomUpdate object in the database. Must be called inside of
/// [transaction].
final Set<String> _ensuredRooms = {};

79
lib/src/database/database.g.dart
View file

@ -5999,35 +5999,6 @@ abstract class _$Database extends GeneratedDatabase {
);
}
DbPresence _rowToDbPresence(QueryRow row) {
return DbPresence(
clientId: row.readInt('client_id'),
type: row.readString('type'),
sender: row.readString('sender'),
content: row.readString('content'),
);
}
Selectable<DbPresence> getAllPresences(int client_id) {
return customSelect('SELECT * FROM presences WHERE client_id = :client_id',
variables: [Variable.withInt(client_id)],
readsFrom: {presences}).map(_rowToDbPresence);
}
Future<int> storePresence(
int client_id, String type, String sender, String content) {
return customInsert(
'INSERT OR REPLACE INTO presences (client_id, type, sender, content) VALUES (:client_id, :type, :sender, :content)',
variables: [
Variable.withInt(client_id),
Variable.withString(type),
Variable.withString(sender),
Variable.withString(content)
],
updates: {presences},
);
}
Future<int> updateEvent(String unsigned, String content, String prev_content,
int client_id, String event_id, String room_id) {
return customUpdate(
@ -6077,6 +6048,22 @@ abstract class _$Database extends GeneratedDatabase {
);
}
Selectable<DbRoomState> getImportantRoomStates(
int client_id, List<String> events) {
var $arrayStartIndex = 2;
final expandedevents = $expandVar($arrayStartIndex, events.length);
$arrayStartIndex += events.length;
return customSelect(
'SELECT * FROM room_states WHERE client_id = :client_id AND type IN ($expandedevents)',
variables: [
Variable.withInt(client_id),
for (var $ in events) Variable.withString($)
],
readsFrom: {
roomStates
}).map(_rowToDbRoomState);
}
Selectable<DbRoomState> getAllRoomStates(int client_id) {
return customSelect(
'SELECT * FROM room_states WHERE client_id = :client_id',
@ -6084,6 +6071,23 @@ abstract class _$Database extends GeneratedDatabase {
readsFrom: {roomStates}).map(_rowToDbRoomState);
}
Selectable<DbRoomState> getUnimportantRoomStatesForRoom(
int client_id, String room_id, List<String> events) {
var $arrayStartIndex = 3;
final expandedevents = $expandVar($arrayStartIndex, events.length);
$arrayStartIndex += events.length;
return customSelect(
'SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type NOT IN ($expandedevents)',
variables: [
Variable.withInt(client_id),
Variable.withString(room_id),
for (var $ in events) Variable.withString($)
],
readsFrom: {
roomStates
}).map(_rowToDbRoomState);
}
Future<int> storeEvent(
int client_id,
String event_id,
@ -6192,6 +6196,23 @@ abstract class _$Database extends GeneratedDatabase {
}).map(_rowToDbRoomState);
}
Selectable<DbRoomState> dbGetUsers(
int client_id, List<String> mxids, String room_id) {
var $arrayStartIndex = 2;
final expandedmxids = $expandVar($arrayStartIndex, mxids.length);
$arrayStartIndex += mxids.length;
return customSelect(
'SELECT * FROM room_states WHERE client_id = :client_id AND type = \'m.room.member\' AND state_key IN ($expandedmxids) AND room_id = :room_id',
variables: [
Variable.withInt(client_id),
for (var $ in mxids) Variable.withString($),
Variable.withString(room_id)
],
readsFrom: {
roomStates
}).map(_rowToDbRoomState);
}
DbEvent _rowToDbEvent(QueryRow row) {
return DbEvent(
clientId: row.readInt('client_id'),

4
lib/src/database/database.moor
View file

@ -206,11 +206,11 @@ setRoomPrevBatch: UPDATE rooms SET prev_batch = :prev_batch WHERE client_id = :c
updateRoomSortOrder: UPDATE rooms SET oldest_sort_order = :oldest_sort_order, newest_sort_order = :newest_sort_order WHERE client_id = :client_id AND room_id = :room_id;
getAllAccountData: SELECT * FROM account_data WHERE client_id = :client_id;
storeAccountData: INSERT OR REPLACE INTO account_data (client_id, type, content) VALUES (:client_id, :type, :content);
getAllPresences: SELECT * FROM presences WHERE client_id = :client_id;
storePresence: INSERT OR REPLACE INTO presences (client_id, type, sender, content) VALUES (:client_id, :type, :sender, :content);
updateEvent: UPDATE events SET unsigned = :unsigned, content = :content, prev_content = :prev_content WHERE client_id = :client_id AND event_id = :event_id AND room_id = :room_id;
updateEventStatus: UPDATE events SET status = :status, event_id = :new_event_id WHERE client_id = :client_id AND event_id = :old_event_id AND room_id = :room_id;
getImportantRoomStates: SELECT * FROM room_states WHERE client_id = :client_id AND type IN :events;
getAllRoomStates: SELECT * FROM room_states WHERE client_id = :client_id;
getUnimportantRoomStatesForRoom: SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type NOT IN :events;
storeEvent: INSERT OR REPLACE INTO events (client_id, event_id, room_id, sort_order, origin_server_ts, sender, type, unsigned, content, prev_content, state_key, status) VALUES (:client_id, :event_id, :room_id, :sort_order, :origin_server_ts, :sender, :type, :unsigned, :content, :prev_content, :state_key, :status);
storeRoomState: INSERT OR REPLACE INTO room_states (client_id, event_id, room_id, sort_order, origin_server_ts, sender, type, unsigned, content, prev_content, state_key) VALUES (:client_id, :event_id, :room_id, :sort_order, :origin_server_ts, :sender, :type, :unsigned, :content, :prev_content, :state_key);
getAllRoomAccountData: SELECT * FROM room_account_data WHERE client_id = :client_id;

29
lib/src/room.dart
View file

@ -101,6 +101,25 @@ class Room {
_oldestSortOrder, _newestSortOrder, client.id, id);
}
/// Flag if the room is partial, meaning not all state events have been loaded yet
bool partial = true;
/// Load all the missing state events for the room from the database. If the room has already been loaded, this does nothing.
Future<void> postLoad() async {
if (!partial || client.database == null) {
return;
}
final allStates = await client.database
.getUnimportantRoomStatesForRoom(
client.id, id, client.importantStateEvents.toList())
.get();
for (final state in allStates) {
final newState = Event.fromDb(state, this);
setState(newState);
}
partial = false;
}
/// Returns the [Event] for the given [typeKey] and optional [stateKey].
/// If no [stateKey] is provided, it defaults to an empty string.
Event getState(String typeKey, [String stateKey = '']) =>
@ -934,6 +953,7 @@ class Room {
Future<Timeline> getTimeline(
{onTimelineUpdateCallback onUpdate,
onTimelineInsertCallback onInsert}) async {
await postLoad();
var events;
if (client.database != null) {
events = await client.database.getEventList(client.id, this);
@ -1033,6 +1053,15 @@ class Room {
if (getState(EventTypes.RoomMember, mxID) != null) {
return getState(EventTypes.RoomMember, mxID).asUser;
}
if (client.database != null) {
// it may be in the database
final user = await client.database.getUser(client.id, mxID, this);
if (user != null) {
setState(user);
if (onUpdate != null) onUpdate.add(id);
return user;
}
}
if (mxID == null || !_requestingMatrixIds.add(mxID)) return null;
Map<String, dynamic> resp;
try {