From c68487ac2180ccddb9aeeb18e53c384122a823e4 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Thu, 23 Jul 2020 08:09:00 +0000 Subject: [PATCH 01/90] fix issue with sending messages --- lib/matrix_api/matrix_api.dart | 26 +-- lib/src/database/database.dart | 29 +++- lib/src/database/database.g.dart | 15 ++ lib/src/database/database.moor | 1 + lib/src/event.dart | 3 +- lib/src/timeline.dart | 74 +++++---- test/encryption/ssss_test.dart | 2 +- test/fake_matrix_api.dart | 56 ++++--- test/matrix_api_test.dart | 2 +- test/matrix_database_test.dart | 184 ++++++++++++++++++++ test/timeline_test.dart | 277 ++++++++++++++++++++++++++++++- 11 files changed, 587 insertions(+), 82 deletions(-) create mode 100644 test/matrix_database_test.dart diff --git a/lib/matrix_api/matrix_api.dart b/lib/matrix_api/matrix_api.dart index 770dd25..569f161 100644 --- a/lib/matrix_api/matrix_api.dart +++ b/lib/matrix_api/matrix_api.dart @@ -787,7 +787,7 @@ class MatrixApi { String stateKey = '', ]) async { final response = await request(RequestType.PUT, - '/client/r0/rooms/${Uri.encodeQueryComponent(roomId)}/state/${Uri.encodeQueryComponent(eventType)}/${Uri.encodeQueryComponent(stateKey)}', + '/client/r0/rooms/${Uri.encodeComponent(roomId)}/state/${Uri.encodeComponent(eventType)}/${Uri.encodeComponent(stateKey)}', data: content); return response['event_id']; } @@ -803,7 +803,7 @@ class MatrixApi { Map content, ) async { final response = await request(RequestType.PUT, - '/client/r0/rooms/${Uri.encodeQueryComponent(roomId)}/send/${Uri.encodeQueryComponent(eventType)}/${Uri.encodeQueryComponent(txnId)}', + '/client/r0/rooms/${Uri.encodeComponent(roomId)}/send/${Uri.encodeComponent(eventType)}/${Uri.encodeComponent(txnId)}', data: content); return response['event_id']; } @@ -818,7 +818,7 @@ class MatrixApi { String reason, }) async { final response = await request(RequestType.PUT, - '/client/r0/rooms/${Uri.encodeQueryComponent(roomId)}/redact/${Uri.encodeQueryComponent(eventId)}/${Uri.encodeQueryComponent(txnId)}', + '/client/r0/rooms/${Uri.encodeComponent(roomId)}/redact/${Uri.encodeComponent(eventId)}/${Uri.encodeComponent(txnId)}', data: { if (reason != null) 'reason': reason, }); @@ -1734,7 +1734,7 @@ class MatrixApi { Future> requestRoomTags(String userId, String roomId) async { final response = await request( RequestType.GET, - '/client/r0/user/${Uri.encodeQueryComponent(userId)}/rooms/${Uri.encodeQueryComponent(roomId)}/tags', + '/client/r0/user/${Uri.encodeComponent(userId)}/rooms/${Uri.encodeComponent(roomId)}/tags', ); return (response['tags'] as Map).map( (k, v) => MapEntry(k, Tag.fromJson(v)), @@ -1750,7 +1750,7 @@ class MatrixApi { double order, }) async { await request(RequestType.PUT, - '/client/r0/user/${Uri.encodeQueryComponent(userId)}/rooms/${Uri.encodeQueryComponent(roomId)}/tags/${Uri.encodeQueryComponent(tag)}', + '/client/r0/user/${Uri.encodeComponent(userId)}/rooms/${Uri.encodeComponent(roomId)}/tags/${Uri.encodeComponent(tag)}', data: { if (order != null) 'order': order, }); @@ -1762,7 +1762,7 @@ class MatrixApi { Future removeRoomTag(String userId, String roomId, String tag) async { await request( RequestType.DELETE, - '/client/r0/user/${Uri.encodeQueryComponent(userId)}/rooms/${Uri.encodeQueryComponent(roomId)}/tags/${Uri.encodeQueryComponent(tag)}', + '/client/r0/user/${Uri.encodeComponent(userId)}/rooms/${Uri.encodeComponent(roomId)}/tags/${Uri.encodeComponent(tag)}', ); return; } @@ -1777,7 +1777,7 @@ class MatrixApi { ) async { await request( RequestType.PUT, - '/client/r0/user/${Uri.encodeQueryComponent(userId)}/account_data/${Uri.encodeQueryComponent(type)}', + '/client/r0/user/${Uri.encodeComponent(userId)}/account_data/${Uri.encodeComponent(type)}', data: content, ); return; @@ -1791,7 +1791,7 @@ class MatrixApi { ) async { return await request( RequestType.GET, - '/client/r0/user/${Uri.encodeQueryComponent(userId)}/account_data/${Uri.encodeQueryComponent(type)}', + '/client/r0/user/${Uri.encodeComponent(userId)}/account_data/${Uri.encodeComponent(type)}', ); } @@ -1806,7 +1806,7 @@ class MatrixApi { ) async { await request( RequestType.PUT, - '/client/r0/user/${Uri.encodeQueryComponent(userId)}/rooms/${Uri.encodeQueryComponent(roomId)}/account_data/${Uri.encodeQueryComponent(type)}', + '/client/r0/user/${Uri.encodeComponent(userId)}/rooms/${Uri.encodeComponent(roomId)}/account_data/${Uri.encodeComponent(type)}', data: content, ); return; @@ -1821,7 +1821,7 @@ class MatrixApi { ) async { return await request( RequestType.GET, - '/client/r0/user/${Uri.encodeQueryComponent(userId)}/rooms/${Uri.encodeQueryComponent(roomId)}/account_data/${Uri.encodeQueryComponent(type)}', + '/client/r0/user/${Uri.encodeComponent(userId)}/rooms/${Uri.encodeComponent(roomId)}/account_data/${Uri.encodeComponent(type)}', ); } @@ -1830,7 +1830,7 @@ class MatrixApi { Future requestWhoIsInfo(String userId) async { final response = await request( RequestType.GET, - '/client/r0/admin/whois/${Uri.encodeQueryComponent(userId)}', + '/client/r0/admin/whois/${Uri.encodeComponent(userId)}', ); return WhoIsInfo.fromJson(response); } @@ -1845,7 +1845,7 @@ class MatrixApi { String filter, }) async { final response = await request(RequestType.GET, - '/client/r0/rooms/${Uri.encodeQueryComponent(roomId)}/context/${Uri.encodeQueryComponent(eventId)}', + '/client/r0/rooms/${Uri.encodeComponent(roomId)}/context/${Uri.encodeComponent(eventId)}', query: { if (filter != null) 'filter': filter, if (limit != null) 'limit': limit.toString(), @@ -1862,7 +1862,7 @@ class MatrixApi { int score, ) async { await request(RequestType.POST, - '/client/r0/rooms/${Uri.encodeQueryComponent(roomId)}/report/${Uri.encodeQueryComponent(eventId)}', + '/client/r0/rooms/${Uri.encodeComponent(roomId)}/report/${Uri.encodeComponent(eventId)}', data: { 'reason': reason, 'score': score, diff --git a/lib/src/database/database.dart b/lib/src/database/database.dart index d339336..adb9fc6 100644 --- a/lib/src/database/database.dart +++ b/lib/src/database/database.dart @@ -361,9 +361,32 @@ class Database extends _$Database { if ((status == 1 || status == -1) && eventContent['unsigned'] is Map && eventContent['unsigned']['transaction_id'] is String) { - // status changed and we have an old transaction id --> update event id and stuffs - await updateEventStatus(status, eventContent['event_id'], clientId, - eventContent['unsigned']['transaction_id'], chatId); + final allOldEvents = + await getEvent(clientId, eventContent['event_id'], chatId).get(); + if (allOldEvents.isNotEmpty) { + // we were likely unable to change transaction_id -> event_id.....because the event ID already exists! + // So, we try to fetch the old event + // the transaction id event will automatically be deleted further down + final oldEvent = allOldEvents.first; + // do we update the status? We should allow 0 -> -1 updates and status increases + if (status > oldEvent.status || + (oldEvent.status == 0 && status == -1)) { + // update the status + await updateEventStatusOnly( + status, clientId, eventContent['event_id'], chatId); + } + } else { + // status changed and we have an old transaction id --> update event id and stuffs + try { + await updateEventStatus(status, eventContent['event_id'], clientId, + eventContent['unsigned']['transaction_id'], chatId); + } catch (err) { + // we could not update the transaction id to the event id....so it already exists + // as we just tried to fetch the event previously this is a race condition if the event comes down sync in the mean time + // that means that the status we already have in the database is likely more accurate + // than our status. So, we just ignore this error + } + } } else { DbEvent oldEvent; if (type == 'history') { diff --git a/lib/src/database/database.g.dart b/lib/src/database/database.g.dart index 203313f..6c3849c 100644 --- a/lib/src/database/database.g.dart +++ b/lib/src/database/database.g.dart @@ -6033,6 +6033,21 @@ abstract class _$Database extends GeneratedDatabase { ); } + Future updateEventStatusOnly( + int status, int client_id, String event_id, String room_id) { + return customUpdate( + 'UPDATE events SET status = :status WHERE client_id = :client_id AND event_id = :event_id AND room_id = :room_id', + variables: [ + Variable.withInt(status), + Variable.withInt(client_id), + Variable.withString(event_id), + Variable.withString(room_id) + ], + updates: {events}, + updateKind: UpdateKind.update, + ); + } + DbRoomState _rowToDbRoomState(QueryRow row) { return DbRoomState( clientId: row.readInt('client_id'), diff --git a/lib/src/database/database.moor b/lib/src/database/database.moor index cade18c..dbcb632 100644 --- a/lib/src/database/database.moor +++ b/lib/src/database/database.moor @@ -208,6 +208,7 @@ getAllAccountData: SELECT * FROM account_data WHERE client_id = :client_id; storeAccountData: INSERT OR REPLACE INTO account_data (client_id, type, content) VALUES (:client_id, :type, :content); updateEvent: UPDATE events SET unsigned = :unsigned, content = :content, prev_content = :prev_content WHERE client_id = :client_id AND event_id = :event_id AND room_id = :room_id; updateEventStatus: UPDATE events SET status = :status, event_id = :new_event_id WHERE client_id = :client_id AND event_id = :old_event_id AND room_id = :room_id; +updateEventStatusOnly: UPDATE events SET status = :status WHERE client_id = :client_id AND event_id = :event_id AND room_id = :room_id; getImportantRoomStates: SELECT * FROM room_states WHERE client_id = :client_id AND type IN :events; getAllRoomStates: SELECT * FROM room_states WHERE client_id = :client_id; getUnimportantRoomStatesForRoom: SELECT * FROM room_states WHERE client_id = :client_id AND room_id = :room_id AND type NOT IN :events; diff --git a/lib/src/event.dart b/lib/src/event.dart index 50c3563..1e55b22 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -312,7 +312,8 @@ class Event extends MatrixEvent { /// Try to send this event again. Only works with events of status -1. Future sendAgain({String txid}) async { if (status != -1) return null; - await remove(); + // we do not remove the event here. It will automatically be updated + // in the `sendEvent` method to transition -1 -> 0 -> 1 -> 2 final eventID = await room.sendEvent( content, txid: txid ?? unsigned['transaction_id'], diff --git a/lib/src/timeline.dart b/lib/src/timeline.dart index e219733..967b830 100644 --- a/lib/src/timeline.dart +++ b/lib/src/timeline.dart @@ -122,10 +122,31 @@ class Timeline { } int _findEvent({String event_id, String unsigned_txid}) { + // we want to find any existing event where either the passed event_id or the passed unsigned_txid + // matches either the event_id or transaction_id of the existing event. + // For that we create two sets, searchNeedle, what we search, and searchHaystack, where we check if there is a match. + // Now, after having these two sets, if the intersect between them is non-empty, we know that we have at least one match in one pair, + // thus meaning we found our element. + final searchNeedle = {}; + if (event_id != null) { + searchNeedle.add(event_id); + } + if (unsigned_txid != null) { + searchNeedle.add(unsigned_txid); + } int i; for (i = 0; i < events.length; i++) { - if (events[i].eventId == event_id || - (unsigned_txid != null && events[i].eventId == unsigned_txid)) break; + final searchHaystack = {}; + if (events[i].eventId != null) { + searchHaystack.add(events[i].eventId); + } + if (events[i].unsigned != null && + events[i].unsigned['transaction_id'] != null) { + searchHaystack.add(events[i].unsigned['transaction_id']); + } + if (searchNeedle.intersection(searchHaystack).isNotEmpty) { + break; + } } return i; } @@ -135,6 +156,7 @@ class Timeline { if (eventUpdate.roomID != room.id) return; if (eventUpdate.type == 'timeline' || eventUpdate.type == 'history') { + var status = eventUpdate.content['status'] ?? 2; // Redaction events are handled as modification for existing events. if (eventUpdate.eventType == EventTypes.Redaction) { final eventId = _findEvent(event_id: eventUpdate.content['redacts']); @@ -142,13 +164,10 @@ class Timeline { events[eventId].setRedactionEvent(Event.fromJson( eventUpdate.content, room, eventUpdate.sortOrder)); } - } else if (eventUpdate.content['status'] == -2) { + } else if (status == -2) { var i = _findEvent(event_id: eventUpdate.content['event_id']); if (i < events.length) events.removeAt(i); - } - // Is this event already in the timeline? - else if (eventUpdate.content['unsigned'] is Map && - eventUpdate.content['unsigned']['transaction_id'] is String) { + } else { var i = _findEvent( event_id: eventUpdate.content['event_id'], unsigned_txid: eventUpdate.content['unsigned'] is Map @@ -156,34 +175,29 @@ class Timeline { : null); if (i < events.length) { + // we want to preserve the old sort order final tempSortOrder = events[i].sortOrder; + // if the old status is larger than the new one, we also want to preserve the old status + final oldStatus = events[i].status; events[i] = Event.fromJson( eventUpdate.content, room, eventUpdate.sortOrder); events[i].sortOrder = tempSortOrder; + // do we preserve the status? we should allow 0 -> -1 updates and status increases + if (status < oldStatus && !(status == -1 && oldStatus == 0)) { + events[i].status = oldStatus; + } + } else { + var newEvent = Event.fromJson( + eventUpdate.content, room, eventUpdate.sortOrder); + + if (eventUpdate.type == 'history' && + events.indexWhere( + (e) => e.eventId == eventUpdate.content['event_id']) != + -1) return; + + events.insert(0, newEvent); + if (onInsert != null) onInsert(0); } - } else { - Event newEvent; - var senderUser = room - .getState( - EventTypes.RoomMember, eventUpdate.content['sender']) - ?.asUser ?? - await room.client.database?.getUser( - room.client.id, eventUpdate.content['sender'], room); - if (senderUser != null) { - eventUpdate.content['displayname'] = senderUser.displayName; - eventUpdate.content['avatar_url'] = senderUser.avatarUrl.toString(); - } - - newEvent = - Event.fromJson(eventUpdate.content, room, eventUpdate.sortOrder); - - if (eventUpdate.type == 'history' && - events.indexWhere( - (e) => e.eventId == eventUpdate.content['event_id']) != - -1) return; - - events.insert(0, newEvent); - if (onInsert != null) onInsert(0); } } sortAndUpdate(); diff --git a/test/encryption/ssss_test.dart b/test/encryption/ssss_test.dart index a0d5b94..7c0c641 100644 --- a/test/encryption/ssss_test.dart +++ b/test/encryption/ssss_test.dart @@ -89,7 +89,7 @@ void main() { // account_data for this test final content = FakeMatrixApi .calledEndpoints[ - '/client/r0/user/%40test%3AfakeServer.notExisting/account_data/best+animal'] + '/client/r0/user/%40test%3AfakeServer.notExisting/account_data/best%20animal'] .first; client.accountData['best animal'] = BasicEvent.fromJson({ 'type': 'best animal', diff --git a/test/fake_matrix_api.dart b/test/fake_matrix_api.dart index 2a149d4..6bbd87e 100644 --- a/test/fake_matrix_api.dart +++ b/test/fake_matrix_api.dart @@ -80,7 +80,7 @@ class FakeMatrixApi extends MockClient { res = {'displayname': ''}; } else if (method == 'PUT' && action.contains( - '/client/r0/rooms/%211234%3AfakeServer.notExisting/send/')) { + '/client/r0/rooms/!1234%3AfakeServer.notExisting/send/')) { res = {'event_id': '\$event${FakeMatrixApi.eventCounter++}'}; } else { res = { @@ -748,7 +748,7 @@ class FakeMatrixApi extends MockClient { 'app_url': 'https://custom.app.example.org' } }, - '/client/r0/user/%40alice%3Aexample.com/rooms/%21localpart%3Aexample.com/tags': + '/client/r0/user/%40alice%3Aexample.com/rooms/!localpart%3Aexample.com/tags': (var req) => { 'tags': { 'm.favourite': {'order': 0.1}, @@ -1982,21 +1982,21 @@ class FakeMatrixApi extends MockClient { (var req) => {}, '/client/r0/pushrules/global/content/nocake/enabled': (var req) => {}, '/client/r0/pushrules/global/content/nocake/actions': (var req) => {}, - '/client/r0/rooms/%21localpart%3Aserver.abc/state/m.room.history_visibility': + '/client/r0/rooms/!localpart%3Aserver.abc/state/m.room.history_visibility': (var req) => {}, - '/client/r0/rooms/%21localpart%3Aserver.abc/state/m.room.join_rules': + '/client/r0/rooms/!localpart%3Aserver.abc/state/m.room.join_rules': (var req) => {}, - '/client/r0/rooms/%21localpart%3Aserver.abc/state/m.room.guest_access': + '/client/r0/rooms/!localpart%3Aserver.abc/state/m.room.guest_access': (var req) => {}, - '/client/r0/rooms/%21localpart%3Aserver.abc/send/m.room.call.invite/1234': + '/client/r0/rooms/!localpart%3Aserver.abc/send/m.room.call.invite/1234': (var req) => {}, - '/client/r0/rooms/%21localpart%3Aserver.abc/send/m.room.call.answer/1234': + '/client/r0/rooms/!localpart%3Aserver.abc/send/m.room.call.answer/1234': (var req) => {}, - '/client/r0/rooms/%21localpart%3Aserver.abc/send/m.room.call.candidates/1234': + '/client/r0/rooms/!localpart%3Aserver.abc/send/m.room.call.candidates/1234': (var req) => {}, - '/client/r0/rooms/%21localpart%3Aserver.abc/send/m.room.call.hangup/1234': + '/client/r0/rooms/!localpart%3Aserver.abc/send/m.room.call.hangup/1234': (var req) => {}, - '/client/r0/rooms/%211234%3Aexample.com/redact/1143273582443PhrSn%3Aexample.org/1234': + '/client/r0/rooms/!1234%3Aexample.com/redact/1143273582443PhrSn%3Aexample.org/1234': (var req) => {'event_id': '1234'}, '/client/r0/pushrules/global/room/!localpart%3Aserver.abc': (var req) => {}, @@ -2006,23 +2006,27 @@ class FakeMatrixApi extends MockClient { (var req) => {}, '/client/r0/devices/QBUAZIFURK': (var req) => {}, '/client/r0/directory/room/%23testalias%3Aexample.com': (var reqI) => {}, - '/client/r0/rooms/%21localpart%3Aserver.abc/send/m.room.message/testtxid': + '/client/r0/rooms/!localpart%3Aserver.abc/send/m.room.message/testtxid': (var reqI) => { 'event_id': '\$event${FakeMatrixApi.eventCounter++}', }, '/client/r0/rooms/!localpart%3Aexample.com/typing/%40alice%3Aexample.com': (var req) => {}, - '/client/r0/rooms/%211234%3Aexample.com/send/m.room.message/1234': + '/client/r0/rooms/!1234%3Aexample.com/send/m.room.message/1234': (var reqI) => { 'event_id': '\$event${FakeMatrixApi.eventCounter++}', }, - '/client/r0/user/%40test%3AfakeServer.notExisting/rooms/%21localpart%3Aserver.abc/tags/m.favourite': + '/client/r0/rooms/!1234%3Aexample.com/send/m.room.message/newresend': + (var reqI) => { + 'event_id': '\$event${FakeMatrixApi.eventCounter++}', + }, + '/client/r0/user/%40test%3AfakeServer.notExisting/rooms/!localpart%3Aserver.abc/tags/m.favourite': (var req) => {}, - '/client/r0/user/%40alice%3Aexample.com/rooms/%21localpart%3Aexample.com/tags/testtag': + '/client/r0/user/%40alice%3Aexample.com/rooms/!localpart%3Aexample.com/tags/testtag': (var req) => {}, '/client/r0/user/%40alice%3Aexample.com/account_data/test.account.data': (var req) => {}, - '/client/r0/user/%40test%3AfakeServer.notExisting/account_data/best+animal': + '/client/r0/user/%40test%3AfakeServer.notExisting/account_data/best%20animal': (var req) => {}, '/client/r0/user/%40alice%3Aexample.com/rooms/1234/account_data/test.account.data': (var req) => {}, @@ -2034,27 +2038,27 @@ class FakeMatrixApi extends MockClient { '/client/r0/profile/%40alice%3Aexample.com/avatar_url': (var reqI) => {}, '/client/r0/profile/%40test%3AfakeServer.notExisting/avatar_url': (var reqI) => {}, - '/client/r0/rooms/%21localpart%3Aserver.abc/state/m.room.encryption': + '/client/r0/rooms/!localpart%3Aserver.abc/state/m.room.encryption': (var reqI) => {'event_id': 'YUwRidLecu:example.com'}, - '/client/r0/rooms/%21localpart%3Aserver.abc/state/m.room.avatar': + '/client/r0/rooms/!localpart%3Aserver.abc/state/m.room.avatar': (var reqI) => {'event_id': 'YUwRidLecu:example.com'}, - '/client/r0/rooms/%21localpart%3Aserver.abc/send/m.room.message/1234': + '/client/r0/rooms/!localpart%3Aserver.abc/send/m.room.message/1234': (var reqI) => {'event_id': 'YUwRidLecu:example.com'}, - '/client/r0/rooms/%21localpart%3Aserver.abc/redact/1234/1234': - (var reqI) => {'event_id': 'YUwRidLecu:example.com'}, - '/client/r0/rooms/%21localpart%3Aserver.abc/state/m.room.name': + '/client/r0/rooms/!localpart%3Aserver.abc/redact/1234/1234': (var reqI) => + {'event_id': 'YUwRidLecu:example.com'}, + '/client/r0/rooms/!localpart%3Aserver.abc/state/m.room.name': (var reqI) => { 'event_id': '42', }, - '/client/r0/rooms/%21localpart%3Aserver.abc/state/m.room.topic': + '/client/r0/rooms/!localpart%3Aserver.abc/state/m.room.topic': (var reqI) => { 'event_id': '42', }, - '/client/r0/rooms/%21localpart%3Aserver.abc/state/m.room.pinned_events': + '/client/r0/rooms/!localpart%3Aserver.abc/state/m.room.pinned_events': (var reqI) => { 'event_id': '42', }, - '/client/r0/rooms/%21localpart%3Aserver.abc/state/m.room.power_levels': + '/client/r0/rooms/!localpart%3Aserver.abc/state/m.room.power_levels': (var reqI) => { 'event_id': '42', }, @@ -2083,9 +2087,9 @@ class FakeMatrixApi extends MockClient { '/client/r0/pushrules/global/content/nocake': (var req) => {}, '/client/r0/pushrules/global/override/!localpart%3Aserver.abc': (var req) => {}, - '/client/r0/user/%40test%3AfakeServer.notExisting/rooms/%21localpart%3Aserver.abc/tags/m.favourite': + '/client/r0/user/%40test%3AfakeServer.notExisting/rooms/!localpart%3Aserver.abc/tags/m.favourite': (var req) => {}, - '/client/r0/user/%40alice%3Aexample.com/rooms/%21localpart%3Aexample.com/tags/testtag': + '/client/r0/user/%40alice%3Aexample.com/rooms/!localpart%3Aexample.com/tags/testtag': (var req) => {}, '/client/unstable/room_keys/version/5': (var req) => {}, '/client/unstable/room_keys/keys/${Uri.encodeComponent('!726s6s6q:example.com')}/${Uri.encodeComponent('ciM/JWTPrmiWPPZNkRLDPQYf9AW/I46bxyLSr+Bx5oU')}?version=5': diff --git a/test/matrix_api_test.dart b/test/matrix_api_test.dart index a165628..7ad8ded 100644 --- a/test/matrix_api_test.dart +++ b/test/matrix_api_test.dart @@ -1377,7 +1377,7 @@ void main() { '@alice:example.com', '!localpart:example.com'); expect( FakeMatrixApi.api['GET'][ - '/client/r0/user/%40alice%3Aexample.com/rooms/%21localpart%3Aexample.com/tags']({}), + '/client/r0/user/%40alice%3Aexample.com/rooms/!localpart%3Aexample.com/tags']({}), {'tags': response.map((k, v) => MapEntry(k, v.toJson()))}, ); diff --git a/test/matrix_database_test.dart b/test/matrix_database_test.dart new file mode 100644 index 0000000..0f16b63 --- /dev/null +++ b/test/matrix_database_test.dart @@ -0,0 +1,184 @@ +/* + * Famedly Matrix SDK + * Copyright (C) 2020 Famedly GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +import 'package:famedlysdk/famedlysdk.dart'; +import 'package:test/test.dart'; +import 'fake_database.dart'; + +void main() { + group('Databse', () { + final database = getDatabase(); + var clientId = -1; + var room = Room(id: '!room:blubb'); + test('setupDatabase', () async { + clientId = await database.insertClient( + 'testclient', + 'https://example.org', + 'blubb', + '@test:example.org', + null, + null, + null, + null); + }); + test('storeEventUpdate', () async { + // store a simple update + var update = EventUpdate( + type: 'timeline', + roomID: room.id, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'origin_server_ts': 100, + 'content': {'blah': 'blubb'}, + 'event_id': '\$event-1', + 'sender': '@blah:blubb', + }, + sortOrder: 0.0, + ); + await database.storeEventUpdate(clientId, update); + var event = await database.getEventById(clientId, '\$event-1', room); + expect(event.eventId, '\$event-1'); + + // insert a transaction id + update = EventUpdate( + type: 'timeline', + roomID: room.id, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'origin_server_ts': 100, + 'content': {'blah': 'blubb'}, + 'event_id': 'transaction-1', + 'sender': '@blah:blubb', + 'status': 0, + }, + sortOrder: 0.0, + ); + await database.storeEventUpdate(clientId, update); + event = await database.getEventById(clientId, 'transaction-1', room); + expect(event.eventId, 'transaction-1'); + update = EventUpdate( + type: 'timeline', + roomID: room.id, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'origin_server_ts': 100, + 'content': {'blah': 'blubb'}, + 'event_id': '\$event-2', + 'sender': '@blah:blubb', + 'unsigned': { + 'transaction_id': 'transaction-1', + }, + 'status': 1, + }, + sortOrder: 0.0, + ); + await database.storeEventUpdate(clientId, update); + event = await database.getEventById(clientId, 'transaction-1', room); + expect(event, null); + event = await database.getEventById(clientId, '\$event-2', room); + + // insert a transaction id if the event id for it already exists + update = EventUpdate( + type: 'timeline', + roomID: room.id, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'origin_server_ts': 100, + 'content': {'blah': 'blubb'}, + 'event_id': '\$event-3', + 'sender': '@blah:blubb', + 'status': 0, + }, + sortOrder: 0.0, + ); + await database.storeEventUpdate(clientId, update); + event = await database.getEventById(clientId, '\$event-3', room); + expect(event.eventId, '\$event-3'); + update = EventUpdate( + type: 'timeline', + roomID: room.id, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'origin_server_ts': 100, + 'content': {'blah': 'blubb'}, + 'event_id': '\$event-3', + 'sender': '@blah:blubb', + 'status': 1, + 'unsigned': { + 'transaction_id': 'transaction-2', + }, + }, + sortOrder: 0.0, + ); + await database.storeEventUpdate(clientId, update); + event = await database.getEventById(clientId, '\$event-3', room); + expect(event.eventId, '\$event-3'); + expect(event.status, 1); + event = await database.getEventById(clientId, 'transaction-2', room); + expect(event, null); + + // insert transaction id and not update status + update = EventUpdate( + type: 'timeline', + roomID: room.id, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'origin_server_ts': 100, + 'content': {'blah': 'blubb'}, + 'event_id': '\$event-4', + 'sender': '@blah:blubb', + 'status': 2, + }, + sortOrder: 0.0, + ); + await database.storeEventUpdate(clientId, update); + event = await database.getEventById(clientId, '\$event-4', room); + expect(event.eventId, '\$event-4'); + update = EventUpdate( + type: 'timeline', + roomID: room.id, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'origin_server_ts': 100, + 'content': {'blah': 'blubb'}, + 'event_id': '\$event-4', + 'sender': '@blah:blubb', + 'status': 1, + 'unsigned': { + 'transaction_id': 'transaction-3', + }, + }, + sortOrder: 0.0, + ); + await database.storeEventUpdate(clientId, update); + event = await database.getEventById(clientId, '\$event-4', room); + expect(event.eventId, '\$event-4'); + expect(event.status, 2); + event = await database.getEventById(clientId, 'transaction-3', room); + expect(event, null); + }); + }); +} diff --git a/test/timeline_test.dart b/test/timeline_test.dart index 005e8b0..385a67d 100644 --- a/test/timeline_test.dart +++ b/test/timeline_test.dart @@ -214,14 +214,29 @@ void main() { }); test('Resend message', () async { - await timeline.events[0].sendAgain(txid: '1234'); + client.onEvent.add(EventUpdate( + type: 'timeline', + roomID: roomID, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'content': {'msgtype': 'm.text', 'body': 'Testcase'}, + 'sender': '@alice:example.com', + 'status': -1, + 'event_id': 'new-test-event', + 'origin_server_ts': testTimeStamp, + 'unsigned': {'transaction_id': 'newresend'}, + }, + sortOrder: room.newSortOrder)); + await Future.delayed(Duration(milliseconds: 50)); + await timeline.events[0].sendAgain(); await Future.delayed(Duration(milliseconds: 50)); expect(updateCount, 17); expect(insertList, [0, 0, 0, 0, 0, 0, 0, 0]); - expect(timeline.events.length, 6); + expect(timeline.events.length, 7); expect(timeline.events[0].status, 1); }); @@ -231,12 +246,12 @@ void main() { await Future.delayed(Duration(milliseconds: 50)); expect(updateCount, 20); - expect(timeline.events.length, 9); - expect(timeline.events[6].eventId, '3143273582443PhrSn:example.org'); - expect(timeline.events[7].eventId, '2143273582443PhrSn:example.org'); - expect(timeline.events[8].eventId, '1143273582443PhrSn:example.org'); + expect(timeline.events.length, 10); + expect(timeline.events[7].eventId, '3143273582443PhrSn:example.org'); + expect(timeline.events[8].eventId, '2143273582443PhrSn:example.org'); + expect(timeline.events[9].eventId, '1143273582443PhrSn:example.org'); expect(room.prev_batch, 't47409-4357353_219380_26003_2265'); - await timeline.events[8].redact(reason: 'test', txid: '1234'); + await timeline.events[9].redact(reason: 'test', txid: '1234'); }); test('Clear cache on limited timeline', () async { @@ -251,5 +266,253 @@ void main() { await Future.delayed(Duration(milliseconds: 50)); expect(timeline.events.isEmpty, true); }); + + test('sending event to failed update', () async { + timeline.events.clear(); + client.onEvent.add(EventUpdate( + type: 'timeline', + roomID: roomID, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'content': {'msgtype': 'm.text', 'body': 'Testcase'}, + 'sender': '@alice:example.com', + 'status': 0, + 'event_id': 'will-fail', + 'origin_server_ts': testTimeStamp + }, + sortOrder: room.newSortOrder)); + await Future.delayed(Duration(milliseconds: 50)); + expect(timeline.events[0].status, 0); + expect(timeline.events.length, 1); + client.onEvent.add(EventUpdate( + type: 'timeline', + roomID: roomID, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'content': {'msgtype': 'm.text', 'body': 'Testcase'}, + 'sender': '@alice:example.com', + 'status': -1, + 'event_id': 'will-fail', + 'origin_server_ts': testTimeStamp + }, + sortOrder: room.newSortOrder)); + await Future.delayed(Duration(milliseconds: 50)); + expect(timeline.events[0].status, -1); + expect(timeline.events.length, 1); + }); + test('sending an event and the http request finishes first, 0 -> 1 -> 2', + () async { + timeline.events.clear(); + client.onEvent.add(EventUpdate( + type: 'timeline', + roomID: roomID, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'content': {'msgtype': 'm.text', 'body': 'Testcase'}, + 'sender': '@alice:example.com', + 'status': 0, + 'event_id': 'transaction', + 'origin_server_ts': testTimeStamp + }, + sortOrder: room.newSortOrder)); + await Future.delayed(Duration(milliseconds: 50)); + expect(timeline.events[0].status, 0); + expect(timeline.events.length, 1); + client.onEvent.add(EventUpdate( + type: 'timeline', + roomID: roomID, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'content': {'msgtype': 'm.text', 'body': 'Testcase'}, + 'sender': '@alice:example.com', + 'status': 1, + 'event_id': '\$event', + 'origin_server_ts': testTimeStamp, + 'unsigned': {'transaction_id': 'transaction'} + }, + sortOrder: room.newSortOrder)); + await Future.delayed(Duration(milliseconds: 50)); + expect(timeline.events[0].status, 1); + expect(timeline.events.length, 1); + client.onEvent.add(EventUpdate( + type: 'timeline', + roomID: roomID, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'content': {'msgtype': 'm.text', 'body': 'Testcase'}, + 'sender': '@alice:example.com', + 'status': 2, + 'event_id': '\$event', + 'origin_server_ts': testTimeStamp, + 'unsigned': {'transaction_id': 'transaction'} + }, + sortOrder: room.newSortOrder)); + await Future.delayed(Duration(milliseconds: 50)); + expect(timeline.events[0].status, 2); + expect(timeline.events.length, 1); + }); + test('sending an event where the sync reply arrives first, 0 -> 2 -> 1', + () async { + timeline.events.clear(); + client.onEvent.add(EventUpdate( + type: 'timeline', + roomID: roomID, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'content': {'msgtype': 'm.text', 'body': 'Testcase'}, + 'sender': '@alice:example.com', + 'status': 0, + 'event_id': 'transaction', + 'origin_server_ts': testTimeStamp + }, + sortOrder: room.newSortOrder)); + await Future.delayed(Duration(milliseconds: 50)); + expect(timeline.events[0].status, 0); + expect(timeline.events.length, 1); + client.onEvent.add(EventUpdate( + type: 'timeline', + roomID: roomID, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'content': {'msgtype': 'm.text', 'body': 'Testcase'}, + 'sender': '@alice:example.com', + 'status': 2, + 'event_id': '\$event', + 'origin_server_ts': testTimeStamp, + 'unsigned': {'transaction_id': 'transaction'} + }, + sortOrder: room.newSortOrder)); + await Future.delayed(Duration(milliseconds: 50)); + expect(timeline.events[0].status, 2); + expect(timeline.events.length, 1); + client.onEvent.add(EventUpdate( + type: 'timeline', + roomID: roomID, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'content': {'msgtype': 'm.text', 'body': 'Testcase'}, + 'sender': '@alice:example.com', + 'status': 1, + 'event_id': '\$event', + 'origin_server_ts': testTimeStamp, + 'unsigned': {'transaction_id': 'transaction'} + }, + sortOrder: room.newSortOrder)); + await Future.delayed(Duration(milliseconds: 50)); + expect(timeline.events[0].status, 2); + expect(timeline.events.length, 1); + }); + test('sending an event 0 -> -1 -> 2', () async { + timeline.events.clear(); + client.onEvent.add(EventUpdate( + type: 'timeline', + roomID: roomID, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'content': {'msgtype': 'm.text', 'body': 'Testcase'}, + 'sender': '@alice:example.com', + 'status': 0, + 'event_id': 'transaction', + 'origin_server_ts': testTimeStamp + }, + sortOrder: room.newSortOrder)); + await Future.delayed(Duration(milliseconds: 50)); + expect(timeline.events[0].status, 0); + expect(timeline.events.length, 1); + client.onEvent.add(EventUpdate( + type: 'timeline', + roomID: roomID, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'content': {'msgtype': 'm.text', 'body': 'Testcase'}, + 'sender': '@alice:example.com', + 'status': -1, + 'origin_server_ts': testTimeStamp, + 'unsigned': {'transaction_id': 'transaction'}, + }, + sortOrder: room.newSortOrder)); + await Future.delayed(Duration(milliseconds: 50)); + expect(timeline.events[0].status, -1); + expect(timeline.events.length, 1); + client.onEvent.add(EventUpdate( + type: 'timeline', + roomID: roomID, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'content': {'msgtype': 'm.text', 'body': 'Testcase'}, + 'sender': '@alice:example.com', + 'status': 2, + 'event_id': '\$event', + 'origin_server_ts': testTimeStamp, + 'unsigned': {'transaction_id': 'transaction'}, + }, + sortOrder: room.newSortOrder)); + await Future.delayed(Duration(milliseconds: 50)); + expect(timeline.events[0].status, 2); + expect(timeline.events.length, 1); + }); + test('sending an event 0 -> 2 -> -1', () async { + timeline.events.clear(); + client.onEvent.add(EventUpdate( + type: 'timeline', + roomID: roomID, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'content': {'msgtype': 'm.text', 'body': 'Testcase'}, + 'sender': '@alice:example.com', + 'status': 0, + 'event_id': 'transaction', + 'origin_server_ts': testTimeStamp + }, + sortOrder: room.newSortOrder)); + await Future.delayed(Duration(milliseconds: 50)); + expect(timeline.events[0].status, 0); + expect(timeline.events.length, 1); + client.onEvent.add(EventUpdate( + type: 'timeline', + roomID: roomID, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'content': {'msgtype': 'm.text', 'body': 'Testcase'}, + 'sender': '@alice:example.com', + 'status': 2, + 'event_id': '\$event', + 'origin_server_ts': testTimeStamp, + 'unsigned': {'transaction_id': 'transaction'}, + }, + sortOrder: room.newSortOrder)); + await Future.delayed(Duration(milliseconds: 50)); + expect(timeline.events[0].status, 2); + expect(timeline.events.length, 1); + client.onEvent.add(EventUpdate( + type: 'timeline', + roomID: roomID, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'content': {'msgtype': 'm.text', 'body': 'Testcase'}, + 'sender': '@alice:example.com', + 'status': -1, + 'origin_server_ts': testTimeStamp, + 'unsigned': {'transaction_id': 'transaction'}, + }, + sortOrder: room.newSortOrder)); + await Future.delayed(Duration(milliseconds: 50)); + expect(timeline.events[0].status, 2); + expect(timeline.events.length, 1); + }); }); } From ff2de35d2842206bb87e87ef79fa977268ec09d0 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Fri, 24 Jul 2020 14:53:06 +0200 Subject: [PATCH 02/90] test web --- lib/matrix_api/model/matrix_event.dart | 7 ++++--- lib/src/event.dart | 18 +++++++++++------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/lib/matrix_api/model/matrix_event.dart b/lib/matrix_api/model/matrix_event.dart index e70f8b5..0a20bf5 100644 --- a/lib/matrix_api/model/matrix_event.dart +++ b/lib/matrix_api/model/matrix_event.dart @@ -41,9 +41,10 @@ class MatrixEvent extends StrippedStateEvent { unsigned = json['unsigned'] != null ? Map.from(json['unsigned']) : null; - prevContent = json['prev_content'] != null - ? Map.from(json['prev_content']) - : null; + prevContent = + json.containsKey('prev_content') && json['prev_content'] != null + ? Map.from(json['prev_content']) + : null; redacts = json['redacts']; } diff --git a/lib/src/event.dart b/lib/src/event.dart index 1e55b22..154f090 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -93,7 +93,9 @@ class Event extends MatrixEvent { // into the unsigned block this.prevContent = prevContent != null && prevContent.isNotEmpty ? prevContent - : (unsigned != null && unsigned['prev_content'] is Map + : (unsigned != null && + unsigned.containsKey('prev_content') && + unsigned['prev_content'] is Map ? unsigned['prev_content'] : null); this.stateKey = stateKey; @@ -481,7 +483,8 @@ class Event extends MatrixEvent { final targetName = stateKeyUser.calcDisplayname(); // Has the membership changed? final newMembership = content['membership'] ?? ''; - final oldMembership = unsigned['prev_content'] is Map + final oldMembership = unsigned.containsKey('prev_content') && + unsigned['prev_content'] is Map ? unsigned['prev_content']['membership'] ?? '' : ''; if (newMembership != oldMembership) { @@ -518,15 +521,16 @@ class Event extends MatrixEvent { } } else if (newMembership == 'join') { final newAvatar = content['avatar_url'] ?? ''; - final oldAvatar = unsigned['prev_content'] is Map + final oldAvatar = unsigned.containsKey('prev_content') && + unsigned['prev_content'] is Map ? unsigned['prev_content']['avatar_url'] ?? '' : ''; final newDisplayname = content['displayname'] ?? ''; - final oldDisplayname = - unsigned['prev_content'] is Map - ? unsigned['prev_content']['displayname'] ?? '' - : ''; + final oldDisplayname = unsigned.containsKey('prev_content') && + unsigned['prev_content'] is Map + ? unsigned['prev_content']['displayname'] ?? '' + : ''; // Has the user avatar changed? if (newAvatar != oldAvatar) { From 9cb4dab9d41756b81e63ef0e8e484bb3ce609479 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Fri, 24 Jul 2020 15:37:00 +0200 Subject: [PATCH 03/90] test web --- lib/matrix_api/model/matrix_event.dart | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/matrix_api/model/matrix_event.dart b/lib/matrix_api/model/matrix_event.dart index 0a20bf5..e70f8b5 100644 --- a/lib/matrix_api/model/matrix_event.dart +++ b/lib/matrix_api/model/matrix_event.dart @@ -41,10 +41,9 @@ class MatrixEvent extends StrippedStateEvent { unsigned = json['unsigned'] != null ? Map.from(json['unsigned']) : null; - prevContent = - json.containsKey('prev_content') && json['prev_content'] != null - ? Map.from(json['prev_content']) - : null; + prevContent = json['prev_content'] != null + ? Map.from(json['prev_content']) + : null; redacts = json['redacts']; } From d4818bd67789cf011ba36b17cefa9b86d5d76bcc Mon Sep 17 00:00:00 2001 From: Sorunome Date: Fri, 24 Jul 2020 15:37:40 +0200 Subject: [PATCH 04/90] forgot to ctrl+s --- lib/src/event.dart | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/lib/src/event.dart b/lib/src/event.dart index 154f090..45f9d03 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -93,9 +93,7 @@ class Event extends MatrixEvent { // into the unsigned block this.prevContent = prevContent != null && prevContent.isNotEmpty ? prevContent - : (unsigned != null && - unsigned.containsKey('prev_content') && - unsigned['prev_content'] is Map + : (unsigned != null && unsigned['prev_content'] is Map ? unsigned['prev_content'] : null); this.stateKey = stateKey; @@ -483,8 +481,7 @@ class Event extends MatrixEvent { final targetName = stateKeyUser.calcDisplayname(); // Has the membership changed? final newMembership = content['membership'] ?? ''; - final oldMembership = unsigned.containsKey('prev_content') && - unsigned['prev_content'] is Map + final oldMembership = unsigned != null && unsigned['prev_content'] is Map ? unsigned['prev_content']['membership'] ?? '' : ''; if (newMembership != oldMembership) { @@ -521,14 +518,12 @@ class Event extends MatrixEvent { } } else if (newMembership == 'join') { final newAvatar = content['avatar_url'] ?? ''; - final oldAvatar = unsigned.containsKey('prev_content') && - unsigned['prev_content'] is Map + final oldAvatar = unsigned != null && unsigned['prev_content'] is Map ? unsigned['prev_content']['avatar_url'] ?? '' : ''; final newDisplayname = content['displayname'] ?? ''; - final oldDisplayname = unsigned.containsKey('prev_content') && - unsigned['prev_content'] is Map + final oldDisplayname = unsigned != null && unsigned['prev_content'] is Map ? unsigned['prev_content']['displayname'] ?? '' : ''; From 84a94f5c9df1aaabe864d7fdb4a149b279f89d14 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Fri, 24 Jul 2020 15:44:55 +0200 Subject: [PATCH 05/90] format --- lib/src/event.dart | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/src/event.dart b/lib/src/event.dart index 45f9d03..47ca3d0 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -481,9 +481,10 @@ class Event extends MatrixEvent { final targetName = stateKeyUser.calcDisplayname(); // Has the membership changed? final newMembership = content['membership'] ?? ''; - final oldMembership = unsigned != null && unsigned['prev_content'] is Map - ? unsigned['prev_content']['membership'] ?? '' - : ''; + final oldMembership = + unsigned != null && unsigned['prev_content'] is Map + ? unsigned['prev_content']['membership'] ?? '' + : ''; if (newMembership != oldMembership) { if (oldMembership == 'invite' && newMembership == 'join') { text = i18n.acceptedTheInvitation(targetName); @@ -518,12 +519,14 @@ class Event extends MatrixEvent { } } else if (newMembership == 'join') { final newAvatar = content['avatar_url'] ?? ''; - final oldAvatar = unsigned != null && unsigned['prev_content'] is Map + final oldAvatar = unsigned != null && + unsigned['prev_content'] is Map ? unsigned['prev_content']['avatar_url'] ?? '' : ''; final newDisplayname = content['displayname'] ?? ''; - final oldDisplayname = unsigned != null && unsigned['prev_content'] is Map + final oldDisplayname = unsigned != null && + unsigned['prev_content'] is Map ? unsigned['prev_content']['displayname'] ?? '' : ''; From 6cd745bd1a322441881dc24d1e69d4bb2705f537 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Fri, 24 Jul 2020 17:59:39 +0200 Subject: [PATCH 06/90] Add data-mx-emote to emotes --- lib/src/utils/markdown.dart | 1 + test/markdown_test.dart | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/src/utils/markdown.dart b/lib/src/utils/markdown.dart index b294473..082513b 100644 --- a/lib/src/utils/markdown.dart +++ b/lib/src/utils/markdown.dart @@ -65,6 +65,7 @@ class EmoteSyntax extends InlineSyntax { return true; } final element = Element.empty('img'); + element.attributes['data-mx-emote'] = ''; element.attributes['src'] = htmlEscape.convert(mxc); element.attributes['alt'] = htmlEscape.convert(emote); element.attributes['title'] = htmlEscape.convert(emote); diff --git a/test/markdown_test.dart b/test/markdown_test.dart index ff7585d..9a4b999 100644 --- a/test/markdown_test.dart +++ b/test/markdown_test.dart @@ -54,11 +54,11 @@ void main() { }); test('emotes', () { expect(markdown(':fox:', emotePacks), - ':fox:'); + ':fox:'); expect(markdown(':user~fox:', emotePacks), - ':fox:'); + ':fox:'); expect(markdown(':raccoon:', emotePacks), - ':raccoon:'); + ':raccoon:'); expect(markdown(':invalid:', emotePacks), ':invalid:'); expect(markdown(':room~invalid:', emotePacks), ':room~invalid:'); }); From 31614364d30b9b9ea6e2b6e8c8f780223f71bbf2 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Sat, 25 Jul 2020 14:46:36 +0000 Subject: [PATCH 07/90] add update filters --- lib/famedlysdk.dart | 1 + lib/src/utils/sync_update_extension.dart | 44 ++++++ test/sync_filter_test.dart | 166 +++++++++++++++++++++++ 3 files changed, 211 insertions(+) create mode 100644 lib/src/utils/sync_update_extension.dart create mode 100644 test/sync_filter_test.dart diff --git a/lib/famedlysdk.dart b/lib/famedlysdk.dart index 39860ab..3a1c51d 100644 --- a/lib/famedlysdk.dart +++ b/lib/famedlysdk.dart @@ -28,6 +28,7 @@ export 'package:famedlysdk/src/utils/uri_extension.dart'; export 'package:famedlysdk/src/utils/matrix_localizations.dart'; export 'package:famedlysdk/src/utils/receipt.dart'; export 'package:famedlysdk/src/utils/states_map.dart'; +export 'package:famedlysdk/src/utils/sync_update_extension.dart'; export 'package:famedlysdk/src/utils/to_device_event.dart'; export 'package:famedlysdk/src/client.dart'; export 'package:famedlysdk/src/event.dart'; diff --git a/lib/src/utils/sync_update_extension.dart b/lib/src/utils/sync_update_extension.dart new file mode 100644 index 0000000..c4b9ecb --- /dev/null +++ b/lib/src/utils/sync_update_extension.dart @@ -0,0 +1,44 @@ +/* + * Famedly Matrix SDK + * Copyright (C) 2020 Famedly GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import 'package:famedlysdk/matrix_api.dart'; + +/// This extension adds easy-to-use filters for the sync update, meant to be used on the `client.onSync` stream, e.g. +/// `client.onSync.stream.where((s) => s.hasRoomUpdate)`. Multiple filters can easily be +/// combind with boolean logic: `client.onSync.stream.where((s) => s.hasRoomUpdate || s.hasPresenceUpdate)` +extension SyncUpdateFilters on SyncUpdate { + /// Returns true if this sync updat has a room update + /// That means there is account data, if there is a room in one of the `join`, `leave` or `invite` blocks of the sync or if there is a to_device event. + bool get hasRoomUpdate { + // if we have an account data change we need to re-render, as `m.direct` might have changed + if (accountData?.isNotEmpty ?? false) { + return true; + } + // check for a to_device event + if (toDevice?.isNotEmpty ?? false) { + return true; + } + // return if there are rooms to update + return (rooms?.join?.isNotEmpty ?? false) || + (rooms?.invite?.isNotEmpty ?? false) || + (rooms?.leave?.isNotEmpty ?? false); + } + + /// Returns if this sync update has presence updates + bool get hasPresenceUpdate => presence != null && presence.isNotEmpty; +} diff --git a/test/sync_filter_test.dart b/test/sync_filter_test.dart new file mode 100644 index 0000000..71ac01b --- /dev/null +++ b/test/sync_filter_test.dart @@ -0,0 +1,166 @@ +/* + * Ansible inventory script used at Famedly GmbH for managing many hosts + * Copyright (C) 2020 Famedly GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import 'package:famedlysdk/famedlysdk.dart'; +import 'package:test/test.dart'; + +const UPDATES = { + 'empty': { + 'next_batch': 'blah', + 'account_data': { + 'events': [], + }, + 'presences': { + 'events': [], + }, + 'rooms': { + 'join': {}, + 'leave': {}, + 'invite': {}, + }, + 'to_device': { + 'events': [], + }, + }, + 'presence': { + 'next_batch': 'blah', + 'presence': { + 'events': [ + { + 'content': { + 'avatar_url': 'mxc://localhost:wefuiwegh8742w', + 'last_active_ago': 2478593, + 'presence': 'online', + 'currently_active': false, + 'status_msg': 'Making cupcakes' + }, + 'type': 'm.presence', + 'sender': '@example:localhost', + }, + ], + }, + }, + 'account_data': { + 'next_batch': 'blah', + 'account_data': { + 'events': [ + { + 'type': 'blah', + 'content': { + 'beep': 'boop', + }, + }, + ], + }, + }, + 'invite': { + 'next_batch': 'blah', + 'rooms': { + 'invite': { + '!room': { + 'invite_state': { + 'events': [], + }, + }, + }, + }, + }, + 'leave': { + 'next_batch': 'blah', + 'rooms': { + 'leave': { + '!room': {}, + }, + }, + }, + 'join': { + 'next_batch': 'blah', + 'rooms': { + 'join': { + '!room': { + 'timeline': { + 'events': [], + }, + 'state': { + 'events': [], + }, + 'account_data': { + 'events': [], + }, + 'ephemeral': { + 'events': [], + }, + 'unread_notifications': {}, + 'summary': {}, + }, + }, + }, + }, + 'to_device': { + 'next_batch': 'blah', + 'to_device': { + 'events': [ + { + 'type': 'beep', + 'content': { + 'blah': 'blubb', + }, + }, + ], + }, + }, +}; + +void testUpdates(bool Function(SyncUpdate s) test, Map expected) { + for (final update in UPDATES.entries) { + var sync = SyncUpdate.fromJson(update.value); + expect(test(sync), expected[update.key]); + } +} + +void main() { + group('Sync Filters', () { + test('room update', () { + var testFn = (SyncUpdate s) => s.hasRoomUpdate; + final expected = { + 'empty': false, + 'presence': false, + 'account_data': true, + 'invite': true, + 'leave': true, + 'join': true, + 'to_device': true, + }; + testUpdates(testFn, expected); + }); + + test('presence update', () { + var testFn = (SyncUpdate s) => s.hasPresenceUpdate; + final expected = { + 'empty': false, + 'presence': true, + 'account_data': false, + 'invite': false, + 'leave': false, + 'join': false, + 'to_device': false, + }; + testUpdates(testFn, expected); + }); + }); +} From 14c8377a2fa3fc42eb46660b66b1dad2e4ff181e Mon Sep 17 00:00:00 2001 From: Sorunome Date: Sun, 26 Jul 2020 07:54:03 +0200 Subject: [PATCH 08/90] make sure that no http requests are done inside of /sync --- lib/encryption/encryption.dart | 10 +++++++--- lib/encryption/key_manager.dart | 2 ++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/encryption/encryption.dart b/lib/encryption/encryption.dart index 940f4fa..4470285 100644 --- a/lib/encryption/encryption.dart +++ b/lib/encryption/encryption.dart @@ -72,12 +72,16 @@ class Encryption { } Future handleToDeviceEvent(ToDeviceEvent event) async { - if (['m.room_key', 'm.room_key_request', 'm.forwarded_room_key'] - .contains(event.type)) { - // a new room key or thelike. We need to handle this asap, before other + if (event.type == 'm.room_key') { + // a new room key. We need to handle this asap, before other // events in /sync are handled await keyManager.handleToDeviceEvent(event); } + if (['m.room_key_request', 'm.forwarded_room_key'].contains(event.type)) { + // "just" room key request things. We don't need these asap, so we handle + // them in the background + unawaited(keyManager.handleToDeviceEvent(event)); + } if (event.type.startsWith('m.key.verification.')) { // some key verification event. No need to handle it now, we can easily // do this in the background diff --git a/lib/encryption/key_manager.dart b/lib/encryption/key_manager.dart index e60cacb..9460827 100644 --- a/lib/encryption/key_manager.dart +++ b/lib/encryption/key_manager.dart @@ -123,6 +123,8 @@ class KeyManager { json.encode(content), json.encode({}), ); + // Note to self: When adding key-backup that needs to be unawaited(), else + // we might accidentally end up with http requests inside of the sync loop // TODO: somehow try to decrypt last message again final room = client.getRoomById(roomId); if (room != null) { From f48f6bca122701d42833ea75d7ce6ca4af369c96 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Mon, 27 Jul 2020 07:39:48 +0000 Subject: [PATCH 09/90] Properly imlement event aggregations --- lib/matrix_api/model/event_types.dart | 1 + lib/matrix_api/model/message_types.dart | 1 - lib/src/event.dart | 80 +++++++++++++++++----- lib/src/room.dart | 38 ++++++++++- lib/src/timeline.dart | 67 +++++++++++++++++- test/event_test.dart | 91 ++++++++++++++++++++++++- test/fake_matrix_api.dart | 4 ++ test/room_test.dart | 80 ++++++++++++++++++++++ 8 files changed, 335 insertions(+), 27 deletions(-) diff --git a/lib/matrix_api/model/event_types.dart b/lib/matrix_api/model/event_types.dart index 4de7608..25f4585 100644 --- a/lib/matrix_api/model/event_types.dart +++ b/lib/matrix_api/model/event_types.dart @@ -19,6 +19,7 @@ abstract class EventTypes { static const String Message = 'm.room.message'; static const String Sticker = 'm.sticker'; + static const String Reaction = 'm.reaction'; static const String Redaction = 'm.room.redaction'; static const String RoomAliases = 'm.room.aliases'; static const String RoomCanonicalAlias = 'm.room.canonical_alias'; diff --git a/lib/matrix_api/model/message_types.dart b/lib/matrix_api/model/message_types.dart index db478bd..90fe2d0 100644 --- a/lib/matrix_api/model/message_types.dart +++ b/lib/matrix_api/model/message_types.dart @@ -25,7 +25,6 @@ abstract class MessageTypes { static const String Audio = 'm.audio'; static const String File = 'm.file'; static const String Location = 'm.location'; - static const String Reply = 'm.relates_to'; static const String Sticker = 'm.sticker'; static const String BadEncrypted = 'm.bad.encrypted'; static const String None = 'm.none'; diff --git a/lib/src/event.dart b/lib/src/event.dart index 47ca3d0..e0e3d5d 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -28,6 +28,12 @@ import './room.dart'; import 'utils/matrix_localizations.dart'; import './database/database.dart' show DbRoomState, DbEvent; +abstract class RelationshipTypes { + static const String Reply = 'm.in_reply_to'; + static const String Edit = 'm.replace'; + static const String Reaction = 'm.annotation'; +} + /// All data exchanged over Matrix is expressed as an "event". Typically each client action (e.g. sending a message) correlates with exactly one event. class Event extends MatrixEvent { User get sender => room.getUserByMXIDSync(senderId ?? '@unknown'); @@ -212,10 +218,7 @@ class Event extends MatrixEvent { unsigned: unsigned, room: room); - String get messageType => (content['m.relates_to'] is Map && - content['m.relates_to']['m.in_reply_to'] != null) - ? MessageTypes.Reply - : content['msgtype'] ?? MessageTypes.Text; + String get messageType => content['msgtype'] ?? MessageTypes.Text; void setRedactionEvent(Event redactedBecause) { unsigned = { @@ -328,20 +331,10 @@ class Event extends MatrixEvent { Future redact({String reason, String txid}) => room.redactEvent(eventId, reason: reason, txid: txid); - /// Whether this event is in reply to another event. - bool get isReply => - content['m.relates_to'] is Map && - content['m.relates_to']['m.in_reply_to'] is Map && - content['m.relates_to']['m.in_reply_to']['event_id'] is String && - (content['m.relates_to']['m.in_reply_to']['event_id'] as String) - .isNotEmpty; - /// Searches for the reply event in the given timeline. Future getReplyEvent(Timeline timeline) async { - if (!isReply) return null; - final String replyEventId = - content['m.relates_to']['m.in_reply_to']['event_id']; - return await timeline.getEventById(replyEventId); + if (relationshipType != RelationshipTypes.Reply) return null; + return await timeline.getEventById(relationshipEventId); } /// If this event is encrypted and the decryption was not successful because @@ -634,7 +627,6 @@ class Event extends MatrixEvent { case MessageTypes.Text: case MessageTypes.Notice: case MessageTypes.None: - case MessageTypes.Reply: localizedBody = body; break; } @@ -663,9 +655,61 @@ class Event extends MatrixEvent { static const Set textOnlyMessageTypes = { MessageTypes.Text, - MessageTypes.Reply, MessageTypes.Notice, MessageTypes.Emote, MessageTypes.None, }; + + /// returns if this event matches the passed event or transaction id + bool matchesEventOrTransactionId(String search) { + if (search == null) { + return false; + } + if (eventId == search) { + return true; + } + return unsigned != null && unsigned['transaction_id'] == search; + } + + /// Get the relationship type of an event. `null` if there is none + String get relationshipType { + if (content == null || !(content['m.relates_to'] is Map)) { + return null; + } + if (content['m.relates_to'].containsKey('rel_type')) { + return content['m.relates_to']['rel_type']; + } + if (content['m.relates_to'].containsKey('m.in_reply_to')) { + return RelationshipTypes.Reply; + } + return null; + } + + /// Get the event ID that this relationship will reference. `null` if there is none + String get relationshipEventId { + if (content == null || !(content['m.relates_to'] is Map)) { + return null; + } + if (content['m.relates_to'].containsKey('event_id')) { + return content['m.relates_to']['event_id']; + } + if (content['m.relates_to']['m.in_reply_to'] is Map && + content['m.relates_to']['m.in_reply_to'].containsKey('event_id')) { + return content['m.relates_to']['m.in_reply_to']['event_id']; + } + return null; + } + + /// Get wether this event has aggregated events from a certain [type] + /// To be able to do that you need to pass a [timeline] + bool hasAggregatedEvents(Timeline timeline, String type) => + timeline.aggregatedEvents.containsKey(eventId) && + timeline.aggregatedEvents[eventId].containsKey(type); + + /// Get all the aggregated event objects for a given [type]. To be able to do this + /// you have to pass a [timeline] + Set aggregatedEvents(Timeline timeline, String type) => + hasAggregatedEvents(timeline, type) + ? timeline.aggregatedEvents[eventId][type] + : {}; } diff --git a/lib/src/room.dart b/lib/src/room.dart index 0113123..be97100 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -104,7 +104,9 @@ class Room { /// Flag if the room is partial, meaning not all state events have been loaded yet bool partial = true; - /// Load all the missing state events for the room from the database. If the room has already been loaded, this does nothing. + /// Post-loads the room. + /// This load all the missing state events for the room from the database + /// If the room has already been loaded, this does nothing. Future postLoad() async { if (!partial || client.database == null) { return; @@ -500,6 +502,7 @@ class Room { Future sendTextEvent(String message, {String txid, Event inReplyTo, + String editEventId, bool parseMarkdown = true, Map> emotePacks}) { final event = { @@ -518,7 +521,20 @@ class Room { event['formatted_body'] = html; } } - return sendEvent(event, txid: txid, inReplyTo: inReplyTo); + return sendEvent(event, + txid: txid, inReplyTo: inReplyTo, editEventId: editEventId); + } + + /// Sends a reaction to an event with an [eventId] and the content [key] into a room. + /// Returns the event ID generated by the server for this reaction. + Future sendReaction(String eventId, String key, {String txid}) { + return sendEvent({ + 'm.relates_to': { + 'rel_type': RelationshipTypes.Reaction, + 'event_id': eventId, + 'key': key, + }, + }, type: EventTypes.Reaction, txid: txid); } /// Sends a [file] to this room after uploading it. Returns the mxc uri of @@ -529,6 +545,7 @@ class Room { MatrixFile file, { String txid, Event inReplyTo, + String editEventId, bool waitUntilSent = false, MatrixImageFile thumbnail, }) async { @@ -605,6 +622,7 @@ class Room { content, txid: txid, inReplyTo: inReplyTo, + editEventId: editEventId, ); if (waitUntilSent) { await sendResponse; @@ -615,7 +633,7 @@ class Room { /// Sends an event to this room with this json as a content. Returns the /// event ID generated from the server. Future sendEvent(Map content, - {String type, String txid, Event inReplyTo}) async { + {String type, String txid, Event inReplyTo, String editEventId}) async { type = type ?? EventTypes.Message; final sendType = (encrypted && client.encryptionEnabled) ? EventTypes.Encrypted : type; @@ -645,6 +663,20 @@ class Room { }, }; } + if (editEventId != null) { + final newContent = Map.from(content); + content['m.new_content'] = newContent; + content['m.relates_to'] = { + 'event_id': editEventId, + 'rel_type': RelationshipTypes.Edit, + }; + if (content['body'] is String) { + content['body'] = '* ' + content['body']; + } + if (content['formatted_body'] is String) { + content['formatted_body'] = '* ' + content['formatted_body']; + } + } final sortOrder = newSortOrder; // Display a *sending* event and store it. diff --git a/lib/src/timeline.dart b/lib/src/timeline.dart index 967b830..5f3bc08 100644 --- a/lib/src/timeline.dart +++ b/lib/src/timeline.dart @@ -35,6 +35,9 @@ class Timeline { final Room room; List events = []; + /// Map of event ID to map of type to set of aggregated events + Map>> aggregatedEvents = {}; + final onTimelineUpdateCallback onUpdate; final onTimelineInsertCallback onInsert; @@ -66,7 +69,10 @@ class Timeline { await room.requestHistory( historyCount: historyCount, onHistoryReceived: () { - if (room.prev_batch.isEmpty || room.prev_batch == null) events = []; + if (room.prev_batch.isEmpty || room.prev_batch == null) { + events.clear(); + aggregatedEvents.clear(); + } }, ); await Future.delayed(const Duration(seconds: 2)); @@ -82,9 +88,17 @@ class Timeline { // to be received via the onEvent stream, it is unneeded to call sortAndUpdate roomSub ??= room.client.onRoomUpdate.stream .where((r) => r.id == room.id && r.limitedTimeline == true) - .listen((r) => events.clear()); + .listen((r) { + events.clear(); + aggregatedEvents.clear(); + }); sessionIdReceivedSub ??= room.onSessionKeyReceived.stream.listen(_sessionKeyReceived); + + // we want to populate our aggregated events + for (final e in events) { + addAggregatedEvent(e); + } } /// Don't forget to call this before you dismiss this object! @@ -151,6 +165,47 @@ class Timeline { return i; } + void _removeEventFromSet(Set eventSet, Event event) { + eventSet.removeWhere((e) => + e.matchesEventOrTransactionId(event.eventId) || + (event.unsigned != null && + e.matchesEventOrTransactionId(event.unsigned['transaction_id']))); + } + + void addAggregatedEvent(Event event) { + // we want to add an event to the aggregation tree + if (event.relationshipType == null || event.relationshipEventId == null) { + return; // nothing to do + } + if (!aggregatedEvents.containsKey(event.relationshipEventId)) { + aggregatedEvents[event.relationshipEventId] = >{}; + } + if (!aggregatedEvents[event.relationshipEventId] + .containsKey(event.relationshipType)) { + aggregatedEvents[event.relationshipEventId] + [event.relationshipType] = {}; + } + // remove a potential old event + _removeEventFromSet( + aggregatedEvents[event.relationshipEventId][event.relationshipType], + event); + // add the new one + aggregatedEvents[event.relationshipEventId][event.relationshipType] + .add(event); + } + + void removeAggregatedEvent(Event event) { + aggregatedEvents.remove(event.eventId); + if (event.unsigned != null) { + aggregatedEvents.remove(event.unsigned['transaction_id']); + } + for (final types in aggregatedEvents.values) { + for (final events in types.values) { + _removeEventFromSet(events, event); + } + } + } + void _handleEventUpdate(EventUpdate eventUpdate) async { try { if (eventUpdate.roomID != room.id) return; @@ -161,12 +216,16 @@ class Timeline { if (eventUpdate.eventType == EventTypes.Redaction) { final eventId = _findEvent(event_id: eventUpdate.content['redacts']); if (eventId != null) { + removeAggregatedEvent(events[eventId]); events[eventId].setRedactionEvent(Event.fromJson( eventUpdate.content, room, eventUpdate.sortOrder)); } } else if (status == -2) { var i = _findEvent(event_id: eventUpdate.content['event_id']); - if (i < events.length) events.removeAt(i); + if (i < events.length) { + removeAggregatedEvent(events[i]); + events.removeAt(i); + } } else { var i = _findEvent( event_id: eventUpdate.content['event_id'], @@ -186,6 +245,7 @@ class Timeline { if (status < oldStatus && !(status == -1 && oldStatus == 0)) { events[i].status = oldStatus; } + addAggregatedEvent(events[i]); } else { var newEvent = Event.fromJson( eventUpdate.content, room, eventUpdate.sortOrder); @@ -196,6 +256,7 @@ class Timeline { -1) return; events.insert(0, newEvent); + addAggregatedEvent(newEvent); if (onInsert != null) onInsert(0); } } diff --git a/test/event_test.dart b/test/event_test.dart index 035b55b..a151c4c 100644 --- a/test/event_test.dart +++ b/test/event_test.dart @@ -67,7 +67,7 @@ void main() { expect(event.formattedText, formatted_body); expect(event.body, body); expect(event.type, EventTypes.Message); - expect(event.isReply, true); + expect(event.relationshipType, RelationshipTypes.Reply); jsonObj['state_key'] = ''; var state = Event.fromJson(jsonObj, null); expect(state.eventId, id); @@ -160,7 +160,43 @@ void main() { 'event_id': '1234', }; event = Event.fromJson(jsonObj, null); - expect(event.messageType, MessageTypes.Reply); + expect(event.messageType, MessageTypes.Text); + expect(event.relationshipType, RelationshipTypes.Reply); + expect(event.relationshipEventId, '1234'); + }); + + test('relationship types', () async { + Event event; + + jsonObj['content'] = { + 'msgtype': 'm.text', + 'text': 'beep', + }; + event = Event.fromJson(jsonObj, null); + expect(event.relationshipType, null); + expect(event.relationshipEventId, null); + + jsonObj['content']['m.relates_to'] = { + 'rel_type': 'm.replace', + 'event_id': 'abc', + }; + event = Event.fromJson(jsonObj, null); + expect(event.relationshipType, RelationshipTypes.Edit); + expect(event.relationshipEventId, 'abc'); + + jsonObj['content']['m.relates_to']['rel_type'] = 'm.annotation'; + event = Event.fromJson(jsonObj, null); + expect(event.relationshipType, RelationshipTypes.Reaction); + expect(event.relationshipEventId, 'abc'); + + jsonObj['content']['m.relates_to'] = { + 'm.in_reply_to': { + 'event_id': 'def', + }, + }; + event = Event.fromJson(jsonObj, null); + expect(event.relationshipType, RelationshipTypes.Reply); + expect(event.relationshipEventId, 'def'); }); test('redact', () async { @@ -790,5 +826,56 @@ void main() { }, room); expect(event.getLocalizedBody(FakeMatrixLocalizations()), null); }); + + test('aggregations', () { + var event = Event.fromJson({ + 'content': { + 'body': 'blah', + 'msgtype': 'm.text', + }, + 'event_id': '\$source', + }, null); + var edit1 = Event.fromJson({ + 'content': { + 'body': 'blah', + 'msgtype': 'm.text', + 'm.relates_to': { + 'event_id': '\$source', + 'rel_type': RelationshipTypes.Edit, + }, + }, + 'event_id': '\$edit1', + }, null); + var edit2 = Event.fromJson({ + 'content': { + 'body': 'blah', + 'msgtype': 'm.text', + 'm.relates_to': { + 'event_id': '\$source', + 'rel_type': RelationshipTypes.Edit, + }, + }, + 'event_id': '\$edit2', + }, null); + var room = Room(client: client); + var timeline = Timeline(events: [event, edit1, edit2], room: room); + expect(event.hasAggregatedEvents(timeline, RelationshipTypes.Edit), true); + expect(event.aggregatedEvents(timeline, RelationshipTypes.Edit), + {edit1, edit2}); + expect(event.aggregatedEvents(timeline, RelationshipTypes.Reaction), + {}); + expect(event.hasAggregatedEvents(timeline, RelationshipTypes.Reaction), + false); + + timeline.removeAggregatedEvent(edit2); + expect(event.aggregatedEvents(timeline, RelationshipTypes.Edit), {edit1}); + timeline.addAggregatedEvent(edit2); + expect(event.aggregatedEvents(timeline, RelationshipTypes.Edit), + {edit1, edit2}); + + timeline.removeAggregatedEvent(event); + expect( + event.aggregatedEvents(timeline, RelationshipTypes.Edit), {}); + }); }); } diff --git a/test/fake_matrix_api.dart b/test/fake_matrix_api.dart index 6bbd87e..760dfcf 100644 --- a/test/fake_matrix_api.dart +++ b/test/fake_matrix_api.dart @@ -2010,6 +2010,10 @@ class FakeMatrixApi extends MockClient { (var reqI) => { 'event_id': '\$event${FakeMatrixApi.eventCounter++}', }, + '/client/r0/rooms/!localpart%3Aserver.abc/send/m.reaction/testtxid': + (var reqI) => { + 'event_id': '\$event${FakeMatrixApi.eventCounter++}', + }, '/client/r0/rooms/!localpart%3Aexample.com/typing/%40alice%3Aexample.com': (var req) => {}, '/client/r0/rooms/!1234%3Aexample.com/send/m.room.message/1234': diff --git a/test/room_test.dart b/test/room_test.dart index b459db5..5bad302 100644 --- a/test/room_test.dart +++ b/test/room_test.dart @@ -27,7 +27,9 @@ import 'package:famedlysdk/src/database/database.dart' import 'package:test/test.dart'; import 'fake_client.dart'; +import 'fake_matrix_api.dart'; +import 'dart:convert'; import 'dart:typed_data'; void main() { @@ -349,9 +351,87 @@ void main() { }); test('sendEvent', () async { + FakeMatrixApi.calledEndpoints.clear(); final dynamic resp = await room.sendTextEvent('Hello world', txid: 'testtxid'); expect(resp.startsWith('\$event'), true); + final entry = FakeMatrixApi.calledEndpoints.entries + .firstWhere((p) => p.key.contains('/send/m.room.message/')); + final content = json.decode(entry.value.first); + expect(content, { + 'body': 'Hello world', + 'msgtype': 'm.text', + }); + }); + + test('send edit', () async { + FakeMatrixApi.calledEndpoints.clear(); + final dynamic resp = await room.sendTextEvent('Hello world', + txid: 'testtxid', editEventId: '\$otherEvent'); + expect(resp.startsWith('\$event'), true); + final entry = FakeMatrixApi.calledEndpoints.entries + .firstWhere((p) => p.key.contains('/send/m.room.message/')); + final content = json.decode(entry.value.first); + expect(content, { + 'body': '* Hello world', + 'msgtype': 'm.text', + 'm.new_content': { + 'body': 'Hello world', + 'msgtype': 'm.text', + }, + 'm.relates_to': { + 'event_id': '\$otherEvent', + 'rel_type': 'm.replace', + }, + }); + }); + + test('send reply', () async { + var event = Event.fromJson({ + 'event_id': '\$replyEvent', + 'content': { + 'body': 'Blah', + 'msgtype': 'm.text', + }, + 'type': 'm.room.message', + 'sender': '@alice:example.org', + }, room); + FakeMatrixApi.calledEndpoints.clear(); + final dynamic resp = await room.sendTextEvent('Hello world', + txid: 'testtxid', inReplyTo: event); + expect(resp.startsWith('\$event'), true); + final entry = FakeMatrixApi.calledEndpoints.entries + .firstWhere((p) => p.key.contains('/send/m.room.message/')); + final content = json.decode(entry.value.first); + expect(content, { + 'body': '> <@alice:example.org> Blah\n\nHello world', + 'msgtype': 'm.text', + 'format': 'org.matrix.custom.html', + 'formatted_body': + '
In reply to @alice:example.org
Blah
Hello world', + 'm.relates_to': { + 'm.in_reply_to': { + 'event_id': '\$replyEvent', + }, + }, + }); + }); + + test('send reaction', () async { + FakeMatrixApi.calledEndpoints.clear(); + final dynamic resp = + await room.sendReaction('\$otherEvent', '🦊', txid: 'testtxid'); + expect(resp.startsWith('\$event'), true); + final entry = FakeMatrixApi.calledEndpoints.entries + .firstWhere((p) => p.key.contains('/send/m.reaction/')); + final content = json.decode(entry.value.first); + expect(content, { + 'm.relates_to': { + 'event_id': '\$otherEvent', + 'rel_type': 'm.annotation', + 'key': '🦊', + }, + }); }); // Not working because there is no real file to test it... From 6696a8b3ca1264ad54d270db6b23302a128f9ae0 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Mon, 27 Jul 2020 07:40:25 +0000 Subject: [PATCH 10/90] Remove trailing slash in checkServer --- lib/src/client.dart | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/src/client.dart b/lib/src/client.dart index 5ae9947..44ad73b 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -247,7 +247,19 @@ class Client { /// Throws FormatException, TimeoutException and MatrixException on error. Future checkServer(dynamic serverUrl) async { try { - api.homeserver = (serverUrl is Uri) ? serverUrl : Uri.parse(serverUrl); + if (serverUrl is Uri) { + api.homeserver = serverUrl; + } else { + // URLs allow to have whitespace surrounding them, see https://www.w3.org/TR/2011/WD-html5-20110525/urls.html + // As we want to strip a trailing slash, though, we have to trim the url ourself + // and thus can't let Uri.parse() deal with it. + serverUrl = serverUrl.trim(); + // strip a trailing slash + if (serverUrl.endsWith('/')) { + serverUrl = serverUrl.substring(0, serverUrl.length - 1); + } + api.homeserver = Uri.parse(serverUrl); + } final versions = await api.requestSupportedVersions(); for (var i = 0; i < versions.versions.length; i++) { From 6915781e6a19fe318cfa8e13a8b9d859fc9c582d Mon Sep 17 00:00:00 2001 From: Sorunome Date: Wed, 29 Jul 2020 11:43:27 +0200 Subject: [PATCH 11/90] Prevent m.relates_to to be removed from the status=1 object in encrypted rooms --- lib/encryption/encryption.dart | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/encryption/encryption.dart b/lib/encryption/encryption.dart index 4470285..62c5a05 100644 --- a/lib/encryption/encryption.dart +++ b/lib/encryption/encryption.dart @@ -267,6 +267,9 @@ class Encryption { if (sess == null) { throw ('Unable to create new outbound group session'); } + // we clone the payload as we do not want to remove 'm.relates_to' from the + // original payload passed into this function + payload = Map.from(payload); final Map mRelatesTo = payload.remove('m.relates_to'); final payloadContent = { 'content': payload, From 69431a1aff83c9bb652a6b99d9a9d79bb134eb63 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Thu, 30 Jul 2020 09:57:45 +0200 Subject: [PATCH 12/90] some key verification fixes and temporarily disable transactions --- lib/encryption/key_verification_manager.dart | 8 +++++ lib/encryption/utils/key_verification.dart | 38 +++++++++++++------- lib/src/client.dart | 13 ++++--- lib/src/database/database.dart | 7 ++++ 4 files changed, 49 insertions(+), 17 deletions(-) diff --git a/lib/encryption/key_verification_manager.dart b/lib/encryption/key_verification_manager.dart index de82074..d02d107 100644 --- a/lib/encryption/key_verification_manager.dart +++ b/lib/encryption/key_verification_manager.dart @@ -67,6 +67,10 @@ class KeyVerificationManager { if (_requests.containsKey(transactionId)) { await _requests[transactionId].handlePayload(event.type, event.content); } else { + if (!['m.key.verification.request', 'm.key.verification.start'] + .contains(event.type)) { + return; // we can only start on these + } final newKeyRequest = KeyVerification(encryption: encryption, userId: event.sender); await newKeyRequest.handlePayload(event.type, event.content); @@ -111,6 +115,10 @@ class KeyVerificationManager { _requests.remove(transactionId); } } else if (event['sender'] != client.userID) { + if (!['m.key.verification.request', 'm.key.verification.start'] + .contains(type)) { + return; // we can only start on these + } final room = client.getRoomById(update.roomID) ?? Room(id: update.roomID, client: client); final newKeyRequest = KeyVerification( diff --git a/lib/encryption/utils/key_verification.dart b/lib/encryption/utils/key_verification.dart index e4a43f3..fbcab32 100644 --- a/lib/encryption/utils/key_verification.dart +++ b/lib/encryption/utils/key_verification.dart @@ -215,7 +215,10 @@ class KeyVerification { DateTime.fromMillisecondsSinceEpoch(payload['timestamp']); if (now.subtract(Duration(minutes: 10)).isAfter(verifyTime) || now.add(Duration(minutes: 5)).isBefore(verifyTime)) { - await cancel('m.timeout'); + // if the request is more than 20min in the past we just silently fail it + // to not generate too many cancels + await cancel('m.timeout', + now.subtract(Duration(minutes: 20)).isAfter(verifyTime)); return; } // verify it has a method we can use @@ -280,6 +283,13 @@ class KeyVerification { } method = _makeVerificationMethod(payload['method'], this); if (lastStep == null) { + // validate the start time + if (room != null) { + // we just silently ignore in-room-verification starts + await cancel('m.unknown_method', true); + return; + } + // validate the specific payload if (!method.validateStart(payload)) { await cancel('m.unknown_method'); return; @@ -301,7 +311,11 @@ class KeyVerification { setState(KeyVerificationState.error); break; default: - await method.handlePayload(type, payload); + if (method != null) { + await method.handlePayload(type, payload); + } else { + await cancel('m.invalid_message'); + } break; } if (lastStep == thisLastStep) { @@ -310,9 +324,7 @@ class KeyVerification { } catch (err, stacktrace) { print('[Key Verification] An error occured: ' + err.toString()); print(stacktrace); - if (deviceId != null) { - await cancel('m.invalid_message'); - } + await cancel('m.invalid_message'); } finally { _handlePayloadLock = false; } @@ -510,11 +522,13 @@ class KeyVerification { return false; } - Future cancel([String code = 'm.unknown']) async { - await send('m.key.verification.cancel', { - 'reason': code, - 'code': code, - }); + Future cancel([String code = 'm.unknown', bool quiet = false]) async { + if (!quiet && (deviceId != null || room != null)) { + await send('m.key.verification.cancel', { + 'reason': code, + 'code': code, + }); + } canceled = true; canceledCode = code; setState(KeyVerificationState.error); @@ -538,7 +552,7 @@ class KeyVerification { makePayload(payload); print('[Key Verification] Sending type ${type}: ' + payload.toString()); if (room != null) { - print('[Key Verification] Sending to ${userId} in room ${room.id}'); + print('[Key Verification] Sending to ${userId} in room ${room.id}...'); if (['m.key.verification.request'].contains(type)) { payload['msgtype'] = type; payload['to'] = userId; @@ -552,7 +566,7 @@ class KeyVerification { encryption.keyVerificationManager.addRequest(this); } } else { - print('[Key Verification] Sending to ${userId} device ${deviceId}'); + print('[Key Verification] Sending to ${userId} device ${deviceId}...'); await client.sendToDevice( [client.userDeviceKeys[userId].deviceKeys[deviceId]], type, payload); } diff --git a/lib/src/client.dart b/lib/src/client.dart index 44ad73b..6243ca4 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -1331,11 +1331,14 @@ class Client { } } } - await database?.transaction(() async { - for (final f in dbActions) { - await f(); - } - }); + + if (dbActions.isNotEmpty) { + await database?.transaction(() async { + for (final f in dbActions) { + await f(); + } + }); + } } catch (e) { print('[LibOlm] Unable to update user device keys: ' + e.toString()); } diff --git a/lib/src/database/database.dart b/lib/src/database/database.dart index adb9fc6..3b148ec 100644 --- a/lib/src/database/database.dart +++ b/lib/src/database/database.dart @@ -22,6 +22,13 @@ class Database extends _$Database { int get maxFileSize => 1 * 1024 * 1024; + // moor transactions are sometimes rather weird and freeze. Until there is a + // proper fix in moor we override that there aren't actually using transactions + @override + Future transaction(Future Function() action) async { + return action(); + } + @override MigrationStrategy get migration => MigrationStrategy( onCreate: (Migrator m) { From dc1ed0c6e21fb3bc895dabb761f9d6f1f99b1d0d Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Thu, 30 Jul 2020 08:48:47 +0000 Subject: [PATCH 13/90] Use SyncUpdate for pending messages --- lib/src/event.dart | 4 ++- lib/src/room.dart | 60 +++++++++++++++++------------------------ test/timeline_test.dart | 4 +++ 3 files changed, 32 insertions(+), 36 deletions(-) diff --git a/lib/src/event.dart b/lib/src/event.dart index e0e3d5d..f3729cf 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -146,7 +146,9 @@ class Event extends MatrixEvent { final unsigned = Event.getMapFromPayload(jsonPayload['unsigned']); final prevContent = Event.getMapFromPayload(jsonPayload['prev_content']); return Event( - status: jsonPayload['status'] ?? defaultStatus, + status: jsonPayload['status'] ?? + unsigned[MessageSendingStatusKey] ?? + defaultStatus, stateKey: jsonPayload['state_key'], prevContent: prevContent, content: content, diff --git a/lib/src/room.dart b/lib/src/room.dart index be97100..234ad39 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -39,6 +39,8 @@ enum PushRuleState { notify, mentions_only, dont_notify } enum JoinRules { public, knock, invite, private } enum GuestAccess { can_join, forbidden } enum HistoryVisibility { invited, joined, shared, world_readable } +const String MessageSendingStatusKey = + 'com.famedly.famedlysdk.message_sending_status'; /// Represents a Matrix room. class Room { @@ -678,27 +680,20 @@ class Room { } } - final sortOrder = newSortOrder; - // Display a *sending* event and store it. - var eventUpdate = EventUpdate( - type: 'timeline', - roomID: id, - eventType: type, - sortOrder: sortOrder, - content: { - 'type': type, - 'event_id': messageID, - 'sender': client.userID, - 'status': 0, - 'origin_server_ts': DateTime.now().millisecondsSinceEpoch, - 'content': content - }, - ); - client.onEvent.add(eventUpdate); - await client.database?.transaction(() async { - await client.database.storeEventUpdate(client.id, eventUpdate); - await updateSortOrder(); - }); + final syncUpdate = SyncUpdate() + ..rooms = (RoomsUpdate() + ..join = ({}..[id] = (JoinedRoomUpdate() + ..timeline = (TimelineUpdate() + ..events = [ + MatrixEvent() + ..content = content + ..type = type + ..eventId = messageID + ..senderId = client.userID + ..originServerTs = DateTime.now() + ..unsigned = {MessageSendingStatusKey: 0}, + ])))); + await client.handleSync(syncUpdate); // Send the text and on success, store and display a *sent* event. try { @@ -712,23 +707,18 @@ class Room { messageID, sendMessageContent, ); - eventUpdate.content['status'] = 1; - eventUpdate.content['unsigned'] = {'transaction_id': messageID}; - eventUpdate.content['event_id'] = res; - client.onEvent.add(eventUpdate); - await client.database?.transaction(() async { - await client.database.storeEventUpdate(client.id, eventUpdate); - }); + syncUpdate.rooms.join.values.first.timeline.events.first + .unsigned[MessageSendingStatusKey] = 1; + syncUpdate.rooms.join.values.first.timeline.events.first + .unsigned['transaction_id'] = messageID; + syncUpdate.rooms.join.values.first.timeline.events.first.eventId = res; + await client.handleSync(syncUpdate); return res; } catch (exception) { print('[Client] Error while sending: ' + exception.toString()); - // On error, set status to -1 - eventUpdate.content['status'] = -1; - eventUpdate.content['unsigned'] = {'transaction_id': messageID}; - client.onEvent.add(eventUpdate); - await client.database?.transaction(() async { - await client.database.storeEventUpdate(client.id, eventUpdate); - }); + syncUpdate.rooms.join.values.first.timeline.events.first + .unsigned[MessageSendingStatusKey] = -1; + await client.handleSync(syncUpdate); } return null; } diff --git a/test/timeline_test.dart b/test/timeline_test.dart index 385a67d..f76eb6c 100644 --- a/test/timeline_test.dart +++ b/test/timeline_test.dart @@ -186,8 +186,12 @@ void main() { }, sortOrder: room.newSortOrder)); await Future.delayed(Duration(milliseconds: 50)); + + expect(updateCount, 7); await room.sendTextEvent('test', txid: 'errortxid'); await Future.delayed(Duration(milliseconds: 50)); + + expect(updateCount, 9); await room.sendTextEvent('test', txid: 'errortxid2'); await Future.delayed(Duration(milliseconds: 50)); await room.sendTextEvent('test', txid: 'errortxid3'); From 18a790be841beaa46e4aa3b0f62fea7c9a054bf5 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Sat, 1 Aug 2020 07:06:39 +0000 Subject: [PATCH 14/90] put key request in try...catch --- lib/encryption/key_manager.dart | 59 ++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/lib/encryption/key_manager.dart b/lib/encryption/key_manager.dart index 9460827..b6bc269 100644 --- a/lib/encryption/key_manager.dart +++ b/lib/encryption/key_manager.dart @@ -417,34 +417,39 @@ class KeyManager { return; // we managed to load the session from online backup, no need to care about it now } } - // while we just send the to-device event to '*', we still need to save the - // devices themself to know where to send the cancel to after receiving a reply - final devices = await room.getUserDeviceKeys(); - final requestId = client.generateUniqueTransactionId(); - final request = KeyManagerKeyShareRequest( - requestId: requestId, - devices: devices, - room: room, - sessionId: sessionId, - senderKey: senderKey, - ); - await client.sendToDevice( - [], - 'm.room_key_request', - { - 'action': 'request', - 'body': { - 'algorithm': 'm.megolm.v1.aes-sha2', - 'room_id': room.id, - 'sender_key': senderKey, - 'session_id': sessionId, + try { + // while we just send the to-device event to '*', we still need to save the + // devices themself to know where to send the cancel to after receiving a reply + final devices = await room.getUserDeviceKeys(); + final requestId = client.generateUniqueTransactionId(); + final request = KeyManagerKeyShareRequest( + requestId: requestId, + devices: devices, + room: room, + sessionId: sessionId, + senderKey: senderKey, + ); + await client.sendToDevice( + [], + 'm.room_key_request', + { + 'action': 'request', + 'body': { + 'algorithm': 'm.megolm.v1.aes-sha2', + 'room_id': room.id, + 'sender_key': senderKey, + 'session_id': sessionId, + }, + 'request_id': requestId, + 'requesting_device_id': client.deviceID, }, - 'request_id': requestId, - 'requesting_device_id': client.deviceID, - }, - encrypted: false, - toUsers: await room.requestParticipants()); - outgoingShareRequests[request.requestId] = request; + encrypted: false, + toUsers: await room.requestParticipants()); + outgoingShareRequests[request.requestId] = request; + } catch (err) { + print('[Key Manager] Sending key verification request failed: ' + + err.toString()); + } } /// Handle an incoming to_device event that is related to key sharing From 938540eca577288720cab7c75ebb8c35133d376c Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Sat, 1 Aug 2020 13:04:03 +0000 Subject: [PATCH 15/90] Detect the file message type --- lib/src/utils/matrix_file.dart | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/src/utils/matrix_file.dart b/lib/src/utils/matrix_file.dart index 1f72c95..f5561bc 100644 --- a/lib/src/utils/matrix_file.dart +++ b/lib/src/utils/matrix_file.dart @@ -1,6 +1,7 @@ /// Workaround until [File] in dart:io and dart:html is unified import 'dart:typed_data'; +import 'package:famedlysdk/matrix_api/model/message_types.dart'; import 'package:matrix_file_e2ee/matrix_file_e2ee.dart'; import 'package:mime/mime.dart'; @@ -22,7 +23,18 @@ class MatrixFile { int get size => bytes.length; - String get msgType => 'm.file'; + String get msgType { + if (mimeType.toLowerCase().startsWith('image/')) { + return MessageTypes.Image; + } + if (mimeType.toLowerCase().startsWith('video/')) { + return MessageTypes.Video; + } + if (mimeType.toLowerCase().startsWith('audio/')) { + return MessageTypes.Audio; + } + return MessageTypes.File; + } Map get info => ({ 'mimetype': mimeType, From f6a253de8822aed9ef5bd0daa18843cb18eb7ee2 Mon Sep 17 00:00:00 2001 From: MTRNord Date: Sat, 1 Aug 2020 15:55:41 +0200 Subject: [PATCH 16/90] repo!: Make CI ready for change from master to main. BREAKING CHANGE: This only works with main as the default instead of master --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 36458ff..33c4799 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -68,7 +68,7 @@ build-api-doc: paths: - doc/api/ only: - - master + - main build-doc: tags: @@ -83,7 +83,7 @@ build-doc: paths: - doc-public only: - - master + - main pages: tags: @@ -101,4 +101,4 @@ pages: paths: - public only: - - master \ No newline at end of file + - main \ No newline at end of file From d4a7345b8a6569dd644041c243dc6241e77d60a3 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Mon, 3 Aug 2020 13:28:30 +0000 Subject: [PATCH 17/90] Enable transactions again to fix web --- lib/src/database/database.dart | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/src/database/database.dart b/lib/src/database/database.dart index 3b148ec..adb9fc6 100644 --- a/lib/src/database/database.dart +++ b/lib/src/database/database.dart @@ -22,13 +22,6 @@ class Database extends _$Database { int get maxFileSize => 1 * 1024 * 1024; - // moor transactions are sometimes rather weird and freeze. Until there is a - // proper fix in moor we override that there aren't actually using transactions - @override - Future transaction(Future Function() action) async { - return action(); - } - @override MigrationStrategy get migration => MigrationStrategy( onCreate: (Migrator m) { From fe700b229cbc2edd079d27f6a67a8b7782481650 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Tue, 4 Aug 2020 10:07:46 +0200 Subject: [PATCH 18/90] Fix prev_content bug --- lib/src/event.dart | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/lib/src/event.dart b/lib/src/event.dart index f3729cf..2ccc8e1 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -99,9 +99,9 @@ class Event extends MatrixEvent { // into the unsigned block this.prevContent = prevContent != null && prevContent.isNotEmpty ? prevContent - : (unsigned != null && unsigned['prev_content'] is Map + : (unsigned != null && unsigned['prev_content'] is Map) ? unsigned['prev_content'] - : null); + : null; this.stateKey = stateKey; this.originServerTs = originServerTs; } @@ -477,9 +477,7 @@ class Event extends MatrixEvent { // Has the membership changed? final newMembership = content['membership'] ?? ''; final oldMembership = - unsigned != null && unsigned['prev_content'] is Map - ? unsigned['prev_content']['membership'] ?? '' - : ''; + prevContent != null ? prevContent['membership'] ?? '' : ''; if (newMembership != oldMembership) { if (oldMembership == 'invite' && newMembership == 'join') { text = i18n.acceptedTheInvitation(targetName); @@ -514,16 +512,12 @@ class Event extends MatrixEvent { } } else if (newMembership == 'join') { final newAvatar = content['avatar_url'] ?? ''; - final oldAvatar = unsigned != null && - unsigned['prev_content'] is Map - ? unsigned['prev_content']['avatar_url'] ?? '' - : ''; + final oldAvatar = + prevContent != null ? prevContent['avatar_url'] ?? '' : ''; final newDisplayname = content['displayname'] ?? ''; - final oldDisplayname = unsigned != null && - unsigned['prev_content'] is Map - ? unsigned['prev_content']['displayname'] ?? '' - : ''; + final oldDisplayname = + prevContent != null ? prevContent['displayname'] ?? '' : ''; // Has the user avatar changed? if (newAvatar != oldAvatar) { From e1fa4983d0b35da90c2bb3d5b51f6e95780432f0 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Sat, 1 Aug 2020 18:18:30 +0200 Subject: [PATCH 19/90] try...catch fetching all encrypted devices, in case we aren't in a room --- lib/src/client.dart | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/src/client.dart b/lib/src/client.dart index 6243ca4..9a0b591 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -1159,11 +1159,16 @@ class Client { var userIds = {}; for (var i = 0; i < rooms.length; i++) { if (rooms[i].encrypted) { - var userList = await rooms[i].requestParticipants(); - for (var user in userList) { - if ([Membership.join, Membership.invite].contains(user.membership)) { - userIds.add(user.id); + try { + var userList = await rooms[i].requestParticipants(); + for (var user in userList) { + if ([Membership.join, Membership.invite] + .contains(user.membership)) { + userIds.add(user.id); + } } + } catch (err) { + print('[E2EE] Failed to fetch participants: ' + err.toString()); } } } From a11a0b592505ed3c9e34a61600bc10bad2042070 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Wed, 5 Aug 2020 05:57:02 +0000 Subject: [PATCH 20/90] Add example --- analysis_options.yaml | 4 +- example/main.dart | 264 ++++++++++++++++++++++++++++++++++++++++++ lib/src/timeline.dart | 5 +- 3 files changed, 269 insertions(+), 4 deletions(-) create mode 100644 example/main.dart diff --git a/analysis_options.yaml b/analysis_options.yaml index c0e32f0..93361f5 100644 --- a/analysis_options.yaml +++ b/analysis_options.yaml @@ -7,5 +7,5 @@ linter: analyzer: errors: todo: ignore -# exclude: -# - path/to/excluded/files/** \ No newline at end of file + exclude: + - example/main.dart \ No newline at end of file diff --git a/example/main.dart b/example/main.dart new file mode 100644 index 0000000..b880ffc --- /dev/null +++ b/example/main.dart @@ -0,0 +1,264 @@ +import 'package:famedlysdk/famedlysdk.dart'; +import 'package:flutter/material.dart'; + +void main() { + runApp(FamedlySdkExampleApp()); +} + +class FamedlySdkExampleApp extends StatelessWidget { + static Client client = Client('Famedly SDK Example Client', debug: true); + @override + Widget build(BuildContext context) { + return MaterialApp( + title: 'Famedly SDK Example App', + home: LoginView(), + ); + } +} + +class LoginView extends StatefulWidget { + @override + _LoginViewState createState() => _LoginViewState(); +} + +class _LoginViewState extends State { + final TextEditingController _homeserverController = TextEditingController(); + final TextEditingController _usernameController = TextEditingController(); + final TextEditingController _passwordController = TextEditingController(); + bool _isLoading = false; + String _error; + + void _loginAction() async { + setState(() => _isLoading = true); + setState(() => _error = null); + try { + if (await FamedlySdkExampleApp.client + .checkServer(_homeserverController.text) == + false) { + throw (Exception('Server not supported')); + } + if (await FamedlySdkExampleApp.client.login( + _usernameController.text, + _passwordController.text, + ) == + false) { + throw (Exception('Username or password incorrect')); + } + Navigator.of(context).pushAndRemoveUntil( + MaterialPageRoute(builder: (_) => ChatListView()), + (route) => false, + ); + } catch (e) { + setState(() => _error = e.toString()); + } + setState(() => _isLoading = false); + } + + @override + Widget build(BuildContext context) { + return Scaffold( + appBar: AppBar(title: Text('Login')), + body: ListView( + padding: const EdgeInsets.all(16), + children: [ + TextField( + controller: _homeserverController, + readOnly: _isLoading, + autocorrect: false, + decoration: InputDecoration( + labelText: 'Homeserver', + hintText: 'https://matrix.org', + ), + ), + SizedBox(height: 8), + TextField( + controller: _usernameController, + readOnly: _isLoading, + autocorrect: false, + decoration: InputDecoration( + labelText: 'Username', + hintText: '@username:domain', + ), + ), + SizedBox(height: 8), + TextField( + controller: _passwordController, + obscureText: true, + readOnly: _isLoading, + autocorrect: false, + decoration: InputDecoration( + labelText: 'Password', + hintText: '****', + errorText: _error, + ), + ), + SizedBox(height: 8), + RaisedButton( + child: _isLoading ? LinearProgressIndicator() : Text('Login'), + onPressed: _isLoading ? null : _loginAction, + ), + ], + ), + ); + } +} + +class ChatListView extends StatefulWidget { + @override + _ChatListViewState createState() => _ChatListViewState(); +} + +class _ChatListViewState extends State { + @override + Widget build(BuildContext context) { + return Scaffold( + appBar: AppBar( + title: Text('Chats'), + ), + body: StreamBuilder( + stream: FamedlySdkExampleApp.client.onSync.stream, + builder: (c, s) => ListView.builder( + itemCount: FamedlySdkExampleApp.client.rooms.length, + itemBuilder: (BuildContext context, int i) { + final room = FamedlySdkExampleApp.client.rooms[i]; + return ListTile( + title: Text(room.displayname + ' (${room.notificationCount})'), + subtitle: Text(room.lastMessage, maxLines: 1), + leading: CircleAvatar( + backgroundImage: NetworkImage(room.avatar.getThumbnail( + FamedlySdkExampleApp.client, + width: 64, + height: 64, + )), + ), + onTap: () => Navigator.of(context).push( + MaterialPageRoute( + builder: (_) => ChatView(room: room), + ), + ), + ); + }, + ), + ), + ); + } +} + +class ChatView extends StatefulWidget { + final Room room; + + const ChatView({Key key, @required this.room}) : super(key: key); + + @override + _ChatViewState createState() => _ChatViewState(); +} + +class _ChatViewState extends State { + final TextEditingController _controller = TextEditingController(); + + void _sendAction() { + print('Send Text'); + widget.room.sendTextEvent(_controller.text); + _controller.clear(); + } + + Timeline timeline; + + Future getTimeline() async { + timeline ??= + await widget.room.getTimeline(onUpdate: () => setState(() => null)); + return true; + } + + @override + void dispose() { + timeline?.cancelSubscriptions(); + super.dispose(); + } + + @override + Widget build(BuildContext context) { + return Scaffold( + appBar: AppBar( + title: StreamBuilder( + stream: widget.room.onUpdate.stream, + builder: (context, snapshot) { + return Text(widget.room.displayname); + }), + ), + body: Column( + children: [ + Expanded( + child: FutureBuilder( + future: getTimeline(), + builder: (context, snapshot) => !snapshot.hasData + ? Center( + child: CircularProgressIndicator(), + ) + : ListView.builder( + reverse: true, + itemCount: timeline.events.length, + itemBuilder: (BuildContext context, int i) => Opacity( + opacity: timeline.events[i].status != 2 ? 0.5 : 1, + child: ListTile( + title: Row( + children: [ + Expanded( + child: Text( + timeline.events[i].sender.calcDisplayname(), + ), + ), + Text( + timeline.events[i].originServerTs + .toIso8601String(), + style: TextStyle(fontSize: 12), + ), + ], + ), + subtitle: Text(timeline.events[i].body), + leading: CircleAvatar( + child: timeline.events[i].sender?.avatarUrl == null + ? Icon(Icons.person) + : null, + backgroundImage: + timeline.events[i].sender?.avatarUrl != null + ? NetworkImage( + timeline.events[i].sender?.avatarUrl + ?.getThumbnail( + FamedlySdkExampleApp.client, + width: 64, + height: 64, + ), + ) + : null, + ), + ), + ), + ), + ), + ), + Container( + height: 60, + child: Row( + children: [ + Expanded( + child: TextField( + controller: _controller, + decoration: InputDecoration( + contentPadding: const EdgeInsets.symmetric(horizontal: 8), + labelText: 'Send a message ...', + ), + ), + ), + IconButton( + icon: Icon(Icons.send), + onPressed: _sendAction, + ) + ], + ), + ), + ], + ), + ); + } +} diff --git a/lib/src/timeline.dart b/lib/src/timeline.dart index 5f3bc08..5a38b39 100644 --- a/lib/src/timeline.dart +++ b/lib/src/timeline.dart @@ -215,7 +215,7 @@ class Timeline { // Redaction events are handled as modification for existing events. if (eventUpdate.eventType == EventTypes.Redaction) { final eventId = _findEvent(event_id: eventUpdate.content['redacts']); - if (eventId != null) { + if (eventId < events.length) { removeAggregatedEvent(events[eventId]); events[eventId].setRedactionEvent(Event.fromJson( eventUpdate.content, room, eventUpdate.sortOrder)); @@ -262,9 +262,10 @@ class Timeline { } } sortAndUpdate(); - } catch (e) { + } catch (e, s) { if (room.client.debug) { print('[WARNING] (_handleEventUpdate) ${e.toString()}'); + print(s); } } } From bbd5749aec7288a412a40ef39c9c0e65a3d91179 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Tue, 4 Aug 2020 15:08:25 +0200 Subject: [PATCH 21/90] Fix storing of event status --- lib/src/database/database.dart | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/src/database/database.dart b/lib/src/database/database.dart index adb9fc6..6caec77 100644 --- a/lib/src/database/database.dart +++ b/lib/src/database/database.dart @@ -6,6 +6,7 @@ import 'package:famedlysdk/matrix_api.dart' as api; import 'package:olm/olm.dart' as olm; import '../../matrix_api.dart'; +import '../room.dart'; part 'database.g.dart'; @@ -357,6 +358,10 @@ class Database extends _$Database { if (type == 'timeline' || type == 'history') { // calculate the status var status = 2; + if (eventContent['unsigned'] is Map && + eventContent['unsigned'][MessageSendingStatusKey] is num) { + status = eventContent['unsigned'][MessageSendingStatusKey]; + } if (eventContent['status'] is num) status = eventContent['status']; if ((status == 1 || status == -1) && eventContent['unsigned'] is Map && From ede4fd1416549f0ff649e7194757926f362b4096 Mon Sep 17 00:00:00 2001 From: Daniel Schaefer Date: Tue, 4 Aug 2020 22:20:22 +0200 Subject: [PATCH 22/90] Implement function to send m.location event Allows to share the location with a room. --- lib/src/room.dart | 11 +++++++++++ test/room_test.dart | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/lib/src/room.dart b/lib/src/room.dart index 234ad39..d5decb8 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -539,6 +539,17 @@ class Room { }, type: EventTypes.Reaction, txid: txid); } + /// Sends the location with description [body] and geo URI [geoUri] into a room. + /// Returns the event ID generated by the server for this message. + Future sendLocation(String body, String geoUri, {String txid}) { + final event = { + 'msgtype': 'm.location', + 'body': body, + 'geo_uri': geoUri, + }; + return sendEvent(event, txid: txid); + } + /// Sends a [file] to this room after uploading it. Returns the mxc uri of /// the uploaded file. If [waitUntilSent] is true, the future will wait until /// the message event has received the server. Otherwise the future will only diff --git a/test/room_test.dart b/test/room_test.dart index 5bad302..df27b80 100644 --- a/test/room_test.dart +++ b/test/room_test.dart @@ -434,6 +434,25 @@ void main() { }); }); + test('send location', () async { + FakeMatrixApi.calledEndpoints.clear(); + + final body = 'Middle of the ocean'; + final geoUri = 'geo:0.0,0.0'; + final dynamic resp = + await room.sendLocation(body, geoUri, txid: 'testtxid'); + expect(resp.startsWith('\$event'), true); + + final entry = FakeMatrixApi.calledEndpoints.entries + .firstWhere((p) => p.key.contains('/send/m.room.message/')); + final content = json.decode(entry.value.first); + expect(content, { + 'msgtype': 'm.location', + 'body': body, + 'geo_uri': geoUri, + }); + }); + // Not working because there is no real file to test it... /*test('sendImageEvent', () async { final File testFile = File.fromUri(Uri.parse("fake/path/file.jpeg")); From 2796ca613ac03e73c86215a56afe614ca58a7ce9 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Wed, 5 Aug 2020 13:24:57 +0200 Subject: [PATCH 23/90] Fix resending messages reusing an existing transaction id --- lib/src/event.dart | 6 +++--- lib/src/room.dart | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/src/event.dart b/lib/src/event.dart index 2ccc8e1..6305a1c 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -319,11 +319,11 @@ class Event extends MatrixEvent { if (status != -1) return null; // we do not remove the event here. It will automatically be updated // in the `sendEvent` method to transition -1 -> 0 -> 1 -> 2 - final eventID = await room.sendEvent( + final newEventId = await room.sendEvent( content, - txid: txid ?? unsigned['transaction_id'], + txid: txid ?? unsigned['transaction_id'] ?? eventId, ); - return eventID; + return newEventId; } /// Whether the client is allowed to redact this event. diff --git a/lib/src/room.dart b/lib/src/room.dart index 234ad39..8ba80b8 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -691,7 +691,10 @@ class Room { ..eventId = messageID ..senderId = client.userID ..originServerTs = DateTime.now() - ..unsigned = {MessageSendingStatusKey: 0}, + ..unsigned = { + MessageSendingStatusKey: 0, + 'transaction_id': messageID, + }, ])))); await client.handleSync(syncUpdate); @@ -709,8 +712,6 @@ class Room { ); syncUpdate.rooms.join.values.first.timeline.events.first .unsigned[MessageSendingStatusKey] = 1; - syncUpdate.rooms.join.values.first.timeline.events.first - .unsigned['transaction_id'] = messageID; syncUpdate.rooms.join.values.first.timeline.events.first.eventId = res; await client.handleSync(syncUpdate); return res; From 6779ab6624351bf954f392186391c8a37b60755a Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Thu, 6 Aug 2020 08:55:35 +0200 Subject: [PATCH 24/90] Deprecate debug mode --- lib/matrix_api/matrix_api.dart | 12 ------- lib/src/client.dart | 32 ++++++++----------- lib/src/timeline.dart | 6 ++-- test/client_test.dart | 10 +++--- .../encrypt_decrypt_to_device_test.dart | 3 +- test/encryption/key_verification_test.dart | 3 +- test/event_test.dart | 16 ++++------ test/fake_client.dart | 2 +- test/matrix_api_test.dart | 1 - test/timeline_test.dart | 2 +- test/user_test.dart | 2 +- test_driver/famedlysdk_test.dart | 7 ++-- 12 files changed, 33 insertions(+), 63 deletions(-) diff --git a/lib/matrix_api/matrix_api.dart b/lib/matrix_api/matrix_api.dart index 569f161..ce528d2 100644 --- a/lib/matrix_api/matrix_api.dart +++ b/lib/matrix_api/matrix_api.dart @@ -88,9 +88,6 @@ class MatrixApi { /// timeout which is usually 30 seconds. int syncTimeoutSec; - /// Whether debug prints should be displayed. - final bool debug; - http.Client httpClient = http.Client(); bool get _testMode => @@ -101,7 +98,6 @@ class MatrixApi { MatrixApi({ this.homeserver, this.accessToken, - this.debug = false, http.Client httpClient, this.syncTimeoutSec = 30, }) { @@ -161,11 +157,6 @@ class MatrixApi { headers['Authorization'] = 'Bearer ${accessToken}'; } - if (debug) { - print( - '[REQUEST ${describeEnum(type)}] $action, Data: ${jsonEncode(data)}'); - } - http.Response resp; var jsonResp = {}; try { @@ -212,8 +203,6 @@ class MatrixApi { throw exception; } - - if (debug) print('[RESPONSE] ${jsonResp.toString()}'); _timeoutFactor = 1; } on TimeoutException catch (_) { _timeoutFactor *= 2; @@ -1300,7 +1289,6 @@ class MatrixApi { streamedRequest.contentLength = await file.length; streamedRequest.sink.add(file); streamedRequest.sink.close(); - if (debug) print('[UPLOADING] $fileName'); var streamedResponse = _testMode ? null : await streamedRequest.send(); Map jsonResponse = json.decode( String.fromCharCodes(_testMode diff --git a/lib/src/client.dart b/lib/src/client.dart index 9a0b591..8eb3907 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -81,14 +81,16 @@ class Client { /// - m.room.canonical_alias /// - m.room.tombstone /// - *some* m.room.member events, where needed - Client(this.clientName, - {this.debug = false, - this.database, - this.enableE2eeRecovery = false, - this.verificationMethods, - http.Client httpClient, - this.importantStateEvents, - this.pinUnreadRooms = false}) { + Client( + this.clientName, { + this.database, + this.enableE2eeRecovery = false, + this.verificationMethods, + http.Client httpClient, + this.importantStateEvents, + this.pinUnreadRooms = false, + @deprecated bool debug, + }) { verificationMethods ??= {}; importantStateEvents ??= {}; importantStateEvents.addAll([ @@ -100,17 +102,9 @@ class Client { EventTypes.RoomCanonicalAlias, EventTypes.RoomTombstone, ]); - api = MatrixApi(debug: debug, httpClient: httpClient); - onLoginStateChanged.stream.listen((loginState) { - if (debug) { - print('[LoginState]: ${loginState.toString()}'); - } - }); + api = MatrixApi(httpClient: httpClient); } - /// Whether debug prints should be displayed. - final bool debug; - /// The required name for this client. final String clientName; @@ -634,8 +628,8 @@ class Client { return; } - encryption = Encryption( - debug: debug, client: this, enableE2eeRecovery: enableE2eeRecovery); + encryption = + Encryption(client: this, enableE2eeRecovery: enableE2eeRecovery); await encryption.init(olmAccount); if (database != null) { diff --git a/lib/src/timeline.dart b/lib/src/timeline.dart index 5a38b39..042e811 100644 --- a/lib/src/timeline.dart +++ b/lib/src/timeline.dart @@ -263,10 +263,8 @@ class Timeline { } sortAndUpdate(); } catch (e, s) { - if (room.client.debug) { - print('[WARNING] (_handleEventUpdate) ${e.toString()}'); - print(s); - } + print('[WARNING] (_handleEventUpdate) ${e.toString()}'); + print(s); } } diff --git a/test/client_test.dart b/test/client_test.dart index 41a9fab..c970708 100644 --- a/test/client_test.dart +++ b/test/client_test.dart @@ -48,7 +48,7 @@ void main() { group('FluffyMatrix', () { /// Check if all Elements get created - matrix = Client('testclient', debug: true, httpClient: FakeMatrixApi()); + matrix = Client('testclient', httpClient: FakeMatrixApi()); roomUpdateListFuture = matrix.onRoomUpdate.stream.toList(); eventUpdateListFuture = matrix.onEvent.stream.toList(); @@ -322,7 +322,7 @@ void main() { }); test('Login', () async { - matrix = Client('testclient', debug: true, httpClient: FakeMatrixApi()); + matrix = Client('testclient', httpClient: FakeMatrixApi()); roomUpdateListFuture = matrix.onRoomUpdate.stream.toList(); eventUpdateListFuture = matrix.onEvent.stream.toList(); @@ -395,8 +395,7 @@ void main() { }); }); test('Test the fake store api', () async { - var client1 = - Client('testclient', debug: true, httpClient: FakeMatrixApi()); + var client1 = Client('testclient', httpClient: FakeMatrixApi()); client1.database = getDatabase(); client1.connect( @@ -413,8 +412,7 @@ void main() { expect(client1.isLogged(), true); expect(client1.rooms.length, 2); - var client2 = - Client('testclient', debug: true, httpClient: FakeMatrixApi()); + var client2 = Client('testclient', httpClient: FakeMatrixApi()); client2.database = client1.database; client2.connect(); diff --git a/test/encryption/encrypt_decrypt_to_device_test.dart b/test/encryption/encrypt_decrypt_to_device_test.dart index 5636e8b..a56adee 100644 --- a/test/encryption/encrypt_decrypt_to_device_test.dart +++ b/test/encryption/encrypt_decrypt_to_device_test.dart @@ -42,8 +42,7 @@ void main() { if (!olmEnabled) return; Client client; - var otherClient = - Client('othertestclient', debug: true, httpClient: FakeMatrixApi()); + var otherClient = Client('othertestclient', httpClient: FakeMatrixApi()); DeviceKeys device; Map payload; diff --git a/test/encryption/key_verification_test.dart b/test/encryption/key_verification_test.dart index 66ccdee..3216b8e 100644 --- a/test/encryption/key_verification_test.dart +++ b/test/encryption/key_verification_test.dart @@ -82,8 +82,7 @@ void main() { test('setupClient', () async { client1 = await getClient(); - client2 = - Client('othertestclient', debug: true, httpClient: FakeMatrixApi()); + client2 = Client('othertestclient', httpClient: FakeMatrixApi()); client2.database = client1.database; await client2.checkServer('https://fakeServer.notExisting'); client2.connect( diff --git a/test/event_test.dart b/test/event_test.dart index a151c4c..5fb041c 100644 --- a/test/event_test.dart +++ b/test/event_test.dart @@ -50,7 +50,7 @@ void main() { 'status': 2, 'content': contentJson, }; - var client = Client('testclient', debug: true, httpClient: FakeMatrixApi()); + var client = Client('testclient', httpClient: FakeMatrixApi()); var event = Event.fromJson( jsonObj, Room(id: '!localpart:server.abc', client: client)); @@ -211,8 +211,7 @@ void main() { ]; for (final testType in testTypes) { redactJsonObj['type'] = testType; - final room = - Room(id: '1234', client: Client('testclient', debug: true)); + final room = Room(id: '1234', client: Client('testclient')); final redactionEventJson = { 'content': {'reason': 'Spamming'}, 'event_id': '143273582443PhrSn:example.org', @@ -236,7 +235,7 @@ void main() { test('remove', () async { var event = Event.fromJson( - jsonObj, Room(id: '1234', client: Client('testclient', debug: true))); + jsonObj, Room(id: '1234', client: Client('testclient'))); final removed1 = await event.remove(); event.status = 0; final removed2 = await event.remove(); @@ -245,8 +244,7 @@ void main() { }); test('sendAgain', () async { - var matrix = - Client('testclient', debug: true, httpClient: FakeMatrixApi()); + var matrix = Client('testclient', httpClient: FakeMatrixApi()); await matrix.checkServer('https://fakeServer.notExisting'); await matrix.login('test', '1234'); @@ -262,8 +260,7 @@ void main() { }); test('requestKey', () async { - var matrix = - Client('testclient', debug: true, httpClient: FakeMatrixApi()); + var matrix = Client('testclient', httpClient: FakeMatrixApi()); await matrix.checkServer('https://fakeServer.notExisting'); await matrix.login('test', '1234'); @@ -310,8 +307,7 @@ void main() { expect(event.canRedact, true); }); test('getLocalizedBody', () async { - final matrix = - Client('testclient', debug: true, httpClient: FakeMatrixApi()); + final matrix = Client('testclient', httpClient: FakeMatrixApi()); final room = Room(id: '!1234:example.com', client: matrix); var event = Event.fromJson({ 'content': { diff --git a/test/fake_client.dart b/test/fake_client.dart index af2c39a..7b1a94a 100644 --- a/test/fake_client.dart +++ b/test/fake_client.dart @@ -29,7 +29,7 @@ const pickledOlmAccount = 'N2v1MkIFGcl0mQpo2OCwSopxPQJ0wnl7oe7PKiT4141AijfdTIhRu+ceXzXKy3Kr00nLqXtRv7kid6hU4a+V0rfJWLL0Y51+3Rp/ORDVnQy+SSeo6Fn4FHcXrxifJEJ0djla5u98fBcJ8BSkhIDmtXRPi5/oJAvpiYn+8zMjFHobOeZUAxYR0VfQ9JzSYBsSovoQ7uFkNks1M4EDUvHtuyg3RxViwdNxs3718fyAqQ/VSwbXsY0Nl+qQbF+nlVGHenGqk5SuNl1P6e1PzZxcR0IfXA94Xij1Ob5gDv5YH4UCn9wRMG0abZsQP0YzpDM0FLaHSCyo9i5JD/vMlhH+nZWrgAzPPCTNGYewNV8/h3c+VyJh8ZTx/fVi6Yq46Fv+27Ga2ETRZ3Qn+Oyx6dLBjnBZ9iUvIhqpe2XqaGA1PopOz8iDnaZitw'; Future getClient() async { - final client = Client('testclient', debug: true, httpClient: FakeMatrixApi()); + final client = Client('testclient', httpClient: FakeMatrixApi()); client.database = getDatabase(); await client.checkServer('https://fakeServer.notExisting'); final resp = await client.api.login( diff --git a/test/matrix_api_test.dart b/test/matrix_api_test.dart index 7ad8ded..5336cce 100644 --- a/test/matrix_api_test.dart +++ b/test/matrix_api_test.dart @@ -33,7 +33,6 @@ void main() { group('Matrix API', () { final matrixApi = MatrixApi( httpClient: FakeMatrixApi(), - debug: true, ); test('MatrixException test', () async { final exception = MatrixException.fromJson({ diff --git a/test/timeline_test.dart b/test/timeline_test.dart index f76eb6c..0725d19 100644 --- a/test/timeline_test.dart +++ b/test/timeline_test.dart @@ -33,7 +33,7 @@ void main() { var updateCount = 0; var insertList = []; - var client = Client('testclient', debug: true, httpClient: FakeMatrixApi()); + var client = Client('testclient', httpClient: FakeMatrixApi()); var room = Room( id: roomID, client: client, prev_batch: '1234', roomAccountData: {}); diff --git a/test/user_test.dart b/test/user_test.dart index 05e415b..35d35d6 100644 --- a/test/user_test.dart +++ b/test/user_test.dart @@ -27,7 +27,7 @@ import 'fake_matrix_api.dart'; void main() { /// All Tests related to the Event group('User', () { - var client = Client('testclient', debug: true, httpClient: FakeMatrixApi()); + var client = Client('testclient', httpClient: FakeMatrixApi()); final user1 = User( '@alice:example.com', membership: 'join', diff --git a/test_driver/famedlysdk_test.dart b/test_driver/famedlysdk_test.dart index 79cabfb..6b40f53 100644 --- a/test_driver/famedlysdk_test.dart +++ b/test_driver/famedlysdk_test.dart @@ -18,14 +18,14 @@ const String testMessage6 = 'Hello mars'; void test() async { print('++++ Login $testUserA ++++'); - var testClientA = Client('TestClientA', debug: false); + var testClientA = Client('TestClientA'); testClientA.database = getDatabase(); await testClientA.checkServer(homeserver); await testClientA.login(testUserA, testPasswordA); assert(testClientA.encryptionEnabled); print('++++ Login $testUserB ++++'); - var testClientB = Client('TestClientB', debug: false); + var testClientB = Client('TestClientB'); testClientB.database = getDatabase(); await testClientB.checkServer(homeserver); await testClientB.login(testUserB, testPasswordA); @@ -212,8 +212,7 @@ void test() async { "++++ ($testUserA) Received decrypted message: '${room.lastMessage}' ++++"); print('++++ Login $testUserB in another client ++++'); - var testClientC = - Client('TestClientC', debug: false, database: getDatabase()); + var testClientC = Client('TestClientC', database: getDatabase()); await testClientC.checkServer(homeserver); await testClientC.login(testUserB, testPasswordA); await Future.delayed(Duration(seconds: 3)); From 6170c79fe16b504f43ace90d5ba52a37edf0f7be Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Thu, 6 Aug 2020 09:35:02 +0000 Subject: [PATCH 25/90] Improve logging --- analysis_options.yaml | 4 +- lib/encryption/key_manager.dart | 60 +++++++++++-------- lib/encryption/olm_manager.dart | 18 +++--- lib/encryption/ssss.dart | 32 +++++----- lib/encryption/utils/key_verification.dart | 24 ++++---- lib/encryption/utils/olm_session.dart | 5 +- .../utils/outbound_group_session.dart | 8 ++- lib/encryption/utils/session_key.dart | 7 ++- lib/src/client.dart | 30 ++++++---- lib/src/database/database.dart | 8 ++- lib/src/event.dart | 19 ++++-- lib/src/room.dart | 10 ++-- lib/src/timeline.dart | 4 +- lib/src/utils/event_update.dart | 5 +- lib/src/utils/logs.dart | 30 ++++++++++ pubspec.lock | 7 +++ pubspec.yaml | 1 + test/client_test.dart | 5 +- test/device_keys_list_test.dart | 5 +- test/encryption/cross_signing_test.dart | 5 +- .../encrypt_decrypt_room_message_test.dart | 5 +- .../encrypt_decrypt_to_device_test.dart | 5 +- test/encryption/key_manager_test.dart | 5 +- test/encryption/key_request_test.dart | 7 ++- test/encryption/key_verification_test.dart | 5 +- test/encryption/olm_manager_test.dart | 5 +- test/encryption/online_key_backup_test.dart | 5 +- test/encryption/ssss_test.dart | 5 +- test_driver/famedlysdk_test.dart | 57 ++++++++++-------- 29 files changed, 241 insertions(+), 145 deletions(-) create mode 100644 lib/src/utils/logs.dart diff --git a/analysis_options.yaml b/analysis_options.yaml index 93361f5..a3efac9 100644 --- a/analysis_options.yaml +++ b/analysis_options.yaml @@ -3,9 +3,11 @@ include: package:pedantic/analysis_options.yaml linter: rules: - camel_case_types + - avoid_print analyzer: errors: todo: ignore exclude: - - example/main.dart \ No newline at end of file + - example/main.dart + - lib/src/utils/logs.dart \ No newline at end of file diff --git a/lib/encryption/key_manager.dart b/lib/encryption/key_manager.dart index b6bc269..a2a3e55 100644 --- a/lib/encryption/key_manager.dart +++ b/lib/encryption/key_manager.dart @@ -18,6 +18,7 @@ import 'dart:convert'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:pedantic/pedantic.dart'; import 'package:famedlysdk/famedlysdk.dart'; import 'package:famedlysdk/matrix_api.dart'; @@ -84,10 +85,11 @@ class KeyManager { } else { inboundGroupSession.create(content['session_key']); } - } catch (e) { + } catch (e, s) { inboundGroupSession.free(); - print( - '[LibOlm] Could not create new InboundGroupSession: ' + e.toString()); + Logs.error( + '[LibOlm] Could not create new InboundGroupSession: ' + e.toString(), + s); return; } final newSession = SessionKey( @@ -263,10 +265,11 @@ class KeyManager { final outboundGroupSession = olm.OutboundGroupSession(); try { outboundGroupSession.create(); - } catch (e) { + } catch (e, s) { outboundGroupSession.free(); - print('[LibOlm] Unable to create new outboundGroupSession: ' + - e.toString()); + Logs.error( + '[LibOlm] Unable to create new outboundGroupSession: ' + e.toString(), + s); return null; } final rawSession = { @@ -289,10 +292,10 @@ class KeyManager { await storeOutboundGroupSession(roomId, sess); _outboundGroupSessions[roomId] = sess; } catch (e, s) { - print( + Logs.error( '[LibOlm] Unable to send the session key to the participating devices: ' + - e.toString()); - print(s); + e.toString(), + s); sess.dispose(); return null; } @@ -365,8 +368,9 @@ class KeyManager { try { decrypted = json.decode(decryption.decrypt(sessionData['ephemeral'], sessionData['mac'], sessionData['ciphertext'])); - } catch (err) { - print('[LibOlm] Error decrypting room key: ' + err.toString()); + } catch (e, s) { + Logs.error( + '[LibOlm] Error decrypting room key: ' + e.toString(), s); } if (decrypted != null) { decrypted['session_id'] = sessionId; @@ -408,9 +412,10 @@ class KeyManager { try { await loadSingleKey(room.id, sessionId); } catch (err, stacktrace) { - print('[KeyManager] Failed to access online key backup: ' + - err.toString()); - print(stacktrace); + Logs.error( + '[KeyManager] Failed to access online key backup: ' + + err.toString(), + stacktrace); } if (!hadPreviously && getInboundGroupSession(room.id, sessionId, senderKey) != null) { @@ -446,9 +451,11 @@ class KeyManager { encrypted: false, toUsers: await room.requestParticipants()); outgoingShareRequests[request.requestId] = request; - } catch (err) { - print('[Key Manager] Sending key verification request failed: ' + - err.toString()); + } catch (e, s) { + Logs.error( + '[Key Manager] Sending key verification request failed: ' + + e.toString(), + s); } } @@ -460,27 +467,27 @@ class KeyManager { } if (event.content['action'] == 'request') { // we are *receiving* a request - print('[KeyManager] Received key sharing request...'); + Logs.info('[KeyManager] Received key sharing request...'); if (!event.content.containsKey('body')) { - print('[KeyManager] No body, doing nothing'); + Logs.info('[KeyManager] No body, doing nothing'); return; // no body } if (!client.userDeviceKeys.containsKey(event.sender) || !client.userDeviceKeys[event.sender].deviceKeys .containsKey(event.content['requesting_device_id'])) { - print('[KeyManager] Device not found, doing nothing'); + Logs.info('[KeyManager] Device not found, doing nothing'); return; // device not found } final device = client.userDeviceKeys[event.sender] .deviceKeys[event.content['requesting_device_id']]; if (device.userId == client.userID && device.deviceId == client.deviceID) { - print('[KeyManager] Request is by ourself, ignoring'); + Logs.info('[KeyManager] Request is by ourself, ignoring'); return; // ignore requests by ourself } final room = client.getRoomById(event.content['body']['room_id']); if (room == null) { - print('[KeyManager] Unknown room, ignoring'); + Logs.info('[KeyManager] Unknown room, ignoring'); return; // unknown room } final sessionId = event.content['body']['session_id']; @@ -488,7 +495,7 @@ class KeyManager { // okay, let's see if we have this session at all if ((await loadInboundGroupSession(room.id, sessionId, senderKey)) == null) { - print('[KeyManager] Unknown session, ignoring'); + Logs.info('[KeyManager] Unknown session, ignoring'); return; // we don't have this session anyways } final request = KeyManagerKeyShareRequest( @@ -499,7 +506,7 @@ class KeyManager { senderKey: senderKey, ); if (incomingShareRequests.containsKey(request.requestId)) { - print('[KeyManager] Already processed this request, ignoring'); + Logs.info('[KeyManager] Already processed this request, ignoring'); return; // we don't want to process one and the same request multiple times } incomingShareRequests[request.requestId] = request; @@ -508,11 +515,12 @@ class KeyManager { if (device.userId == client.userID && device.verified && !device.blocked) { - print('[KeyManager] All checks out, forwarding key...'); + Logs.info('[KeyManager] All checks out, forwarding key...'); // alright, we can forward the key await roomKeyRequest.forwardKey(); } else { - print('[KeyManager] Asking client, if the key should be forwarded'); + Logs.info( + '[KeyManager] Asking client, if the key should be forwarded'); client.onRoomKeyRequest .add(roomKeyRequest); // let the client handle this } diff --git a/lib/encryption/olm_manager.dart b/lib/encryption/olm_manager.dart index 743e432..e068208 100644 --- a/lib/encryption/olm_manager.dart +++ b/lib/encryption/olm_manager.dart @@ -18,6 +18,7 @@ import 'dart:convert'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:pedantic/pedantic.dart'; import 'package:canonical_json/canonical_json.dart'; import 'package:famedlysdk/famedlysdk.dart'; @@ -119,9 +120,9 @@ class OlmManager { try { olmutil.ed25519_verify(key, message, signature); isValid = true; - } catch (e) { + } catch (e, s) { isValid = false; - print('[LibOlm] Signature check failed: ' + e.toString()); + Logs.error('[LibOlm] Signature check failed: ' + e.toString(), s); } finally { olmutil.free(); } @@ -408,10 +409,12 @@ class OlmManager { lastReceived: DateTime.now(), // we want to use a newly created session )); - } catch (e) { + } catch (e, s) { session.free(); - print('[LibOlm] Could not create new outbound olm session: ' + - e.toString()); + Logs.error( + '[LibOlm] Could not create new outbound olm session: ' + + e.toString(), + s); } } } @@ -483,8 +486,9 @@ class OlmManager { try { data[device.userId][device.deviceId] = await encryptToDeviceMessagePayload(device, type, payload); - } catch (e) { - print('[LibOlm] Error encrypting to-device event: ' + e.toString()); + } catch (e, s) { + Logs.error( + '[LibOlm] Error encrypting to-device event: ' + e.toString(), s); continue; } } diff --git a/lib/encryption/ssss.dart b/lib/encryption/ssss.dart index 9349a14..49b35ef 100644 --- a/lib/encryption/ssss.dart +++ b/lib/encryption/ssss.dart @@ -22,6 +22,7 @@ import 'dart:convert'; import 'package:encrypt/encrypt.dart'; import 'package:crypto/crypto.dart'; import 'package:base58check/base58.dart'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:password_hash/password_hash.dart'; import 'package:famedlysdk/famedlysdk.dart'; import 'package:famedlysdk/matrix_api.dart'; @@ -253,14 +254,14 @@ class SSSS { Future request(String type, List devices) async { // only send to own, verified devices - print('[SSSS] Requesting type ${type}...'); + Logs.info('[SSSS] Requesting type ${type}...'); devices.removeWhere((DeviceKeys d) => d.userId != client.userID || !d.verified || d.blocked || d.deviceId == client.deviceID); if (devices.isEmpty) { - print('[SSSS] Warn: No devices'); + Logs.warning('[SSSS] No devices'); return; } final requestId = client.generateUniqueTransactionId(); @@ -281,31 +282,32 @@ class SSSS { Future handleToDeviceEvent(ToDeviceEvent event) async { if (event.type == 'm.secret.request') { // got a request to share a secret - print('[SSSS] Received sharing request...'); + Logs.info('[SSSS] Received sharing request...'); if (event.sender != client.userID || !client.userDeviceKeys.containsKey(client.userID)) { - print('[SSSS] Not sent by us'); + Logs.info('[SSSS] Not sent by us'); return; // we aren't asking for it ourselves, so ignore } if (event.content['action'] != 'request') { - print('[SSSS] it is actually a cancelation'); + Logs.info('[SSSS] it is actually a cancelation'); return; // not actually requesting, so ignore } final device = client.userDeviceKeys[client.userID] .deviceKeys[event.content['requesting_device_id']]; if (device == null || !device.verified || device.blocked) { - print('[SSSS] Unknown / unverified devices, ignoring'); + Logs.info('[SSSS] Unknown / unverified devices, ignoring'); return; // nope....unknown or untrusted device } // alright, all seems fine...let's check if we actually have the secret they are asking for final type = event.content['name']; final secret = await getCached(type); if (secret == null) { - print('[SSSS] We don\'t have the secret for ${type} ourself, ignoring'); + Logs.info( + '[SSSS] We don\'t have the secret for ${type} ourself, ignoring'); return; // seems like we don't have this, either } // okay, all checks out...time to share this secret! - print('[SSSS] Replying with secret for ${type}'); + Logs.info('[SSSS] Replying with secret for ${type}'); await client.sendToDevice( [device], 'm.secret.send', @@ -315,11 +317,11 @@ class SSSS { }); } else if (event.type == 'm.secret.send') { // receiving a secret we asked for - print('[SSSS] Received shared secret...'); + Logs.info('[SSSS] Received shared secret...'); if (event.sender != client.userID || !pendingShareRequests.containsKey(event.content['request_id']) || event.encryptedContent == null) { - print('[SSSS] Not by us or unknown request'); + Logs.info('[SSSS] Not by us or unknown request'); return; // we have no idea what we just received } final request = pendingShareRequests[event.content['request_id']]; @@ -330,26 +332,26 @@ class SSSS { d.curve25519Key == event.encryptedContent['sender_key'], orElse: () => null); if (device == null) { - print('[SSSS] Someone else replied?'); + Logs.info('[SSSS] Someone else replied?'); return; // someone replied whom we didn't send the share request to } final secret = event.content['secret']; if (!(event.content['secret'] is String)) { - print('[SSSS] Secret wasn\'t a string'); + Logs.info('[SSSS] Secret wasn\'t a string'); return; // the secret wasn't a string....wut? } // let's validate if the secret is, well, valid if (_validators.containsKey(request.type) && !(await _validators[request.type](secret))) { - print('[SSSS] The received secret was invalid'); + Logs.info('[SSSS] The received secret was invalid'); return; // didn't pass the validator } pendingShareRequests.remove(request.requestId); if (request.start.add(Duration(minutes: 15)).isBefore(DateTime.now())) { - print('[SSSS] Request is too far in the past'); + Logs.info('[SSSS] Request is too far in the past'); return; // our request is more than 15min in the past...better not trust it anymore } - print('[SSSS] Secret for type ${request.type} is ok, storing it'); + Logs.info('[SSSS] Secret for type ${request.type} is ok, storing it'); if (client.database != null) { final keyId = keyIdFromType(request.type); if (keyId != null) { diff --git a/lib/encryption/utils/key_verification.dart b/lib/encryption/utils/key_verification.dart index fbcab32..8250068 100644 --- a/lib/encryption/utils/key_verification.dart +++ b/lib/encryption/utils/key_verification.dart @@ -19,6 +19,7 @@ import 'dart:async'; import 'dart:typed_data'; import 'package:canonical_json/canonical_json.dart'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:pedantic/pedantic.dart'; import 'package:olm/olm.dart' as olm; import 'package:famedlysdk/famedlysdk.dart'; @@ -150,7 +151,7 @@ class KeyVerification { } void dispose() { - print('[Key Verification] disposing object...'); + Logs.info('[Key Verification] disposing object...'); method?.dispose(); } @@ -202,7 +203,8 @@ class KeyVerification { await Future.delayed(Duration(milliseconds: 50)); } _handlePayloadLock = true; - print('[Key Verification] Received type ${type}: ' + payload.toString()); + Logs.info( + '[Key Verification] Received type ${type}: ' + payload.toString()); try { var thisLastStep = lastStep; switch (type) { @@ -297,7 +299,7 @@ class KeyVerification { startPaylaod = payload; setState(KeyVerificationState.askAccept); } else { - print('handling start in method.....'); + Logs.info('handling start in method.....'); await method.handlePayload(type, payload); } break; @@ -322,8 +324,8 @@ class KeyVerification { lastStep = type; } } catch (err, stacktrace) { - print('[Key Verification] An error occured: ' + err.toString()); - print(stacktrace); + Logs.error( + '[Key Verification] An error occured: ' + err.toString(), stacktrace); await cancel('m.invalid_message'); } finally { _handlePayloadLock = false; @@ -550,9 +552,10 @@ class KeyVerification { Future send(String type, Map payload) async { makePayload(payload); - print('[Key Verification] Sending type ${type}: ' + payload.toString()); + Logs.info('[Key Verification] Sending type ${type}: ' + payload.toString()); if (room != null) { - print('[Key Verification] Sending to ${userId} in room ${room.id}...'); + Logs.info( + '[Key Verification] Sending to ${userId} in room ${room.id}...'); if (['m.key.verification.request'].contains(type)) { payload['msgtype'] = type; payload['to'] = userId; @@ -566,7 +569,8 @@ class KeyVerification { encryption.keyVerificationManager.addRequest(this); } } else { - print('[Key Verification] Sending to ${userId} device ${deviceId}...'); + Logs.info( + '[Key Verification] Sending to ${userId} device ${deviceId}...'); await client.sendToDevice( [client.userDeviceKeys[userId].deviceKeys[deviceId]], type, payload); } @@ -693,8 +697,8 @@ class _KeyVerificationMethodSas extends _KeyVerificationMethod { break; } } catch (err, stacktrace) { - print('[Key Verification SAS] An error occured: ' + err.toString()); - print(stacktrace); + Logs.error('[Key Verification SAS] An error occured: ' + err.toString(), + stacktrace); if (request.deviceId != null) { await request.cancel('m.invalid_message'); } diff --git a/lib/encryption/utils/olm_session.dart b/lib/encryption/utils/olm_session.dart index 1dca413..73d8a98 100644 --- a/lib/encryption/utils/olm_session.dart +++ b/lib/encryption/utils/olm_session.dart @@ -16,6 +16,7 @@ * along with this program. If not, see . */ +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:olm/olm.dart' as olm; import '../../src/database/database.dart' show DbOlmSessions; @@ -46,8 +47,8 @@ class OlmSession { lastReceived = dbEntry.lastReceived ?? DateTime.fromMillisecondsSinceEpoch(0); assert(sessionId == session.session_id()); - } catch (e) { - print('[LibOlm] Could not unpickle olm session: ' + e.toString()); + } catch (e, s) { + Logs.error('[LibOlm] Could not unpickle olm session: ' + e.toString(), s); dispose(); } } diff --git a/lib/encryption/utils/outbound_group_session.dart b/lib/encryption/utils/outbound_group_session.dart index 2a1c617..bf10818 100644 --- a/lib/encryption/utils/outbound_group_session.dart +++ b/lib/encryption/utils/outbound_group_session.dart @@ -18,6 +18,7 @@ import 'dart:convert'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:olm/olm.dart' as olm; import '../../src/database/database.dart' show DbOutboundGroupSession; @@ -44,10 +45,11 @@ class OutboundGroupSession { devices = List.from(json.decode(dbEntry.deviceIds)); creationTime = dbEntry.creationTime; sentMessages = dbEntry.sentMessages; - } catch (e) { + } catch (e, s) { dispose(); - print( - '[LibOlm] Unable to unpickle outboundGroupSession: ' + e.toString()); + Logs.error( + '[LibOlm] Unable to unpickle outboundGroupSession: ' + e.toString(), + s); } } diff --git a/lib/encryption/utils/session_key.dart b/lib/encryption/utils/session_key.dart index 5c6f0b2..176c9e0 100644 --- a/lib/encryption/utils/session_key.dart +++ b/lib/encryption/utils/session_key.dart @@ -18,6 +18,7 @@ import 'dart:convert'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:olm/olm.dart' as olm; import 'package:famedlysdk/famedlysdk.dart'; @@ -48,9 +49,11 @@ class SessionKey { inboundGroupSession = olm.InboundGroupSession(); try { inboundGroupSession.unpickle(key, dbEntry.pickle); - } catch (e) { + } catch (e, s) { dispose(); - print('[LibOlm] Unable to unpickle inboundGroupSession: ' + e.toString()); + Logs.error( + '[LibOlm] Unable to unpickle inboundGroupSession: ' + e.toString(), + s); } } diff --git a/lib/src/client.dart b/lib/src/client.dart index 8eb3907..74d75c1 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -25,6 +25,7 @@ import 'package:famedlysdk/famedlysdk.dart'; import 'package:famedlysdk/matrix_api.dart'; import 'package:famedlysdk/src/room.dart'; import 'package:famedlysdk/src/utils/device_keys_list.dart'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:famedlysdk/src/utils/matrix_file.dart'; import 'package:famedlysdk/src/utils/to_device_event.dart'; import 'package:http/http.dart' as http; @@ -147,7 +148,7 @@ class Client { /// Warning! This endpoint is for testing only! set rooms(List newList) { - print('Warning! This endpoint is for testing only!'); + Logs.warning('Warning! This endpoint is for testing only!'); _rooms = newList; } @@ -368,8 +369,8 @@ class Client { Future logout() async { try { await api.logout(); - } catch (exception) { - print(exception); + } catch (e, s) { + Logs.error(e, s); rethrow; } finally { await clear(); @@ -664,6 +665,9 @@ class Client { } onLoginStateChanged.add(LoginState.logged); + Logs.success( + 'Successfully connected as ${userID.localpart} with ${api.homeserver.toString()}', + ); return _sync(); } @@ -734,8 +738,7 @@ class Client { if (isLogged() == false || _disposed) { return; } - print('Error during processing events: ' + e.toString()); - print(s); + Logs.error('Error during processing events: ' + e.toString(), s); onSyncError.add(SyncError( exception: e is Exception ? e : Exception(e), stackTrace: s)); await Future.delayed(Duration(seconds: syncErrorTimeoutSec), _sync); @@ -814,10 +817,10 @@ class Client { try { toDeviceEvent = await encryption.decryptToDeviceEvent(toDeviceEvent); } catch (e, s) { - print( - '[LibOlm] Could not decrypt to device event from ${toDeviceEvent.sender} with content: ${toDeviceEvent.content}'); - print(e); - print(s); + Logs.error( + '[LibOlm] Could not decrypt to device event from ${toDeviceEvent.sender} with content: ${toDeviceEvent.content}\n${e.toString()}', + s); + onOlmError.add( ToDeviceEventDecryptionError( exception: e is Exception ? e : Exception(e), @@ -1161,8 +1164,8 @@ class Client { userIds.add(user.id); } } - } catch (err) { - print('[E2EE] Failed to fetch participants: ' + err.toString()); + } catch (e, s) { + Logs.error('[E2EE] Failed to fetch participants: ' + e.toString(), s); } } } @@ -1338,8 +1341,9 @@ class Client { } }); } - } catch (e) { - print('[LibOlm] Unable to update user device keys: ' + e.toString()); + } catch (e, s) { + Logs.error( + '[LibOlm] Unable to update user device keys: ' + e.toString(), s); } } diff --git a/lib/src/database/database.dart b/lib/src/database/database.dart index 6caec77..793fd48 100644 --- a/lib/src/database/database.dart +++ b/lib/src/database/database.dart @@ -1,3 +1,4 @@ +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:moor/moor.dart'; import 'dart:convert'; @@ -66,7 +67,7 @@ class Database extends _$Database { if (executor.dialect == SqlDialect.sqlite) { final ret = await customSelect('PRAGMA journal_mode=WAL').get(); if (ret.isNotEmpty) { - print('[Moor] Switched database to mode ' + + Logs.info('[Moor] Switched database to mode ' + ret.first.data['journal_mode'].toString()); } } @@ -113,8 +114,9 @@ class Database extends _$Database { var session = olm.Session(); session.unpickle(userId, row.pickle); res[row.identityKey].add(session); - } catch (e) { - print('[LibOlm] Could not unpickle olm session: ' + e.toString()); + } catch (e, s) { + Logs.error( + '[LibOlm] Could not unpickle olm session: ' + e.toString(), s); } } return res; diff --git a/lib/src/event.dart b/lib/src/event.dart index 6305a1c..df0aa03 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -20,6 +20,7 @@ import 'dart:convert'; import 'dart:typed_data'; import 'package:famedlysdk/famedlysdk.dart'; import 'package:famedlysdk/encryption.dart'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:famedlysdk/src/utils/receipt.dart'; import 'package:http/http.dart' as http; import 'package:matrix_file_e2ee/matrix_file_e2ee.dart'; @@ -96,12 +97,18 @@ class Event extends MatrixEvent { this.senderId = senderId; this.unsigned = unsigned; // synapse unfortunatley isn't following the spec and tosses the prev_content - // into the unsigned block - this.prevContent = prevContent != null && prevContent.isNotEmpty - ? prevContent - : (unsigned != null && unsigned['prev_content'] is Map) - ? unsigned['prev_content'] - : null; + // into the unsigned block. + // Currently we are facing a very strange bug in web which is impossible to debug. + // It may be because of this line so we put this in try-catch until we can fix it. + try { + this.prevContent = (prevContent != null && prevContent.isNotEmpty) + ? prevContent + : (unsigned != null && unsigned['prev_content'] is Map) + ? unsigned['prev_content'] + : null; + } catch (e, s) { + Logs.error('Event constructor crashed: ${e.toString()}', s); + } this.stateKey = stateKey; this.originServerTs = originServerTs; } diff --git a/lib/src/room.dart b/lib/src/room.dart index 8ba80b8..b085d50 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -23,6 +23,7 @@ import 'package:famedlysdk/famedlysdk.dart'; import 'package:famedlysdk/src/client.dart'; import 'package:famedlysdk/src/event.dart'; import 'package:famedlysdk/src/utils/event_update.dart'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:famedlysdk/src/utils/room_update.dart'; import 'package:famedlysdk/src/utils/matrix_file.dart'; import 'package:matrix_file_e2ee/matrix_file_e2ee.dart'; @@ -136,8 +137,8 @@ class Room { if (state.type == EventTypes.Encrypted && client.encryptionEnabled) { try { state = client.encryption.decryptRoomEventSync(id, state); - } catch (e) { - print('[LibOlm] Could not decrypt room state: ' + e.toString()); + } catch (e, s) { + Logs.error('[LibOlm] Could not decrypt room state: ' + e.toString(), s); } } if (!(state.stateKey is String) && @@ -715,8 +716,9 @@ class Room { syncUpdate.rooms.join.values.first.timeline.events.first.eventId = res; await client.handleSync(syncUpdate); return res; - } catch (exception) { - print('[Client] Error while sending: ' + exception.toString()); + } catch (e, s) { + Logs.warning( + '[Client] Problem while sending message: ' + e.toString(), s); syncUpdate.rooms.join.values.first.timeline.events.first .unsigned[MessageSendingStatusKey] = -1; await client.handleSync(syncUpdate); diff --git a/lib/src/timeline.dart b/lib/src/timeline.dart index 042e811..18fe0bf 100644 --- a/lib/src/timeline.dart +++ b/lib/src/timeline.dart @@ -19,6 +19,7 @@ import 'dart:async'; import 'package:famedlysdk/matrix_api.dart'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'event.dart'; import 'room.dart'; @@ -263,8 +264,7 @@ class Timeline { } sortAndUpdate(); } catch (e, s) { - print('[WARNING] (_handleEventUpdate) ${e.toString()}'); - print(s); + Logs.warning('Handle event update failed: ${e.toString()}', s); } } diff --git a/lib/src/utils/event_update.dart b/lib/src/utils/event_update.dart index 0be1c4e..514966b 100644 --- a/lib/src/utils/event_update.dart +++ b/lib/src/utils/event_update.dart @@ -18,6 +18,7 @@ import '../../famedlysdk.dart'; import '../../matrix_api.dart'; +import 'logs.dart'; /// Represents a new event (e.g. a message in a room) or an update for an /// already known event. @@ -57,8 +58,8 @@ class EventUpdate { content: decrpytedEvent.toJson(), sortOrder: sortOrder, ); - } catch (e) { - print('[LibOlm] Could not decrypt megolm event: ' + e.toString()); + } catch (e, s) { + Logs.error('[LibOlm] Could not decrypt megolm event: ' + e.toString(), s); return this; } } diff --git a/lib/src/utils/logs.dart b/lib/src/utils/logs.dart new file mode 100644 index 0000000..f774de3 --- /dev/null +++ b/lib/src/utils/logs.dart @@ -0,0 +1,30 @@ +import 'package:ansicolor/ansicolor.dart'; + +abstract class Logs { + static final AnsiPen _infoPen = AnsiPen()..blue(); + static final AnsiPen _warningPen = AnsiPen()..yellow(); + static final AnsiPen _successPen = AnsiPen()..green(); + static final AnsiPen _errorPen = AnsiPen()..red(); + + static const String _prefixText = '[Famedly Matrix SDK] '; + + static void info(dynamic info) => print( + _prefixText + _infoPen(info.toString()), + ); + + static void success(dynamic obj, [dynamic stackTrace]) => print( + _prefixText + _successPen(obj.toString()), + ); + + static void warning(dynamic warning, [dynamic stackTrace]) => print( + _prefixText + + _warningPen(warning.toString()) + + (stackTrace != null ? '\n${stackTrace.toString()}' : ''), + ); + + static void error(dynamic obj, [dynamic stackTrace]) => print( + _prefixText + + _errorPen(obj.toString()) + + (stackTrace != null ? '\n${stackTrace.toString()}' : ''), + ); +} diff --git a/pubspec.lock b/pubspec.lock index faec33f..25d83fe 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -22,6 +22,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "0.2.2" + ansicolor: + dependency: "direct main" + description: + name: ansicolor + url: "https://pub.dartlang.org" + source: hosted + version: "1.0.2" args: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index a82843e..d594d4c 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -21,6 +21,7 @@ dependencies: password_hash: ^2.0.0 olm: ^1.2.1 matrix_file_e2ee: ^1.0.4 + ansicolor: ^1.0.2 dev_dependencies: test: ^1.0.0 diff --git a/test/client_test.dart b/test/client_test.dart index c970708..8d0242f 100644 --- a/test/client_test.dart +++ b/test/client_test.dart @@ -23,6 +23,7 @@ import 'package:famedlysdk/famedlysdk.dart'; import 'package:famedlysdk/matrix_api.dart'; import 'package:famedlysdk/src/client.dart'; import 'package:famedlysdk/src/utils/event_update.dart'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:famedlysdk/src/utils/room_update.dart'; import 'package:famedlysdk/src/utils/matrix_file.dart'; import 'package:olm/olm.dart' as olm; @@ -59,9 +60,9 @@ void main() { olm.Account(); } catch (_) { olmEnabled = false; - print('[LibOlm] Failed to load LibOlm: ' + _.toString()); + Logs.warning('[LibOlm] Failed to load LibOlm: ' + _.toString()); } - print('[LibOlm] Enabled: $olmEnabled'); + Logs.success('[LibOlm] Enabled: $olmEnabled'); test('Login', () async { var presenceCounter = 0; diff --git a/test/device_keys_list_test.dart b/test/device_keys_list_test.dart index a98ff1b..4b870d3 100644 --- a/test/device_keys_list_test.dart +++ b/test/device_keys_list_test.dart @@ -19,6 +19,7 @@ import 'dart:convert'; import 'package:famedlysdk/famedlysdk.dart'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:test/test.dart'; import 'package:olm/olm.dart' as olm; @@ -74,9 +75,9 @@ void main() { olm.Account(); } catch (_) { olmEnabled = false; - print('[LibOlm] Failed to load LibOlm: ' + _.toString()); + Logs.error('[LibOlm] Failed to load LibOlm: ' + _.toString()); } - print('[LibOlm] Enabled: $olmEnabled'); + Logs.success('[LibOlm] Enabled: $olmEnabled'); if (!olmEnabled) return; diff --git a/test/encryption/cross_signing_test.dart b/test/encryption/cross_signing_test.dart index 4ec212b..fe22982 100644 --- a/test/encryption/cross_signing_test.dart +++ b/test/encryption/cross_signing_test.dart @@ -19,6 +19,7 @@ import 'dart:convert'; import 'package:famedlysdk/famedlysdk.dart'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:test/test.dart'; import 'package:olm/olm.dart' as olm; @@ -33,9 +34,9 @@ void main() { olm.Account(); } catch (_) { olmEnabled = false; - print('[LibOlm] Failed to load LibOlm: ' + _.toString()); + Logs.warning('[LibOlm] Failed to load LibOlm: ' + _.toString()); } - print('[LibOlm] Enabled: $olmEnabled'); + Logs.success('[LibOlm] Enabled: $olmEnabled'); if (!olmEnabled) return; diff --git a/test/encryption/encrypt_decrypt_room_message_test.dart b/test/encryption/encrypt_decrypt_room_message_test.dart index 70d75fa..711a4a4 100644 --- a/test/encryption/encrypt_decrypt_room_message_test.dart +++ b/test/encryption/encrypt_decrypt_room_message_test.dart @@ -17,6 +17,7 @@ */ import 'package:famedlysdk/famedlysdk.dart'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:test/test.dart'; import 'package:olm/olm.dart' as olm; @@ -30,9 +31,9 @@ void main() { olm.Account(); } catch (_) { olmEnabled = false; - print('[LibOlm] Failed to load LibOlm: ' + _.toString()); + Logs.warning('[LibOlm] Failed to load LibOlm: ' + _.toString()); } - print('[LibOlm] Enabled: $olmEnabled'); + Logs.success('[LibOlm] Enabled: $olmEnabled'); if (!olmEnabled) return; diff --git a/test/encryption/encrypt_decrypt_to_device_test.dart b/test/encryption/encrypt_decrypt_to_device_test.dart index a56adee..b98ef22 100644 --- a/test/encryption/encrypt_decrypt_to_device_test.dart +++ b/test/encryption/encrypt_decrypt_to_device_test.dart @@ -17,6 +17,7 @@ */ import 'package:famedlysdk/famedlysdk.dart'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:test/test.dart'; import 'package:olm/olm.dart' as olm; @@ -35,9 +36,9 @@ void main() { olm.Account(); } catch (_) { olmEnabled = false; - print('[LibOlm] Failed to load LibOlm: ' + _.toString()); + Logs.warning('[LibOlm] Failed to load LibOlm: ' + _.toString()); } - print('[LibOlm] Enabled: $olmEnabled'); + Logs.success('[LibOlm] Enabled: $olmEnabled'); if (!olmEnabled) return; diff --git a/test/encryption/key_manager_test.dart b/test/encryption/key_manager_test.dart index 5b3025f..bdd0304 100644 --- a/test/encryption/key_manager_test.dart +++ b/test/encryption/key_manager_test.dart @@ -17,6 +17,7 @@ */ import 'package:famedlysdk/famedlysdk.dart'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:test/test.dart'; import 'package:olm/olm.dart' as olm; @@ -30,9 +31,9 @@ void main() { olm.Account(); } catch (_) { olmEnabled = false; - print('[LibOlm] Failed to load LibOlm: ' + _.toString()); + Logs.warning('[LibOlm] Failed to load LibOlm: ' + _.toString()); } - print('[LibOlm] Enabled: $olmEnabled'); + Logs.success('[LibOlm] Enabled: $olmEnabled'); if (!olmEnabled) return; diff --git a/test/encryption/key_request_test.dart b/test/encryption/key_request_test.dart index c7dbb9f..b780d2b 100644 --- a/test/encryption/key_request_test.dart +++ b/test/encryption/key_request_test.dart @@ -18,6 +18,7 @@ import 'dart:convert'; import 'package:famedlysdk/famedlysdk.dart'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:test/test.dart'; import 'package:olm/olm.dart' as olm; @@ -45,9 +46,9 @@ void main() { olm.Account(); } catch (_) { olmEnabled = false; - print('[LibOlm] Failed to load LibOlm: ' + _.toString()); + Logs.warning('[LibOlm] Failed to load LibOlm: ' + _.toString()); } - print('[LibOlm] Enabled: $olmEnabled'); + Logs.success('[LibOlm] Enabled: $olmEnabled'); if (!olmEnabled) return; @@ -106,7 +107,7 @@ void main() { 'requesting_device_id': 'OTHERDEVICE', }); await matrix.encryption.keyManager.handleToDeviceEvent(event); - print(FakeMatrixApi.calledEndpoints.keys.toString()); + Logs.info(FakeMatrixApi.calledEndpoints.keys.toString()); expect( FakeMatrixApi.calledEndpoints.keys.any( (k) => k.startsWith('/client/r0/sendToDevice/m.room.encrypted')), diff --git a/test/encryption/key_verification_test.dart b/test/encryption/key_verification_test.dart index 3216b8e..5ae9689 100644 --- a/test/encryption/key_verification_test.dart +++ b/test/encryption/key_verification_test.dart @@ -20,6 +20,7 @@ import 'dart:convert'; import 'package:famedlysdk/famedlysdk.dart'; import 'package:famedlysdk/encryption.dart'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:test/test.dart'; import 'package:olm/olm.dart' as olm; @@ -67,9 +68,9 @@ void main() { olm.Account(); } catch (_) { olmEnabled = false; - print('[LibOlm] Failed to load LibOlm: ' + _.toString()); + Logs.warning('[LibOlm] Failed to load LibOlm: ' + _.toString()); } - print('[LibOlm] Enabled: $olmEnabled'); + Logs.success('[LibOlm] Enabled: $olmEnabled'); if (!olmEnabled) return; diff --git a/test/encryption/olm_manager_test.dart b/test/encryption/olm_manager_test.dart index bf0e7a3..78e7068 100644 --- a/test/encryption/olm_manager_test.dart +++ b/test/encryption/olm_manager_test.dart @@ -18,6 +18,7 @@ import 'dart:convert'; import 'package:famedlysdk/famedlysdk.dart'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:test/test.dart'; import 'package:olm/olm.dart' as olm; @@ -32,9 +33,9 @@ void main() { olm.Account(); } catch (_) { olmEnabled = false; - print('[LibOlm] Failed to load LibOlm: ' + _.toString()); + Logs.warning('[LibOlm] Failed to load LibOlm: ' + _.toString()); } - print('[LibOlm] Enabled: $olmEnabled'); + Logs.success('[LibOlm] Enabled: $olmEnabled'); if (!olmEnabled) return; diff --git a/test/encryption/online_key_backup_test.dart b/test/encryption/online_key_backup_test.dart index 0a3b842..12b9ae0 100644 --- a/test/encryption/online_key_backup_test.dart +++ b/test/encryption/online_key_backup_test.dart @@ -17,6 +17,7 @@ */ import 'package:famedlysdk/famedlysdk.dart'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:test/test.dart'; import 'package:olm/olm.dart' as olm; @@ -30,9 +31,9 @@ void main() { olm.Account(); } catch (_) { olmEnabled = false; - print('[LibOlm] Failed to load LibOlm: ' + _.toString()); + Logs.warning('[LibOlm] Failed to load LibOlm: ' + _.toString()); } - print('[LibOlm] Enabled: $olmEnabled'); + Logs.success('[LibOlm] Enabled: $olmEnabled'); if (!olmEnabled) return; diff --git a/test/encryption/ssss_test.dart b/test/encryption/ssss_test.dart index 7c0c641..d213248 100644 --- a/test/encryption/ssss_test.dart +++ b/test/encryption/ssss_test.dart @@ -22,6 +22,7 @@ import 'dart:convert'; import 'package:famedlysdk/famedlysdk.dart'; import 'package:famedlysdk/matrix_api.dart'; import 'package:famedlysdk/encryption.dart'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:test/test.dart'; import 'package:encrypt/encrypt.dart'; import 'package:olm/olm.dart' as olm; @@ -37,9 +38,9 @@ void main() { olm.Account(); } catch (_) { olmEnabled = false; - print('[LibOlm] Failed to load LibOlm: ' + _.toString()); + Logs.warning('[LibOlm] Failed to load LibOlm: ' + _.toString()); } - print('[LibOlm] Enabled: $olmEnabled'); + Logs.success('[LibOlm] Enabled: $olmEnabled'); if (!olmEnabled) return; diff --git a/test_driver/famedlysdk_test.dart b/test_driver/famedlysdk_test.dart index 6b40f53..61f3cdc 100644 --- a/test_driver/famedlysdk_test.dart +++ b/test_driver/famedlysdk_test.dart @@ -1,5 +1,6 @@ import 'package:famedlysdk/famedlysdk.dart'; import 'package:famedlysdk/matrix_api.dart'; +import 'package:famedlysdk/src/utils/logs.dart'; import '../test/fake_database.dart'; void main() => test(); @@ -17,21 +18,21 @@ const String testMessage5 = 'Hello earth'; const String testMessage6 = 'Hello mars'; void test() async { - print('++++ Login $testUserA ++++'); + Logs.success('++++ Login $testUserA ++++'); var testClientA = Client('TestClientA'); testClientA.database = getDatabase(); await testClientA.checkServer(homeserver); await testClientA.login(testUserA, testPasswordA); assert(testClientA.encryptionEnabled); - print('++++ Login $testUserB ++++'); + Logs.success('++++ Login $testUserB ++++'); var testClientB = Client('TestClientB'); testClientB.database = getDatabase(); await testClientB.checkServer(homeserver); await testClientB.login(testUserB, testPasswordA); assert(testClientB.encryptionEnabled); - print('++++ ($testUserA) Leave all rooms ++++'); + Logs.success('++++ ($testUserA) Leave all rooms ++++'); while (testClientA.rooms.isNotEmpty) { var room = testClientA.rooms.first; if (room.canonicalAlias?.isNotEmpty ?? false) { @@ -43,7 +44,7 @@ void test() async { } catch (_) {} } - print('++++ ($testUserB) Leave all rooms ++++'); + Logs.success('++++ ($testUserB) Leave all rooms ++++'); for (var i = 0; i < 3; i++) { if (testClientB.rooms.isNotEmpty) { var room = testClientB.rooms.first; @@ -54,7 +55,7 @@ void test() async { } } - print('++++ Check if own olm device is verified by default ++++'); + Logs.success('++++ Check if own olm device is verified by default ++++'); assert(testClientA.userDeviceKeys.containsKey(testUserA)); assert(testClientA.userDeviceKeys[testUserA].deviceKeys .containsKey(testClientA.deviceID)); @@ -70,20 +71,20 @@ void test() async { assert(!testClientB .userDeviceKeys[testUserB].deviceKeys[testClientB.deviceID].blocked); - print('++++ ($testUserA) Create room and invite $testUserB ++++'); + Logs.success('++++ ($testUserA) Create room and invite $testUserB ++++'); await testClientA.api.createRoom(invite: [testUserB]); await Future.delayed(Duration(seconds: 1)); var room = testClientA.rooms.first; assert(room != null); final roomId = room.id; - print('++++ ($testUserB) Join room ++++'); + Logs.success('++++ ($testUserB) Join room ++++'); var inviteRoom = testClientB.getRoomById(roomId); await inviteRoom.join(); await Future.delayed(Duration(seconds: 1)); assert(inviteRoom.membership == Membership.join); - print('++++ ($testUserA) Enable encryption ++++'); + Logs.success('++++ ($testUserA) Enable encryption ++++'); assert(room.encrypted == false); await room.enableEncryption(); await Future.delayed(Duration(seconds: 5)); @@ -91,7 +92,7 @@ void test() async { assert(room.client.encryption.keyManager.getOutboundGroupSession(room.id) == null); - print('++++ ($testUserA) Check known olm devices ++++'); + Logs.success('++++ ($testUserA) Check known olm devices ++++'); assert(testClientA.userDeviceKeys.containsKey(testUserB)); assert(testClientA.userDeviceKeys[testUserB].deviceKeys .containsKey(testClientB.deviceID)); @@ -109,7 +110,7 @@ void test() async { await testClientA.userDeviceKeys[testUserB].deviceKeys[testClientB.deviceID] .setVerified(true); - print('++++ Check if own olm device is verified by default ++++'); + Logs.success('++++ Check if own olm device is verified by default ++++'); assert(testClientA.userDeviceKeys.containsKey(testUserA)); assert(testClientA.userDeviceKeys[testUserA].deviceKeys .containsKey(testClientA.deviceID)); @@ -121,7 +122,7 @@ void test() async { assert(testClientB .userDeviceKeys[testUserB].deviceKeys[testClientB.deviceID].verified); - print("++++ ($testUserA) Send encrypted message: '$testMessage' ++++"); + Logs.success("++++ ($testUserA) Send encrypted message: '$testMessage' ++++"); await room.sendTextEvent(testMessage); await Future.delayed(Duration(seconds: 5)); assert(room.client.encryption.keyManager.getOutboundGroupSession(room.id) != @@ -148,10 +149,11 @@ void test() async { null); assert(room.lastMessage == testMessage); assert(inviteRoom.lastMessage == testMessage); - print( + Logs.success( "++++ ($testUserB) Received decrypted message: '${inviteRoom.lastMessage}' ++++"); - print("++++ ($testUserA) Send again encrypted message: '$testMessage2' ++++"); + Logs.success( + "++++ ($testUserA) Send again encrypted message: '$testMessage2' ++++"); await room.sendTextEvent(testMessage2); await Future.delayed(Duration(seconds: 5)); assert(testClientA @@ -175,10 +177,11 @@ void test() async { null); assert(room.lastMessage == testMessage2); assert(inviteRoom.lastMessage == testMessage2); - print( + Logs.success( "++++ ($testUserB) Received decrypted message: '${inviteRoom.lastMessage}' ++++"); - print("++++ ($testUserB) Send again encrypted message: '$testMessage3' ++++"); + Logs.success( + "++++ ($testUserB) Send again encrypted message: '$testMessage3' ++++"); await inviteRoom.sendTextEvent(testMessage3); await Future.delayed(Duration(seconds: 5)); assert(testClientA @@ -208,16 +211,17 @@ void test() async { null); assert(inviteRoom.lastMessage == testMessage3); assert(room.lastMessage == testMessage3); - print( + Logs.success( "++++ ($testUserA) Received decrypted message: '${room.lastMessage}' ++++"); - print('++++ Login $testUserB in another client ++++'); + Logs.success('++++ Login $testUserB in another client ++++'); var testClientC = Client('TestClientC', database: getDatabase()); await testClientC.checkServer(homeserver); await testClientC.login(testUserB, testPasswordA); await Future.delayed(Duration(seconds: 3)); - print("++++ ($testUserA) Send again encrypted message: '$testMessage4' ++++"); + Logs.success( + "++++ ($testUserA) Send again encrypted message: '$testMessage4' ++++"); await room.sendTextEvent(testMessage4); await Future.delayed(Duration(seconds: 5)); assert(testClientA @@ -254,16 +258,17 @@ void test() async { null); assert(room.lastMessage == testMessage4); assert(inviteRoom.lastMessage == testMessage4); - print( + Logs.success( "++++ ($testUserB) Received decrypted message: '${inviteRoom.lastMessage}' ++++"); - print('++++ Logout $testUserB another client ++++'); + Logs.success('++++ Logout $testUserB another client ++++'); await testClientC.dispose(); await testClientC.logout(); testClientC = null; await Future.delayed(Duration(seconds: 5)); - print("++++ ($testUserA) Send again encrypted message: '$testMessage6' ++++"); + Logs.success( + "++++ ($testUserA) Send again encrypted message: '$testMessage6' ++++"); await room.sendTextEvent(testMessage6); await Future.delayed(Duration(seconds: 5)); assert(testClientA @@ -290,10 +295,10 @@ void test() async { null); assert(room.lastMessage == testMessage6); assert(inviteRoom.lastMessage == testMessage6); - print( + Logs.success( "++++ ($testUserB) Received decrypted message: '${inviteRoom.lastMessage}' ++++"); -/* print('++++ ($testUserA) Restore user ++++'); +/* Logs.success('++++ ($testUserA) Restore user ++++'); await testClientA.dispose(); testClientA = null; testClientA = Client( @@ -320,7 +325,7 @@ void test() async { assert(testClientA.encryption.olmManager.olmSessions[testClientB.identityKey].first.session_id() == testClientB.encryption.olmManager.olmSessions[testClientA.identityKey].first.session_id()); - print("++++ ($testUserA) Send again encrypted message: '$testMessage5' ++++"); + Logs.success("++++ ($testUserA) Send again encrypted message: '$testMessage5' ++++"); await restoredRoom.sendTextEvent(testMessage5); await Future.delayed(Duration(seconds: 5)); assert(testClientA.encryption.olmManager.olmSessions[testClientB.identityKey].length == 1); @@ -330,10 +335,10 @@ void test() async { assert(restoredRoom.lastMessage == testMessage5); assert(inviteRoom.lastMessage == testMessage5); assert(testClientB.getRoomById(roomId).lastMessage == testMessage5); - print( + Logs.success( "++++ ($testUserB) Received decrypted message: '${inviteRoom.lastMessage}' ++++");*/ - print('++++ Logout $testUserA and $testUserB ++++'); + Logs.success('++++ Logout $testUserA and $testUserB ++++'); await room.leave(); await room.forget(); await inviteRoom.leave(); From c184dfba6b97a1cdef2cfd779f4ad11e0f1a15fe Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Mon, 10 Aug 2020 10:42:14 +0200 Subject: [PATCH 26/90] Don't show potential session keys in logs --- lib/encryption/olm_manager.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/encryption/olm_manager.dart b/lib/encryption/olm_manager.dart index e068208..a71bf32 100644 --- a/lib/encryption/olm_manager.dart +++ b/lib/encryption/olm_manager.dart @@ -232,7 +232,7 @@ class OlmManager { return event; } if (event.content['algorithm'] != 'm.olm.v1.curve25519-aes-sha2') { - throw ('Unknown algorithm: ${event.content}'); + throw ('Unknown algorithm: ${event.content['algorithm']}'); } if (!event.content['ciphertext'].containsKey(identityKey)) { throw ("The message isn't sent for this device"); From 574fe27101bb03c8c18c776e98f7f44668e6d159 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Tue, 11 Aug 2020 10:12:26 +0200 Subject: [PATCH 27/90] feat: Add Event.getDisplayEvent, which fetches an event based on all edits etc. --- lib/src/event.dart | 24 ++++++++++++ test/event_test.dart | 87 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/lib/src/event.dart b/lib/src/event.dart index df0aa03..23c53ca 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -715,4 +715,28 @@ class Event extends MatrixEvent { hasAggregatedEvents(timeline, type) ? timeline.aggregatedEvents[eventId][type] : {}; + + /// Fetches the event to be rendered, taking into account all the edits and the like. + /// It needs a [timeline] for that. + Event getDisplayEvent(Timeline timeline) { + if (hasAggregatedEvents(timeline, RelationshipTypes.Edit)) { + // alright, we have an edit + final allEditEvents = aggregatedEvents(timeline, RelationshipTypes.Edit) + // we only allow edits made by the original author themself + .where((e) => e.senderId == senderId && e.type == EventTypes.Message) + .toList(); + // we need to check again if it isn't empty, as we potentially removed all + // aggregated edits + if (allEditEvents.isNotEmpty) { + allEditEvents.sort((a, b) => a.sortOrder - b.sortOrder > 0 ? 1 : -1); + var rawEvent = allEditEvents.last.toJson(); + // update the content of the new event to render + if (rawEvent['content']['m.new_content'] is Map) { + rawEvent['content'] = rawEvent['content']['m.new_content']; + } + return Event.fromJson(rawEvent, room); + } + } + return this; + } } diff --git a/test/event_test.dart b/test/event_test.dart index 5fb041c..e57be26 100644 --- a/test/event_test.dart +++ b/test/event_test.dart @@ -873,5 +873,92 @@ void main() { expect( event.aggregatedEvents(timeline, RelationshipTypes.Edit), {}); }); + test('getDisplayEvent', () { + var event = Event.fromJson({ + 'type': EventTypes.Message, + 'content': { + 'body': 'blah', + 'msgtype': 'm.text', + }, + 'event_id': '\$source', + 'sender': '@alice:example.org', + }, null); + event.sortOrder = 0; + var edit1 = Event.fromJson({ + 'type': EventTypes.Message, + 'content': { + 'body': '* edit 1', + 'msgtype': 'm.text', + 'm.new_content': { + 'body': 'edit 1', + 'msgtype': 'm.text', + }, + 'm.relates_to': { + 'event_id': '\$source', + 'rel_type': RelationshipTypes.Edit, + }, + }, + 'event_id': '\$edit1', + 'sender': '@alice:example.org', + }, null); + edit1.sortOrder = 1; + var edit2 = Event.fromJson({ + 'type': EventTypes.Message, + 'content': { + 'body': '* edit 2', + 'msgtype': 'm.text', + 'm.new_content': { + 'body': 'edit 2', + 'msgtype': 'm.text', + }, + 'm.relates_to': { + 'event_id': '\$source', + 'rel_type': RelationshipTypes.Edit, + }, + }, + 'event_id': '\$edit2', + 'sender': '@alice:example.org', + }, null); + edit2.sortOrder = 2; + var edit3 = Event.fromJson({ + 'type': EventTypes.Message, + 'content': { + 'body': '* edit 3', + 'msgtype': 'm.text', + 'm.new_content': { + 'body': 'edit 3', + 'msgtype': 'm.text', + }, + 'm.relates_to': { + 'event_id': '\$source', + 'rel_type': RelationshipTypes.Edit, + }, + }, + 'event_id': '\$edit3', + 'sender': '@bob:example.org', + }, null); + edit3.sortOrder = 3; + var room = Room(client: client); + // no edits + var displayEvent = + event.getDisplayEvent(Timeline(events: [event], room: room)); + expect(displayEvent.body, 'blah'); + // one edit + displayEvent = event + .getDisplayEvent(Timeline(events: [event, edit1], room: room)); + expect(displayEvent.body, 'edit 1'); + // two edits + displayEvent = event.getDisplayEvent( + Timeline(events: [event, edit1, edit2], room: room)); + expect(displayEvent.body, 'edit 2'); + // foreign edit + displayEvent = event + .getDisplayEvent(Timeline(events: [event, edit3], room: room)); + expect(displayEvent.body, 'blah'); + // mixed foreign and non-foreign + displayEvent = event.getDisplayEvent( + Timeline(events: [event, edit1, edit2, edit3], room: room)); + expect(displayEvent.body, 'edit 2'); + }); }); } From fb9b505988a801a115e1987906c325a007819efd Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Tue, 11 Aug 2020 16:11:51 +0000 Subject: [PATCH 28/90] Krille/make client extend matrixapi --- lib/encryption/cross_signing.dart | 2 +- lib/encryption/key_manager.dart | 67 +++--- lib/encryption/olm_manager.dart | 6 +- lib/encryption/ssss.dart | 6 +- lib/encryption/utils/key_verification.dart | 2 +- lib/matrix_api/matrix_api.dart | 7 +- lib/src/client.dart | 217 ++++++++---------- lib/src/room.dart | 86 +++---- lib/src/user.dart | 2 +- lib/src/utils/uri_extension.dart | 8 +- test/client_test.dart | 50 ++-- .../encrypt_decrypt_to_device_test.dart | 2 +- test/encryption/key_verification_test.dart | 2 +- test/event_test.dart | 4 +- test/fake_client.dart | 14 +- test/fake_matrix_api.dart | 4 + test/mxc_uri_extension_test.dart | 6 +- test/user_test.dart | 2 +- test_driver/famedlysdk_test.dart | 12 +- 19 files changed, 230 insertions(+), 269 deletions(-) diff --git a/lib/encryption/cross_signing.dart b/lib/encryption/cross_signing.dart index 92cbb86..c8e2249 100644 --- a/lib/encryption/cross_signing.dart +++ b/lib/encryption/cross_signing.dart @@ -167,7 +167,7 @@ class CrossSigning { } if (signedKeys.isNotEmpty) { // post our new keys! - await client.api.uploadKeySignatures(signedKeys); + await client.uploadKeySignatures(signedKeys); } } diff --git a/lib/encryption/key_manager.dart b/lib/encryption/key_manager.dart index a2a3e55..a0e1809 100644 --- a/lib/encryption/key_manager.dart +++ b/lib/encryption/key_manager.dart @@ -44,7 +44,7 @@ class KeyManager { encryption.ssss.setValidator(MEGOLM_KEY, (String secret) async { final keyObj = olm.PkDecryption(); try { - final info = await client.api.getRoomKeysBackup(); + final info = await client.getRoomKeysBackup(); if (info.algorithm != RoomKeysAlgorithmType.v1Curve25519AesSha2) { return false; } @@ -288,7 +288,7 @@ class KeyManager { key: client.userID, ); try { - await client.sendToDevice(deviceKeys, 'm.room_key', rawSession); + await client.sendToDeviceEncrypted(deviceKeys, 'm.room_key', rawSession); await storeOutboundGroupSession(roomId, sess); _outboundGroupSessions[roomId] = sess; } catch (e, s) { @@ -339,7 +339,7 @@ class KeyManager { final privateKey = base64.decode(await encryption.ssss.getCached(MEGOLM_KEY)); final decryption = olm.PkDecryption(); - final info = await client.api.getRoomKeysBackup(); + final info = await client.getRoomKeysBackup(); String backupPubKey; try { backupPubKey = decryption.init_with_private_key(privateKey); @@ -387,9 +387,9 @@ class KeyManager { } Future loadSingleKey(String roomId, String sessionId) async { - final info = await client.api.getRoomKeysBackup(); + final info = await client.getRoomKeysBackup(); final ret = - await client.api.getRoomKeysSingleKey(roomId, sessionId, info.version); + await client.getRoomKeysSingleKey(roomId, sessionId, info.version); final keys = RoomKeys.fromJson({ 'rooms': { roomId: { @@ -434,22 +434,22 @@ class KeyManager { sessionId: sessionId, senderKey: senderKey, ); - await client.sendToDevice( - [], - 'm.room_key_request', - { - 'action': 'request', - 'body': { - 'algorithm': 'm.megolm.v1.aes-sha2', - 'room_id': room.id, - 'sender_key': senderKey, - 'session_id': sessionId, - }, - 'request_id': requestId, - 'requesting_device_id': client.deviceID, + final userList = await room.requestParticipants(); + await client.sendToDevicesOfUserIds( + userList.map((u) => u.id).toSet(), + 'm.room_key_request', + { + 'action': 'request', + 'body': { + 'algorithm': 'm.megolm.v1.aes-sha2', + 'room_id': room.id, + 'sender_key': senderKey, + 'session_id': sessionId, }, - encrypted: false, - toUsers: await room.requestParticipants()); + 'request_id': requestId, + 'requesting_device_id': client.deviceID, + }, + ); outgoingShareRequests[request.requestId] = request; } catch (e, s) { Logs.error( @@ -568,15 +568,24 @@ class KeyManager { if (request.devices.isEmpty) { return; // no need to send any cancellation } + // Send with send-to-device messaging + final sendToDeviceMessage = { + 'action': 'request_cancellation', + 'request_id': request.requestId, + 'requesting_device_id': client.deviceID, + }; + var data = >>{}; + for (final device in request.devices) { + if (!data.containsKey(device.userId)) { + data[device.userId] = {}; + } + data[device.userId][device.deviceId] = sendToDeviceMessage; + } await client.sendToDevice( - request.devices, - 'm.room_key_request', - { - 'action': 'request_cancellation', - 'request_id': request.requestId, - 'requesting_device_id': client.deviceID, - }, - encrypted: false); + 'm.room_key_request', + client.generateUniqueTransactionId(), + data, + ); } else if (event.type == 'm.room_key') { if (event.encryptedContent == null) { return; // the event wasn't encrypted, this is a security risk; @@ -675,7 +684,7 @@ class RoomKeyRequest extends ToDeviceEvent { message['session_key'] = session.inboundGroupSession .export_session(session.inboundGroupSession.first_known_index()); // send the actual reply of the key back to the requester - await keyManager.client.sendToDevice( + await keyManager.client.sendToDeviceEncrypted( [requestingDevice], 'm.forwarded_room_key', message, diff --git a/lib/encryption/olm_manager.dart b/lib/encryption/olm_manager.dart index a71bf32..6bb9493 100644 --- a/lib/encryption/olm_manager.dart +++ b/lib/encryption/olm_manager.dart @@ -183,7 +183,7 @@ class OlmManager { signJson(keysContent['device_keys'] as Map); } - final response = await client.api.uploadDeviceKeys( + final response = await client.uploadDeviceKeys( deviceKeys: uploadDeviceKeys ? MatrixDeviceKeys.fromJson(keysContent['device_keys']) : null, @@ -335,7 +335,7 @@ class OlmManager { return; } await startOutgoingOlmSessions([device]); - await client.sendToDevice([device], 'm.dummy', {}); + await client.sendToDeviceEncrypted([device], 'm.dummy', {}); } Future decryptToDeviceEvent(ToDeviceEvent event) async { @@ -383,7 +383,7 @@ class OlmManager { } final response = - await client.api.requestOneTimeKeys(requestingKeysFrom, timeout: 10000); + await client.requestOneTimeKeys(requestingKeysFrom, timeout: 10000); for (var userKeysEntry in response.oneTimeKeys.entries) { final userId = userKeysEntry.key; diff --git a/lib/encryption/ssss.dart b/lib/encryption/ssss.dart index 49b35ef..fb53ae2 100644 --- a/lib/encryption/ssss.dart +++ b/lib/encryption/ssss.dart @@ -222,7 +222,7 @@ class SSSS { 'mac': encrypted.mac, }; // store the thing in your account data - await client.api.setAccountData(client.userID, type, content); + await client.setAccountData(client.userID, type, content); if (CACHE_TYPES.contains(type) && client.database != null) { // cache the thing await client.database @@ -271,7 +271,7 @@ class SSSS { devices: devices, ); pendingShareRequests[requestId] = request; - await client.sendToDevice(devices, 'm.secret.request', { + await client.sendToDeviceEncrypted(devices, 'm.secret.request', { 'action': 'request', 'requesting_device_id': client.deviceID, 'request_id': requestId, @@ -308,7 +308,7 @@ class SSSS { } // okay, all checks out...time to share this secret! Logs.info('[SSSS] Replying with secret for ${type}'); - await client.sendToDevice( + await client.sendToDeviceEncrypted( [device], 'm.secret.send', { diff --git a/lib/encryption/utils/key_verification.dart b/lib/encryption/utils/key_verification.dart index 8250068..4b89f06 100644 --- a/lib/encryption/utils/key_verification.dart +++ b/lib/encryption/utils/key_verification.dart @@ -571,7 +571,7 @@ class KeyVerification { } else { Logs.info( '[Key Verification] Sending to ${userId} device ${deviceId}...'); - await client.sendToDevice( + await client.sendToDeviceEncrypted( [client.userDeviceKeys[userId].deviceKeys[deviceId]], type, payload); } } diff --git a/lib/matrix_api/matrix_api.dart b/lib/matrix_api/matrix_api.dart index ce528d2..50b66c0 100644 --- a/lib/matrix_api/matrix_api.dart +++ b/lib/matrix_api/matrix_api.dart @@ -1329,8 +1329,11 @@ class MatrixApi { /// This endpoint is used to send send-to-device events to a set of client devices. /// https://matrix.org/docs/spec/client_server/r0.6.1#put-matrix-client-r0-sendtodevice-eventtype-txnid - Future sendToDevice(String eventType, String txnId, - Map>> messages) async { + Future sendToDevice( + String eventType, + String txnId, + Map>> messages, + ) async { await request( RequestType.PUT, '/client/r0/sendToDevice/${Uri.encodeComponent(eventType)}/${Uri.encodeComponent(txnId)}', diff --git a/lib/src/client.dart b/lib/src/client.dart index 74d75c1..806f9a0 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -22,7 +22,6 @@ import 'dart:core'; import 'package:famedlysdk/encryption.dart'; import 'package:famedlysdk/famedlysdk.dart'; -import 'package:famedlysdk/matrix_api.dart'; import 'package:famedlysdk/src/room.dart'; import 'package:famedlysdk/src/utils/device_keys_list.dart'; import 'package:famedlysdk/src/utils/logs.dart'; @@ -45,7 +44,7 @@ enum LoginState { logged, loggedOut } /// Represents a Matrix client to communicate with a /// [Matrix](https://matrix.org) homeserver and is the entry point for this /// SDK. -class Client { +class Client extends MatrixApi { int _id; int get id => _id; @@ -53,7 +52,8 @@ class Client { bool enableE2eeRecovery; - MatrixApi api; + @deprecated + MatrixApi get api => this; Encryption encryption; @@ -103,7 +103,7 @@ class Client { EventTypes.RoomCanonicalAlias, EventTypes.RoomTombstone, ]); - api = MatrixApi(httpClient: httpClient); + this.httpClient = httpClient; } /// The required name for this client. @@ -125,7 +125,7 @@ class Client { String _deviceName; /// Returns the current login state. - bool isLogged() => api.accessToken != null; + bool isLogged() => accessToken != null; /// A list of all rooms the user is participating or invited. List get rooms => _rooms; @@ -160,21 +160,6 @@ class Client { int _transactionCounter = 0; - @Deprecated('Use [api.request()] instead') - Future> jsonRequest( - {RequestType type, - String action, - dynamic data = '', - int timeout, - String contentType = 'application/json'}) => - api.request( - type, - action, - data: data, - timeout: timeout, - contentType: contentType, - ); - String generateUniqueTransactionId() { _transactionCounter++; return '${clientName}-${_transactionCounter}-${DateTime.now().millisecondsSinceEpoch}'; @@ -243,7 +228,7 @@ class Client { Future checkServer(dynamic serverUrl) async { try { if (serverUrl is Uri) { - api.homeserver = serverUrl; + homeserver = serverUrl; } else { // URLs allow to have whitespace surrounding them, see https://www.w3.org/TR/2011/WD-html5-20110525/urls.html // As we want to strip a trailing slash, though, we have to trim the url ourself @@ -253,9 +238,9 @@ class Client { if (serverUrl.endsWith('/')) { serverUrl = serverUrl.substring(0, serverUrl.length - 1); } - api.homeserver = Uri.parse(serverUrl); + homeserver = Uri.parse(serverUrl); } - final versions = await api.requestSupportedVersions(); + final versions = await requestSupportedVersions(); for (var i = 0; i < versions.versions.length; i++) { if (versions.versions[i] == 'r0.5.0' || @@ -266,7 +251,7 @@ class Client { } } - final loginTypes = await api.requestLoginTypes(); + final loginTypes = await requestLoginTypes(); if (loginTypes.flows.indexWhere((f) => f.type == 'm.login.password') == -1) { return false; @@ -274,7 +259,7 @@ class Client { return true; } catch (_) { - api.homeserver = null; + homeserver = null; rethrow; } } @@ -282,16 +267,17 @@ class Client { /// Checks to see if a username is available, and valid, for the server. /// Returns the fully-qualified Matrix user ID (MXID) that has been registered. /// You have to call [checkServer] first to set a homeserver. - Future register({ - String kind, + @override + Future register({ String username, String password, - Map auth, String deviceId, String initialDeviceDisplayName, bool inhibitLogin, + Map auth, + String kind, }) async { - final response = await api.register( + final response = await super.register( username: username, password: password, auth: auth, @@ -309,66 +295,62 @@ class Client { await connect( newToken: response.accessToken, newUserID: response.userId, - newHomeserver: api.homeserver, + newHomeserver: homeserver, newDeviceName: initialDeviceDisplayName ?? '', newDeviceID: response.deviceId); - return; + return response; } /// Handles the login and allows the client to call all APIs which require /// authentication. Returns false if the login was not successful. Throws /// MatrixException if login was not successful. /// You have to call [checkServer] first to set a homeserver. - Future login( - String username, - String password, { - String initialDeviceDisplayName, + @override + Future login({ + String type = 'm.login.password', + String userIdentifierType = 'm.id.user', + String user, + String medium, + String address, + String password, + String token, String deviceId, + String initialDeviceDisplayName, }) async { - var data = { - 'type': 'm.login.password', - 'user': username, - 'identifier': { - 'type': 'm.id.user', - 'user': username, - }, - 'password': password, - }; - if (deviceId != null) data['device_id'] = deviceId; - if (initialDeviceDisplayName != null) { - data['initial_device_display_name'] = initialDeviceDisplayName; - } - - final loginResp = await api.login( - type: 'm.login.password', - userIdentifierType: 'm.id.user', - user: username, + final loginResp = await super.login( + type: type, + userIdentifierType: userIdentifierType, + user: user, password: password, deviceId: deviceId, initialDeviceDisplayName: initialDeviceDisplayName, + medium: medium, + address: address, + token: token, ); // Connect if there is an access token in the response. if (loginResp.accessToken == null || loginResp.deviceId == null || loginResp.userId == null) { - throw 'Registered but token, device ID or user ID is null.'; + throw Exception('Registered but token, device ID or user ID is null.'); } await connect( newToken: loginResp.accessToken, newUserID: loginResp.userId, - newHomeserver: api.homeserver, + newHomeserver: homeserver, newDeviceName: initialDeviceDisplayName ?? '', newDeviceID: loginResp.deviceId, ); - return true; + return loginResp; } /// Sends a logout command to the homeserver and clears all local data, /// including all persistent data from the store. + @override Future logout() async { try { - await api.logout(); + await super.logout(); } catch (e, s) { Logs.error(e, s); rethrow; @@ -421,19 +403,19 @@ class Client { if (cache && _profileCache.containsKey(userId)) { return _profileCache[userId]; } - final profile = await api.requestProfile(userId); + final profile = await requestProfile(userId); _profileCache[userId] = profile; return profile; } Future> get archive async { var archiveList = []; - final sync = await api.sync( + final syncResp = await sync( filter: '{"room":{"include_leave":true,"timeline":{"limit":10}}}', timeout: 0, ); - if (sync.rooms.leave is Map) { - for (var entry in sync.rooms.leave.entries) { + if (syncResp.rooms.leave is Map) { + for (var entry in syncResp.rooms.leave.entries) { final id = entry.key; final room = entry.value; var leftRoom = Room( @@ -460,14 +442,10 @@ class Client { return archiveList; } - /// Changes the user's displayname. - Future setDisplayname(String displayname) => - api.setDisplayname(userID, displayname); - /// Uploads a new user avatar for this user. Future setAvatar(MatrixFile file) async { - final uploadResp = await api.upload(file.bytes, file.name); - await api.setAvatarUrl(userID, Uri.parse(uploadResp)); + final uploadResp = await upload(file.bytes, file.name); + await setAvatarUrl(userID, Uri.parse(uploadResp)); return; } @@ -550,10 +528,6 @@ class Client { final StreamController onKeyVerificationRequest = StreamController.broadcast(); - /// Matrix synchronisation is done with https long polling. This needs a - /// timeout which is usually 30 seconds. - int syncTimeoutSec = 30; - /// How long should the app wait until it retrys the synchronisation after /// an error? int syncErrorTimeoutSec = 3; @@ -575,7 +549,7 @@ class Client { /// "type": "m.login.password", /// "user": "test", /// "password": "1234", - /// "initial_device_display_name": "Fluffy Matrix Client" + /// "initial_device_display_name": "Matrix Client" /// }); /// ``` /// @@ -604,8 +578,8 @@ class Client { final account = await database.getClient(clientName); if (account != null) { _id = account.clientId; - api.homeserver = Uri.parse(account.homeserverUrl); - api.accessToken = account.token; + homeserver = Uri.parse(account.homeserverUrl); + accessToken = account.token; _userID = account.userId; _deviceID = account.deviceId; _deviceName = account.deviceName; @@ -613,15 +587,15 @@ class Client { olmAccount = account.olmAccount; } } - api.accessToken = newToken ?? api.accessToken; - api.homeserver = newHomeserver ?? api.homeserver; + accessToken = newToken ?? accessToken; + homeserver = newHomeserver ?? homeserver; _userID = newUserID ?? _userID; _deviceID = newDeviceID ?? _deviceID; _deviceName = newDeviceName ?? _deviceName; prevBatch = newPrevBatch ?? prevBatch; olmAccount = newOlmAccount ?? olmAccount; - if (api.accessToken == null || api.homeserver == null || _userID == null) { + if (accessToken == null || homeserver == null || _userID == null) { // we aren't logged in encryption?.dispose(); encryption = null; @@ -636,8 +610,8 @@ class Client { if (database != null) { if (id != null) { await database.updateClient( - api.homeserver.toString(), - api.accessToken, + homeserver.toString(), + accessToken, _userID, _deviceID, _deviceName, @@ -648,8 +622,8 @@ class Client { } else { _id = await database.insertClient( clientName, - api.homeserver.toString(), - api.accessToken, + homeserver.toString(), + accessToken, _userID, _deviceID, _deviceName, @@ -666,7 +640,7 @@ class Client { onLoginStateChanged.add(LoginState.logged); Logs.success( - 'Successfully connected as ${userID.localpart} with ${api.homeserver.toString()}', + 'Successfully connected as ${userID.localpart} with ${homeserver.toString()}', ); return _sync(); @@ -680,8 +654,8 @@ class Client { /// Resets all settings and stops the synchronisation. void clear() { database?.clear(id); - _id = api.accessToken = - api.homeserver = _userID = _deviceID = _deviceName = prevBatch = null; + _id = accessToken = + homeserver = _userID = _deviceID = _deviceName = prevBatch = null; _rooms = []; encryption?.dispose(); encryption = null; @@ -694,13 +668,11 @@ class Client { Future _sync() async { if (isLogged() == false || _disposed) return; try { - _syncRequest = api - .sync( + _syncRequest = sync( filter: syncFilters, since: prevBatch, timeout: prevBatch != null ? 30000 : null, - ) - .catchError((e) { + ).catchError((e) { _lastSyncError = e; return null; }); @@ -1197,8 +1169,7 @@ class Client { if (outdatedLists.isNotEmpty) { // Request the missing device key lists from the server. - final response = - await api.requestDeviceKeys(outdatedLists, timeout: 10000); + final response = await requestDeviceKeys(outdatedLists, timeout: 10000); for (final rawDeviceKeyListEntry in response.deviceKeys.entries) { final userId = rawDeviceKeyListEntry.key; @@ -1347,17 +1318,35 @@ class Client { } } + /// Send an (unencrypted) to device [message] of a specific [eventType] to all + /// devices of a set of [users]. + Future sendToDevicesOfUserIds( + Set users, + String eventType, + Map message, { + String messageId, + }) async { + // Send with send-to-device messaging + var data = >>{}; + for (var user in users) { + data[user] = {}; + data[user]['*'] = message; + } + await sendToDevice( + eventType, messageId ?? generateUniqueTransactionId(), data); + return; + } + /// Sends an encrypted [message] of this [type] to these [deviceKeys]. To send /// the request to all devices of the current user, pass an empty list to [deviceKeys]. - Future sendToDevice( + Future sendToDeviceEncrypted( List deviceKeys, - String type, + String eventType, Map message, { - bool encrypted = true, - List toUsers, + String messageId, bool onlyVerified = false, }) async { - if (encrypted && !encryptionEnabled) return; + if (!encryptionEnabled) return; // Don't send this message to blocked devices, and if specified onlyVerified // then only send it to verified devices if (deviceKeys.isNotEmpty) { @@ -1368,36 +1357,13 @@ class Client { if (deviceKeys.isEmpty) return; } - var sendToDeviceMessage = message; - // Send with send-to-device messaging var data = >>{}; - if (deviceKeys.isEmpty) { - if (toUsers == null) { - data[userID] = {}; - data[userID]['*'] = sendToDeviceMessage; - } else { - for (var user in toUsers) { - data[user.id] = {}; - data[user.id]['*'] = sendToDeviceMessage; - } - } - } else { - if (encrypted) { - data = - await encryption.encryptToDeviceMessage(deviceKeys, type, message); - } else { - for (final device in deviceKeys) { - if (!data.containsKey(device.userId)) { - data[device.userId] = {}; - } - data[device.userId][device.deviceId] = sendToDeviceMessage; - } - } - } - if (encrypted) type = EventTypes.Encrypted; - final messageID = generateUniqueTransactionId(); - await api.sendToDevice(type, messageID, data); + data = + await encryption.encryptToDeviceMessage(deviceKeys, eventType, message); + eventType = EventTypes.Encrypted; + await sendToDevice( + eventType, messageId ?? generateUniqueTransactionId(), data); } /// Whether all push notifications are muted using the [.m.rule.master] @@ -1422,7 +1388,7 @@ class Client { } Future setMuteAllPushNotifications(bool muted) async { - await api.enablePushRule( + await enablePushRule( 'global', PushRuleKind.override, '.m.rule.master', @@ -1432,6 +1398,7 @@ class Client { } /// Changes the password. You should either set oldPasswort or another authentication flow. + @override Future changePassword(String newPassword, {String oldPassword, Map auth}) async { try { @@ -1442,7 +1409,7 @@ class Client { 'password': oldPassword, }; } - await api.changePassword(newPassword, auth: auth); + await super.changePassword(newPassword, auth: auth); } on MatrixException catch (matrixException) { if (!matrixException.requireAdditionalAuthentication) { rethrow; diff --git a/lib/src/room.dart b/lib/src/room.dart index b085d50..f8d25c1 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -374,21 +374,21 @@ class Room { /// Call the Matrix API to change the name of this room. Returns the event ID of the /// new m.room.name event. - Future setName(String newName) => client.api.sendState( + Future setName(String newName) => client.sendState( id, EventTypes.RoomName, {'name': newName}, ); /// Call the Matrix API to change the topic of this room. - Future setDescription(String newName) => client.api.sendState( + Future setDescription(String newName) => client.sendState( id, EventTypes.RoomTopic, {'topic': newName}, ); /// Add a tag to the room. - Future addTag(String tag, {double order}) => client.api.addRoomTag( + Future addTag(String tag, {double order}) => client.addRoomTag( client.userID, id, tag, @@ -396,7 +396,7 @@ class Room { ); /// Removes a tag from the room. - Future removeTag(String tag) => client.api.removeRoomTag( + Future removeTag(String tag) => client.removeRoomTag( client.userID, id, tag, @@ -423,7 +423,7 @@ class Room { /// Call the Matrix API to change the pinned events of this room. Future setPinnedEvents(List pinnedEventIds) => - client.api.sendState( + client.sendState( id, EventTypes.RoomPinnedEvents, {'pinned': pinnedEventIds}, @@ -565,13 +565,13 @@ class Room { uploadThumbnail = encryptedThumbnail.toMatrixFile(); } } - final uploadResp = await client.api.upload( + final uploadResp = await client.upload( uploadFile.bytes, uploadFile.name, contentType: uploadFile.mimeType, ); final thumbnailUploadResp = uploadThumbnail != null - ? await client.api.upload( + ? await client.upload( uploadThumbnail.bytes, uploadThumbnail.name, contentType: uploadThumbnail.mimeType, @@ -705,7 +705,7 @@ class Room { ? await client.encryption .encryptGroupMessagePayload(id, content, type: type) : content; - final res = await client.api.sendMessage( + final res = await client.sendMessage( id, sendType, messageID, @@ -731,7 +731,7 @@ class Room { /// automatically be set. Future join() async { try { - await client.api.joinRoom(id); + await client.joinRoom(id); final invitation = getState(EventTypes.RoomMember, client.userID); if (invitation != null && invitation.content['is_direct'] is bool && @@ -757,25 +757,25 @@ class Room { /// chat, this will be removed too. Future leave() async { if (directChatMatrixID != '') await removeFromDirectChat(); - await client.api.leaveRoom(id); + await client.leaveRoom(id); return; } /// Call the Matrix API to forget this room if you already left it. Future forget() async { await client.database?.forgetRoom(client.id, id); - await client.api.forgetRoom(id); + await client.forgetRoom(id); return; } /// Call the Matrix API to kick a user from this room. - Future kick(String userID) => client.api.kickFromRoom(id, userID); + Future kick(String userID) => client.kickFromRoom(id, userID); /// Call the Matrix API to ban a user from this room. - Future ban(String userID) => client.api.banFromRoom(id, userID); + Future ban(String userID) => client.banFromRoom(id, userID); /// Call the Matrix API to unban a banned user from this room. - Future unban(String userID) => client.api.unbanInRoom(id, userID); + Future unban(String userID) => client.unbanInRoom(id, userID); /// Set the power level of the user with the [userID] to the value [power]. /// Returns the event ID of the new state event. If there is no known @@ -787,7 +787,7 @@ class Room { if (powerMap['users'] == null) powerMap['users'] = {}; powerMap['users'][userID] = power; - return await client.api.sendState( + return await client.sendState( id, EventTypes.RoomPowerLevels, powerMap, @@ -795,14 +795,14 @@ class Room { } /// Call the Matrix API to invite a user to this room. - Future invite(String userID) => client.api.inviteToRoom(id, userID); + Future invite(String userID) => client.inviteToRoom(id, userID); /// Request more previous events from the server. [historyCount] defines how much events should /// be received maximum. When the request is answered, [onHistoryReceived] will be triggered **before** /// the historical events will be published in the onEvent stream. Future requestHistory( {int historyCount = DefaultHistoryCount, onHistoryReceived}) async { - final resp = await client.api.requestMessages( + final resp = await client.requestMessages( id, prev_batch, Direction.b, @@ -853,7 +853,7 @@ class Room { directChats[userID] = [id]; } - await client.api.setAccountData( + await client.setAccountData( client.userID, 'm.direct', directChats, @@ -871,7 +871,7 @@ class Room { return; } // Nothing to do here - await client.api.setRoomAccountData( + await client.setRoomAccountData( client.userID, id, 'm.direct', @@ -884,7 +884,7 @@ class Room { Future sendReadReceipt(String eventID) async { notificationCount = 0; await client.database?.resetNotificationCount(client.id, id); - await client.api.sendReadMarker( + await client.sendReadMarker( id, eventID, readReceiptLocationEventId: eventID, @@ -1017,7 +1017,7 @@ class Room { } } if (participantListComplete) return getParticipants(); - final matrixEvents = await client.api.requestMembers(id); + final matrixEvents = await client.requestMembers(id); final users = matrixEvents.map((e) => Event.fromMatrixEvent(e, this).asUser).toList(); for (final user in users) { @@ -1080,7 +1080,7 @@ class Room { if (mxID == null || !_requestingMatrixIds.add(mxID)) return null; Map resp; try { - resp = await client.api.requestStateContent( + resp = await client.requestStateContent( id, EventTypes.RoomMember, mxID, @@ -1093,7 +1093,7 @@ class Room { } if (resp == null && requestProfile) { try { - final profile = await client.api.requestProfile(mxID); + final profile = await client.requestProfile(mxID); resp = { 'displayname': profile.displayname, 'avatar_url': profile.avatarUrl, @@ -1135,7 +1135,7 @@ class Room { /// Searches for the event on the server. Returns null if not found. Future getEventById(String eventID) async { - final matrixEvent = await client.api.requestEvent(id, eventID); + final matrixEvent = await client.requestEvent(id, eventID); return Event.fromMatrixEvent(matrixEvent, this); } @@ -1169,8 +1169,8 @@ class Room { /// Uploads a new user avatar for this room. Returns the event ID of the new /// m.room.avatar event. Future setAvatar(MatrixFile file) async { - final uploadResp = await client.api.upload(file.bytes, file.name); - return await client.api.sendState( + final uploadResp = await client.upload(file.bytes, file.name); + return await client.sendState( id, EventTypes.RoomAvatar, {'url': uploadResp}, @@ -1267,23 +1267,23 @@ class Room { // All push notifications should be sent to the user case PushRuleState.notify: if (pushRuleState == PushRuleState.dont_notify) { - await client.api.deletePushRule('global', PushRuleKind.override, id); + await client.deletePushRule('global', PushRuleKind.override, id); } else if (pushRuleState == PushRuleState.mentions_only) { - await client.api.deletePushRule('global', PushRuleKind.room, id); + await client.deletePushRule('global', PushRuleKind.room, id); } break; // Only when someone mentions the user, a push notification should be sent case PushRuleState.mentions_only: if (pushRuleState == PushRuleState.dont_notify) { - await client.api.deletePushRule('global', PushRuleKind.override, id); - await client.api.setPushRule( + await client.deletePushRule('global', PushRuleKind.override, id); + await client.setPushRule( 'global', PushRuleKind.room, id, [PushRuleAction.dont_notify], ); } else if (pushRuleState == PushRuleState.notify) { - await client.api.setPushRule( + await client.setPushRule( 'global', PushRuleKind.room, id, @@ -1294,9 +1294,9 @@ class Room { // No push notification should be ever sent for this room. case PushRuleState.dont_notify: if (pushRuleState == PushRuleState.mentions_only) { - await client.api.deletePushRule('global', PushRuleKind.room, id); + await client.deletePushRule('global', PushRuleKind.room, id); } - await client.api.setPushRule( + await client.setPushRule( 'global', PushRuleKind.override, id, @@ -1322,7 +1322,7 @@ class Room { } var data = {}; if (reason != null) data['reason'] = reason; - return await client.api.redact( + return await client.redact( id, eventId, messageID, @@ -1335,7 +1335,7 @@ class Room { 'typing': isTyping, }; if (timeout != null) data['timeout'] = timeout; - return client.api.sendTypingNotification(client.userID, id, isTyping); + return client.sendTypingNotification(client.userID, id, isTyping); } /// This is sent by the caller when they wish to establish a call. @@ -1349,7 +1349,7 @@ class Room { {String type = 'offer', int version = 0, String txid}) async { txid ??= 'txid${DateTime.now().millisecondsSinceEpoch}'; - return await client.api.sendMessage( + return await client.sendMessage( id, EventTypes.CallInvite, txid, @@ -1387,7 +1387,7 @@ class Room { String txid, }) async { txid ??= 'txid${DateTime.now().millisecondsSinceEpoch}'; - return await client.api.sendMessage( + return await client.sendMessage( id, EventTypes.CallCandidates, txid, @@ -1407,7 +1407,7 @@ class Room { Future answerCall(String callId, String sdp, {String type = 'answer', int version = 0, String txid}) async { txid ??= 'txid${DateTime.now().millisecondsSinceEpoch}'; - return await client.api.sendMessage( + return await client.sendMessage( id, EventTypes.CallAnswer, txid, @@ -1425,7 +1425,7 @@ class Room { Future hangupCall(String callId, {int version = 0, String txid}) async { txid ??= 'txid${DateTime.now().millisecondsSinceEpoch}'; - return await client.api.sendMessage( + return await client.sendMessage( id, EventTypes.CallHangup, txid, @@ -1461,7 +1461,7 @@ class Room { /// Changes the join rules. You should check first if the user is able to change it. Future setJoinRules(JoinRules joinRules) async { - await client.api.sendState( + await client.sendState( id, EventTypes.RoomJoinRules, { @@ -1486,7 +1486,7 @@ class Room { /// Changes the guest access. You should check first if the user is able to change it. Future setGuestAccess(GuestAccess guestAccess) async { - await client.api.sendState( + await client.sendState( id, EventTypes.GuestAccess, { @@ -1512,7 +1512,7 @@ class Room { /// Changes the history visibility. You should check first if the user is able to change it. Future setHistoryVisibility(HistoryVisibility historyVisibility) async { - await client.api.sendState( + await client.sendState( id, EventTypes.HistoryVisibility, { @@ -1539,7 +1539,7 @@ class Room { Future enableEncryption({int algorithmIndex = 0}) async { if (encrypted) throw ('Encryption is already enabled!'); final algorithm = Client.supportedGroupEncryptionAlgorithms[algorithmIndex]; - await client.api.sendState( + await client.sendState( id, EventTypes.Encryption, { diff --git a/lib/src/user.dart b/lib/src/user.dart index 3efedf4..ea1ce42 100644 --- a/lib/src/user.dart +++ b/lib/src/user.dart @@ -146,7 +146,7 @@ class User extends Event { if (roomID != null) return roomID; // Start a new direct chat - final newRoomID = await room.client.api.createRoom( + final newRoomID = await room.client.createRoom( invite: [id], isDirect: true, preset: CreateRoomPreset.trusted_private_chat, diff --git a/lib/src/utils/uri_extension.dart b/lib/src/utils/uri_extension.dart index edecdcf..804e385 100644 --- a/lib/src/utils/uri_extension.dart +++ b/lib/src/utils/uri_extension.dart @@ -22,8 +22,8 @@ import 'dart:core'; extension MxcUriExtension on Uri { /// Returns a download Link to this content. String getDownloadLink(Client matrix) => isScheme('mxc') - ? matrix.api.homeserver != null - ? '${matrix.api.homeserver.toString()}/_matrix/media/r0/download/$host$path' + ? matrix.homeserver != null + ? '${matrix.homeserver.toString()}/_matrix/media/r0/download/$host$path' : '' : toString(); @@ -36,8 +36,8 @@ extension MxcUriExtension on Uri { final methodStr = method.toString().split('.').last; width = width.round(); height = height.round(); - return matrix.api.homeserver != null - ? '${matrix.api.homeserver.toString()}/_matrix/media/r0/thumbnail/$host$path?width=$width&height=$height&method=$methodStr' + return matrix.homeserver != null + ? '${matrix.homeserver.toString()}/_matrix/media/r0/thumbnail/$host$path?width=$width&height=$height&method=$methodStr' : ''; } } diff --git a/test/client_test.dart b/test/client_test.dart index 8d0242f..2fa4178 100644 --- a/test/client_test.dart +++ b/test/client_test.dart @@ -46,7 +46,7 @@ void main() { const fingerprintKey = 'gjL//fyaFHADt9KBADGag8g7F8Up78B/K1zXeiEPLJo'; /// All Tests related to the Login - group('FluffyMatrix', () { + group('Client', () { /// Check if all Elements get created matrix = Client('testclient', httpClient: FakeMatrixApi()); @@ -74,7 +74,7 @@ void main() { accountDataCounter++; }); - expect(matrix.api.homeserver, null); + expect(matrix.homeserver, null); try { await matrix.checkServer('https://fakeserver.wrongaddress'); @@ -82,17 +82,9 @@ void main() { expect(exception != null, true); } await matrix.checkServer('https://fakeserver.notexisting'); - expect( - matrix.api.homeserver.toString(), 'https://fakeserver.notexisting'); + expect(matrix.homeserver.toString(), 'https://fakeserver.notexisting'); - final resp = await matrix.api.login( - type: 'm.login.password', - user: 'test', - password: '1234', - initialDeviceDisplayName: 'Fluffy Matrix Client', - ); - - final available = await matrix.api.usernameAvailable('testuser'); + final available = await matrix.usernameAvailable('testuser'); expect(available, true); var loginStateFuture = matrix.onLoginStateChanged.stream.first; @@ -100,21 +92,16 @@ void main() { var syncFuture = matrix.onSync.stream.first; matrix.connect( - newToken: resp.accessToken, - newUserID: resp.userId, - newHomeserver: matrix.api.homeserver, + newToken: 'abcd', + newUserID: '@test:fakeServer.notExisting', + newHomeserver: matrix.homeserver, newDeviceName: 'Text Matrix Client', - newDeviceID: resp.deviceId, + newDeviceID: 'GHTYAJCE', newOlmAccount: pickledOlmAccount, ); await Future.delayed(Duration(milliseconds: 50)); - expect(matrix.api.accessToken == resp.accessToken, true); - expect(matrix.deviceName == 'Text Matrix Client', true); - expect(matrix.deviceID == resp.deviceId, true); - expect(matrix.userID == resp.userId, true); - var loginState = await loginStateFuture; var firstSync = await firstSyncFuture; var sync = await syncFuture; @@ -208,14 +195,11 @@ void main() { }); test('Logout', () async { - await matrix.api.logout(); - var loginStateFuture = matrix.onLoginStateChanged.stream.first; + await matrix.logout(); - matrix.clear(); - - expect(matrix.api.accessToken == null, true); - expect(matrix.api.homeserver == null, true); + expect(matrix.accessToken == null, true); + expect(matrix.homeserver == null, true); expect(matrix.userID == null, true); expect(matrix.deviceID == null, true); expect(matrix.deviceName == null, true); @@ -330,10 +314,10 @@ void main() { final checkResp = await matrix.checkServer('https://fakeServer.notExisting'); - final loginResp = await matrix.login('test', '1234'); + final loginResp = await matrix.login(user: 'test', password: '1234'); expect(checkResp, true); - expect(loginResp, true); + expect(loginResp != null, true); }); test('setAvatar', () async { @@ -386,8 +370,8 @@ void main() { } } }, matrix); - test('sendToDevice', () async { - await matrix.sendToDevice( + test('sendToDeviceEncrypted', () async { + await matrix.sendToDeviceEncrypted( [deviceKeys], 'm.message', { @@ -420,9 +404,9 @@ void main() { await Future.delayed(Duration(milliseconds: 100)); expect(client2.isLogged(), true); - expect(client2.api.accessToken, client1.api.accessToken); + expect(client2.accessToken, client1.accessToken); expect(client2.userID, client1.userID); - expect(client2.api.homeserver, client1.api.homeserver); + expect(client2.homeserver, client1.homeserver); expect(client2.deviceID, client1.deviceID); expect(client2.deviceName, client1.deviceName); if (client2.encryptionEnabled) { diff --git a/test/encryption/encrypt_decrypt_to_device_test.dart b/test/encryption/encrypt_decrypt_to_device_test.dart index b98ef22..4fbde05 100644 --- a/test/encryption/encrypt_decrypt_to_device_test.dart +++ b/test/encryption/encrypt_decrypt_to_device_test.dart @@ -54,7 +54,7 @@ void main() { otherClient.connect( newToken: 'abc', newUserID: '@othertest:fakeServer.notExisting', - newHomeserver: otherClient.api.homeserver, + newHomeserver: otherClient.homeserver, newDeviceName: 'Text Matrix Client', newDeviceID: 'FOXDEVICE', newOlmAccount: otherPickledOlmAccount, diff --git a/test/encryption/key_verification_test.dart b/test/encryption/key_verification_test.dart index 5ae9689..2612207 100644 --- a/test/encryption/key_verification_test.dart +++ b/test/encryption/key_verification_test.dart @@ -89,7 +89,7 @@ void main() { client2.connect( newToken: 'abc', newUserID: '@othertest:fakeServer.notExisting', - newHomeserver: client2.api.homeserver, + newHomeserver: client2.homeserver, newDeviceName: 'Text Matrix Client', newDeviceID: 'FOXDEVICE', newOlmAccount: otherPickledOlmAccount, diff --git a/test/event_test.dart b/test/event_test.dart index e57be26..f40f8ec 100644 --- a/test/event_test.dart +++ b/test/event_test.dart @@ -246,7 +246,7 @@ void main() { test('sendAgain', () async { var matrix = Client('testclient', httpClient: FakeMatrixApi()); await matrix.checkServer('https://fakeServer.notExisting'); - await matrix.login('test', '1234'); + await matrix.login(user: 'test', password: '1234'); var event = Event.fromJson( jsonObj, Room(id: '!1234:example.com', client: matrix)); @@ -262,7 +262,7 @@ void main() { test('requestKey', () async { var matrix = Client('testclient', httpClient: FakeMatrixApi()); await matrix.checkServer('https://fakeServer.notExisting'); - await matrix.login('test', '1234'); + await matrix.login(user: 'test', password: '1234'); var event = Event.fromJson( jsonObj, Room(id: '!1234:example.com', client: matrix)); diff --git a/test/fake_client.dart b/test/fake_client.dart index 7b1a94a..5f30487 100644 --- a/test/fake_client.dart +++ b/test/fake_client.dart @@ -32,18 +32,12 @@ Future getClient() async { final client = Client('testclient', httpClient: FakeMatrixApi()); client.database = getDatabase(); await client.checkServer('https://fakeServer.notExisting'); - final resp = await client.api.login( - type: 'm.login.password', - user: 'test', - password: '1234', - initialDeviceDisplayName: 'Fluffy Matrix Client', - ); client.connect( - newToken: resp.accessToken, - newUserID: resp.userId, - newHomeserver: client.api.homeserver, + newToken: 'abcd', + newUserID: '@test:fakeServer.notExisting', + newHomeserver: client.homeserver, newDeviceName: 'Text Matrix Client', - newDeviceID: resp.deviceId, + newDeviceID: 'GHTYAJCE', newOlmAccount: pickledOlmAccount, ); await Future.delayed(Duration(milliseconds: 10)); diff --git a/test/fake_matrix_api.dart b/test/fake_matrix_api.dart index 760dfcf..2dec135 100644 --- a/test/fake_matrix_api.dart +++ b/test/fake_matrix_api.dart @@ -82,6 +82,10 @@ class FakeMatrixApi extends MockClient { action.contains( '/client/r0/rooms/!1234%3AfakeServer.notExisting/send/')) { res = {'event_id': '\$event${FakeMatrixApi.eventCounter++}'}; + } else if (action.contains('/client/r0/sync')) { + res = { + 'next_batch': DateTime.now().millisecondsSinceEpoch.toString + }; } else { res = { 'errcode': 'M_UNRECOGNIZED', diff --git a/test/mxc_uri_extension_test.dart b/test/mxc_uri_extension_test.dart index 798854c..30e6f84 100644 --- a/test/mxc_uri_extension_test.dart +++ b/test/mxc_uri_extension_test.dart @@ -33,13 +33,13 @@ void main() { expect(content.isScheme('mxc'), true); expect(content.getDownloadLink(client), - '${client.api.homeserver.toString()}/_matrix/media/r0/download/exampleserver.abc/abcdefghijklmn'); + '${client.homeserver.toString()}/_matrix/media/r0/download/exampleserver.abc/abcdefghijklmn'); expect(content.getThumbnail(client, width: 50, height: 50), - '${client.api.homeserver.toString()}/_matrix/media/r0/thumbnail/exampleserver.abc/abcdefghijklmn?width=50&height=50&method=crop'); + '${client.homeserver.toString()}/_matrix/media/r0/thumbnail/exampleserver.abc/abcdefghijklmn?width=50&height=50&method=crop'); expect( content.getThumbnail(client, width: 50, height: 50, method: ThumbnailMethod.scale), - '${client.api.homeserver.toString()}/_matrix/media/r0/thumbnail/exampleserver.abc/abcdefghijklmn?width=50&height=50&method=scale'); + '${client.homeserver.toString()}/_matrix/media/r0/thumbnail/exampleserver.abc/abcdefghijklmn?width=50&height=50&method=scale'); }); }); } diff --git a/test/user_test.dart b/test/user_test.dart index 35d35d6..0e7de24 100644 --- a/test/user_test.dart +++ b/test/user_test.dart @@ -102,7 +102,7 @@ void main() { }); test('startDirectChat', () async { await client.checkServer('https://fakeserver.notexisting'); - await client.login('test', '1234'); + await client.login(user: 'test', password: '1234'); await user1.startDirectChat(); }); test('getPresence', () async { diff --git a/test_driver/famedlysdk_test.dart b/test_driver/famedlysdk_test.dart index 61f3cdc..f2f3988 100644 --- a/test_driver/famedlysdk_test.dart +++ b/test_driver/famedlysdk_test.dart @@ -22,14 +22,14 @@ void test() async { var testClientA = Client('TestClientA'); testClientA.database = getDatabase(); await testClientA.checkServer(homeserver); - await testClientA.login(testUserA, testPasswordA); + await testClientA.login(user: testUserA, password: testPasswordA); assert(testClientA.encryptionEnabled); Logs.success('++++ Login $testUserB ++++'); var testClientB = Client('TestClientB'); testClientB.database = getDatabase(); await testClientB.checkServer(homeserver); - await testClientB.login(testUserB, testPasswordA); + await testClientB.login(user: testUserB, password: testPasswordA); assert(testClientB.encryptionEnabled); Logs.success('++++ ($testUserA) Leave all rooms ++++'); @@ -72,7 +72,7 @@ void test() async { .userDeviceKeys[testUserB].deviceKeys[testClientB.deviceID].blocked); Logs.success('++++ ($testUserA) Create room and invite $testUserB ++++'); - await testClientA.api.createRoom(invite: [testUserB]); + await testClientA.createRoom(invite: [testUserB]); await Future.delayed(Duration(seconds: 1)); var room = testClientA.rooms.first; assert(room != null); @@ -217,7 +217,7 @@ void test() async { Logs.success('++++ Login $testUserB in another client ++++'); var testClientC = Client('TestClientC', database: getDatabase()); await testClientC.checkServer(homeserver); - await testClientC.login(testUserB, testPasswordA); + await testClientC.login(user: testUserB, password: testPasswordA); await Future.delayed(Duration(seconds: 3)); Logs.success( @@ -346,8 +346,8 @@ void test() async { await Future.delayed(Duration(seconds: 1)); await testClientA.dispose(); await testClientB.dispose(); - await testClientA.api.logoutAll(); - await testClientB.api.logoutAll(); + await testClientA.logoutAll(); + await testClientB.logoutAll(); testClientA = null; testClientB = null; return; From 26586b6f020e0ab0d25bcadf6b5a38ae0e27867a Mon Sep 17 00:00:00 2001 From: MTRNord Date: Thu, 13 Aug 2020 10:40:39 +0200 Subject: [PATCH 29/90] style: Change package:famedlysdk imports to relative imports Changing the imports from `package:famedlysdk` to relative imports allows us to easier move the files Took 2 minutes --- lib/encryption.dart | 8 +- lib/encryption/cross_signing.dart | 4 +- lib/encryption/encryption.dart | 11 ++- lib/encryption/key_manager.dart | 10 +- lib/encryption/key_verification_manager.dart | 6 +- lib/encryption/olm_manager.dart | 9 +- lib/encryption/ssss.dart | 12 +-- lib/encryption/utils/key_verification.dart | 13 +-- lib/encryption/utils/olm_session.dart | 3 +- .../utils/outbound_group_session.dart | 3 +- lib/encryption/utils/session_key.dart | 4 +- lib/famedlysdk.dart | 34 +++---- lib/matrix_api.dart | 92 +++++++++---------- lib/matrix_api/matrix_api.dart | 26 +++--- lib/matrix_api/model/basic_room_event.dart | 2 +- lib/matrix_api/model/matrix_event.dart | 2 +- .../model/stripped_state_event.dart | 2 +- lib/src/client.dart | 13 ++- lib/src/database/database.dart | 8 +- lib/src/event.dart | 14 +-- lib/src/room.dart | 24 ++--- lib/src/timeline.dart | 5 +- lib/src/user.dart | 8 +- lib/src/utils/device_keys_list.dart | 10 +- lib/src/utils/matrix_file.dart | 4 +- lib/src/utils/room_update.dart | 2 +- lib/src/utils/states_map.dart | 3 +- lib/src/utils/sync_update_extension.dart | 2 +- lib/src/utils/to_device_event.dart | 2 +- lib/src/utils/uri_extension.dart | 3 +- 30 files changed, 173 insertions(+), 166 deletions(-) diff --git a/lib/encryption.dart b/lib/encryption.dart index 2239ee2..6ebb4f7 100644 --- a/lib/encryption.dart +++ b/lib/encryption.dart @@ -18,7 +18,7 @@ library encryption; -export './encryption/encryption.dart'; -export './encryption/key_manager.dart'; -export './encryption/ssss.dart'; -export './encryption/utils/key_verification.dart'; +export 'encryption/encryption.dart'; +export 'encryption/key_manager.dart'; +export 'encryption/ssss.dart'; +export 'encryption/utils/key_verification.dart'; diff --git a/lib/encryption/cross_signing.dart b/lib/encryption/cross_signing.dart index c8e2249..a44a85e 100644 --- a/lib/encryption/cross_signing.dart +++ b/lib/encryption/cross_signing.dart @@ -16,12 +16,12 @@ * along with this program. If not, see . */ -import 'dart:typed_data'; import 'dart:convert'; +import 'dart:typed_data'; import 'package:olm/olm.dart' as olm; -import 'package:famedlysdk/famedlysdk.dart'; +import '../famedlysdk.dart'; import 'encryption.dart'; const SELF_SIGNING_KEY = 'm.cross_signing.self_signing'; diff --git a/lib/encryption/encryption.dart b/lib/encryption/encryption.dart index 62c5a05..6b78c95 100644 --- a/lib/encryption/encryption.dart +++ b/lib/encryption/encryption.dart @@ -18,13 +18,14 @@ import 'dart:convert'; -import 'package:famedlysdk/famedlysdk.dart'; -import 'package:famedlysdk/matrix_api.dart'; import 'package:pedantic/pedantic.dart'; -import 'key_manager.dart'; -import 'olm_manager.dart'; -import 'key_verification_manager.dart'; + +import '../famedlysdk.dart'; +import '../matrix_api.dart'; import 'cross_signing.dart'; +import 'key_manager.dart'; +import 'key_verification_manager.dart'; +import 'olm_manager.dart'; import 'ssss.dart'; class Encryption { diff --git a/lib/encryption/key_manager.dart b/lib/encryption/key_manager.dart index a0e1809..59efb64 100644 --- a/lib/encryption/key_manager.dart +++ b/lib/encryption/key_manager.dart @@ -18,15 +18,15 @@ import 'dart:convert'; -import 'package:famedlysdk/src/utils/logs.dart'; -import 'package:pedantic/pedantic.dart'; -import 'package:famedlysdk/famedlysdk.dart'; -import 'package:famedlysdk/matrix_api.dart'; import 'package:olm/olm.dart' as olm; +import 'package:pedantic/pedantic.dart'; import './encryption.dart'; -import './utils/session_key.dart'; import './utils/outbound_group_session.dart'; +import './utils/session_key.dart'; +import '../famedlysdk.dart'; +import '../matrix_api.dart'; +import '../src/utils/logs.dart'; const MEGOLM_KEY = 'm.megolm_backup.v1'; diff --git a/lib/encryption/key_verification_manager.dart b/lib/encryption/key_verification_manager.dart index d02d107..387836d 100644 --- a/lib/encryption/key_verification_manager.dart +++ b/lib/encryption/key_verification_manager.dart @@ -16,9 +16,9 @@ * along with this program. If not, see . */ -import 'package:famedlysdk/famedlysdk.dart'; -import './encryption.dart'; -import './utils/key_verification.dart'; +import '../famedlysdk.dart'; +import 'encryption.dart'; +import 'utils/key_verification.dart'; class KeyVerificationManager { final Encryption encryption; diff --git a/lib/encryption/olm_manager.dart b/lib/encryption/olm_manager.dart index 6bb9493..a0078ec 100644 --- a/lib/encryption/olm_manager.dart +++ b/lib/encryption/olm_manager.dart @@ -18,14 +18,15 @@ import 'dart:convert'; -import 'package:famedlysdk/src/utils/logs.dart'; -import 'package:pedantic/pedantic.dart'; import 'package:canonical_json/canonical_json.dart'; import 'package:famedlysdk/famedlysdk.dart'; import 'package:famedlysdk/matrix_api.dart'; import 'package:olm/olm.dart' as olm; -import './encryption.dart'; -import './utils/olm_session.dart'; +import 'package:pedantic/pedantic.dart'; + +import '../src/utils/logs.dart'; +import 'encryption.dart'; +import 'utils/olm_session.dart'; class OlmManager { final Encryption encryption; diff --git a/lib/encryption/ssss.dart b/lib/encryption/ssss.dart index fb53ae2..d646397 100644 --- a/lib/encryption/ssss.dart +++ b/lib/encryption/ssss.dart @@ -16,17 +16,17 @@ * along with this program. If not, see . */ -import 'dart:typed_data'; import 'dart:convert'; +import 'dart:typed_data'; -import 'package:encrypt/encrypt.dart'; -import 'package:crypto/crypto.dart'; import 'package:base58check/base58.dart'; -import 'package:famedlysdk/src/utils/logs.dart'; +import 'package:crypto/crypto.dart'; +import 'package:encrypt/encrypt.dart'; import 'package:password_hash/password_hash.dart'; -import 'package:famedlysdk/famedlysdk.dart'; -import 'package:famedlysdk/matrix_api.dart'; +import '../famedlysdk.dart'; +import '../matrix_api.dart'; +import '../src/utils/logs.dart'; import 'encryption.dart'; const CACHE_TYPES = [ diff --git a/lib/encryption/utils/key_verification.dart b/lib/encryption/utils/key_verification.dart index 4b89f06..8075a21 100644 --- a/lib/encryption/utils/key_verification.dart +++ b/lib/encryption/utils/key_verification.dart @@ -18,13 +18,14 @@ import 'dart:async'; import 'dart:typed_data'; -import 'package:canonical_json/canonical_json.dart'; -import 'package:famedlysdk/src/utils/logs.dart'; -import 'package:pedantic/pedantic.dart'; -import 'package:olm/olm.dart' as olm; -import 'package:famedlysdk/famedlysdk.dart'; -import 'package:famedlysdk/matrix_api.dart'; +import 'package:canonical_json/canonical_json.dart'; +import 'package:olm/olm.dart' as olm; +import 'package:pedantic/pedantic.dart'; + +import '../../famedlysdk.dart'; +import '../../matrix_api.dart'; +import '../../src/utils/logs.dart'; import '../encryption.dart'; /* diff --git a/lib/encryption/utils/olm_session.dart b/lib/encryption/utils/olm_session.dart index 73d8a98..2edf4cb 100644 --- a/lib/encryption/utils/olm_session.dart +++ b/lib/encryption/utils/olm_session.dart @@ -16,9 +16,10 @@ * along with this program. If not, see . */ -import 'package:famedlysdk/src/utils/logs.dart'; import 'package:olm/olm.dart' as olm; + import '../../src/database/database.dart' show DbOlmSessions; +import '../../src/utils/logs.dart'; class OlmSession { String identityKey; diff --git a/lib/encryption/utils/outbound_group_session.dart b/lib/encryption/utils/outbound_group_session.dart index bf10818..d3da8cb 100644 --- a/lib/encryption/utils/outbound_group_session.dart +++ b/lib/encryption/utils/outbound_group_session.dart @@ -18,9 +18,10 @@ import 'dart:convert'; -import 'package:famedlysdk/src/utils/logs.dart'; import 'package:olm/olm.dart' as olm; + import '../../src/database/database.dart' show DbOutboundGroupSession; +import '../../src/utils/logs.dart'; class OutboundGroupSession { List devices; diff --git a/lib/encryption/utils/session_key.dart b/lib/encryption/utils/session_key.dart index 176c9e0..523ff7a 100644 --- a/lib/encryption/utils/session_key.dart +++ b/lib/encryption/utils/session_key.dart @@ -18,11 +18,11 @@ import 'dart:convert'; -import 'package:famedlysdk/src/utils/logs.dart'; import 'package:olm/olm.dart' as olm; -import 'package:famedlysdk/famedlysdk.dart'; +import '../../famedlysdk.dart'; import '../../src/database/database.dart' show DbInboundGroupSession; +import '../../src/utils/logs.dart'; class SessionKey { Map content; diff --git a/lib/famedlysdk.dart b/lib/famedlysdk.dart index 3a1c51d..a6617e9 100644 --- a/lib/famedlysdk.dart +++ b/lib/famedlysdk.dart @@ -19,20 +19,20 @@ library famedlysdk; export 'matrix_api.dart'; -export 'package:famedlysdk/src/utils/room_update.dart'; -export 'package:famedlysdk/src/utils/event_update.dart'; -export 'package:famedlysdk/src/utils/device_keys_list.dart'; -export 'package:famedlysdk/src/utils/matrix_file.dart'; -export 'package:famedlysdk/src/utils/matrix_id_string_extension.dart'; -export 'package:famedlysdk/src/utils/uri_extension.dart'; -export 'package:famedlysdk/src/utils/matrix_localizations.dart'; -export 'package:famedlysdk/src/utils/receipt.dart'; -export 'package:famedlysdk/src/utils/states_map.dart'; -export 'package:famedlysdk/src/utils/sync_update_extension.dart'; -export 'package:famedlysdk/src/utils/to_device_event.dart'; -export 'package:famedlysdk/src/client.dart'; -export 'package:famedlysdk/src/event.dart'; -export 'package:famedlysdk/src/room.dart'; -export 'package:famedlysdk/src/timeline.dart'; -export 'package:famedlysdk/src/user.dart'; -export 'package:famedlysdk/src/database/database.dart' show Database; +export 'src/utils/room_update.dart'; +export 'src/utils/event_update.dart'; +export 'src/utils/device_keys_list.dart'; +export 'src/utils/matrix_file.dart'; +export 'src/utils/matrix_id_string_extension.dart'; +export 'src/utils/uri_extension.dart'; +export 'src/utils/matrix_localizations.dart'; +export 'src/utils/receipt.dart'; +export 'src/utils/states_map.dart'; +export 'src/utils/sync_update_extension.dart'; +export 'src/utils/to_device_event.dart'; +export 'src/client.dart'; +export 'src/event.dart'; +export 'src/room.dart'; +export 'src/timeline.dart'; +export 'src/user.dart'; +export 'src/database/database.dart' show Database; diff --git a/lib/matrix_api.dart b/lib/matrix_api.dart index be120a3..e918827 100644 --- a/lib/matrix_api.dart +++ b/lib/matrix_api.dart @@ -18,49 +18,49 @@ library matrix_api; -export 'package:famedlysdk/matrix_api/matrix_api.dart'; -export 'package:famedlysdk/matrix_api/model/basic_event_with_sender.dart'; -export 'package:famedlysdk/matrix_api/model/basic_event.dart'; -export 'package:famedlysdk/matrix_api/model/device.dart'; -export 'package:famedlysdk/matrix_api/model/basic_room_event.dart'; -export 'package:famedlysdk/matrix_api/model/event_context.dart'; -export 'package:famedlysdk/matrix_api/model/matrix_event.dart'; -export 'package:famedlysdk/matrix_api/model/event_types.dart'; -export 'package:famedlysdk/matrix_api/model/events_sync_update.dart'; -export 'package:famedlysdk/matrix_api/model/filter.dart'; -export 'package:famedlysdk/matrix_api/model/keys_query_response.dart'; -export 'package:famedlysdk/matrix_api/model/login_response.dart'; -export 'package:famedlysdk/matrix_api/model/login_types.dart'; -export 'package:famedlysdk/matrix_api/model/matrix_exception.dart'; -export 'package:famedlysdk/matrix_api/model/matrix_keys.dart'; -export 'package:famedlysdk/matrix_api/model/message_types.dart'; -export 'package:famedlysdk/matrix_api/model/presence_content.dart'; -export 'package:famedlysdk/matrix_api/model/notifications_query_response.dart'; -export 'package:famedlysdk/matrix_api/model/one_time_keys_claim_response.dart'; -export 'package:famedlysdk/matrix_api/model/open_graph_data.dart'; -export 'package:famedlysdk/matrix_api/model/open_id_credentials.dart'; -export 'package:famedlysdk/matrix_api/model/presence.dart'; -export 'package:famedlysdk/matrix_api/model/profile.dart'; -export 'package:famedlysdk/matrix_api/model/public_rooms_response.dart'; -export 'package:famedlysdk/matrix_api/model/push_rule_set.dart'; -export 'package:famedlysdk/matrix_api/model/pusher.dart'; -export 'package:famedlysdk/matrix_api/model/request_token_response.dart'; -export 'package:famedlysdk/matrix_api/model/room_alias_informations.dart'; -export 'package:famedlysdk/matrix_api/model/room_keys_info.dart'; -export 'package:famedlysdk/matrix_api/model/room_keys_keys.dart'; -export 'package:famedlysdk/matrix_api/model/room_summary.dart'; -export 'package:famedlysdk/matrix_api/model/server_capabilities.dart'; -export 'package:famedlysdk/matrix_api/model/stripped_state_event.dart'; -export 'package:famedlysdk/matrix_api/model/supported_protocol.dart'; -export 'package:famedlysdk/matrix_api/model/supported_versions.dart'; -export 'package:famedlysdk/matrix_api/model/sync_update.dart'; -export 'package:famedlysdk/matrix_api/model/tag.dart'; -export 'package:famedlysdk/matrix_api/model/third_party_identifier.dart'; -export 'package:famedlysdk/matrix_api/model/third_party_location.dart'; -export 'package:famedlysdk/matrix_api/model/third_party_user.dart'; -export 'package:famedlysdk/matrix_api/model/timeline_history_response.dart'; -export 'package:famedlysdk/matrix_api/model/turn_server_credentials.dart'; -export 'package:famedlysdk/matrix_api/model/upload_key_signatures_response.dart'; -export 'package:famedlysdk/matrix_api/model/user_search_result.dart'; -export 'package:famedlysdk/matrix_api/model/well_known_informations.dart'; -export 'package:famedlysdk/matrix_api/model/who_is_info.dart'; +export 'matrix_api/matrix_api.dart'; +export 'matrix_api/model/basic_event.dart'; +export 'matrix_api/model/basic_event_with_sender.dart'; +export 'matrix_api/model/basic_room_event.dart'; +export 'matrix_api/model/device.dart'; +export 'matrix_api/model/event_context.dart'; +export 'matrix_api/model/event_types.dart'; +export 'matrix_api/model/events_sync_update.dart'; +export 'matrix_api/model/filter.dart'; +export 'matrix_api/model/keys_query_response.dart'; +export 'matrix_api/model/login_response.dart'; +export 'matrix_api/model/login_types.dart'; +export 'matrix_api/model/matrix_event.dart'; +export 'matrix_api/model/matrix_exception.dart'; +export 'matrix_api/model/matrix_keys.dart'; +export 'matrix_api/model/message_types.dart'; +export 'matrix_api/model/notifications_query_response.dart'; +export 'matrix_api/model/one_time_keys_claim_response.dart'; +export 'matrix_api/model/open_graph_data.dart'; +export 'matrix_api/model/open_id_credentials.dart'; +export 'matrix_api/model/presence.dart'; +export 'matrix_api/model/presence_content.dart'; +export 'matrix_api/model/profile.dart'; +export 'matrix_api/model/public_rooms_response.dart'; +export 'matrix_api/model/push_rule_set.dart'; +export 'matrix_api/model/pusher.dart'; +export 'matrix_api/model/request_token_response.dart'; +export 'matrix_api/model/room_alias_informations.dart'; +export 'matrix_api/model/room_keys_info.dart'; +export 'matrix_api/model/room_keys_keys.dart'; +export 'matrix_api/model/room_summary.dart'; +export 'matrix_api/model/server_capabilities.dart'; +export 'matrix_api/model/stripped_state_event.dart'; +export 'matrix_api/model/supported_protocol.dart'; +export 'matrix_api/model/supported_versions.dart'; +export 'matrix_api/model/sync_update.dart'; +export 'matrix_api/model/tag.dart'; +export 'matrix_api/model/third_party_identifier.dart'; +export 'matrix_api/model/third_party_location.dart'; +export 'matrix_api/model/third_party_user.dart'; +export 'matrix_api/model/timeline_history_response.dart'; +export 'matrix_api/model/turn_server_credentials.dart'; +export 'matrix_api/model/upload_key_signatures_response.dart'; +export 'matrix_api/model/user_search_result.dart'; +export 'matrix_api/model/well_known_informations.dart'; +export 'matrix_api/model/who_is_info.dart'; diff --git a/lib/matrix_api/matrix_api.dart b/lib/matrix_api/matrix_api.dart index 50b66c0..f738a83 100644 --- a/lib/matrix_api/matrix_api.dart +++ b/lib/matrix_api/matrix_api.dart @@ -19,19 +19,6 @@ import 'dart:async'; import 'dart:convert'; -import 'package:famedlysdk/matrix_api/model/filter.dart'; -import 'package:famedlysdk/matrix_api/model/keys_query_response.dart'; -import 'package:famedlysdk/matrix_api/model/login_types.dart'; -import 'package:famedlysdk/matrix_api/model/notifications_query_response.dart'; -import 'package:famedlysdk/matrix_api/model/open_graph_data.dart'; -import 'package:famedlysdk/matrix_api/model/profile.dart'; -import 'package:famedlysdk/matrix_api/model/request_token_response.dart'; -import 'package:famedlysdk/matrix_api/model/server_capabilities.dart'; -import 'package:famedlysdk/matrix_api/model/supported_versions.dart'; -import 'package:famedlysdk/matrix_api/model/sync_update.dart'; -import 'package:famedlysdk/matrix_api/model/third_party_location.dart'; -import 'package:famedlysdk/matrix_api/model/timeline_history_response.dart'; -import 'package:famedlysdk/matrix_api/model/user_search_result.dart'; import 'package:http/http.dart' as http; import 'package:mime/mime.dart'; import 'package:moor/moor.dart'; @@ -39,25 +26,38 @@ import 'package:moor/moor.dart'; import 'model/device.dart'; import 'model/event_context.dart'; import 'model/events_sync_update.dart'; +import 'model/filter.dart'; +import 'model/keys_query_response.dart'; import 'model/login_response.dart'; +import 'model/login_types.dart'; import 'model/matrix_event.dart'; import 'model/matrix_exception.dart'; import 'model/matrix_keys.dart'; +import 'model/notifications_query_response.dart'; import 'model/one_time_keys_claim_response.dart'; +import 'model/open_graph_data.dart'; import 'model/open_id_credentials.dart'; import 'model/presence_content.dart'; +import 'model/profile.dart'; import 'model/public_rooms_response.dart'; import 'model/push_rule_set.dart'; import 'model/pusher.dart'; +import 'model/request_token_response.dart'; import 'model/room_alias_informations.dart'; import 'model/room_keys_info.dart'; import 'model/room_keys_keys.dart'; +import 'model/server_capabilities.dart'; import 'model/supported_protocol.dart'; +import 'model/supported_versions.dart'; +import 'model/sync_update.dart'; import 'model/tag.dart'; import 'model/third_party_identifier.dart'; +import 'model/third_party_location.dart'; import 'model/third_party_user.dart'; +import 'model/timeline_history_response.dart'; import 'model/turn_server_credentials.dart'; import 'model/upload_key_signatures_response.dart'; +import 'model/user_search_result.dart'; import 'model/well_known_informations.dart'; import 'model/who_is_info.dart'; diff --git a/lib/matrix_api/model/basic_room_event.dart b/lib/matrix_api/model/basic_room_event.dart index c8f7564..de8ee75 100644 --- a/lib/matrix_api/model/basic_room_event.dart +++ b/lib/matrix_api/model/basic_room_event.dart @@ -16,7 +16,7 @@ * along with this program. If not, see . */ -import 'package:famedlysdk/matrix_api/model/basic_event.dart'; +import 'basic_event.dart'; class BasicRoomEvent extends BasicEvent { String roomId; diff --git a/lib/matrix_api/model/matrix_event.dart b/lib/matrix_api/model/matrix_event.dart index e70f8b5..2f5f35f 100644 --- a/lib/matrix_api/model/matrix_event.dart +++ b/lib/matrix_api/model/matrix_event.dart @@ -16,7 +16,7 @@ * along with this program. If not, see . */ -import 'package:famedlysdk/matrix_api/model/stripped_state_event.dart'; +import 'stripped_state_event.dart'; class MatrixEvent extends StrippedStateEvent { String eventId; diff --git a/lib/matrix_api/model/stripped_state_event.dart b/lib/matrix_api/model/stripped_state_event.dart index 86511a5..29c0740 100644 --- a/lib/matrix_api/model/stripped_state_event.dart +++ b/lib/matrix_api/model/stripped_state_event.dart @@ -16,7 +16,7 @@ * along with this program. If not, see . */ -import 'package:famedlysdk/matrix_api/model/basic_event_with_sender.dart'; +import 'basic_event_with_sender.dart'; class StrippedStateEvent extends BasicEventWithSender { String stateKey; diff --git a/lib/src/client.dart b/lib/src/client.dart index 806f9a0..40b2bd7 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -20,22 +20,21 @@ import 'dart:async'; import 'dart:convert'; import 'dart:core'; -import 'package:famedlysdk/encryption.dart'; -import 'package:famedlysdk/famedlysdk.dart'; -import 'package:famedlysdk/src/room.dart'; -import 'package:famedlysdk/src/utils/device_keys_list.dart'; -import 'package:famedlysdk/src/utils/logs.dart'; -import 'package:famedlysdk/src/utils/matrix_file.dart'; -import 'package:famedlysdk/src/utils/to_device_event.dart'; import 'package:http/http.dart' as http; import 'package:pedantic/pedantic.dart'; +import '../encryption.dart'; +import '../famedlysdk.dart'; import 'database/database.dart' show Database; import 'event.dart'; import 'room.dart'; import 'user.dart'; +import 'utils/device_keys_list.dart'; import 'utils/event_update.dart'; +import 'utils/logs.dart'; +import 'utils/matrix_file.dart'; import 'utils/room_update.dart'; +import 'utils/to_device_event.dart'; typedef RoomSorter = int Function(Room a, Room b); diff --git a/lib/src/database/database.dart b/lib/src/database/database.dart index 793fd48..a1ad0cb 100644 --- a/lib/src/database/database.dart +++ b/lib/src/database/database.dart @@ -1,13 +1,13 @@ -import 'package:famedlysdk/src/utils/logs.dart'; -import 'package:moor/moor.dart'; import 'dart:convert'; -import 'package:famedlysdk/famedlysdk.dart' as sdk; -import 'package:famedlysdk/matrix_api.dart' as api; +import 'package:moor/moor.dart'; import 'package:olm/olm.dart' as olm; +import '../../famedlysdk.dart' as sdk; +import '../../matrix_api.dart' as api; import '../../matrix_api.dart'; import '../room.dart'; +import '../utils/logs.dart'; part 'database.g.dart'; diff --git a/lib/src/event.dart b/lib/src/event.dart index 23c53ca..92cafeb 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -18,16 +18,18 @@ import 'dart:convert'; import 'dart:typed_data'; -import 'package:famedlysdk/famedlysdk.dart'; -import 'package:famedlysdk/encryption.dart'; -import 'package:famedlysdk/src/utils/logs.dart'; -import 'package:famedlysdk/src/utils/receipt.dart'; + import 'package:http/http.dart' as http; import 'package:matrix_file_e2ee/matrix_file_e2ee.dart'; + +import '../encryption.dart'; +import '../famedlysdk.dart'; import '../matrix_api.dart'; -import './room.dart'; +import 'database/database.dart' show DbRoomState, DbEvent; +import 'room.dart'; +import 'utils/logs.dart'; import 'utils/matrix_localizations.dart'; -import './database/database.dart' show DbRoomState, DbEvent; +import 'utils/receipt.dart'; abstract class RelationshipTypes { static const String Reply = 'm.in_reply_to'; diff --git a/lib/src/room.dart b/lib/src/room.dart index f8d25c1..943fe89 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -18,23 +18,23 @@ import 'dart:async'; -import 'package:famedlysdk/matrix_api.dart'; -import 'package:famedlysdk/famedlysdk.dart'; -import 'package:famedlysdk/src/client.dart'; -import 'package:famedlysdk/src/event.dart'; -import 'package:famedlysdk/src/utils/event_update.dart'; -import 'package:famedlysdk/src/utils/logs.dart'; -import 'package:famedlysdk/src/utils/room_update.dart'; -import 'package:famedlysdk/src/utils/matrix_file.dart'; -import 'package:matrix_file_e2ee/matrix_file_e2ee.dart'; import 'package:html_unescape/html_unescape.dart'; +import 'package:matrix_file_e2ee/matrix_file_e2ee.dart'; -import './user.dart'; +import '../famedlysdk.dart'; +import '../matrix_api.dart'; +import 'client.dart'; +import 'database/database.dart' show DbRoom; +import 'event.dart'; import 'timeline.dart'; +import 'user.dart'; +import 'utils/event_update.dart'; +import 'utils/logs.dart'; +import 'utils/markdown.dart'; +import 'utils/matrix_file.dart'; import 'utils/matrix_localizations.dart'; +import 'utils/room_update.dart'; import 'utils/states_map.dart'; -import './utils/markdown.dart'; -import './database/database.dart' show DbRoom; enum PushRuleState { notify, mentions_only, dont_notify } enum JoinRules { public, knock, invite, private } diff --git a/lib/src/timeline.dart b/lib/src/timeline.dart index 18fe0bf..4f21dc6 100644 --- a/lib/src/timeline.dart +++ b/lib/src/timeline.dart @@ -18,12 +18,11 @@ import 'dart:async'; -import 'package:famedlysdk/matrix_api.dart'; -import 'package:famedlysdk/src/utils/logs.dart'; - +import '../matrix_api.dart'; import 'event.dart'; import 'room.dart'; import 'utils/event_update.dart'; +import 'utils/logs.dart'; import 'utils/room_update.dart'; typedef onTimelineUpdateCallback = void Function(); diff --git a/lib/src/user.dart b/lib/src/user.dart index ea1ce42..f0d2a88 100644 --- a/lib/src/user.dart +++ b/lib/src/user.dart @@ -16,10 +16,10 @@ * along with this program. If not, see . */ -import 'package:famedlysdk/famedlysdk.dart'; -import 'package:famedlysdk/matrix_api.dart'; -import 'package:famedlysdk/src/room.dart'; -import 'package:famedlysdk/src/event.dart'; +import '../famedlysdk.dart'; +import '../matrix_api.dart'; +import 'event.dart'; +import 'room.dart'; /// Represents a Matrix User which may be a participant in a Matrix Room. class User extends Event { diff --git a/lib/src/utils/device_keys_list.dart b/lib/src/utils/device_keys_list.dart index c8430b7..cc8b79c 100644 --- a/lib/src/utils/device_keys_list.dart +++ b/lib/src/utils/device_keys_list.dart @@ -1,16 +1,16 @@ import 'dart:convert'; + import 'package:canonical_json/canonical_json.dart'; import 'package:olm/olm.dart' as olm; -import 'package:famedlysdk/matrix_api.dart'; -import 'package:famedlysdk/encryption.dart'; - +import '../../encryption.dart'; +import '../../matrix_api.dart'; import '../client.dart'; -import '../user.dart'; -import '../room.dart'; import '../database/database.dart' show DbUserDeviceKey, DbUserDeviceKeysKey, DbUserCrossSigningKey; import '../event.dart'; +import '../room.dart'; +import '../user.dart'; enum UserVerifiedStatus { verified, unknown, unknownDevice } diff --git a/lib/src/utils/matrix_file.dart b/lib/src/utils/matrix_file.dart index f5561bc..3c728f2 100644 --- a/lib/src/utils/matrix_file.dart +++ b/lib/src/utils/matrix_file.dart @@ -1,10 +1,12 @@ /// Workaround until [File] in dart:io and dart:html is unified import 'dart:typed_data'; -import 'package:famedlysdk/matrix_api/model/message_types.dart'; + import 'package:matrix_file_e2ee/matrix_file_e2ee.dart'; import 'package:mime/mime.dart'; +import '../../matrix_api/model/message_types.dart'; + class MatrixFile { Uint8List bytes; String name; diff --git a/lib/src/utils/room_update.dart b/lib/src/utils/room_update.dart index 43f5e22..bdb918f 100644 --- a/lib/src/utils/room_update.dart +++ b/lib/src/utils/room_update.dart @@ -16,7 +16,7 @@ * along with this program. If not, see . */ -import 'package:famedlysdk/matrix_api.dart'; +import '../../matrix_api.dart'; /// Represents a new room or an update for an /// already known room. diff --git a/lib/src/utils/states_map.dart b/lib/src/utils/states_map.dart index 75fbcb8..4af4322 100644 --- a/lib/src/utils/states_map.dart +++ b/lib/src/utils/states_map.dart @@ -1,5 +1,4 @@ -import 'package:famedlysdk/famedlysdk.dart'; - +import '../../famedlysdk.dart'; import '../../matrix_api.dart'; /// Matrix room states are addressed by a tuple of the [type] and an diff --git a/lib/src/utils/sync_update_extension.dart b/lib/src/utils/sync_update_extension.dart index c4b9ecb..a14150f 100644 --- a/lib/src/utils/sync_update_extension.dart +++ b/lib/src/utils/sync_update_extension.dart @@ -16,7 +16,7 @@ * along with this program. If not, see . */ -import 'package:famedlysdk/matrix_api.dart'; +import '../../matrix_api.dart'; /// This extension adds easy-to-use filters for the sync update, meant to be used on the `client.onSync` stream, e.g. /// `client.onSync.stream.where((s) => s.hasRoomUpdate)`. Multiple filters can easily be diff --git a/lib/src/utils/to_device_event.dart b/lib/src/utils/to_device_event.dart index 729124a..96ff91b 100644 --- a/lib/src/utils/to_device_event.dart +++ b/lib/src/utils/to_device_event.dart @@ -1,4 +1,4 @@ -import 'package:famedlysdk/matrix_api.dart'; +import '../../matrix_api.dart'; class ToDeviceEvent extends BasicEventWithSender { Map encryptedContent; diff --git a/lib/src/utils/uri_extension.dart b/lib/src/utils/uri_extension.dart index 804e385..69ad82a 100644 --- a/lib/src/utils/uri_extension.dart +++ b/lib/src/utils/uri_extension.dart @@ -16,9 +16,10 @@ * along with this program. If not, see . */ -import 'package:famedlysdk/src/client.dart'; import 'dart:core'; +import '../client.dart'; + extension MxcUriExtension on Uri { /// Returns a download Link to this content. String getDownloadLink(Client matrix) => isScheme('mxc') From 3d2476cfdb4dd442fde58d32ddedec07005094d4 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Tue, 11 Aug 2020 17:27:11 +0200 Subject: [PATCH 30/90] fix: Have matrix id string extension obay the proper grammar --- lib/src/utils/matrix_id_string_extension.dart | 8 +++++++- test/matrix_id_string_extension_test.dart | 3 ++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/src/utils/matrix_id_string_extension.dart b/lib/src/utils/matrix_id_string_extension.dart index 296a780..685e12e 100644 --- a/lib/src/utils/matrix_id_string_extension.dart +++ b/lib/src/utils/matrix_id_string_extension.dart @@ -9,8 +9,14 @@ extension MatrixIdExtension on String { if (!VALID_SIGILS.contains(substring(0, 1))) { return false; } + // event IDs do not have to have a domain + if (substring(0, 1) == '\$') { + return true; + } + // all other matrix IDs have to have a domain final parts = substring(1).split(':'); - if (parts.length != 2 || parts[0].isEmpty || parts[1].isEmpty) { + // the localpart can be an empty string, e.g. for aliases + if (parts.length != 2 || parts[1].isEmpty) { return false; } return true; diff --git a/test/matrix_id_string_extension_test.dart b/test/matrix_id_string_extension_test.dart index 5cacdd8..0ee37c3 100644 --- a/test/matrix_id_string_extension_test.dart +++ b/test/matrix_id_string_extension_test.dart @@ -29,9 +29,10 @@ void main() { expect('!test:example.com'.isValidMatrixId, true); expect('+test:example.com'.isValidMatrixId, true); expect('\$test:example.com'.isValidMatrixId, true); + expect('\$testevent'.isValidMatrixId, true); expect('test:example.com'.isValidMatrixId, false); expect('@testexample.com'.isValidMatrixId, false); - expect('@:example.com'.isValidMatrixId, false); + expect('@:example.com'.isValidMatrixId, true); expect('@test:'.isValidMatrixId, false); expect(mxId.sigil, '@'); expect('#test:example.com'.sigil, '#'); From a861ceed5f12ec23ec8853f19bc56ba44dff2775 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Fri, 14 Aug 2020 14:45:26 +0200 Subject: [PATCH 31/90] Fix turn server credentials type --- lib/matrix_api/model/turn_server_credentials.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/matrix_api/model/turn_server_credentials.dart b/lib/matrix_api/model/turn_server_credentials.dart index d7c4520..bf350f5 100644 --- a/lib/matrix_api/model/turn_server_credentials.dart +++ b/lib/matrix_api/model/turn_server_credentials.dart @@ -20,7 +20,7 @@ class TurnServerCredentials { String username; String password; List uris; - int ttl; + num ttl; TurnServerCredentials.fromJson(Map json) { username = json['username']; From 61b32e0bd9da041fb8b72bc9878346b81a698d75 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Fri, 14 Aug 2020 18:22:31 +0200 Subject: [PATCH 32/90] Hotfix client --- lib/src/client.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/client.dart b/lib/src/client.dart index 40b2bd7..cddb65e 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -102,7 +102,7 @@ class Client extends MatrixApi { EventTypes.RoomCanonicalAlias, EventTypes.RoomTombstone, ]); - this.httpClient = httpClient; + this.httpClient = httpClient ?? http.Client(); } /// The required name for this client. From a288216e03ce1bb46c101711cfdd1626fae951a1 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Sat, 15 Aug 2020 14:24:44 +0200 Subject: [PATCH 33/90] Add call state localizations --- lib/src/event.dart | 12 ++++++++++++ lib/src/utils/matrix_localizations.dart | 8 ++++++++ test/fake_matrix_localizations.dart | 24 ++++++++++++++++++++++++ test/matrix_default_localizations.dart | 20 ++++++++++++++++++++ 4 files changed, 64 insertions(+) diff --git a/lib/src/event.dart b/lib/src/event.dart index 92cafeb..f813ec8 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -584,6 +584,18 @@ class Event extends MatrixEvent { localizedBody += '. ' + i18n.needPantalaimonWarning; } break; + case EventTypes.CallAnswer: + localizedBody = i18n.answeredTheCall(senderName); + break; + case EventTypes.CallHangup: + localizedBody = i18n.endedTheCall(senderName); + break; + case EventTypes.CallInvite: + localizedBody = i18n.startedACall(senderName); + break; + case EventTypes.CallCandidates: + localizedBody = i18n.sentCallInformations(senderName); + break; case EventTypes.Encrypted: case EventTypes.Message: switch (messageType) { diff --git a/lib/src/utils/matrix_localizations.dart b/lib/src/utils/matrix_localizations.dart index ad70da9..78cc720 100644 --- a/lib/src/utils/matrix_localizations.dart +++ b/lib/src/utils/matrix_localizations.dart @@ -109,6 +109,14 @@ abstract class MatrixLocalizations { String couldNotDecryptMessage(String errorText); String unknownEvent(String typeKey); + + String startedACall(String senderName); + + String endedTheCall(String senderName); + + String answeredTheCall(String senderName); + + String sentCallInformations(String senderName); } extension HistoryVisibilityDisplayString on HistoryVisibility { diff --git a/test/fake_matrix_localizations.dart b/test/fake_matrix_localizations.dart index c5e7046..7342d69 100644 --- a/test/fake_matrix_localizations.dart +++ b/test/fake_matrix_localizations.dart @@ -306,4 +306,28 @@ class FakeMatrixLocalizations extends MatrixLocalizations { @override // TODO: implement you String get you => null; + + @override + String answeredTheCall(String senderName) { + // TODO: implement answeredTheCall + return null; + } + + @override + String endedTheCall(String senderName) { + // TODO: implement endedTheCall + return null; + } + + @override + String sentCallInformations(String senderName) { + // TODO: implement sentCallInformations + return null; + } + + @override + String startedACall(String senderName) { + // TODO: implement startedACall + return null; + } } diff --git a/test/matrix_default_localizations.dart b/test/matrix_default_localizations.dart index 7a2f187..0b5b5cc 100644 --- a/test/matrix_default_localizations.dart +++ b/test/matrix_default_localizations.dart @@ -205,4 +205,24 @@ class MatrixDefaultLocalizations extends MatrixLocalizations { @override String get you => 'You'; + + @override + String answeredTheCall(String senderName) { + return 'answeredTheCall'; + } + + @override + String endedTheCall(String senderName) { + return 'endedTheCall'; + } + + @override + String sentCallInformations(String senderName) { + return 'sentCallInformations'; + } + + @override + String startedACall(String senderName) { + return 'startedACall'; + } } From 215563ab9261ba3e6873625ee36e15724c1100da Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Sat, 15 Aug 2020 15:10:36 +0200 Subject: [PATCH 34/90] Fix wrong call types --- lib/matrix_api/model/event_types.dart | 8 ++++---- lib/src/client.dart | 8 ++++---- test/fake_matrix_api.dart | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/matrix_api/model/event_types.dart b/lib/matrix_api/model/event_types.dart index 25f4585..f08927a 100644 --- a/lib/matrix_api/model/event_types.dart +++ b/lib/matrix_api/model/event_types.dart @@ -36,9 +36,9 @@ abstract class EventTypes { static const String HistoryVisibility = 'm.room.history_visibility'; static const String Encryption = 'm.room.encryption'; static const String Encrypted = 'm.room.encrypted'; - static const String CallInvite = 'm.room.call.invite'; - static const String CallAnswer = 'm.room.call.answer'; - static const String CallCandidates = 'm.room.call.candidates'; - static const String CallHangup = 'm.room.call.hangup'; + static const String CallInvite = 'm.call.invite'; + static const String CallAnswer = 'm.call.answer'; + static const String CallCandidates = 'm.call.candidates'; + static const String CallHangup = 'm.call.hangup'; static const String Unknown = 'm.unknown'; } diff --git a/lib/src/client.dart b/lib/src/client.dart index cddb65e..e625d45 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -981,13 +981,13 @@ class Client extends MatrixApi { } onEvent.add(update); - if (event['type'] == 'm.call.invite') { + if (event['type'] == EventTypes.CallInvite) { onCallInvite.add(Event.fromJson(event, room, sortOrder)); - } else if (event['type'] == 'm.call.hangup') { + } else if (event['type'] == EventTypes.CallHangup) { onCallHangup.add(Event.fromJson(event, room, sortOrder)); - } else if (event['type'] == 'm.call.answer') { + } else if (event['type'] == EventTypes.CallAnswer) { onCallAnswer.add(Event.fromJson(event, room, sortOrder)); - } else if (event['type'] == 'm.call.candidates') { + } else if (event['type'] == EventTypes.CallCandidates) { onCallCandidates.add(Event.fromJson(event, room, sortOrder)); } } diff --git a/test/fake_matrix_api.dart b/test/fake_matrix_api.dart index 2dec135..2400de2 100644 --- a/test/fake_matrix_api.dart +++ b/test/fake_matrix_api.dart @@ -1992,13 +1992,13 @@ class FakeMatrixApi extends MockClient { (var req) => {}, '/client/r0/rooms/!localpart%3Aserver.abc/state/m.room.guest_access': (var req) => {}, - '/client/r0/rooms/!localpart%3Aserver.abc/send/m.room.call.invite/1234': + '/client/r0/rooms/!localpart%3Aserver.abc/send/m.call.invite/1234': (var req) => {}, - '/client/r0/rooms/!localpart%3Aserver.abc/send/m.room.call.answer/1234': + '/client/r0/rooms/!localpart%3Aserver.abc/send/m.call.answer/1234': (var req) => {}, - '/client/r0/rooms/!localpart%3Aserver.abc/send/m.room.call.candidates/1234': + '/client/r0/rooms/!localpart%3Aserver.abc/send/m.call.candidates/1234': (var req) => {}, - '/client/r0/rooms/!localpart%3Aserver.abc/send/m.room.call.hangup/1234': + '/client/r0/rooms/!localpart%3Aserver.abc/send/m.call.hangup/1234': (var req) => {}, '/client/r0/rooms/!1234%3Aexample.com/redact/1143273582443PhrSn%3Aexample.org/1234': (var req) => {'event_id': '1234'}, From 50d97ebeb224dcbfa8e306b8cb50adcaee12d4db Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Sat, 15 Aug 2020 16:05:11 +0200 Subject: [PATCH 35/90] Fix unencrypted call events --- lib/src/client.dart | 19 +++++++------ lib/src/room.dart | 65 +++++++++++++++++++++++++++++++-------------- 2 files changed, 56 insertions(+), 28 deletions(-) diff --git a/lib/src/client.dart b/lib/src/client.dart index e625d45..f234ac8 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -981,14 +981,17 @@ class Client extends MatrixApi { } onEvent.add(update); - if (event['type'] == EventTypes.CallInvite) { - onCallInvite.add(Event.fromJson(event, room, sortOrder)); - } else if (event['type'] == EventTypes.CallHangup) { - onCallHangup.add(Event.fromJson(event, room, sortOrder)); - } else if (event['type'] == EventTypes.CallAnswer) { - onCallAnswer.add(Event.fromJson(event, room, sortOrder)); - } else if (event['type'] == EventTypes.CallCandidates) { - onCallCandidates.add(Event.fromJson(event, room, sortOrder)); + final rawUnencryptedEvent = update.content; + + if (rawUnencryptedEvent['type'] == EventTypes.CallInvite) { + onCallInvite.add(Event.fromJson(rawUnencryptedEvent, room, sortOrder)); + } else if (rawUnencryptedEvent['type'] == EventTypes.CallHangup) { + onCallHangup.add(Event.fromJson(rawUnencryptedEvent, room, sortOrder)); + } else if (rawUnencryptedEvent['type'] == EventTypes.CallAnswer) { + onCallAnswer.add(Event.fromJson(rawUnencryptedEvent, room, sortOrder)); + } else if (rawUnencryptedEvent['type'] == EventTypes.CallCandidates) { + onCallCandidates + .add(Event.fromJson(rawUnencryptedEvent, room, sortOrder)); } } } diff --git a/lib/src/room.dart b/lib/src/room.dart index 943fe89..2ebcfd1 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -1349,16 +1349,22 @@ class Room { {String type = 'offer', int version = 0, String txid}) async { txid ??= 'txid${DateTime.now().millisecondsSinceEpoch}'; + final content = { + 'call_id': callId, + 'lifetime': lifetime, + 'offer': {'sdp': sdp, 'type': type}, + 'version': version, + }; + + final sendMessageContent = encrypted && client.encryptionEnabled + ? await client.encryption.encryptGroupMessagePayload(id, content, + type: EventTypes.CallInvite) + : content; return await client.sendMessage( id, EventTypes.CallInvite, txid, - { - 'call_id': callId, - 'lifetime': lifetime, - 'offer': {'sdp': sdp, 'type': type}, - 'version': version, - }, + sendMessageContent, ); } @@ -1387,15 +1393,21 @@ class Room { String txid, }) async { txid ??= 'txid${DateTime.now().millisecondsSinceEpoch}'; + final content = { + 'call_id': callId, + 'candidates': candidates, + 'version': version, + }; + + final sendMessageContent = encrypted && client.encryptionEnabled + ? await client.encryption.encryptGroupMessagePayload(id, content, + type: EventTypes.CallCandidates) + : content; return await client.sendMessage( id, EventTypes.CallCandidates, txid, - { - 'call_id': callId, - 'candidates': candidates, - 'version': version, - }, + sendMessageContent, ); } @@ -1407,15 +1419,21 @@ class Room { Future answerCall(String callId, String sdp, {String type = 'answer', int version = 0, String txid}) async { txid ??= 'txid${DateTime.now().millisecondsSinceEpoch}'; + final content = { + 'call_id': callId, + 'answer': {'sdp': sdp, 'type': type}, + 'version': version, + }; + + final sendMessageContent = encrypted && client.encryptionEnabled + ? await client.encryption.encryptGroupMessagePayload(id, content, + type: EventTypes.CallAnswer) + : content; return await client.sendMessage( id, EventTypes.CallAnswer, txid, - { - 'call_id': callId, - 'answer': {'sdp': sdp, 'type': type}, - 'version': version, - }, + sendMessageContent, ); } @@ -1425,14 +1443,21 @@ class Room { Future hangupCall(String callId, {int version = 0, String txid}) async { txid ??= 'txid${DateTime.now().millisecondsSinceEpoch}'; + + final content = { + 'call_id': callId, + 'version': version, + }; + + final sendMessageContent = encrypted && client.encryptionEnabled + ? await client.encryption.encryptGroupMessagePayload(id, content, + type: EventTypes.CallHangup) + : content; return await client.sendMessage( id, EventTypes.CallHangup, txid, - { - 'call_id': callId, - 'version': version, - }, + sendMessageContent, ); } From cbc66ea308dfe0cd8ea6a65d42faf22d9c83cd52 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Sat, 15 Aug 2020 18:11:21 +0200 Subject: [PATCH 36/90] Fix unencrypted calls --- lib/src/room.dart | 87 +++++++++++++++++++++------------------------ test/room_test.dart | 22 ++++++------ 2 files changed, 52 insertions(+), 57 deletions(-) diff --git a/lib/src/room.dart b/lib/src/room.dart index 2ebcfd1..74664a8 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -633,10 +633,34 @@ class Room { return uploadResp; } + Future _sendContent( + String type, + Map content, { + String txid, + }) async { + txid ??= client.generateUniqueTransactionId(); + final mustEncrypt = encrypted && client.encryptionEnabled; + final sendMessageContent = mustEncrypt + ? await client.encryption + .encryptGroupMessagePayload(id, content, type: type) + : content; + return await client.sendMessage( + id, + mustEncrypt ? EventTypes.Encrypted : type, + txid, + sendMessageContent, + ); + } + /// Sends an event to this room with this json as a content. Returns the /// event ID generated from the server. - Future sendEvent(Map content, - {String type, String txid, Event inReplyTo, String editEventId}) async { + Future sendEvent( + Map content, { + String type, + String txid, + Event inReplyTo, + String editEventId, + }) async { type = type ?? EventTypes.Message; final sendType = (encrypted && client.encryptionEnabled) ? EventTypes.Encrypted : type; @@ -701,15 +725,10 @@ class Room { // Send the text and on success, store and display a *sent* event. try { - final sendMessageContent = encrypted && client.encryptionEnabled - ? await client.encryption - .encryptGroupMessagePayload(id, content, type: type) - : content; - final res = await client.sendMessage( - id, + final res = await _sendContent( sendType, - messageID, - sendMessageContent, + content, + txid: messageID, ); syncUpdate.rooms.join.values.first.timeline.events.first .unsigned[MessageSendingStatusKey] = 1; @@ -1355,16 +1374,10 @@ class Room { 'offer': {'sdp': sdp, 'type': type}, 'version': version, }; - - final sendMessageContent = encrypted && client.encryptionEnabled - ? await client.encryption.encryptGroupMessagePayload(id, content, - type: EventTypes.CallInvite) - : content; - return await client.sendMessage( - id, + return await _sendContent( EventTypes.CallInvite, - txid, - sendMessageContent, + content, + txid: txid, ); } @@ -1398,16 +1411,10 @@ class Room { 'candidates': candidates, 'version': version, }; - - final sendMessageContent = encrypted && client.encryptionEnabled - ? await client.encryption.encryptGroupMessagePayload(id, content, - type: EventTypes.CallCandidates) - : content; - return await client.sendMessage( - id, + return await _sendContent( EventTypes.CallCandidates, - txid, - sendMessageContent, + content, + txid: txid, ); } @@ -1424,16 +1431,10 @@ class Room { 'answer': {'sdp': sdp, 'type': type}, 'version': version, }; - - final sendMessageContent = encrypted && client.encryptionEnabled - ? await client.encryption.encryptGroupMessagePayload(id, content, - type: EventTypes.CallAnswer) - : content; - return await client.sendMessage( - id, + return await _sendContent( EventTypes.CallAnswer, - txid, - sendMessageContent, + content, + txid: txid, ); } @@ -1448,16 +1449,10 @@ class Room { 'call_id': callId, 'version': version, }; - - final sendMessageContent = encrypted && client.encryptionEnabled - ? await client.encryption.encryptGroupMessagePayload(id, content, - type: EventTypes.CallHangup) - : content; - return await client.sendMessage( - id, + return await _sendContent( EventTypes.CallHangup, - txid, - sendMessageContent, + content, + txid: txid, ); } diff --git a/test/room_test.dart b/test/room_test.dart index 5bad302..4ef6fa2 100644 --- a/test/room_test.dart +++ b/test/room_test.dart @@ -183,10 +183,6 @@ void main() { await room.sendReadReceipt('§1234:fakeServer.notExisting'); }); - test('enableEncryption', () async { - await room.enableEncryption(); - }); - test('requestParticipants', () async { final participants = await room.requestParticipants(); expect(participants.length, 1); @@ -455,6 +451,17 @@ void main() { expect(room.pushRuleState, PushRuleState.dont_notify); }); + test('Test call methods', () async { + await room.inviteToCall('1234', 1234, 'sdp', txid: '1234'); + await room.answerCall('1234', 'sdp', txid: '1234'); + await room.hangupCall('1234', txid: '1234'); + await room.sendCallCandidates('1234', [], txid: '1234'); + }); + + test('enableEncryption', () async { + await room.enableEncryption(); + }); + test('Enable encryption', () async { room.setState( Event( @@ -482,13 +489,6 @@ void main() { await room.setPushRuleState(PushRuleState.notify); }); - test('Test call methods', () async { - await room.inviteToCall('1234', 1234, 'sdp', txid: '1234'); - await room.answerCall('1234', 'sdp', txid: '1234'); - await room.hangupCall('1234', txid: '1234'); - await room.sendCallCandidates('1234', [], txid: '1234'); - }); - test('Test tag methods', () async { await room.addTag(TagType.Favourite, order: 0.1); await room.removeTag(TagType.Favourite); From 20d72eb8d7b53fda0ceb7940191064ba19f99ad8 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Fri, 14 Aug 2020 16:09:20 +0200 Subject: [PATCH 37/90] fix: Event statuses progress and are saved correctly --- lib/src/database/database.dart | 1 + lib/src/timeline.dart | 6 +++++- test/timeline_test.dart | 19 +++++++++++++------ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/lib/src/database/database.dart b/lib/src/database/database.dart index a1ad0cb..a338805 100644 --- a/lib/src/database/database.dart +++ b/lib/src/database/database.dart @@ -424,6 +424,7 @@ class Database extends _$Database { // is there a transaction id? Then delete the event with this id. if (status != -1 && + status != 0 && eventUpdate.content['unsigned'] is Map && eventUpdate.content['unsigned']['transaction_id'] is String) { await removeEvent(clientId, diff --git a/lib/src/timeline.dart b/lib/src/timeline.dart index 4f21dc6..fcca957 100644 --- a/lib/src/timeline.dart +++ b/lib/src/timeline.dart @@ -211,7 +211,11 @@ class Timeline { if (eventUpdate.roomID != room.id) return; if (eventUpdate.type == 'timeline' || eventUpdate.type == 'history') { - var status = eventUpdate.content['status'] ?? 2; + var status = eventUpdate.content['status'] ?? + (eventUpdate.content['unsigned'] is Map + ? eventUpdate.content['unsigned'][MessageSendingStatusKey] + : null) ?? + 2; // Redaction events are handled as modification for existing events. if (eventUpdate.eventType == EventTypes.Redaction) { final eventId = _findEvent(event_id: eventUpdate.content['redacts']); diff --git a/test/timeline_test.dart b/test/timeline_test.dart index 0725d19..5b308f0 100644 --- a/test/timeline_test.dart +++ b/test/timeline_test.dart @@ -371,9 +371,12 @@ void main() { 'type': 'm.room.message', 'content': {'msgtype': 'm.text', 'body': 'Testcase'}, 'sender': '@alice:example.com', - 'status': 0, 'event_id': 'transaction', - 'origin_server_ts': testTimeStamp + 'origin_server_ts': testTimeStamp, + 'unsigned': { + MessageSendingStatusKey: 0, + 'transaction_id': 'transaction', + }, }, sortOrder: room.newSortOrder)); await Future.delayed(Duration(milliseconds: 50)); @@ -387,10 +390,12 @@ void main() { 'type': 'm.room.message', 'content': {'msgtype': 'm.text', 'body': 'Testcase'}, 'sender': '@alice:example.com', - 'status': 2, 'event_id': '\$event', 'origin_server_ts': testTimeStamp, - 'unsigned': {'transaction_id': 'transaction'} + 'unsigned': { + 'transaction_id': 'transaction', + MessageSendingStatusKey: 2, + }, }, sortOrder: room.newSortOrder)); await Future.delayed(Duration(milliseconds: 50)); @@ -404,10 +409,12 @@ void main() { 'type': 'm.room.message', 'content': {'msgtype': 'm.text', 'body': 'Testcase'}, 'sender': '@alice:example.com', - 'status': 1, 'event_id': '\$event', 'origin_server_ts': testTimeStamp, - 'unsigned': {'transaction_id': 'transaction'} + 'unsigned': { + 'transaction_id': 'transaction', + MessageSendingStatusKey: 1, + }, }, sortOrder: room.newSortOrder)); await Future.delayed(Duration(milliseconds: 50)); From ea59c4bd940c7a2cd5ae45d135ce02683cded6b5 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Tue, 11 Aug 2020 13:38:50 +0200 Subject: [PATCH 38/90] refactor(keybackup): Update database for stored megolm keys to prepare for proper online key backup --- lib/encryption/key_manager.dart | 85 +++++++++---- lib/encryption/utils/session_key.dart | 44 ++++++- lib/src/database/database.dart | 11 +- lib/src/database/database.g.dart | 167 ++++++++++++++++++++++++-- lib/src/database/database.moor | 5 +- test/encryption/key_manager_test.dart | 26 +++- test/encryption/key_request_test.dart | 2 +- 7 files changed, 291 insertions(+), 49 deletions(-) diff --git a/lib/encryption/key_manager.dart b/lib/encryption/key_manager.dart index 59efb64..bb39da7 100644 --- a/lib/encryption/key_manager.dart +++ b/lib/encryption/key_manager.dart @@ -71,7 +71,20 @@ class KeyManager { void setInboundGroupSession(String roomId, String sessionId, String senderKey, Map content, - {bool forwarded = false}) { + {bool forwarded = false, Map senderClaimedKeys}) { + senderClaimedKeys ??= {}; + if (!senderClaimedKeys.containsKey('ed25519')) { + DeviceKeys device; + for (final user in client.userDeviceKeys.values) { + device = user.deviceKeys.values.firstWhere( + (e) => e.curve25519Key == senderKey, + orElse: () => null); + if (device != null) { + senderClaimedKeys['ed25519'] = device.ed25519Key; + break; + } + } + } final oldSession = getInboundGroupSession(roomId, sessionId, senderKey, otherRooms: false); if (content['algorithm'] != 'm.megolm.v1.aes-sha2') { @@ -97,6 +110,8 @@ class KeyManager { inboundGroupSession: inboundGroupSession, indexes: {}, key: client.userID, + senderKey: senderKey, + senderClaimedKeys: senderClaimedKeys, ); final oldFirstIndex = oldSession?.inboundGroupSession?.first_known_index() ?? 0; @@ -124,6 +139,8 @@ class KeyManager { inboundGroupSession.pickle(client.userID), json.encode(content), json.encode({}), + senderKey, + json.encode(senderClaimedKeys), ); // Note to self: When adding key-backup that needs to be unawaited(), else // we might accidentally end up with http requests inside of the sync loop @@ -139,7 +156,11 @@ class KeyManager { {bool otherRooms = true}) { if (_inboundGroupSessions.containsKey(roomId) && _inboundGroupSessions[roomId].containsKey(sessionId)) { - return _inboundGroupSessions[roomId][sessionId]; + final sess = _inboundGroupSessions[roomId][sessionId]; + if (sess.senderKey != senderKey && sess.senderKey.isNotEmpty) { + return null; + } + return sess; } if (!otherRooms) { return null; @@ -147,7 +168,11 @@ class KeyManager { // search if this session id is *somehow* found in another room for (final val in _inboundGroupSessions.values) { if (val.containsKey(sessionId)) { - return val[sessionId]; + final sess = val[sessionId]; + if (sess.senderKey != senderKey && sess.senderKey.isNotEmpty) { + return null; + } + return sess; } } return null; @@ -161,7 +186,11 @@ class KeyManager { } if (_inboundGroupSessions.containsKey(roomId) && _inboundGroupSessions[roomId].containsKey(sessionId)) { - return _inboundGroupSessions[roomId][sessionId]; // nothing to do + final sess = _inboundGroupSessions[roomId][sessionId]; + if (sess.senderKey != senderKey && sess.senderKey.isNotEmpty) { + return null; // sender keys do not match....better not do anything + } + return sess; // nothing to do } final session = await client.database ?.getDbInboundGroupSession(client.id, roomId, sessionId); @@ -181,7 +210,8 @@ class KeyManager { _inboundGroupSessions[roomId] = {}; } final sess = SessionKey.fromDb(session, client.userID); - if (!sess.isValid) { + if (!sess.isValid || + (sess.senderKey.isNotEmpty && sess.senderKey != senderKey)) { return null; } _inboundGroupSessions[roomId][sessionId] = sess; @@ -377,7 +407,10 @@ class KeyManager { decrypted['room_id'] = roomId; setInboundGroupSession( roomId, sessionId, decrypted['sender_key'], decrypted, - forwarded: true); + forwarded: true, + senderClaimedKeys: decrypted['sender_claimed_keys'] != null + ? Map.from(decrypted['sender_claimed_keys']) + : null); } } } @@ -556,11 +589,20 @@ class KeyManager { if (device == null) { return; // someone we didn't send our request to replied....better ignore this } + // we add the sender key to the forwarded key chain + if (!(event.content['forwarding_curve25519_key_chain'] is List)) { + event.content['forwarding_curve25519_key_chain'] = []; + } + event.content['forwarding_curve25519_key_chain'] + .add(event.encryptedContent['sender_key']); // TODO: verify that the keys work to decrypt a message // alright, all checks out, let's go ahead and store this session setInboundGroupSession( request.room.id, request.sessionId, request.senderKey, event.content, - forwarded: true); + forwarded: true, + senderClaimedKeys: { + 'ed25519': event.content['sender_claimed_ed25519_key'], + }); request.devices.removeWhere( (k) => k.userId == device.userId && k.deviceId == device.deviceId); outgoingShareRequests.remove(request.requestId); @@ -659,28 +701,19 @@ class RoomKeyRequest extends ToDeviceEvent { var room = this.room; final session = await keyManager.loadInboundGroupSession( room.id, request.sessionId, request.senderKey); - var forwardedKeys = [keyManager.encryption.identityKey]; - for (final key in session.forwardingCurve25519KeyChain) { - forwardedKeys.add(key); - } var message = session.content; - message['forwarding_curve25519_key_chain'] = forwardedKeys; + message['forwarding_curve25519_key_chain'] = + List.from(session.forwardingCurve25519KeyChain); - message['sender_key'] = request.senderKey; + message['sender_key'] = + (session.senderKey != null && session.senderKey.isNotEmpty) + ? session.senderKey + : request.senderKey; message['sender_claimed_ed25519_key'] = - forwardedKeys.isEmpty ? keyManager.encryption.fingerprintKey : null; - if (message['sender_claimed_ed25519_key'] == null) { - for (final value in keyManager.client.userDeviceKeys.values) { - for (final key in value.deviceKeys.values) { - if (key.curve25519Key == forwardedKeys.first) { - message['sender_claimed_ed25519_key'] = key.ed25519Key; - } - } - if (message['sender_claimed_ed25519_key'] != null) { - break; - } - } - } + session.senderClaimedKeys['ed25519'] ?? + (session.forwardingCurve25519KeyChain.isEmpty + ? keyManager.encryption.fingerprintKey + : null); message['session_key'] = session.inboundGroupSession .export_session(session.inboundGroupSession.first_known_index()); // send the actual reply of the key back to the requester diff --git a/lib/encryption/utils/session_key.dart b/lib/encryption/utils/session_key.dart index 523ff7a..0477935 100644 --- a/lib/encryption/utils/session_key.dart +++ b/lib/encryption/utils/session_key.dart @@ -29,23 +29,39 @@ class SessionKey { Map indexes; olm.InboundGroupSession inboundGroupSession; final String key; - List get forwardingCurve25519KeyChain => - content['forwarding_curve25519_key_chain'] ?? []; - String get senderClaimedEd25519Key => - content['sender_claimed_ed25519_key'] ?? ''; - String get senderKey => content['sender_key'] ?? ''; + List get forwardingCurve25519KeyChain => + (content['forwarding_curve25519_key_chain'] != null + ? List.from(content['forwarding_curve25519_key_chain']) + : null) ?? + []; + Map senderClaimedKeys; + String senderKey; bool get isValid => inboundGroupSession != null; - SessionKey({this.content, this.inboundGroupSession, this.key, this.indexes}); + SessionKey( + {this.content, + this.inboundGroupSession, + this.key, + this.indexes, + String senderKey, + Map senderClaimedKeys}) { + _setSenderKey(senderKey); + _setSenderClaimedKeys(senderClaimedKeys); + } SessionKey.fromDb(DbInboundGroupSession dbEntry, String key) : key = key { final parsedContent = Event.getMapFromPayload(dbEntry.content); final parsedIndexes = Event.getMapFromPayload(dbEntry.indexes); + final parsedSenderClaimedKeys = + Event.getMapFromPayload(dbEntry.senderClaimedKeys); content = parsedContent != null ? Map.from(parsedContent) : null; indexes = parsedIndexes != null ? Map.from(parsedIndexes) : {}; + _setSenderKey(dbEntry.senderKey); + _setSenderClaimedKeys(Map.from(parsedSenderClaimedKeys)); + inboundGroupSession = olm.InboundGroupSession(); try { inboundGroupSession.unpickle(key, dbEntry.pickle); @@ -57,6 +73,22 @@ class SessionKey { } } + void _setSenderKey(String key) { + senderKey = key ?? content['sender_key'] ?? ''; + } + + void _setSenderClaimedKeys(Map keys) { + senderClaimedKeys = (keys != null && keys.isNotEmpty) + ? keys + : (content['sender_claimed_keys'] is Map + ? Map.from(content['sender_claimed_keys']) + : (content['sender_claimed_ed25519_key'] is String + ? { + 'ed25519': content['sender_claimed_ed25519_key'] + } + : {})); + } + Map toJson() { final data = {}; if (content != null) { diff --git a/lib/src/database/database.dart b/lib/src/database/database.dart index a338805..af30acc 100644 --- a/lib/src/database/database.dart +++ b/lib/src/database/database.dart @@ -20,7 +20,7 @@ class Database extends _$Database { Database.connect(DatabaseConnection connection) : super.connect(connection); @override - int get schemaVersion => 5; + int get schemaVersion => 6; int get maxFileSize => 1 * 1024 * 1024; @@ -62,6 +62,15 @@ class Database extends _$Database { await m.addColumn(olmSessions, olmSessions.lastReceived); from++; } + if (from == 5) { + await m.addColumn( + inboundGroupSessions, inboundGroupSessions.uploaded); + await m.addColumn( + inboundGroupSessions, inboundGroupSessions.senderKey); + await m.addColumn( + inboundGroupSessions, inboundGroupSessions.senderClaimedKeys); + from++; + } }, beforeOpen: (_) async { if (executor.dialect == SqlDialect.sqlite) { diff --git a/lib/src/database/database.g.dart b/lib/src/database/database.g.dart index 6c3849c..2ee1edb 100644 --- a/lib/src/database/database.g.dart +++ b/lib/src/database/database.g.dart @@ -2062,19 +2062,26 @@ class DbInboundGroupSession extends DataClass final String pickle; final String content; final String indexes; + final bool uploaded; + final String senderKey; + final String senderClaimedKeys; DbInboundGroupSession( {@required this.clientId, @required this.roomId, @required this.sessionId, @required this.pickle, this.content, - this.indexes}); + this.indexes, + this.uploaded, + this.senderKey, + this.senderClaimedKeys}); factory DbInboundGroupSession.fromData( Map data, GeneratedDatabase db, {String prefix}) { final effectivePrefix = prefix ?? ''; final intType = db.typeSystem.forDartType(); final stringType = db.typeSystem.forDartType(); + final boolType = db.typeSystem.forDartType(); return DbInboundGroupSession( clientId: intType.mapFromDatabaseResponse(data['${effectivePrefix}client_id']), @@ -2088,6 +2095,12 @@ class DbInboundGroupSession extends DataClass stringType.mapFromDatabaseResponse(data['${effectivePrefix}content']), indexes: stringType.mapFromDatabaseResponse(data['${effectivePrefix}indexes']), + uploaded: + boolType.mapFromDatabaseResponse(data['${effectivePrefix}uploaded']), + senderKey: stringType + .mapFromDatabaseResponse(data['${effectivePrefix}sender_key']), + senderClaimedKeys: stringType.mapFromDatabaseResponse( + data['${effectivePrefix}sender_claimed_keys']), ); } @override @@ -2111,6 +2124,15 @@ class DbInboundGroupSession extends DataClass if (!nullToAbsent || indexes != null) { map['indexes'] = Variable(indexes); } + if (!nullToAbsent || uploaded != null) { + map['uploaded'] = Variable(uploaded); + } + if (!nullToAbsent || senderKey != null) { + map['sender_key'] = Variable(senderKey); + } + if (!nullToAbsent || senderClaimedKeys != null) { + map['sender_claimed_keys'] = Variable(senderClaimedKeys); + } return map; } @@ -2124,6 +2146,10 @@ class DbInboundGroupSession extends DataClass pickle: serializer.fromJson(json['pickle']), content: serializer.fromJson(json['content']), indexes: serializer.fromJson(json['indexes']), + uploaded: serializer.fromJson(json['uploaded']), + senderKey: serializer.fromJson(json['sender_key']), + senderClaimedKeys: + serializer.fromJson(json['sender_claimed_keys']), ); } @override @@ -2136,6 +2162,9 @@ class DbInboundGroupSession extends DataClass 'pickle': serializer.toJson(pickle), 'content': serializer.toJson(content), 'indexes': serializer.toJson(indexes), + 'uploaded': serializer.toJson(uploaded), + 'sender_key': serializer.toJson(senderKey), + 'sender_claimed_keys': serializer.toJson(senderClaimedKeys), }; } @@ -2145,7 +2174,10 @@ class DbInboundGroupSession extends DataClass String sessionId, String pickle, String content, - String indexes}) => + String indexes, + bool uploaded, + String senderKey, + String senderClaimedKeys}) => DbInboundGroupSession( clientId: clientId ?? this.clientId, roomId: roomId ?? this.roomId, @@ -2153,6 +2185,9 @@ class DbInboundGroupSession extends DataClass pickle: pickle ?? this.pickle, content: content ?? this.content, indexes: indexes ?? this.indexes, + uploaded: uploaded ?? this.uploaded, + senderKey: senderKey ?? this.senderKey, + senderClaimedKeys: senderClaimedKeys ?? this.senderClaimedKeys, ); @override String toString() { @@ -2162,7 +2197,10 @@ class DbInboundGroupSession extends DataClass ..write('sessionId: $sessionId, ') ..write('pickle: $pickle, ') ..write('content: $content, ') - ..write('indexes: $indexes') + ..write('indexes: $indexes, ') + ..write('uploaded: $uploaded, ') + ..write('senderKey: $senderKey, ') + ..write('senderClaimedKeys: $senderClaimedKeys') ..write(')')) .toString(); } @@ -2174,8 +2212,16 @@ class DbInboundGroupSession extends DataClass roomId.hashCode, $mrjc( sessionId.hashCode, - $mrjc(pickle.hashCode, - $mrjc(content.hashCode, indexes.hashCode)))))); + $mrjc( + pickle.hashCode, + $mrjc( + content.hashCode, + $mrjc( + indexes.hashCode, + $mrjc( + uploaded.hashCode, + $mrjc(senderKey.hashCode, + senderClaimedKeys.hashCode))))))))); @override bool operator ==(dynamic other) => identical(this, other) || @@ -2185,7 +2231,10 @@ class DbInboundGroupSession extends DataClass other.sessionId == this.sessionId && other.pickle == this.pickle && other.content == this.content && - other.indexes == this.indexes); + other.indexes == this.indexes && + other.uploaded == this.uploaded && + other.senderKey == this.senderKey && + other.senderClaimedKeys == this.senderClaimedKeys); } class InboundGroupSessionsCompanion @@ -2196,6 +2245,9 @@ class InboundGroupSessionsCompanion final Value pickle; final Value content; final Value indexes; + final Value uploaded; + final Value senderKey; + final Value senderClaimedKeys; const InboundGroupSessionsCompanion({ this.clientId = const Value.absent(), this.roomId = const Value.absent(), @@ -2203,6 +2255,9 @@ class InboundGroupSessionsCompanion this.pickle = const Value.absent(), this.content = const Value.absent(), this.indexes = const Value.absent(), + this.uploaded = const Value.absent(), + this.senderKey = const Value.absent(), + this.senderClaimedKeys = const Value.absent(), }); InboundGroupSessionsCompanion.insert({ @required int clientId, @@ -2211,6 +2266,9 @@ class InboundGroupSessionsCompanion @required String pickle, this.content = const Value.absent(), this.indexes = const Value.absent(), + this.uploaded = const Value.absent(), + this.senderKey = const Value.absent(), + this.senderClaimedKeys = const Value.absent(), }) : clientId = Value(clientId), roomId = Value(roomId), sessionId = Value(sessionId), @@ -2222,6 +2280,9 @@ class InboundGroupSessionsCompanion Expression pickle, Expression content, Expression indexes, + Expression uploaded, + Expression senderKey, + Expression senderClaimedKeys, }) { return RawValuesInsertable({ if (clientId != null) 'client_id': clientId, @@ -2230,6 +2291,9 @@ class InboundGroupSessionsCompanion if (pickle != null) 'pickle': pickle, if (content != null) 'content': content, if (indexes != null) 'indexes': indexes, + if (uploaded != null) 'uploaded': uploaded, + if (senderKey != null) 'sender_key': senderKey, + if (senderClaimedKeys != null) 'sender_claimed_keys': senderClaimedKeys, }); } @@ -2239,7 +2303,10 @@ class InboundGroupSessionsCompanion Value sessionId, Value pickle, Value content, - Value indexes}) { + Value indexes, + Value uploaded, + Value senderKey, + Value senderClaimedKeys}) { return InboundGroupSessionsCompanion( clientId: clientId ?? this.clientId, roomId: roomId ?? this.roomId, @@ -2247,6 +2314,9 @@ class InboundGroupSessionsCompanion pickle: pickle ?? this.pickle, content: content ?? this.content, indexes: indexes ?? this.indexes, + uploaded: uploaded ?? this.uploaded, + senderKey: senderKey ?? this.senderKey, + senderClaimedKeys: senderClaimedKeys ?? this.senderClaimedKeys, ); } @@ -2271,6 +2341,15 @@ class InboundGroupSessionsCompanion if (indexes.present) { map['indexes'] = Variable(indexes.value); } + if (uploaded.present) { + map['uploaded'] = Variable(uploaded.value); + } + if (senderKey.present) { + map['sender_key'] = Variable(senderKey.value); + } + if (senderClaimedKeys.present) { + map['sender_claimed_keys'] = Variable(senderClaimedKeys.value); + } return map; } } @@ -2328,9 +2407,45 @@ class InboundGroupSessions extends Table $customConstraints: ''); } + final VerificationMeta _uploadedMeta = const VerificationMeta('uploaded'); + GeneratedBoolColumn _uploaded; + GeneratedBoolColumn get uploaded => _uploaded ??= _constructUploaded(); + GeneratedBoolColumn _constructUploaded() { + return GeneratedBoolColumn('uploaded', $tableName, true, + $customConstraints: 'DEFAULT false', + defaultValue: const CustomExpression('false')); + } + + final VerificationMeta _senderKeyMeta = const VerificationMeta('senderKey'); + GeneratedTextColumn _senderKey; + GeneratedTextColumn get senderKey => _senderKey ??= _constructSenderKey(); + GeneratedTextColumn _constructSenderKey() { + return GeneratedTextColumn('sender_key', $tableName, true, + $customConstraints: ''); + } + + final VerificationMeta _senderClaimedKeysMeta = + const VerificationMeta('senderClaimedKeys'); + GeneratedTextColumn _senderClaimedKeys; + GeneratedTextColumn get senderClaimedKeys => + _senderClaimedKeys ??= _constructSenderClaimedKeys(); + GeneratedTextColumn _constructSenderClaimedKeys() { + return GeneratedTextColumn('sender_claimed_keys', $tableName, true, + $customConstraints: ''); + } + @override - List get $columns => - [clientId, roomId, sessionId, pickle, content, indexes]; + List get $columns => [ + clientId, + roomId, + sessionId, + pickle, + content, + indexes, + uploaded, + senderKey, + senderClaimedKeys + ]; @override InboundGroupSessions get asDslTable => this; @override @@ -2375,6 +2490,20 @@ class InboundGroupSessions extends Table context.handle(_indexesMeta, indexes.isAcceptableOrUnknown(data['indexes'], _indexesMeta)); } + if (data.containsKey('uploaded')) { + context.handle(_uploadedMeta, + uploaded.isAcceptableOrUnknown(data['uploaded'], _uploadedMeta)); + } + if (data.containsKey('sender_key')) { + context.handle(_senderKeyMeta, + senderKey.isAcceptableOrUnknown(data['sender_key'], _senderKeyMeta)); + } + if (data.containsKey('sender_claimed_keys')) { + context.handle( + _senderClaimedKeysMeta, + senderClaimedKeys.isAcceptableOrUnknown( + data['sender_claimed_keys'], _senderClaimedKeysMeta)); + } return context; } @@ -5669,6 +5798,9 @@ abstract class _$Database extends GeneratedDatabase { pickle: row.readString('pickle'), content: row.readString('content'), indexes: row.readString('indexes'), + uploaded: row.readBool('uploaded'), + senderKey: row.readString('sender_key'), + senderClaimedKeys: row.readString('sender_claimed_keys'), ); } @@ -5701,17 +5833,26 @@ abstract class _$Database extends GeneratedDatabase { readsFrom: {inboundGroupSessions}).map(_rowToDbInboundGroupSession); } - Future storeInboundGroupSession(int client_id, String room_id, - String session_id, String pickle, String content, String indexes) { + Future storeInboundGroupSession( + int client_id, + String room_id, + String session_id, + String pickle, + String content, + String indexes, + String sender_key, + String sender_claimed_keys) { return customInsert( - 'INSERT OR REPLACE INTO inbound_group_sessions (client_id, room_id, session_id, pickle, content, indexes) VALUES (:client_id, :room_id, :session_id, :pickle, :content, :indexes)', + 'INSERT OR REPLACE INTO inbound_group_sessions (client_id, room_id, session_id, pickle, content, indexes, sender_key, sender_claimed_keys) VALUES (:client_id, :room_id, :session_id, :pickle, :content, :indexes, :sender_key, :sender_claimed_keys)', variables: [ Variable.withInt(client_id), Variable.withString(room_id), Variable.withString(session_id), Variable.withString(pickle), Variable.withString(content), - Variable.withString(indexes) + Variable.withString(indexes), + Variable.withString(sender_key), + Variable.withString(sender_claimed_keys) ], updates: {inboundGroupSessions}, ); diff --git a/lib/src/database/database.moor b/lib/src/database/database.moor index dbcb632..13de47e 100644 --- a/lib/src/database/database.moor +++ b/lib/src/database/database.moor @@ -71,6 +71,9 @@ CREATE TABLE inbound_group_sessions ( pickle TEXT NOT NULL, content TEXT, indexes TEXT, + uploaded BOOLEAN DEFAULT false, + sender_key TEXT, + sender_claimed_keys TEXT, UNIQUE(client_id, room_id, session_id) ) AS DbInboundGroupSession; CREATE INDEX inbound_group_sessions_index ON inbound_group_sessions(client_id); @@ -186,7 +189,7 @@ removeOutboundGroupSession: DELETE FROM outbound_group_sessions WHERE client_id dbGetInboundGroupSessionKey: SELECT * FROM inbound_group_sessions WHERE client_id = :client_id AND room_id = :room_id AND session_id = :session_id; dbGetInboundGroupSessionKeys: SELECT * FROM inbound_group_sessions WHERE client_id = :client_id AND room_id = :room_id; getAllInboundGroupSessions: SELECT * FROM inbound_group_sessions WHERE client_id = :client_id; -storeInboundGroupSession: INSERT OR REPLACE INTO inbound_group_sessions (client_id, room_id, session_id, pickle, content, indexes) VALUES (:client_id, :room_id, :session_id, :pickle, :content, :indexes); +storeInboundGroupSession: INSERT OR REPLACE INTO inbound_group_sessions (client_id, room_id, session_id, pickle, content, indexes, sender_key, sender_claimed_keys) VALUES (:client_id, :room_id, :session_id, :pickle, :content, :indexes, :sender_key, :sender_claimed_keys); updateInboundGroupSessionIndexes: UPDATE inbound_group_sessions SET indexes = :indexes WHERE client_id = :client_id AND room_id = :room_id AND session_id = :session_id; storeUserDeviceKeysInfo: INSERT OR REPLACE INTO user_device_keys (client_id, user_id, outdated) VALUES (:client_id, :user_id, :outdated); setVerifiedUserDeviceKey: UPDATE user_device_keys_key SET verified = :verified WHERE client_id = :client_id AND user_id = :user_id AND device_id = :device_id; diff --git a/test/encryption/key_manager_test.dart b/test/encryption/key_manager_test.dart index bdd0304..1dea9c4 100644 --- a/test/encryption/key_manager_test.dart +++ b/test/encryption/key_manager_test.dart @@ -60,7 +60,7 @@ void main() { 'session_key': sessionKey, }, encryptedContent: { - 'sender_key': validSessionId, + 'sender_key': validSenderKey, }); await client.encryption.keyManager.handleToDeviceEvent(event); expect( @@ -185,6 +185,11 @@ void main() { .getInboundGroupSession(roomId, sessionId, senderKey) != null, true); + expect( + client.encryption.keyManager + .getInboundGroupSession(roomId, sessionId, 'invalid') != + null, + false); expect( client.encryption.keyManager @@ -196,6 +201,11 @@ void main() { .getInboundGroupSession('otherroom', sessionId, senderKey) != null, true); + expect( + client.encryption.keyManager + .getInboundGroupSession('otherroom', sessionId, 'invalid') != + null, + false); expect( client.encryption.keyManager .getInboundGroupSession('otherroom', 'invalid', senderKey) != @@ -215,6 +225,20 @@ void main() { .getInboundGroupSession(roomId, sessionId, senderKey) != null, true); + + client.encryption.keyManager.clearInboundGroupSessions(); + expect( + client.encryption.keyManager + .getInboundGroupSession(roomId, sessionId, senderKey) != + null, + false); + await client.encryption.keyManager + .loadInboundGroupSession(roomId, sessionId, 'invalid'); + expect( + client.encryption.keyManager + .getInboundGroupSession(roomId, sessionId, 'invalid') != + null, + false); }); test('setInboundGroupSession', () async { diff --git a/test/encryption/key_request_test.dart b/test/encryption/key_request_test.dart index b780d2b..b6747e3 100644 --- a/test/encryption/key_request_test.dart +++ b/test/encryption/key_request_test.dart @@ -53,7 +53,7 @@ void main() { if (!olmEnabled) return; final validSessionId = 'ciM/JWTPrmiWPPZNkRLDPQYf9AW/I46bxyLSr+Bx5oU'; - final validSenderKey = '3C5BFWi2Y8MaVvjM8M22DBmh24PmgR0nPvJOIArzgyI'; + final validSenderKey = 'JBG7ZaPn54OBC7TuIEiylW3BZ+7WcGQhFBPB9pogbAg'; test('Create Request', () async { var matrix = await getClient(); final requestRoom = matrix.getRoomById('!726s6s6q:example.com'); From 84c27129d2b256b041674e18ab8f58f707340996 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Mon, 17 Aug 2020 16:25:57 +0200 Subject: [PATCH 39/90] Hotfix: Send correct message type --- lib/src/room.dart | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/src/room.dart b/lib/src/room.dart index 74664a8..54f389b 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -662,8 +662,6 @@ class Room { String editEventId, }) async { type = type ?? EventTypes.Message; - final sendType = - (encrypted && client.encryptionEnabled) ? EventTypes.Encrypted : type; // Create new transaction id String messageID; @@ -726,7 +724,7 @@ class Room { // Send the text and on success, store and display a *sent* event. try { final res = await _sendContent( - sendType, + type, content, txid: messageID, ); From 631b28eab20132bd66ef0aa042197c1db549af3d Mon Sep 17 00:00:00 2001 From: Sorunome Date: Tue, 18 Aug 2020 14:00:42 +0200 Subject: [PATCH 40/90] fix: Migrations don't fail anymore if they were partly completed --- lib/src/database/database.dart | 67 +++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 17 deletions(-) diff --git a/lib/src/database/database.dart b/lib/src/database/database.dart index af30acc..9ac84df 100644 --- a/lib/src/database/database.dart +++ b/lib/src/database/database.dart @@ -11,6 +11,39 @@ import '../utils/logs.dart'; part 'database.g.dart'; +extension MigratorExtension on Migrator { + Future createIndexIfNotExists(Index index) async { + try { + await createIndex(index); + } catch (err) { + if (!err.toString().toLowerCase().contains('already exists')) { + rethrow; + } + } + } + + Future createTableIfNotExists(TableInfo table) async { + try { + await createTable(table); + } catch (err) { + if (!err.toString().toLowerCase().contains('already exists')) { + rethrow; + } + } + } + + Future addColumnIfNotExists( + TableInfo table, GeneratedColumn column) async { + try { + await addColumn(table, column); + } catch (err) { + if (!err.toString().toLowerCase().contains('duplicate column name')) { + rethrow; + } + } + } +} + @UseMoor( include: {'database.moor'}, ) @@ -32,17 +65,17 @@ class Database extends _$Database { onUpgrade: (Migrator m, int from, int to) async { // this appears to be only called once, so multiple consecutive upgrades have to be handled appropriately in here if (from == 1) { - await m.createIndex(userDeviceKeysIndex); - await m.createIndex(userDeviceKeysKeyIndex); - await m.createIndex(olmSessionsIndex); - await m.createIndex(outboundGroupSessionsIndex); - await m.createIndex(inboundGroupSessionsIndex); - await m.createIndex(roomsIndex); - await m.createIndex(eventsIndex); - await m.createIndex(roomStatesIndex); - await m.createIndex(accountDataIndex); - await m.createIndex(roomAccountDataIndex); - await m.createIndex(presencesIndex); + await m.createIndexIfNotExists(userDeviceKeysIndex); + await m.createIndexIfNotExists(userDeviceKeysKeyIndex); + await m.createIndexIfNotExists(olmSessionsIndex); + await m.createIndexIfNotExists(outboundGroupSessionsIndex); + await m.createIndexIfNotExists(inboundGroupSessionsIndex); + await m.createIndexIfNotExists(roomsIndex); + await m.createIndexIfNotExists(eventsIndex); + await m.createIndexIfNotExists(roomStatesIndex); + await m.createIndexIfNotExists(accountDataIndex); + await m.createIndexIfNotExists(roomAccountDataIndex); + await m.createIndexIfNotExists(presencesIndex); from++; } if (from == 2) { @@ -51,23 +84,23 @@ class Database extends _$Database { from++; } if (from == 3) { - await m.createTable(userCrossSigningKeys); - await m.createTable(ssssCache); + await m.createTableIfNotExists(userCrossSigningKeys); + await m.createTableIfNotExists(ssssCache); // mark all keys as outdated so that the cross signing keys will be fetched await m.issueCustomQuery( 'UPDATE user_device_keys SET outdated = true'); from++; } if (from == 4) { - await m.addColumn(olmSessions, olmSessions.lastReceived); + await m.addColumnIfNotExists(olmSessions, olmSessions.lastReceived); from++; } if (from == 5) { - await m.addColumn( + await m.addColumnIfNotExists( inboundGroupSessions, inboundGroupSessions.uploaded); - await m.addColumn( + await m.addColumnIfNotExists( inboundGroupSessions, inboundGroupSessions.senderKey); - await m.addColumn( + await m.addColumnIfNotExists( inboundGroupSessions, inboundGroupSessions.senderClaimedKeys); from++; } From 0d159c2db466b8087e14465a5741277c2ea92cfd Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Tue, 18 Aug 2020 10:52:18 +0200 Subject: [PATCH 41/90] Fix: Send messages in web delay --- lib/src/room.dart | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/src/room.dart b/lib/src/room.dart index 54f389b..fd9999c 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -719,7 +719,7 @@ class Room { 'transaction_id': messageID, }, ])))); - await client.handleSync(syncUpdate); + await _handleFakeSync(syncUpdate); // Send the text and on success, store and display a *sent* event. try { @@ -731,14 +731,15 @@ class Room { syncUpdate.rooms.join.values.first.timeline.events.first .unsigned[MessageSendingStatusKey] = 1; syncUpdate.rooms.join.values.first.timeline.events.first.eventId = res; - await client.handleSync(syncUpdate); + await _handleFakeSync(syncUpdate); + return res; } catch (e, s) { Logs.warning( '[Client] Problem while sending message: ' + e.toString(), s); syncUpdate.rooms.join.values.first.timeline.events.first .unsigned[MessageSendingStatusKey] = -1; - await client.handleSync(syncUpdate); + await _handleFakeSync(syncUpdate); } return null; } @@ -1588,4 +1589,15 @@ class Room { } await client.encryption.keyManager.request(this, sessionId, senderKey); } + + Future _handleFakeSync(SyncUpdate syncUpdate, + {bool sortAtTheEnd = false}) async { + if (client.database != null) { + await client.database.transaction(() async { + await client.handleSync(syncUpdate, sortAtTheEnd: sortAtTheEnd); + }); + } else { + await client.handleSync(syncUpdate, sortAtTheEnd: sortAtTheEnd); + } + } } From 09ffa0940484862139c3fe431d4c3ef3392f2232 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Tue, 18 Aug 2020 10:07:47 +0200 Subject: [PATCH 42/90] Ignore old webrtc invites --- lib/src/client.dart | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/src/client.dart b/lib/src/client.dart index f234ac8..73f495a 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -983,15 +983,20 @@ class Client extends MatrixApi { final rawUnencryptedEvent = update.content; - if (rawUnencryptedEvent['type'] == EventTypes.CallInvite) { - onCallInvite.add(Event.fromJson(rawUnencryptedEvent, room, sortOrder)); - } else if (rawUnencryptedEvent['type'] == EventTypes.CallHangup) { - onCallHangup.add(Event.fromJson(rawUnencryptedEvent, room, sortOrder)); - } else if (rawUnencryptedEvent['type'] == EventTypes.CallAnswer) { - onCallAnswer.add(Event.fromJson(rawUnencryptedEvent, room, sortOrder)); - } else if (rawUnencryptedEvent['type'] == EventTypes.CallCandidates) { - onCallCandidates - .add(Event.fromJson(rawUnencryptedEvent, room, sortOrder)); + if (prevBatch != null && type == 'timeline') { + if (rawUnencryptedEvent['type'] == EventTypes.CallInvite) { + onCallInvite + .add(Event.fromJson(rawUnencryptedEvent, room, sortOrder)); + } else if (rawUnencryptedEvent['type'] == EventTypes.CallHangup) { + onCallHangup + .add(Event.fromJson(rawUnencryptedEvent, room, sortOrder)); + } else if (rawUnencryptedEvent['type'] == EventTypes.CallAnswer) { + onCallAnswer + .add(Event.fromJson(rawUnencryptedEvent, room, sortOrder)); + } else if (rawUnencryptedEvent['type'] == EventTypes.CallCandidates) { + onCallCandidates + .add(Event.fromJson(rawUnencryptedEvent, room, sortOrder)); + } } } } From d6b97b8e787f7aa8e68c9a041ab6a4f1061395a9 Mon Sep 17 00:00:00 2001 From: Lukas Lihotzki Date: Fri, 21 Aug 2020 17:20:26 +0200 Subject: [PATCH 43/90] feat: safe dispose while _sync --- lib/src/client.dart | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/src/client.dart b/lib/src/client.dart index 73f495a..e5623c9 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -675,18 +675,19 @@ class Client extends MatrixApi { _lastSyncError = e; return null; }); - if (_disposed) return; final hash = _syncRequest.hashCode; final syncResp = await _syncRequest; + if (_disposed) return; if (syncResp == null) throw _lastSyncError; if (hash != _syncRequest.hashCode) return; if (database != null) { - await database.transaction(() async { + _currentTransaction = database.transaction(() async { await handleSync(syncResp); if (prevBatch != syncResp.nextBatch) { await database.storePrevBatch(syncResp.nextBatch, id); } }); + await _currentTransaction; } else { await handleSync(syncResp); } @@ -1445,11 +1446,17 @@ class Client extends MatrixApi { } bool _disposed = false; + Future _currentTransaction = Future.sync(() => {}); /// Stops the synchronization and closes the database. After this /// you can safely make this Client instance null. Future dispose({bool closeDatabase = false}) async { _disposed = true; + try { + await _currentTransaction; + } catch (_) { + // No-OP + } if (closeDatabase) await database?.close(); database = null; return; From c46f4ba066d8d195d8ac6bacd9294882d115a9c8 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Fri, 21 Aug 2020 07:56:39 +0200 Subject: [PATCH 44/90] refactor: timeline --- lib/src/timeline.dart | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/lib/src/timeline.dart b/lib/src/timeline.dart index fcca957..df7e091 100644 --- a/lib/src/timeline.dart +++ b/lib/src/timeline.dart @@ -238,13 +238,10 @@ class Timeline { : null); if (i < events.length) { - // we want to preserve the old sort order - final tempSortOrder = events[i].sortOrder; // if the old status is larger than the new one, we also want to preserve the old status final oldStatus = events[i].status; events[i] = Event.fromJson( eventUpdate.content, room, eventUpdate.sortOrder); - events[i].sortOrder = tempSortOrder; // do we preserve the status? we should allow 0 -> -1 updates and status increases if (status < oldStatus && !(status == -1 && oldStatus == 0)) { events[i].status = oldStatus; @@ -265,23 +262,19 @@ class Timeline { } } } - sortAndUpdate(); + _sort(); + if (onUpdate != null) onUpdate(); } catch (e, s) { Logs.warning('Handle event update failed: ${e.toString()}', s); } } - bool sortLock = false; + bool _sortLock = false; - void sort() { - if (sortLock || events.length < 2) return; - sortLock = true; + void _sort() { + if (_sortLock || events.length < 2) return; + _sortLock = true; events?.sort((a, b) => b.sortOrder - a.sortOrder > 0 ? 1 : -1); - sortLock = false; - } - - void sortAndUpdate() async { - sort(); - if (onUpdate != null) onUpdate(); + _sortLock = false; } } From 35e48f9641fa16559000f14ec90b09f357293a22 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Wed, 19 Aug 2020 11:02:00 +0200 Subject: [PATCH 45/90] Fix: prev_content error message --- lib/src/event.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/src/event.dart b/lib/src/event.dart index f813ec8..97a5391 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -105,8 +105,8 @@ class Event extends MatrixEvent { try { this.prevContent = (prevContent != null && prevContent.isNotEmpty) ? prevContent - : (unsigned != null && unsigned['prev_content'] is Map) - ? unsigned['prev_content'] + : (this.unsigned != null && this.unsigned['prev_content'] is Map) + ? this.unsigned['prev_content'] : null; } catch (e, s) { Logs.error('Event constructor crashed: ${e.toString()}', s); From 6fbee4ee0586ade6efdaaf1814261685afe45a46 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Fri, 21 Aug 2020 11:02:20 +0200 Subject: [PATCH 46/90] test: Integrate E2EE tests --- .gitlab-ci.yml | 32 +- lib/src/client.dart | 15 + test_driver.sh | 2 + test_driver/famedlysdk_test.dart | 572 +++++++++++++++---------------- test_driver/test_config.dart | 6 + 5 files changed, 323 insertions(+), 304 deletions(-) create mode 100644 test_driver.sh create mode 100644 test_driver/test_config.dart diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 33c4799..5285d35 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,6 +46,30 @@ coverage_without_olm: - chmod +x ./test.sh - pub get - pub run test + +e2ee_test: + tags: + - linux + stage: coverage + image: debian:testing + dependencies: [] + script: + - apt update + - apt install -y curl gnupg2 git + - curl https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - + - curl https://storage.googleapis.com/download.dartlang.org/linux/debian/dart_stable.list > /etc/apt/sources.list.d/dart_stable.list + - apt update + - apt install -y dart chromium lcov libolm3 sqlite3 libsqlite3-dev + - ln -s /usr/lib/dart/bin/pub /usr/bin/ + - useradd -m test + - chown -R 'test:' '.' + - chmod +x ./prepare.sh + - chmod +x ./test_driver.sh + - printf "abstract class TestUser {\n static const String homeserver = '$TEST_HOMESERVER';\n static const String username = '$TEST_USER1';\n static const String username2 = '$TEST_USER2';\n static const String password = '$TEST_USER_PASSWORD';\n}" > ./test_driver/test_config.dart + - su -c ./prepare.sh test + - su -c ./test_driver.sh test + timeout: 16m + resource_group: e2ee_test code_analyze: tags: @@ -57,7 +81,7 @@ code_analyze: - flutter format lib/ test/ test_driver/ --set-exit-if-changed - flutter analyze -build-api-doc: +build_api_doc: tags: - docker stage: builddocs @@ -70,7 +94,7 @@ build-api-doc: only: - main -build-doc: +build_doc: tags: - docker stage: builddocs @@ -95,8 +119,8 @@ pages: - mv doc-public ./home/doc - mv home public dependencies: - - build-api-doc - - build-doc + - build_api_doc + - build_doc artifacts: paths: - public diff --git a/lib/src/client.dart b/lib/src/client.dart index e5623c9..a01173c 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -358,6 +358,20 @@ class Client extends MatrixApi { } } + /// Sends a logout command to the homeserver and clears all local data, + /// including all persistent data from the store. + @override + Future logoutAll() async { + try { + await super.logoutAll(); + } catch (e, s) { + Logs.error(e, s); + rethrow; + } finally { + await clear(); + } + } + /// Returns the user's own displayname and avatar url. In Matrix it is possible that /// one user can have different displaynames and avatar urls in different rooms. So /// this endpoint first checks if the profile is the same in all rooms. If not, the @@ -1177,6 +1191,7 @@ class Client extends MatrixApi { if (outdatedLists.isNotEmpty) { // Request the missing device key lists from the server. + if (!isLogged()) return; final response = await requestDeviceKeys(outdatedLists, timeout: 10000); for (final rawDeviceKeyListEntry in response.deviceKeys.entries) { diff --git a/test_driver.sh b/test_driver.sh new file mode 100644 index 0000000..30f0150 --- /dev/null +++ b/test_driver.sh @@ -0,0 +1,2 @@ +#!/bin/sh -e +pub run test_driver/famedlysdk_test.dart -p vm \ No newline at end of file diff --git a/test_driver/famedlysdk_test.dart b/test_driver/famedlysdk_test.dart index f2f3988..1c19612 100644 --- a/test_driver/famedlysdk_test.dart +++ b/test_driver/famedlysdk_test.dart @@ -2,14 +2,10 @@ import 'package:famedlysdk/famedlysdk.dart'; import 'package:famedlysdk/matrix_api.dart'; import 'package:famedlysdk/src/utils/logs.dart'; import '../test/fake_database.dart'; +import 'test_config.dart'; +import 'package:olm/olm.dart' as olm; void main() => test(); - -const String homeserver = 'https://matrix.test.famedly.de'; -const String testUserA = '@tick:test.famedly.de'; -const String testPasswordA = 'test'; -const String testUserB = '@trick:test.famedly.de'; -const String testPasswordB = 'test'; const String testMessage = 'Hello world'; const String testMessage2 = 'Hello moon'; const String testMessage3 = 'Hello sun'; @@ -18,188 +14,198 @@ const String testMessage5 = 'Hello earth'; const String testMessage6 = 'Hello mars'; void test() async { - Logs.success('++++ Login $testUserA ++++'); - var testClientA = Client('TestClientA'); - testClientA.database = getDatabase(); - await testClientA.checkServer(homeserver); - await testClientA.login(user: testUserA, password: testPasswordA); - assert(testClientA.encryptionEnabled); + Client testClientA, testClientB; - Logs.success('++++ Login $testUserB ++++'); - var testClientB = Client('TestClientB'); - testClientB.database = getDatabase(); - await testClientB.checkServer(homeserver); - await testClientB.login(user: testUserB, password: testPasswordA); - assert(testClientB.encryptionEnabled); + try { + await olm.init(); + olm.Account(); + Logs.success('[LibOlm] Enabled'); - Logs.success('++++ ($testUserA) Leave all rooms ++++'); - while (testClientA.rooms.isNotEmpty) { - var room = testClientA.rooms.first; - if (room.canonicalAlias?.isNotEmpty ?? false) { - break; - } - try { - await room.leave(); - await room.forget(); - } catch (_) {} - } + Logs.success('++++ Login Alice at ++++'); + testClientA = Client('TestClientA'); + testClientA.database = getDatabase(); + await testClientA.checkServer(TestUser.homeserver); + await testClientA.login( + user: TestUser.username, password: TestUser.password); + assert(testClientA.encryptionEnabled); - Logs.success('++++ ($testUserB) Leave all rooms ++++'); - for (var i = 0; i < 3; i++) { - if (testClientB.rooms.isNotEmpty) { - var room = testClientB.rooms.first; + Logs.success('++++ Login Bob ++++'); + testClientB = Client('TestClientB'); + testClientB.database = getDatabase(); + await testClientB.checkServer(TestUser.homeserver); + await testClientB.login( + user: TestUser.username2, password: TestUser.password); + assert(testClientB.encryptionEnabled); + + Logs.success('++++ (Alice) Leave all rooms ++++'); + while (testClientA.rooms.isNotEmpty) { + var room = testClientA.rooms.first; + if (room.canonicalAlias?.isNotEmpty ?? false) { + break; + } try { await room.leave(); await room.forget(); } catch (_) {} } - } - Logs.success('++++ Check if own olm device is verified by default ++++'); - assert(testClientA.userDeviceKeys.containsKey(testUserA)); - assert(testClientA.userDeviceKeys[testUserA].deviceKeys - .containsKey(testClientA.deviceID)); - assert(testClientA - .userDeviceKeys[testUserA].deviceKeys[testClientA.deviceID].verified); - assert(!testClientA - .userDeviceKeys[testUserA].deviceKeys[testClientA.deviceID].blocked); - assert(testClientB.userDeviceKeys.containsKey(testUserB)); - assert(testClientB.userDeviceKeys[testUserB].deviceKeys - .containsKey(testClientB.deviceID)); - assert(testClientB - .userDeviceKeys[testUserB].deviceKeys[testClientB.deviceID].verified); - assert(!testClientB - .userDeviceKeys[testUserB].deviceKeys[testClientB.deviceID].blocked); + Logs.success('++++ (Bob) Leave all rooms ++++'); + for (var i = 0; i < 3; i++) { + if (testClientB.rooms.isNotEmpty) { + var room = testClientB.rooms.first; + try { + await room.leave(); + await room.forget(); + } catch (_) {} + } + } - Logs.success('++++ ($testUserA) Create room and invite $testUserB ++++'); - await testClientA.createRoom(invite: [testUserB]); - await Future.delayed(Duration(seconds: 1)); - var room = testClientA.rooms.first; - assert(room != null); - final roomId = room.id; + Logs.success('++++ Check if own olm device is verified by default ++++'); + assert(testClientA.userDeviceKeys.containsKey(TestUser.username)); + assert(testClientA.userDeviceKeys[TestUser.username].deviceKeys + .containsKey(testClientA.deviceID)); + assert(testClientA.userDeviceKeys[TestUser.username] + .deviceKeys[testClientA.deviceID].verified); + assert(!testClientA.userDeviceKeys[TestUser.username] + .deviceKeys[testClientA.deviceID].blocked); + assert(testClientB.userDeviceKeys.containsKey(TestUser.username2)); + assert(testClientB.userDeviceKeys[TestUser.username2].deviceKeys + .containsKey(testClientB.deviceID)); + assert(testClientB.userDeviceKeys[TestUser.username2] + .deviceKeys[testClientB.deviceID].verified); + assert(!testClientB.userDeviceKeys[TestUser.username2] + .deviceKeys[testClientB.deviceID].blocked); - Logs.success('++++ ($testUserB) Join room ++++'); - var inviteRoom = testClientB.getRoomById(roomId); - await inviteRoom.join(); - await Future.delayed(Duration(seconds: 1)); - assert(inviteRoom.membership == Membership.join); + Logs.success('++++ (Alice) Create room and invite Bob ++++'); + await testClientA.createRoom(invite: [TestUser.username2]); + await Future.delayed(Duration(seconds: 1)); + var room = testClientA.rooms.first; + assert(room != null); + final roomId = room.id; - Logs.success('++++ ($testUserA) Enable encryption ++++'); - assert(room.encrypted == false); - await room.enableEncryption(); - await Future.delayed(Duration(seconds: 5)); - assert(room.encrypted == true); - assert(room.client.encryption.keyManager.getOutboundGroupSession(room.id) == - null); + Logs.success('++++ (Bob) Join room ++++'); + var inviteRoom = testClientB.getRoomById(roomId); + await inviteRoom.join(); + await Future.delayed(Duration(seconds: 1)); + assert(inviteRoom.membership == Membership.join); - Logs.success('++++ ($testUserA) Check known olm devices ++++'); - assert(testClientA.userDeviceKeys.containsKey(testUserB)); - assert(testClientA.userDeviceKeys[testUserB].deviceKeys - .containsKey(testClientB.deviceID)); - assert(!testClientA - .userDeviceKeys[testUserB].deviceKeys[testClientB.deviceID].verified); - assert(!testClientA - .userDeviceKeys[testUserB].deviceKeys[testClientB.deviceID].blocked); - assert(testClientB.userDeviceKeys.containsKey(testUserA)); - assert(testClientB.userDeviceKeys[testUserA].deviceKeys - .containsKey(testClientA.deviceID)); - assert(!testClientB - .userDeviceKeys[testUserA].deviceKeys[testClientA.deviceID].verified); - assert(!testClientB - .userDeviceKeys[testUserA].deviceKeys[testClientA.deviceID].blocked); - await testClientA.userDeviceKeys[testUserB].deviceKeys[testClientB.deviceID] - .setVerified(true); + Logs.success('++++ (Alice) Enable encryption ++++'); + assert(room.encrypted == false); + await room.enableEncryption(); + await Future.delayed(Duration(seconds: 5)); + assert(room.encrypted == true); + assert(room.client.encryption.keyManager.getOutboundGroupSession(room.id) == + null); - Logs.success('++++ Check if own olm device is verified by default ++++'); - assert(testClientA.userDeviceKeys.containsKey(testUserA)); - assert(testClientA.userDeviceKeys[testUserA].deviceKeys - .containsKey(testClientA.deviceID)); - assert(testClientA - .userDeviceKeys[testUserA].deviceKeys[testClientA.deviceID].verified); - assert(testClientB.userDeviceKeys.containsKey(testUserB)); - assert(testClientB.userDeviceKeys[testUserB].deviceKeys - .containsKey(testClientB.deviceID)); - assert(testClientB - .userDeviceKeys[testUserB].deviceKeys[testClientB.deviceID].verified); + Logs.success('++++ (Alice) Check known olm devices ++++'); + assert(testClientA.userDeviceKeys.containsKey(TestUser.username2)); + assert(testClientA.userDeviceKeys[TestUser.username2].deviceKeys + .containsKey(testClientB.deviceID)); + assert(!testClientA.userDeviceKeys[TestUser.username2] + .deviceKeys[testClientB.deviceID].verified); + assert(!testClientA.userDeviceKeys[TestUser.username2] + .deviceKeys[testClientB.deviceID].blocked); + assert(testClientB.userDeviceKeys.containsKey(TestUser.username)); + assert(testClientB.userDeviceKeys[TestUser.username].deviceKeys + .containsKey(testClientA.deviceID)); + assert(!testClientB.userDeviceKeys[TestUser.username] + .deviceKeys[testClientA.deviceID].verified); + assert(!testClientB.userDeviceKeys[TestUser.username] + .deviceKeys[testClientA.deviceID].blocked); + await testClientA + .userDeviceKeys[TestUser.username2].deviceKeys[testClientB.deviceID] + .setVerified(true); - Logs.success("++++ ($testUserA) Send encrypted message: '$testMessage' ++++"); - await room.sendTextEvent(testMessage); - await Future.delayed(Duration(seconds: 5)); - assert(room.client.encryption.keyManager.getOutboundGroupSession(room.id) != - null); - var currentSessionIdA = room.client.encryption.keyManager - .getOutboundGroupSession(room.id) - .outboundGroupSession - .session_id(); - assert(room.client.encryption.keyManager + Logs.success('++++ Check if own olm device is verified by default ++++'); + assert(testClientA.userDeviceKeys.containsKey(TestUser.username)); + assert(testClientA.userDeviceKeys[TestUser.username].deviceKeys + .containsKey(testClientA.deviceID)); + assert(testClientA.userDeviceKeys[TestUser.username] + .deviceKeys[testClientA.deviceID].verified); + assert(testClientB.userDeviceKeys.containsKey(TestUser.username2)); + assert(testClientB.userDeviceKeys[TestUser.username2].deviceKeys + .containsKey(testClientB.deviceID)); + assert(testClientB.userDeviceKeys[TestUser.username2] + .deviceKeys[testClientB.deviceID].verified); + + Logs.success("++++ (Alice) Send encrypted message: '$testMessage' ++++"); + await room.sendTextEvent(testMessage); + await Future.delayed(Duration(seconds: 5)); + assert(room.client.encryption.keyManager.getOutboundGroupSession(room.id) != + null); + var currentSessionIdA = room.client.encryption.keyManager + .getOutboundGroupSession(room.id) + .outboundGroupSession + .session_id(); + /*assert(room.client.encryption.keyManager .getInboundGroupSession(room.id, currentSessionIdA, '') != - null); - assert(testClientA - .encryption.olmManager.olmSessions[testClientB.identityKey].length == - 1); - assert(testClientB - .encryption.olmManager.olmSessions[testClientA.identityKey].length == - 1); - assert(testClientA.encryption.olmManager.olmSessions[testClientB.identityKey] - .first.sessionId == - testClientB.encryption.olmManager.olmSessions[testClientA.identityKey] - .first.sessionId); - assert(inviteRoom.client.encryption.keyManager + null);*/ + assert(testClientA.encryption.olmManager + .olmSessions[testClientB.identityKey].length == + 1); + assert(testClientB.encryption.olmManager + .olmSessions[testClientA.identityKey].length == + 1); + assert(testClientA.encryption.olmManager + .olmSessions[testClientB.identityKey].first.sessionId == + testClientB.encryption.olmManager.olmSessions[testClientA.identityKey] + .first.sessionId); + /*assert(inviteRoom.client.encryption.keyManager .getInboundGroupSession(inviteRoom.id, currentSessionIdA, '') != - null); - assert(room.lastMessage == testMessage); - assert(inviteRoom.lastMessage == testMessage); - Logs.success( - "++++ ($testUserB) Received decrypted message: '${inviteRoom.lastMessage}' ++++"); + null);*/ + assert(room.lastMessage == testMessage); + assert(inviteRoom.lastMessage == testMessage); + Logs.success( + "++++ (Bob) Received decrypted message: '${inviteRoom.lastMessage}' ++++"); - Logs.success( - "++++ ($testUserA) Send again encrypted message: '$testMessage2' ++++"); - await room.sendTextEvent(testMessage2); - await Future.delayed(Duration(seconds: 5)); - assert(testClientA - .encryption.olmManager.olmSessions[testClientB.identityKey].length == - 1); - assert(testClientB - .encryption.olmManager.olmSessions[testClientA.identityKey].length == - 1); - assert(testClientA.encryption.olmManager.olmSessions[testClientB.identityKey] - .first.sessionId == - testClientB.encryption.olmManager.olmSessions[testClientA.identityKey] - .first.sessionId); + Logs.success( + "++++ (Alice) Send again encrypted message: '$testMessage2' ++++"); + await room.sendTextEvent(testMessage2); + await Future.delayed(Duration(seconds: 5)); + assert(testClientA.encryption.olmManager + .olmSessions[testClientB.identityKey].length == + 1); + assert(testClientB.encryption.olmManager + .olmSessions[testClientA.identityKey].length == + 1); + assert(testClientA.encryption.olmManager + .olmSessions[testClientB.identityKey].first.sessionId == + testClientB.encryption.olmManager.olmSessions[testClientA.identityKey] + .first.sessionId); - assert(room.client.encryption.keyManager - .getOutboundGroupSession(room.id) - .outboundGroupSession - .session_id() == - currentSessionIdA); - assert(room.client.encryption.keyManager + assert(room.client.encryption.keyManager + .getOutboundGroupSession(room.id) + .outboundGroupSession + .session_id() == + currentSessionIdA); + /*assert(room.client.encryption.keyManager .getInboundGroupSession(room.id, currentSessionIdA, '') != - null); - assert(room.lastMessage == testMessage2); - assert(inviteRoom.lastMessage == testMessage2); - Logs.success( - "++++ ($testUserB) Received decrypted message: '${inviteRoom.lastMessage}' ++++"); + null);*/ + assert(room.lastMessage == testMessage2); + assert(inviteRoom.lastMessage == testMessage2); + Logs.success( + "++++ (Bob) Received decrypted message: '${inviteRoom.lastMessage}' ++++"); - Logs.success( - "++++ ($testUserB) Send again encrypted message: '$testMessage3' ++++"); - await inviteRoom.sendTextEvent(testMessage3); - await Future.delayed(Duration(seconds: 5)); - assert(testClientA - .encryption.olmManager.olmSessions[testClientB.identityKey].length == - 1); - assert(testClientB - .encryption.olmManager.olmSessions[testClientA.identityKey].length == - 1); - assert(room.client.encryption.keyManager - .getOutboundGroupSession(room.id) - .outboundGroupSession - .session_id() == - currentSessionIdA); - var inviteRoomOutboundGroupSession = inviteRoom.client.encryption.keyManager - .getOutboundGroupSession(inviteRoom.id); + Logs.success( + "++++ (Bob) Send again encrypted message: '$testMessage3' ++++"); + await inviteRoom.sendTextEvent(testMessage3); + await Future.delayed(Duration(seconds: 5)); + assert(testClientA.encryption.olmManager + .olmSessions[testClientB.identityKey].length == + 1); + assert(testClientB.encryption.olmManager + .olmSessions[testClientA.identityKey].length == + 1); + assert(room.client.encryption.keyManager + .getOutboundGroupSession(room.id) + .outboundGroupSession + .session_id() == + currentSessionIdA); + var inviteRoomOutboundGroupSession = inviteRoom.client.encryption.keyManager + .getOutboundGroupSession(inviteRoom.id); - assert(inviteRoomOutboundGroupSession != null); - assert(inviteRoom.client.encryption.keyManager.getInboundGroupSession( + assert(inviteRoomOutboundGroupSession != null); + /*assert(inviteRoom.client.encryption.keyManager.getInboundGroupSession( inviteRoom.id, inviteRoomOutboundGroupSession.outboundGroupSession.session_id(), '') != @@ -208,147 +214,113 @@ void test() async { room.id, inviteRoomOutboundGroupSession.outboundGroupSession.session_id(), '') != - null); - assert(inviteRoom.lastMessage == testMessage3); - assert(room.lastMessage == testMessage3); - Logs.success( - "++++ ($testUserA) Received decrypted message: '${room.lastMessage}' ++++"); + null);*/ + assert(inviteRoom.lastMessage == testMessage3); + assert(room.lastMessage == testMessage3); + Logs.success( + "++++ (Alice) Received decrypted message: '${room.lastMessage}' ++++"); - Logs.success('++++ Login $testUserB in another client ++++'); - var testClientC = Client('TestClientC', database: getDatabase()); - await testClientC.checkServer(homeserver); - await testClientC.login(user: testUserB, password: testPasswordA); - await Future.delayed(Duration(seconds: 3)); + Logs.success('++++ Login Bob in another client ++++'); + var testClientC = Client('TestClientC', database: getDatabase()); + await testClientC.checkServer(TestUser.homeserver); + await testClientC.login( + user: TestUser.username2, password: TestUser.password); + await Future.delayed(Duration(seconds: 3)); - Logs.success( - "++++ ($testUserA) Send again encrypted message: '$testMessage4' ++++"); - await room.sendTextEvent(testMessage4); - await Future.delayed(Duration(seconds: 5)); - assert(testClientA - .encryption.olmManager.olmSessions[testClientB.identityKey].length == - 1); - assert(testClientB - .encryption.olmManager.olmSessions[testClientA.identityKey].length == - 1); - assert(testClientA.encryption.olmManager.olmSessions[testClientB.identityKey] - .first.sessionId == - testClientB.encryption.olmManager.olmSessions[testClientA.identityKey] - .first.sessionId); - assert(testClientA - .encryption.olmManager.olmSessions[testClientC.identityKey].length == - 1); - assert(testClientC - .encryption.olmManager.olmSessions[testClientA.identityKey].length == - 1); - assert(testClientA.encryption.olmManager.olmSessions[testClientC.identityKey] - .first.sessionId == - testClientC.encryption.olmManager.olmSessions[testClientA.identityKey] - .first.sessionId); - assert(room.client.encryption.keyManager - .getOutboundGroupSession(room.id) - .outboundGroupSession - .session_id() != - currentSessionIdA); - currentSessionIdA = room.client.encryption.keyManager - .getOutboundGroupSession(room.id) - .outboundGroupSession - .session_id(); - assert(inviteRoom.client.encryption.keyManager + Logs.success( + "++++ (Alice) Send again encrypted message: '$testMessage4' ++++"); + await room.sendTextEvent(testMessage4); + await Future.delayed(Duration(seconds: 5)); + assert(testClientA.encryption.olmManager + .olmSessions[testClientB.identityKey].length == + 1); + assert(testClientB.encryption.olmManager + .olmSessions[testClientA.identityKey].length == + 1); + assert(testClientA.encryption.olmManager + .olmSessions[testClientB.identityKey].first.sessionId == + testClientB.encryption.olmManager.olmSessions[testClientA.identityKey] + .first.sessionId); + assert(testClientA.encryption.olmManager + .olmSessions[testClientC.identityKey].length == + 1); + assert(testClientC.encryption.olmManager + .olmSessions[testClientA.identityKey].length == + 1); + assert(testClientA.encryption.olmManager + .olmSessions[testClientC.identityKey].first.sessionId == + testClientC.encryption.olmManager.olmSessions[testClientA.identityKey] + .first.sessionId); + assert(room.client.encryption.keyManager + .getOutboundGroupSession(room.id) + .outboundGroupSession + .session_id() != + currentSessionIdA); + currentSessionIdA = room.client.encryption.keyManager + .getOutboundGroupSession(room.id) + .outboundGroupSession + .session_id(); + /*assert(inviteRoom.client.encryption.keyManager .getInboundGroupSession(inviteRoom.id, currentSessionIdA, '') != - null); - assert(room.lastMessage == testMessage4); - assert(inviteRoom.lastMessage == testMessage4); - Logs.success( - "++++ ($testUserB) Received decrypted message: '${inviteRoom.lastMessage}' ++++"); + null);*/ + assert(room.lastMessage == testMessage4); + assert(inviteRoom.lastMessage == testMessage4); + Logs.success( + "++++ (Bob) Received decrypted message: '${inviteRoom.lastMessage}' ++++"); - Logs.success('++++ Logout $testUserB another client ++++'); - await testClientC.dispose(); - await testClientC.logout(); - testClientC = null; - await Future.delayed(Duration(seconds: 5)); + Logs.success('++++ Logout Bob another client ++++'); + await testClientC.dispose(); + await testClientC.logout(); + testClientC = null; + await Future.delayed(Duration(seconds: 5)); - Logs.success( - "++++ ($testUserA) Send again encrypted message: '$testMessage6' ++++"); - await room.sendTextEvent(testMessage6); - await Future.delayed(Duration(seconds: 5)); - assert(testClientA - .encryption.olmManager.olmSessions[testClientB.identityKey].length == - 1); - assert(testClientB - .encryption.olmManager.olmSessions[testClientA.identityKey].length == - 1); - assert(testClientA.encryption.olmManager.olmSessions[testClientB.identityKey] - .first.sessionId == - testClientB.encryption.olmManager.olmSessions[testClientA.identityKey] - .first.sessionId); - assert(room.client.encryption.keyManager - .getOutboundGroupSession(room.id) - .outboundGroupSession - .session_id() != - currentSessionIdA); - currentSessionIdA = room.client.encryption.keyManager - .getOutboundGroupSession(room.id) - .outboundGroupSession - .session_id(); - assert(inviteRoom.client.encryption.keyManager + Logs.success( + "++++ (Alice) Send again encrypted message: '$testMessage6' ++++"); + await room.sendTextEvent(testMessage6); + await Future.delayed(Duration(seconds: 5)); + assert(testClientA.encryption.olmManager + .olmSessions[testClientB.identityKey].length == + 1); + assert(testClientB.encryption.olmManager + .olmSessions[testClientA.identityKey].length == + 1); + assert(testClientA.encryption.olmManager + .olmSessions[testClientB.identityKey].first.sessionId == + testClientB.encryption.olmManager.olmSessions[testClientA.identityKey] + .first.sessionId); + assert(room.client.encryption.keyManager + .getOutboundGroupSession(room.id) + .outboundGroupSession + .session_id() != + currentSessionIdA); + currentSessionIdA = room.client.encryption.keyManager + .getOutboundGroupSession(room.id) + .outboundGroupSession + .session_id(); + /*assert(inviteRoom.client.encryption.keyManager .getInboundGroupSession(inviteRoom.id, currentSessionIdA, '') != - null); - assert(room.lastMessage == testMessage6); - assert(inviteRoom.lastMessage == testMessage6); - Logs.success( - "++++ ($testUserB) Received decrypted message: '${inviteRoom.lastMessage}' ++++"); + null);*/ + assert(room.lastMessage == testMessage6); + assert(inviteRoom.lastMessage == testMessage6); + Logs.success( + "++++ (Bob) Received decrypted message: '${inviteRoom.lastMessage}' ++++"); -/* Logs.success('++++ ($testUserA) Restore user ++++'); - await testClientA.dispose(); - testClientA = null; - testClientA = Client( - 'TestClientA', - debug: false, - database: getDatabase(), - ); - testClientA.connect(); - await Future.delayed(Duration(seconds: 3)); - var restoredRoom = testClientA.rooms.first; - assert(room != null); - assert(restoredRoom.id == room.id); - assert(restoredRoom.outboundGroupSession.session_id() == - room.outboundGroupSession.session_id()); - assert(restoredRoom.inboundGroupSessions.length == 4); - assert(restoredRoom.inboundGroupSessions.length == - room.inboundGroupSessions.length); - for (var i = 0; i < restoredRoom.inboundGroupSessions.length; i++) { - assert(restoredRoom.inboundGroupSessions.keys.toList()[i] == - room.inboundGroupSessions.keys.toList()[i]); + await room.leave(); + await room.forget(); + await inviteRoom.leave(); + await inviteRoom.forget(); + await Future.delayed(Duration(seconds: 1)); + } catch (e, s) { + Logs.error('Test failed: ${e.toString()}', s); + rethrow; + } finally { + Logs.success('++++ Logout Alice and Bob ++++'); + if (testClientA?.isLogged() ?? false) await testClientA.logoutAll(); + if (testClientA?.isLogged() ?? false) await testClientB.logoutAll(); + await testClientA?.dispose(); + await testClientB?.dispose(); + testClientA = null; + testClientB = null; } - assert(testClientA.encryption.olmManager.olmSessions[testClientB.identityKey].length == 1); - assert(testClientB.encryption.olmManager.olmSessions[testClientA.identityKey].length == 1); - assert(testClientA.encryption.olmManager.olmSessions[testClientB.identityKey].first.session_id() == - testClientB.encryption.olmManager.olmSessions[testClientA.identityKey].first.session_id()); - - Logs.success("++++ ($testUserA) Send again encrypted message: '$testMessage5' ++++"); - await restoredRoom.sendTextEvent(testMessage5); - await Future.delayed(Duration(seconds: 5)); - assert(testClientA.encryption.olmManager.olmSessions[testClientB.identityKey].length == 1); - assert(testClientB.encryption.olmManager.olmSessions[testClientA.identityKey].length == 1); - assert(testClientA.encryption.olmManager.olmSessions[testClientB.identityKey].first.session_id() == - testClientB.encryption.olmManager.olmSessions[testClientA.identityKey].first.session_id()); - assert(restoredRoom.lastMessage == testMessage5); - assert(inviteRoom.lastMessage == testMessage5); - assert(testClientB.getRoomById(roomId).lastMessage == testMessage5); - Logs.success( - "++++ ($testUserB) Received decrypted message: '${inviteRoom.lastMessage}' ++++");*/ - - Logs.success('++++ Logout $testUserA and $testUserB ++++'); - await room.leave(); - await room.forget(); - await inviteRoom.leave(); - await inviteRoom.forget(); - await Future.delayed(Duration(seconds: 1)); - await testClientA.dispose(); - await testClientB.dispose(); - await testClientA.logoutAll(); - await testClientB.logoutAll(); - testClientA = null; - testClientB = null; return; } diff --git a/test_driver/test_config.dart b/test_driver/test_config.dart new file mode 100644 index 0000000..8014255 --- /dev/null +++ b/test_driver/test_config.dart @@ -0,0 +1,6 @@ +class TestUser { + static const String homeserver = 'https://enter-your-server.here'; + static const String username = 'alice'; + static const String username2 = 'bob'; + static const String password = '1234'; +} From 9142dcbeec0d106429df8ab1117d0c4110faf31d Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Wed, 26 Aug 2020 09:38:14 +0200 Subject: [PATCH 47/90] fix: Database error handling --- lib/src/client.dart | 8 +-- lib/src/database/database.dart | 118 ++++++++++++++++++++------------- 2 files changed, 75 insertions(+), 51 deletions(-) diff --git a/lib/src/client.dart b/lib/src/client.dart index a01173c..56e6aa8 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -502,7 +502,7 @@ class Client extends MatrixApi { StreamController.broadcast(); /// Synchronization erros are coming here. - final StreamController onSyncError = StreamController.broadcast(); + final StreamController onSyncError = StreamController.broadcast(); /// Synchronization erros are coming here. final StreamController onOlmError = @@ -725,7 +725,7 @@ class Client extends MatrixApi { return; } Logs.error('Error during processing events: ' + e.toString(), s); - onSyncError.add(SyncError( + onSyncError.add(SdkError( exception: e is Exception ? e : Exception(e), stackTrace: s)); await Future.delayed(Duration(seconds: syncErrorTimeoutSec), _sync); } @@ -1478,8 +1478,8 @@ class Client extends MatrixApi { } } -class SyncError { +class SdkError { Exception exception; StackTrace stackTrace; - SyncError({this.exception, this.stackTrace}); + SdkError({this.exception, this.stackTrace}); } diff --git a/lib/src/database/database.dart b/lib/src/database/database.dart index 9ac84df..81dff83 100644 --- a/lib/src/database/database.dart +++ b/lib/src/database/database.dart @@ -1,3 +1,4 @@ +import 'dart:async'; import 'dart:convert'; import 'package:moor/moor.dart'; @@ -6,6 +7,7 @@ import 'package:olm/olm.dart' as olm; import '../../famedlysdk.dart' as sdk; import '../../matrix_api.dart' as api; import '../../matrix_api.dart'; +import '../client.dart'; import '../room.dart'; import '../utils/logs.dart'; @@ -57,61 +59,83 @@ class Database extends _$Database { int get maxFileSize => 1 * 1024 * 1024; + /// Update errors are coming here. + final StreamController onError = StreamController.broadcast(); + @override MigrationStrategy get migration => MigrationStrategy( - onCreate: (Migrator m) { - return m.createAll(); + onCreate: (Migrator m) async { + try { + await m.createAll(); + } catch (e, s) { + Logs.error(e, s); + onError.add(SdkError(exception: e, stackTrace: s)); + rethrow; + } }, onUpgrade: (Migrator m, int from, int to) async { - // this appears to be only called once, so multiple consecutive upgrades have to be handled appropriately in here - if (from == 1) { - await m.createIndexIfNotExists(userDeviceKeysIndex); - await m.createIndexIfNotExists(userDeviceKeysKeyIndex); - await m.createIndexIfNotExists(olmSessionsIndex); - await m.createIndexIfNotExists(outboundGroupSessionsIndex); - await m.createIndexIfNotExists(inboundGroupSessionsIndex); - await m.createIndexIfNotExists(roomsIndex); - await m.createIndexIfNotExists(eventsIndex); - await m.createIndexIfNotExists(roomStatesIndex); - await m.createIndexIfNotExists(accountDataIndex); - await m.createIndexIfNotExists(roomAccountDataIndex); - await m.createIndexIfNotExists(presencesIndex); - from++; - } - if (from == 2) { - await m.deleteTable('outbound_group_sessions'); - await m.createTable(outboundGroupSessions); - from++; - } - if (from == 3) { - await m.createTableIfNotExists(userCrossSigningKeys); - await m.createTableIfNotExists(ssssCache); - // mark all keys as outdated so that the cross signing keys will be fetched - await m.issueCustomQuery( - 'UPDATE user_device_keys SET outdated = true'); - from++; - } - if (from == 4) { - await m.addColumnIfNotExists(olmSessions, olmSessions.lastReceived); - from++; - } - if (from == 5) { - await m.addColumnIfNotExists( - inboundGroupSessions, inboundGroupSessions.uploaded); - await m.addColumnIfNotExists( - inboundGroupSessions, inboundGroupSessions.senderKey); - await m.addColumnIfNotExists( - inboundGroupSessions, inboundGroupSessions.senderClaimedKeys); - from++; + try { + // this appears to be only called once, so multiple consecutive upgrades have to be handled appropriately in here + if (from == 1) { + await m.createIndexIfNotExists(userDeviceKeysIndex); + await m.createIndexIfNotExists(userDeviceKeysKeyIndex); + await m.createIndexIfNotExists(olmSessionsIndex); + await m.createIndexIfNotExists(outboundGroupSessionsIndex); + await m.createIndexIfNotExists(inboundGroupSessionsIndex); + await m.createIndexIfNotExists(roomsIndex); + await m.createIndexIfNotExists(eventsIndex); + await m.createIndexIfNotExists(roomStatesIndex); + await m.createIndexIfNotExists(accountDataIndex); + await m.createIndexIfNotExists(roomAccountDataIndex); + await m.createIndexIfNotExists(presencesIndex); + from++; + } + if (from == 2) { + await m.deleteTable('outbound_group_sessions'); + await m.createTable(outboundGroupSessions); + from++; + } + if (from == 3) { + await m.createTableIfNotExists(userCrossSigningKeys); + await m.createTableIfNotExists(ssssCache); + // mark all keys as outdated so that the cross signing keys will be fetched + await m.issueCustomQuery( + 'UPDATE user_device_keys SET outdated = true'); + from++; + } + if (from == 4) { + await m.addColumnIfNotExists( + olmSessions, olmSessions.lastReceived); + from++; + } + if (from == 5) { + await m.addColumnIfNotExists( + inboundGroupSessions, inboundGroupSessions.uploaded); + await m.addColumnIfNotExists( + inboundGroupSessions, inboundGroupSessions.senderKey); + await m.addColumnIfNotExists( + inboundGroupSessions, inboundGroupSessions.senderClaimedKeys); + from++; + } + } catch (e, s) { + Logs.error(e, s); + onError.add(SdkError(exception: e, stackTrace: s)); + rethrow; } }, beforeOpen: (_) async { - if (executor.dialect == SqlDialect.sqlite) { - final ret = await customSelect('PRAGMA journal_mode=WAL').get(); - if (ret.isNotEmpty) { - Logs.info('[Moor] Switched database to mode ' + - ret.first.data['journal_mode'].toString()); + try { + if (executor.dialect == SqlDialect.sqlite) { + final ret = await customSelect('PRAGMA journal_mode=WAL').get(); + if (ret.isNotEmpty) { + Logs.info('[Moor] Switched database to mode ' + + ret.first.data['journal_mode'].toString()); + } } + } catch (e, s) { + Logs.error(e, s); + onError.add(SdkError(exception: e, stackTrace: s)); + rethrow; } }, ); From e90793bef14bc034717319fcff636289275b5dce Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Fri, 4 Sep 2020 09:48:35 +0200 Subject: [PATCH 48/90] fix: Last message sort order --- lib/src/client.dart | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/src/client.dart b/lib/src/client.dart index 56e6aa8..1a458eb 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -1104,9 +1104,9 @@ class Client extends MatrixApi { ); } else { var prevState = rooms[j].getState(stateEvent.type, stateEvent.stateKey); - if (prevState != null && - prevState.originServerTs.millisecondsSinceEpoch > - stateEvent.originServerTs.millisecondsSinceEpoch) return; + if (prevState != null && prevState.sortOrder > stateEvent.sortOrder) { + return; + } rooms[j].setState(stateEvent); } } else if (eventUpdate.type == 'account_data') { From 089ce88b570c18e8fe89f3bf2ba0f1cad42d5a65 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Fri, 4 Sep 2020 11:00:56 +0200 Subject: [PATCH 49/90] chore: Add tests to Event.downloadAndDecryptAttachment --- lib/src/event.dart | 8 +- test/event_test.dart | 194 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 200 insertions(+), 2 deletions(-) diff --git a/lib/src/event.dart b/lib/src/event.dart index 97a5391..a82c2b6 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -372,7 +372,8 @@ class Event extends MatrixEvent { /// contain an attachment, this throws an error. Set [getThumbnail] to /// true to download the thumbnail instead. Future downloadAndDecryptAttachment( - {bool getThumbnail = false}) async { + {bool getThumbnail = false, + Future Function(String) downloadCallback}) async { if (![EventTypes.Message, EventTypes.Sticker].contains(type)) { throw ("This event has the type '$type' and so it can't contain an attachment."); } @@ -413,8 +414,11 @@ class Event extends MatrixEvent { // Download the file if (uint8list == null) { + downloadCallback ??= (String url) async { + return (await http.get(url)).bodyBytes; + }; uint8list = - (await http.get(mxContent.getDownloadLink(room.client))).bodyBytes; + await downloadCallback(mxContent.getDownloadLink(room.client)); if (storeable) { await room.client.database .storeFile(mxContent.toString(), uint8list, DateTime.now()); diff --git a/test/event_test.dart b/test/event_test.dart index f40f8ec..c2b3fcd 100644 --- a/test/event_test.dart +++ b/test/event_test.dart @@ -17,19 +17,33 @@ */ import 'dart:convert'; +import 'dart:typed_data'; import 'package:famedlysdk/famedlysdk.dart'; import 'package:famedlysdk/matrix_api.dart'; import 'package:famedlysdk/encryption.dart'; import 'package:famedlysdk/src/event.dart'; +import 'package:famedlysdk/src/utils/logs.dart'; import 'package:test/test.dart'; +import 'package:olm/olm.dart' as olm; +import 'fake_client.dart'; import 'fake_matrix_api.dart'; import 'fake_matrix_localizations.dart'; void main() { /// All Tests related to the Event group('Event', () { + var olmEnabled = true; + try { + olm.init(); + olm.Account(); + } catch (_) { + olmEnabled = false; + Logs.warning('[LibOlm] Failed to load LibOlm: ' + _.toString()); + } + Logs.success('[LibOlm] Enabled: $olmEnabled'); + final timestamp = DateTime.now().millisecondsSinceEpoch; final id = '!4fsdfjisjf:server.abc'; final senderID = '@alice:server.abc'; @@ -960,5 +974,185 @@ void main() { Timeline(events: [event, edit1, edit2, edit3], room: room)); expect(displayEvent.body, 'edit 2'); }); + test('downloadAndDecryptAttachment', () async { + final FILE_BUFF = Uint8List.fromList([0]); + final THUMBNAIL_BUFF = Uint8List.fromList([2]); + final downloadCallback = (String url) async { + return { + 'https://fakeserver.notexisting/_matrix/media/r0/download/example.org/file': + FILE_BUFF, + 'https://fakeserver.notexisting/_matrix/media/r0/download/example.org/thumb': + THUMBNAIL_BUFF, + }[url]; + }; + await client.checkServer('https://fakeServer.notExisting'); + final room = Room(id: '!localpart:server.abc', client: client); + var event = Event.fromJson({ + 'type': EventTypes.Message, + 'content': { + 'body': 'image', + 'msgtype': 'm.image', + 'url': 'mxc://example.org/file', + }, + 'event_id': '\$edit2', + 'sender': '@alice:example.org', + }, room); + var buffer = await event.downloadAndDecryptAttachment( + downloadCallback: downloadCallback); + expect(buffer.bytes, FILE_BUFF); + + event = Event.fromJson({ + 'type': EventTypes.Message, + 'content': { + 'body': 'image', + 'msgtype': 'm.image', + 'url': 'mxc://example.org/file', + 'info': { + 'thumbnail_url': 'mxc://example.org/thumb', + }, + }, + 'event_id': '\$edit2', + 'sender': '@alice:example.org', + }, room); + buffer = await event.downloadAndDecryptAttachment( + downloadCallback: downloadCallback); + expect(buffer.bytes, FILE_BUFF); + + buffer = await event.downloadAndDecryptAttachment( + getThumbnail: true, downloadCallback: downloadCallback); + expect(buffer.bytes, THUMBNAIL_BUFF); + }); + test('downloadAndDecryptAttachment encrypted', () async { + if (!olmEnabled) return; + + final FILE_BUFF_ENC = Uint8List.fromList([0x3B, 0x6B, 0xB2, 0x8C, 0xAF]); + final FILE_BUFF_DEC = Uint8List.fromList([0x74, 0x65, 0x73, 0x74, 0x0A]); + final THUMB_BUFF_ENC = + Uint8List.fromList([0x55, 0xD7, 0xEB, 0x72, 0x05, 0x13]); + final THUMB_BUFF_DEC = + Uint8List.fromList([0x74, 0x68, 0x75, 0x6D, 0x62, 0x0A]); + final downloadCallback = (String url) async { + return { + 'https://fakeserver.notexisting/_matrix/media/r0/download/example.com/file': + FILE_BUFF_ENC, + 'https://fakeserver.notexisting/_matrix/media/r0/download/example.com/thumb': + THUMB_BUFF_ENC, + }[url]; + }; + final room = Room(id: '!localpart:server.abc', client: await getClient()); + var event = Event.fromJson({ + 'type': EventTypes.Message, + 'content': { + 'body': 'image', + 'msgtype': 'm.image', + 'file': { + 'v': 'v2', + 'key': { + 'alg': 'A256CTR', + 'ext': true, + 'k': '7aPRNIDPeUAUqD6SPR3vVX5W9liyMG98NexVJ9udnCc', + 'key_ops': ['encrypt', 'decrypt'], + 'kty': 'oct' + }, + 'iv': 'Wdsf+tnOHIoAAAAAAAAAAA', + 'hashes': {'sha256': 'WgC7fw2alBC5t+xDx+PFlZxfFJXtIstQCg+j0WDaXxE'}, + 'url': 'mxc://example.com/file', + 'mimetype': 'text/plain' + }, + }, + 'event_id': '\$edit2', + 'sender': '@alice:example.org', + }, room); + var buffer = await event.downloadAndDecryptAttachment( + downloadCallback: downloadCallback); + expect(buffer.bytes, FILE_BUFF_DEC); + + event = Event.fromJson({ + 'type': EventTypes.Message, + 'content': { + 'body': 'image', + 'msgtype': 'm.image', + 'file': { + 'v': 'v2', + 'key': { + 'alg': 'A256CTR', + 'ext': true, + 'k': '7aPRNIDPeUAUqD6SPR3vVX5W9liyMG98NexVJ9udnCc', + 'key_ops': ['encrypt', 'decrypt'], + 'kty': 'oct' + }, + 'iv': 'Wdsf+tnOHIoAAAAAAAAAAA', + 'hashes': {'sha256': 'WgC7fw2alBC5t+xDx+PFlZxfFJXtIstQCg+j0WDaXxE'}, + 'url': 'mxc://example.com/file', + 'mimetype': 'text/plain' + }, + 'info': { + 'thumbnail_file': { + 'v': 'v2', + 'key': { + 'alg': 'A256CTR', + 'ext': true, + 'k': 'TmF-rZYetZbxpL5yjDPE21UALQJcpEE6X-nvUDD5rA0', + 'key_ops': ['encrypt', 'decrypt'], + 'kty': 'oct' + }, + 'iv': '41ZqNRZSLFUAAAAAAAAAAA', + 'hashes': { + 'sha256': 'zccOwXiOTAYhGXyk0Fra7CRreBF6itjiCKdd+ov8mO4' + }, + 'url': 'mxc://example.com/thumb', + 'mimetype': 'text/plain' + } + }, + }, + 'event_id': '\$edit2', + 'sender': '@alice:example.org', + }, room); + buffer = await event.downloadAndDecryptAttachment( + downloadCallback: downloadCallback); + expect(buffer.bytes, FILE_BUFF_DEC); + + buffer = await event.downloadAndDecryptAttachment( + getThumbnail: true, downloadCallback: downloadCallback); + expect(buffer.bytes, THUMB_BUFF_DEC); + + await room.client.dispose(closeDatabase: true); + }); + test('downloadAndDecryptAttachment store', () async { + final FILE_BUFF = Uint8List.fromList([0]); + var serverHits = 0; + final downloadCallback = (String url) async { + serverHits++; + return { + 'https://fakeserver.notexisting/_matrix/media/r0/download/example.org/newfile': + FILE_BUFF, + }[url]; + }; + await client.checkServer('https://fakeServer.notExisting'); + final room = Room(id: '!localpart:server.abc', client: await getClient()); + var event = Event.fromJson({ + 'type': EventTypes.Message, + 'content': { + 'body': 'image', + 'msgtype': 'm.image', + 'url': 'mxc://example.org/newfile', + 'info': { + 'size': 5, + }, + }, + 'event_id': '\$edit2', + 'sender': '@alice:example.org', + }, room); + var buffer = await event.downloadAndDecryptAttachment( + downloadCallback: downloadCallback); + expect(buffer.bytes, FILE_BUFF); + expect(serverHits, 1); + buffer = await event.downloadAndDecryptAttachment( + downloadCallback: downloadCallback); + expect(buffer.bytes, FILE_BUFF); + expect(serverHits, 1); + + await room.client.dispose(closeDatabase: true); + }); }); } From 5863c8e168d796bcda3556409c46e27b71986862 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Sun, 6 Sep 2020 14:48:06 +0200 Subject: [PATCH 50/90] fix: Run advanced things in database handling in their own separate zone --- lib/encryption/encryption.dart | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/encryption/encryption.dart b/lib/encryption/encryption.dart index 6b78c95..4b9e0fc 100644 --- a/lib/encryption/encryption.dart +++ b/lib/encryption/encryption.dart @@ -17,6 +17,7 @@ */ import 'dart:convert'; +import 'dart:async'; import 'package:pedantic/pedantic.dart'; @@ -81,16 +82,17 @@ class Encryption { if (['m.room_key_request', 'm.forwarded_room_key'].contains(event.type)) { // "just" room key request things. We don't need these asap, so we handle // them in the background - unawaited(keyManager.handleToDeviceEvent(event)); + unawaited(Zone.root.run(() => keyManager.handleToDeviceEvent(event))); } if (event.type.startsWith('m.key.verification.')) { // some key verification event. No need to handle it now, we can easily // do this in the background - unawaited(keyVerificationManager.handleToDeviceEvent(event)); + unawaited(Zone.root + .run(() => keyVerificationManager.handleToDeviceEvent(event))); } if (event.type.startsWith('m.secret.')) { // some ssss thing. We can do this in the background - unawaited(ssss.handleToDeviceEvent(event)); + unawaited(Zone.root.run(() => ssss.handleToDeviceEvent(event))); } } @@ -104,7 +106,8 @@ class Encryption { update.content['content']['msgtype'] .startsWith('m.key.verification.'))) { // "just" key verification, no need to do this in sync - unawaited(keyVerificationManager.handleEventUpdate(update)); + unawaited(Zone.root + .run(() => keyVerificationManager.handleEventUpdate(update))); } } From 54a128d2c5cddc5b9837c7fe5dc60a841bd30b84 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Sat, 5 Sep 2020 13:54:43 +0200 Subject: [PATCH 51/90] fix: Properly detect sicket message types --- lib/src/event.dart | 4 +++- test/event_test.dart | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/src/event.dart b/lib/src/event.dart index a82c2b6..0299acd 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -229,7 +229,9 @@ class Event extends MatrixEvent { unsigned: unsigned, room: room); - String get messageType => content['msgtype'] ?? MessageTypes.Text; + String get messageType => type == EventTypes.Sticker + ? MessageTypes.Sticker + : content['msgtype'] ?? MessageTypes.Text; void setRedactionEvent(Event redactedBecause) { unsigned = { diff --git a/test/event_test.dart b/test/event_test.dart index c2b3fcd..9649b7c 100644 --- a/test/event_test.dart +++ b/test/event_test.dart @@ -167,6 +167,11 @@ void main() { event = Event.fromJson(jsonObj, null); expect(event.messageType, MessageTypes.Location); + jsonObj['type'] = 'm.sticker'; + jsonObj['content']['msgtype'] = null; + event = Event.fromJson(jsonObj, null); + expect(event.messageType, MessageTypes.Sticker); + jsonObj['type'] = 'm.room.message'; jsonObj['content']['msgtype'] = 'm.text'; jsonObj['content']['m.relates_to'] = {}; From e6d96ad8edd48c3780c9db4811c8f42d29f64166 Mon Sep 17 00:00:00 2001 From: Lukas Lihotzki Date: Fri, 4 Sep 2020 13:10:09 +0200 Subject: [PATCH 52/90] feat(sync): configurable sync --- lib/src/client.dart | 63 +++++++++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/lib/src/client.dart b/lib/src/client.dart index 1a458eb..76b38b3 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -21,7 +21,6 @@ import 'dart:convert'; import 'dart:core'; import 'package:http/http.dart' as http; -import 'package:pedantic/pedantic.dart'; import '../encryption.dart'; import '../famedlysdk.dart'; @@ -656,6 +655,7 @@ class Client extends MatrixApi { 'Successfully connected as ${userID.localpart} with ${homeserver.toString()}', ); + // Always do a _sync after login, even if backgroundSync is set to off return _sync(); } @@ -675,25 +675,48 @@ class Client extends MatrixApi { onLoginStateChanged.add(LoginState.loggedOut); } - Future _syncRequest; - Exception _lastSyncError; + bool _backgroundSync = true; + Future _currentSync, _retryDelay = Future.value(); + bool get syncPending => _currentSync != null; - Future _sync() async { - if (isLogged() == false || _disposed) return; + /// Controls the background sync (automatically looping forever if turned on). + set backgroundSync(bool enabled) { + _backgroundSync = enabled; + if (_backgroundSync) { + _sync(); + } + } + + /// Immediately start a sync and wait for completion. + /// If there is an active sync already, wait for the active sync instead. + Future oneShotSync() { + return _sync(); + } + + Future _sync() { + if (_currentSync == null) { + _currentSync = _innerSync(); + _currentSync.whenComplete(() { + _currentSync = null; + if (_backgroundSync && isLogged() && !_disposed) { + _sync(); + } + }); + } + return _currentSync; + } + + Future _innerSync() async { + await _retryDelay; + _retryDelay = Future.delayed(Duration(seconds: syncErrorTimeoutSec)); + if (!isLogged() || _disposed) return null; try { - _syncRequest = sync( + final syncResp = await sync( filter: syncFilters, since: prevBatch, timeout: prevBatch != null ? 30000 : null, - ).catchError((e) { - _lastSyncError = e; - return null; - }); - final hash = _syncRequest.hashCode; - final syncResp = await _syncRequest; + ); if (_disposed) return; - if (syncResp == null) throw _lastSyncError; - if (hash != _syncRequest.hashCode) return; if (database != null) { _currentTransaction = database.transaction(() async { await handleSync(syncResp); @@ -716,18 +739,14 @@ class Client extends MatrixApi { if (encryptionEnabled) { encryption.onSync(); } - if (hash == _syncRequest.hashCode) unawaited(_sync()); - } on MatrixException catch (exception) { - onError.add(exception); - await Future.delayed(Duration(seconds: syncErrorTimeoutSec), _sync); + _retryDelay = Future.value(); + } on MatrixException catch (e) { + onError.add(e); } catch (e, s) { - if (isLogged() == false || _disposed) { - return; - } + if (!isLogged() || _disposed) return; Logs.error('Error during processing events: ' + e.toString(), s); onSyncError.add(SdkError( exception: e is Exception ? e : Exception(e), stackTrace: s)); - await Future.delayed(Duration(seconds: syncErrorTimeoutSec), _sync); } } From 8a0cc70cfec2fa03baab4c7abf4e4f4d1be88dc6 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Mon, 7 Sep 2020 11:16:52 +0200 Subject: [PATCH 53/90] fix: prev content error log in web --- lib/src/event.dart | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/src/event.dart b/lib/src/event.dart index 0299acd..3c183e5 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -105,8 +105,10 @@ class Event extends MatrixEvent { try { this.prevContent = (prevContent != null && prevContent.isNotEmpty) ? prevContent - : (this.unsigned != null && this.unsigned['prev_content'] is Map) - ? this.unsigned['prev_content'] + : (unsigned != null && + unsigned.containsKey('prev_content') && + unsigned['prev_content'] is Map) + ? unsigned['prev_content'] : null; } catch (e, s) { Logs.error('Event constructor crashed: ${e.toString()}', s); From f7e63097b4e4578f801b59ef1f56ea0ae2f4735b Mon Sep 17 00:00:00 2001 From: Sorunome Date: Fri, 4 Sep 2020 13:52:59 +0200 Subject: [PATCH 54/90] chore: Update emotes to match MSC --- lib/src/room.dart | 27 +++++++++++++++------------ lib/src/utils/markdown.dart | 2 +- test/markdown_test.dart | 6 +++--- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/lib/src/room.dart b/lib/src/room.dart index fd9999c..8f528e5 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -437,6 +437,7 @@ class Room { name = name.replaceAll(RegExp(r'[^\w-]'), ''); return name.toLowerCase(); }; + final allMxcs = {}; // for easy dedupint final addEmotePack = (String packName, Map content, [String packNameOverride]) { if (!(content['short'] is Map)) { @@ -454,25 +455,18 @@ class Room { } content['short'].forEach((key, value) { if (key is String && value is String && value.startsWith('mxc://')) { - packs[packName][key] = value; + if (allMxcs.add(value)) { + packs[packName][key] = value; + } } }); }; - // first add all the room emotes - final allRoomEmotes = states.states['im.ponies.room_emotes']; - if (allRoomEmotes != null) { - for (final entry in allRoomEmotes.entries) { - final stateKey = entry.key; - final event = entry.value; - addEmotePack(stateKey.isEmpty ? 'room' : stateKey, event.content); - } - } - // next add all the user emotes + // first add all the user emotes final userEmotes = client.accountData['im.ponies.user_emotes']; if (userEmotes != null) { addEmotePack('user', userEmotes.content); } - // finally add all the external emote rooms + // next add all the external emote rooms final emoteRooms = client.accountData['im.ponies.emote_rooms']; if (emoteRooms != null && emoteRooms.content['rooms'] is Map) { for (final roomEntry in emoteRooms.content['rooms'].entries) { @@ -497,6 +491,15 @@ class Room { } } } + // finally add all the room emotes + final allRoomEmotes = states.states['im.ponies.room_emotes']; + if (allRoomEmotes != null) { + for (final entry in allRoomEmotes.entries) { + final stateKey = entry.key; + final event = entry.value; + addEmotePack(stateKey.isEmpty ? 'room' : stateKey, event.content); + } + } return packs; } diff --git a/lib/src/utils/markdown.dart b/lib/src/utils/markdown.dart index 082513b..745601b 100644 --- a/lib/src/utils/markdown.dart +++ b/lib/src/utils/markdown.dart @@ -65,7 +65,7 @@ class EmoteSyntax extends InlineSyntax { return true; } final element = Element.empty('img'); - element.attributes['data-mx-emote'] = ''; + element.attributes['data-mx-emoticon'] = ''; element.attributes['src'] = htmlEscape.convert(mxc); element.attributes['alt'] = htmlEscape.convert(emote); element.attributes['title'] = htmlEscape.convert(emote); diff --git a/test/markdown_test.dart b/test/markdown_test.dart index 9a4b999..7690d4b 100644 --- a/test/markdown_test.dart +++ b/test/markdown_test.dart @@ -54,11 +54,11 @@ void main() { }); test('emotes', () { expect(markdown(':fox:', emotePacks), - ':fox:'); + ':fox:'); expect(markdown(':user~fox:', emotePacks), - ':fox:'); + ':fox:'); expect(markdown(':raccoon:', emotePacks), - ':raccoon:'); + ':raccoon:'); expect(markdown(':invalid:', emotePacks), ':invalid:'); expect(markdown(':room~invalid:', emotePacks), ':room~invalid:'); }); From 8a104b34ffbe6e25482b5f00a905344c547dea40 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Mon, 7 Sep 2020 16:19:19 +0200 Subject: [PATCH 55/90] fix: potentially fix SSSS passphrase not working for some accounts --- lib/encryption/ssss.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/encryption/ssss.dart b/lib/encryption/ssss.dart index d646397..8386f03 100644 --- a/lib/encryption/ssss.dart +++ b/lib/encryption/ssss.dart @@ -16,6 +16,7 @@ * along with this program. If not, see . */ +import 'dart:core'; import 'dart:convert'; import 'dart:typed_data'; @@ -133,7 +134,7 @@ class SSSS { } final generator = PBKDF2(hashAlgorithm: sha512); return Uint8List.fromList(generator.generateKey(passphrase, info.salt, - info.iterations, info.bits != null ? info.bits / 8 : 32)); + info.iterations, info.bits != null ? (info.bits / 8).ceil() : 32)); } void setValidator(String type, Future Function(String) validator) { From 8899f4c677f6f255db62f34a6ae46f92deaa1c88 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Mon, 7 Sep 2020 12:53:35 +0200 Subject: [PATCH 56/90] fix: Remove logs in event constructor trycatch --- lib/src/event.dart | 5 ++--- olm | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) create mode 160000 olm diff --git a/lib/src/event.dart b/lib/src/event.dart index 3c183e5..29dfb7b 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -27,7 +27,6 @@ import '../famedlysdk.dart'; import '../matrix_api.dart'; import 'database/database.dart' show DbRoomState, DbEvent; import 'room.dart'; -import 'utils/logs.dart'; import 'utils/matrix_localizations.dart'; import 'utils/receipt.dart'; @@ -110,8 +109,8 @@ class Event extends MatrixEvent { unsigned['prev_content'] is Map) ? unsigned['prev_content'] : null; - } catch (e, s) { - Logs.error('Event constructor crashed: ${e.toString()}', s); + } catch (_) { + // A strange bug in dart web makes this crash } this.stateKey = stateKey; this.originServerTs = originServerTs; diff --git a/olm b/olm new file mode 160000 index 0000000..efd1763 --- /dev/null +++ b/olm @@ -0,0 +1 @@ +Subproject commit efd17631b16d1271a029e0af8f7d8e5ae795cc5d From 99d536b14fa42baf8ecd9b96523d5200238f125d Mon Sep 17 00:00:00 2001 From: Sorunome Date: Mon, 17 Aug 2020 14:25:48 +0200 Subject: [PATCH 57/90] feature: Upload to online key backup --- lib/encryption/encryption.dart | 17 ++ lib/encryption/key_manager.dart | 173 ++++++++++++++++++-- lib/encryption/ssss.dart | 26 ++- lib/encryption/utils/session_key.dart | 6 + lib/matrix_api/matrix_api.dart | 2 +- lib/matrix_api/model/room_keys_keys.dart | 14 ++ lib/src/client.dart | 15 ++ lib/src/database/database.g.dart | 21 +++ lib/src/database/database.moor | 2 + lib/src/utils/logs.dart | 18 ++ lib/src/utils/run_in_background.dart | 30 ++++ pubspec.lock | 9 +- pubspec.yaml | 3 +- test.sh | 4 +- test/encryption/key_verification_test.dart | 6 +- test/encryption/online_key_backup_test.dart | 47 ++++++ test/encryption/ssss_test.dart | 16 +- test/user_test.dart | 4 +- 18 files changed, 375 insertions(+), 38 deletions(-) create mode 100644 lib/src/utils/run_in_background.dart diff --git a/lib/encryption/encryption.dart b/lib/encryption/encryption.dart index 4b9e0fc..f6f3a3c 100644 --- a/lib/encryption/encryption.dart +++ b/lib/encryption/encryption.dart @@ -63,6 +63,8 @@ class Encryption { Future init(String olmAccount) async { await olmManager.init(olmAccount); + _backgroundTasksRunning = true; + _backgroundTasks(); // start the background tasks } void handleDeviceOneTimeKeysCount(Map countJson) { @@ -307,10 +309,25 @@ class Encryption { return await olmManager.encryptToDeviceMessage(deviceKeys, type, payload); } + // this method is responsible for all background tasks, such as uploading online key backups + bool _backgroundTasksRunning = true; + void _backgroundTasks() { + if (!_backgroundTasksRunning) { + return; + } + + keyManager.backgroundTasks(); + + if (_backgroundTasksRunning) { + Timer(Duration(seconds: 10), _backgroundTasks); + } + } + void dispose() { keyManager.dispose(); olmManager.dispose(); keyVerificationManager.dispose(); + _backgroundTasksRunning = false; } } diff --git a/lib/encryption/key_manager.dart b/lib/encryption/key_manager.dart index bb39da7..d032704 100644 --- a/lib/encryption/key_manager.dart +++ b/lib/encryption/key_manager.dart @@ -26,7 +26,9 @@ import './utils/outbound_group_session.dart'; import './utils/session_key.dart'; import '../famedlysdk.dart'; import '../matrix_api.dart'; +import '../src/database/database.dart'; import '../src/utils/logs.dart'; +import '../src/utils/run_in_background.dart'; const MEGOLM_KEY = 'm.megolm_backup.v1'; @@ -71,18 +73,14 @@ class KeyManager { void setInboundGroupSession(String roomId, String sessionId, String senderKey, Map content, - {bool forwarded = false, Map senderClaimedKeys}) { + {bool forwarded = false, + Map senderClaimedKeys, + bool uploaded = false}) { senderClaimedKeys ??= {}; if (!senderClaimedKeys.containsKey('ed25519')) { - DeviceKeys device; - for (final user in client.userDeviceKeys.values) { - device = user.deviceKeys.values.firstWhere( - (e) => e.curve25519Key == senderKey, - orElse: () => null); - if (device != null) { - senderClaimedKeys['ed25519'] = device.ed25519Key; - break; - } + final device = client.getUserDeviceKeysByCurve25519Key(senderKey); + if (device != null) { + senderClaimedKeys['ed25519'] = device.ed25519Key; } } final oldSession = @@ -109,6 +107,8 @@ class KeyManager { content: content, inboundGroupSession: inboundGroupSession, indexes: {}, + roomId: roomId, + sessionId: sessionId, key: client.userID, senderKey: senderKey, senderClaimedKeys: senderClaimedKeys, @@ -132,7 +132,8 @@ class KeyManager { _inboundGroupSessions[roomId] = {}; } _inboundGroupSessions[roomId][sessionId] = newSession; - client.database?.storeInboundGroupSession( + client.database + ?.storeInboundGroupSession( client.id, roomId, sessionId, @@ -141,9 +142,13 @@ class KeyManager { json.encode({}), senderKey, json.encode(senderClaimedKeys), - ); - // Note to self: When adding key-backup that needs to be unawaited(), else - // we might accidentally end up with http requests inside of the sync loop + ) + ?.then((_) { + if (uploaded) { + client.database + .markInboundGroupSessionAsUploaded(client.id, roomId, sessionId); + } + }); // TODO: somehow try to decrypt last message again final room = client.getRoomById(roomId); if (room != null) { @@ -410,7 +415,8 @@ class KeyManager { forwarded: true, senderClaimedKeys: decrypted['sender_claimed_keys'] != null ? Map.from(decrypted['sender_claimed_keys']) - : null); + : null, + uploaded: true); } } } @@ -492,6 +498,79 @@ class KeyManager { } } + bool _isUploadingKeys = false; + Future backgroundTasks() async { + if (_isUploadingKeys || client.database == null) { + return; + } + _isUploadingKeys = true; + try { + if (!(await isCached())) { + return; // we can't backup anyways + } + final dbSessions = + await client.database.getInboundGroupSessionsToUpload().get(); + if (dbSessions.isEmpty) { + return; // nothing to do + } + final privateKey = + base64.decode(await encryption.ssss.getCached(MEGOLM_KEY)); + // decryption is needed to calculate the public key and thus see if the claimed information is in fact valid + final decryption = olm.PkDecryption(); + final info = await client.getRoomKeysBackup(); + String backupPubKey; + try { + backupPubKey = decryption.init_with_private_key(privateKey); + + if (backupPubKey == null || + info.algorithm != RoomKeysAlgorithmType.v1Curve25519AesSha2 || + info.authData['public_key'] != backupPubKey) { + return; + } + final args = _GenerateUploadKeysArgs( + pubkey: backupPubKey, + dbSessions: <_DbInboundGroupSessionBundle>[], + userId: client.userID, + ); + // we need to calculate verified beforehand, as else we pass a closure to an isolate + // with 500 keys they do, however, noticably block the UI, which is why we give brief async suspentions in here + // so that the event loop can progress + var i = 0; + for (final dbSession in dbSessions) { + final device = + client.getUserDeviceKeysByCurve25519Key(dbSession.senderKey); + args.dbSessions.add(_DbInboundGroupSessionBundle( + dbSession: dbSession, + verified: device?.verified ?? false, + )); + i++; + if (i > 10) { + await Future.delayed(Duration(milliseconds: 1)); + i = 0; + } + } + final roomKeys = + await runInBackground( + _generateUploadKeys, args); + Logs.info('[Key Manager] Uploading ${dbSessions.length} room keys...'); + // upload the payload... + await client.storeRoomKeys(info.version, roomKeys); + // and now finally mark all the keys as uploaded + // no need to optimze this, as we only run it so seldomly and almost never with many keys at once + for (final dbSession in dbSessions) { + await client.database.markInboundGroupSessionAsUploaded( + client.id, dbSession.roomId, dbSession.sessionId); + } + } finally { + decryption.free(); + } + } catch (e, s) { + Logs.error('[Key Manager] Error uploading room keys: ' + e.toString(), s); + } finally { + _isUploadingKeys = false; + } + } + /// Handle an incoming to_device event that is related to key sharing Future handleToDeviceEvent(ToDeviceEvent event) async { if (event.type == 'm.room_key_request') { @@ -725,3 +804,67 @@ class RoomKeyRequest extends ToDeviceEvent { keyManager.incomingShareRequests.remove(request.requestId); } } + +RoomKeys _generateUploadKeys(_GenerateUploadKeysArgs args) { + final enc = olm.PkEncryption(); + try { + enc.set_recipient_key(args.pubkey); + // first we generate the payload to upload all the session keys in this chunk + final roomKeys = RoomKeys(); + for (final dbSession in args.dbSessions) { + final sess = SessionKey.fromDb(dbSession.dbSession, args.userId); + if (!sess.isValid) { + continue; + } + // create the room if it doesn't exist + if (!roomKeys.rooms.containsKey(sess.roomId)) { + roomKeys.rooms[sess.roomId] = RoomKeysRoom(); + } + // generate the encrypted content + final payload = { + 'algorithm': 'm.megolm.v1.aes-sha2', + 'forwarding_curve25519_key_chain': sess.forwardingCurve25519KeyChain, + 'sender_key': sess.senderKey, + 'sender_clencaimed_keys': sess.senderClaimedKeys, + 'session_key': sess.inboundGroupSession + .export_session(sess.inboundGroupSession.first_known_index()), + }; + // encrypt the content + final encrypted = enc.encrypt(json.encode(payload)); + // fetch the device, if available... + //final device = args.client.getUserDeviceKeysByCurve25519Key(sess.senderKey); + // aaaand finally add the session key to our payload + roomKeys.rooms[sess.roomId].sessions[sess.sessionId] = RoomKeysSingleKey( + firstMessageIndex: sess.inboundGroupSession.first_known_index(), + forwardedCount: sess.forwardingCurve25519KeyChain.length, + isVerified: dbSession.verified, //device?.verified ?? false, + sessionData: { + 'ephemeral': encrypted.ephemeral, + 'ciphertext': encrypted.ciphertext, + 'mac': encrypted.mac, + }, + ); + } + return roomKeys; + } catch (e, s) { + Logs.error('[Key Manager] Error generating payload ' + e.toString(), s); + rethrow; + } finally { + enc.free(); + } +} + +class _DbInboundGroupSessionBundle { + _DbInboundGroupSessionBundle({this.dbSession, this.verified}); + + DbInboundGroupSession dbSession; + bool verified; +} + +class _GenerateUploadKeysArgs { + _GenerateUploadKeysArgs({this.pubkey, this.dbSessions, this.userId}); + + String pubkey; + List<_DbInboundGroupSessionBundle> dbSessions; + String userId; +} diff --git a/lib/encryption/ssss.dart b/lib/encryption/ssss.dart index 8386f03..f6be44d 100644 --- a/lib/encryption/ssss.dart +++ b/lib/encryption/ssss.dart @@ -27,6 +27,7 @@ import 'package:password_hash/password_hash.dart'; import '../famedlysdk.dart'; import '../matrix_api.dart'; +import '../src/database/database.dart'; import '../src/utils/logs.dart'; import 'encryption.dart'; @@ -48,8 +49,15 @@ class SSSS { Client get client => encryption.client; final pendingShareRequests = {}; final _validators = Function(String)>{}; + final Map _cache = {}; SSSS(this.encryption); + // for testing + Future clearCache() async { + await client.database?.clearSSSSCache(client.id); + _cache.clear(); + } + static _DerivedKeys deriveKeys(Uint8List key, String name) { final zerosalt = Uint8List(8); final prk = Hmac(sha256, zerosalt).convert(key); @@ -173,16 +181,22 @@ class SSSS { if (client.database == null) { return null; } + // check if it is still valid + final keys = keyIdsFromType(type); + final isValid = (dbEntry) => + keys.contains(dbEntry.keyId) && + client.accountData[type].content['encrypted'][dbEntry.keyId] + ['ciphertext'] == + dbEntry.ciphertext; + if (_cache.containsKey(type) && isValid(_cache[type])) { + return _cache[type].content; + } final ret = await client.database.getSSSSCache(client.id, type); if (ret == null) { return null; } - // check if it is still valid - final keys = keyIdsFromType(type); - if (keys.contains(ret.keyId) && - client.accountData[type].content['encrypted'][ret.keyId] - ['ciphertext'] == - ret.ciphertext) { + if (isValid(ret)) { + _cache[type] = ret; return ret.content; } return null; diff --git a/lib/encryption/utils/session_key.dart b/lib/encryption/utils/session_key.dart index 0477935..b54ca32 100644 --- a/lib/encryption/utils/session_key.dart +++ b/lib/encryption/utils/session_key.dart @@ -37,12 +37,16 @@ class SessionKey { Map senderClaimedKeys; String senderKey; bool get isValid => inboundGroupSession != null; + String roomId; + String sessionId; SessionKey( {this.content, this.inboundGroupSession, this.key, this.indexes, + this.roomId, + this.sessionId, String senderKey, Map senderClaimedKeys}) { _setSenderKey(senderKey); @@ -59,6 +63,8 @@ class SessionKey { indexes = parsedIndexes != null ? Map.from(parsedIndexes) : {}; + roomId = dbEntry.roomId; + sessionId = dbEntry.sessionId; _setSenderKey(dbEntry.senderKey); _setSenderClaimedKeys(Map.from(parsedSenderClaimedKeys)); diff --git a/lib/matrix_api/matrix_api.dart b/lib/matrix_api/matrix_api.dart index f738a83..4e372e1 100644 --- a/lib/matrix_api/matrix_api.dart +++ b/lib/matrix_api/matrix_api.dart @@ -2062,7 +2062,7 @@ class MatrixApi { return RoomKeysRoom.fromJson(ret); } - /// Deletes room ekys for a room + /// Deletes room keys for a room /// https://matrix.org/docs/spec/client_server/unstable#delete-matrix-client-r0-room-keys-keys-roomid Future deleteRoomKeysRoom( String roomId, String version) async { diff --git a/lib/matrix_api/model/room_keys_keys.dart b/lib/matrix_api/model/room_keys_keys.dart index 3b2c88e..6987066 100644 --- a/lib/matrix_api/model/room_keys_keys.dart +++ b/lib/matrix_api/model/room_keys_keys.dart @@ -22,6 +22,12 @@ class RoomKeysSingleKey { bool isVerified; Map sessionData; + RoomKeysSingleKey( + {this.firstMessageIndex, + this.forwardedCount, + this.isVerified, + this.sessionData}); + RoomKeysSingleKey.fromJson(Map json) { firstMessageIndex = json['first_message_index']; forwardedCount = json['forwarded_count']; @@ -42,6 +48,10 @@ class RoomKeysSingleKey { class RoomKeysRoom { Map sessions; + RoomKeysRoom({this.sessions}) { + sessions ??= {}; + } + RoomKeysRoom.fromJson(Map json) { sessions = (json['sessions'] as Map) .map((k, v) => MapEntry(k, RoomKeysSingleKey.fromJson(v))); @@ -57,6 +67,10 @@ class RoomKeysRoom { class RoomKeys { Map rooms; + RoomKeys({this.rooms}) { + rooms ??= {}; + } + RoomKeys.fromJson(Map json) { rooms = (json['rooms'] as Map) .map((k, v) => MapEntry(k, RoomKeysRoom.fromJson(v))); diff --git a/lib/src/client.dart b/lib/src/client.dart index 76b38b3..8e67d42 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -615,6 +615,7 @@ class Client extends MatrixApi { return; } + encryption?.dispose(); encryption = Encryption(client: this, enableE2eeRecovery: enableE2eeRecovery); await encryption.init(olmAccount); @@ -1165,6 +1166,18 @@ class Client extends MatrixApi { Map get userDeviceKeys => _userDeviceKeys; Map _userDeviceKeys = {}; + /// Gets user device keys by its curve25519 key. Returns null if it isn't found + DeviceKeys getUserDeviceKeysByCurve25519Key(String senderKey) { + for (final user in userDeviceKeys.values) { + final device = user.deviceKeys.values + .firstWhere((e) => e.curve25519Key == senderKey, orElse: () => null); + if (device != null) { + return device; + } + } + return null; + } + Future> _getUserIdsInEncryptedRooms() async { var userIds = {}; for (var i = 0; i < rooms.length; i++) { @@ -1493,6 +1506,8 @@ class Client extends MatrixApi { } if (closeDatabase) await database?.close(); database = null; + encryption?.dispose(); + encryption = null; return; } } diff --git a/lib/src/database/database.g.dart b/lib/src/database/database.g.dart index 2ee1edb..283679e 100644 --- a/lib/src/database/database.g.dart +++ b/lib/src/database/database.g.dart @@ -5873,6 +5873,27 @@ abstract class _$Database extends GeneratedDatabase { ); } + Selectable getInboundGroupSessionsToUpload() { + return customSelect( + 'SELECT * FROM inbound_group_sessions WHERE uploaded = false LIMIT 500', + variables: [], + readsFrom: {inboundGroupSessions}).map(_rowToDbInboundGroupSession); + } + + Future markInboundGroupSessionAsUploaded( + int client_id, String room_id, String session_id) { + return customUpdate( + 'UPDATE inbound_group_sessions SET uploaded = true WHERE client_id = :client_id AND room_id = :room_id AND session_id = :session_id', + variables: [ + Variable.withInt(client_id), + Variable.withString(room_id), + Variable.withString(session_id) + ], + updates: {inboundGroupSessions}, + updateKind: UpdateKind.update, + ); + } + Future storeUserDeviceKeysInfo( int client_id, String user_id, bool outdated) { return customInsert( diff --git a/lib/src/database/database.moor b/lib/src/database/database.moor index 13de47e..7ce4425 100644 --- a/lib/src/database/database.moor +++ b/lib/src/database/database.moor @@ -191,6 +191,8 @@ dbGetInboundGroupSessionKeys: SELECT * FROM inbound_group_sessions WHERE client_ getAllInboundGroupSessions: SELECT * FROM inbound_group_sessions WHERE client_id = :client_id; storeInboundGroupSession: INSERT OR REPLACE INTO inbound_group_sessions (client_id, room_id, session_id, pickle, content, indexes, sender_key, sender_claimed_keys) VALUES (:client_id, :room_id, :session_id, :pickle, :content, :indexes, :sender_key, :sender_claimed_keys); updateInboundGroupSessionIndexes: UPDATE inbound_group_sessions SET indexes = :indexes WHERE client_id = :client_id AND room_id = :room_id AND session_id = :session_id; +getInboundGroupSessionsToUpload: SELECT * FROM inbound_group_sessions WHERE uploaded = false LIMIT 500; +markInboundGroupSessionAsUploaded: UPDATE inbound_group_sessions SET uploaded = true WHERE client_id = :client_id AND room_id = :room_id AND session_id = :session_id; storeUserDeviceKeysInfo: INSERT OR REPLACE INTO user_device_keys (client_id, user_id, outdated) VALUES (:client_id, :user_id, :outdated); setVerifiedUserDeviceKey: UPDATE user_device_keys_key SET verified = :verified WHERE client_id = :client_id AND user_id = :user_id AND device_id = :device_id; setBlockedUserDeviceKey: UPDATE user_device_keys_key SET blocked = :blocked WHERE client_id = :client_id AND user_id = :user_id AND device_id = :device_id; diff --git a/lib/src/utils/logs.dart b/lib/src/utils/logs.dart index f774de3..fa2609a 100644 --- a/lib/src/utils/logs.dart +++ b/lib/src/utils/logs.dart @@ -1,3 +1,21 @@ +/* + * Famedly Matrix SDK + * Copyright (C) 2020 Famedly GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + import 'package:ansicolor/ansicolor.dart'; abstract class Logs { diff --git a/lib/src/utils/run_in_background.dart b/lib/src/utils/run_in_background.dart new file mode 100644 index 0000000..cbc5a40 --- /dev/null +++ b/lib/src/utils/run_in_background.dart @@ -0,0 +1,30 @@ +/* + * Famedly Matrix SDK + * Copyright (C) 2020 Famedly GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import 'package:isolate/isolate.dart'; +import 'dart:async'; + +Future runInBackground( + FutureOr Function(U arg) function, U arg) async { + final isolate = await IsolateRunner.spawn(); + try { + return await isolate.run(function, arg); + } finally { + await isolate.close(); + } +} diff --git a/pubspec.lock b/pubspec.lock index 25d83fe..ebc2d2f 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -281,6 +281,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "0.3.4" + isolate: + dependency: "direct main" + description: + name: isolate + url: "https://pub.dartlang.org" + source: hosted + version: "2.0.3" js: dependency: transitive description: @@ -609,7 +616,7 @@ packages: name: test_coverage url: "https://pub.dartlang.org" source: hosted - version: "0.4.1" + version: "0.4.3" timing: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index d594d4c..8880a3c 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -22,10 +22,11 @@ dependencies: olm: ^1.2.1 matrix_file_e2ee: ^1.0.4 ansicolor: ^1.0.2 + isolate: ^2.0.3 dev_dependencies: test: ^1.0.0 - test_coverage: ^0.4.1 + test_coverage: ^0.4.3 moor_generator: ^3.0.0 build_runner: ^1.5.2 pedantic: ^1.9.0 diff --git a/test.sh b/test.sh index 2496656..f6300dd 100644 --- a/test.sh +++ b/test.sh @@ -1,6 +1,6 @@ #!/bin/sh -e -pub run test -p vm -pub run test_coverage +# pub run test -p vm +pub run test_coverage --print-test-output pub global activate remove_from_coverage pub global run remove_from_coverage:remove_from_coverage -f coverage/lcov.info -r '\.g\.dart$' genhtml -o coverage coverage/lcov.info || true diff --git a/test/encryption/key_verification_test.dart b/test/encryption/key_verification_test.dart index 2612207..319b8f5 100644 --- a/test/encryption/key_verification_test.dart +++ b/test/encryption/key_verification_test.dart @@ -207,7 +207,7 @@ void main() { test('ask SSSS start', () async { client1.userDeviceKeys[client1.userID].masterKey.setDirectVerified(true); - await client1.database.clearSSSSCache(client1.id); + await client1.encryption.ssss.clearCache(); final req1 = await client1.userDeviceKeys[client2.userID].startVerification(); expect(req1.state, KeyVerificationState.askSSSS); @@ -288,7 +288,7 @@ void main() { // alright, they match client1.userDeviceKeys[client1.userID].masterKey.setDirectVerified(true); - await client1.database.clearSSSSCache(client1.id); + await client1.encryption.ssss.clearCache(); // send mac FakeMatrixApi.calledEndpoints.clear(); @@ -312,7 +312,7 @@ void main() { client1.encryption.ssss = MockSSSS(client1.encryption); (client1.encryption.ssss as MockSSSS).requestedSecrets = false; - await client1.database.clearSSSSCache(client1.id); + await client1.encryption.ssss.clearCache(); await req1.maybeRequestSSSSSecrets(); await Future.delayed(Duration(milliseconds: 10)); expect((client1.encryption.ssss as MockSSSS).requestedSecrets, true); diff --git a/test/encryption/online_key_backup_test.dart b/test/encryption/online_key_backup_test.dart index 12b9ae0..9218466 100644 --- a/test/encryption/online_key_backup_test.dart +++ b/test/encryption/online_key_backup_test.dart @@ -16,12 +16,16 @@ * along with this program. If not, see . */ +import 'dart:convert'; + import 'package:famedlysdk/famedlysdk.dart'; import 'package:famedlysdk/src/utils/logs.dart'; +import 'package:famedlysdk/matrix_api.dart'; import 'package:test/test.dart'; import 'package:olm/olm.dart' as olm; import '../fake_client.dart'; +import '../fake_matrix_api.dart'; void main() { group('Online Key Backup', () { @@ -67,6 +71,49 @@ void main() { true); }); + test('upload key', () async { + final session = olm.OutboundGroupSession(); + session.create(); + final inbound = olm.InboundGroupSession(); + inbound.create(session.session_key()); + final senderKey = client.identityKey; + final roomId = '!someroom:example.org'; + final sessionId = inbound.session_id(); + // set a payload... + var sessionPayload = { + 'algorithm': 'm.megolm.v1.aes-sha2', + 'room_id': roomId, + 'forwarding_curve25519_key_chain': [client.identityKey], + 'session_id': sessionId, + 'session_key': inbound.export_session(1), + 'sender_key': senderKey, + 'sender_claimed_ed25519_key': client.fingerprintKey, + }; + FakeMatrixApi.calledEndpoints.clear(); + client.encryption.keyManager.setInboundGroupSession( + roomId, sessionId, senderKey, sessionPayload, + forwarded: true); + var dbSessions = + await client.database.getInboundGroupSessionsToUpload().get(); + expect(dbSessions.isNotEmpty, true); + await client.encryption.keyManager.backgroundTasks(); + final payload = FakeMatrixApi + .calledEndpoints['/client/unstable/room_keys/keys?version=5'].first; + dbSessions = + await client.database.getInboundGroupSessionsToUpload().get(); + expect(dbSessions.isEmpty, true); + + final onlineKeys = RoomKeys.fromJson(json.decode(payload)); + client.encryption.keyManager.clearInboundGroupSessions(); + var ret = client.encryption.keyManager + .getInboundGroupSession(roomId, sessionId, senderKey); + expect(ret, null); + await client.encryption.keyManager.loadFromResponse(onlineKeys); + ret = client.encryption.keyManager + .getInboundGroupSession(roomId, sessionId, senderKey); + expect(ret != null, true); + }); + test('dispose client', () async { await client.dispose(closeDatabase: true); }); diff --git a/test/encryption/ssss_test.dart b/test/encryption/ssss_test.dart index d213248..e698e09 100644 --- a/test/encryption/ssss_test.dart +++ b/test/encryption/ssss_test.dart @@ -248,7 +248,7 @@ void main() { client.encryption.ssss.open('m.cross_signing.self_signing'); handle.unlock(recoveryKey: SSSS_KEY); - await client.database.clearSSSSCache(client.id); + await client.encryption.ssss.clearCache(); client.encryption.ssss.pendingShareRequests.clear(); await client.encryption.ssss.request('best animal', [key]); var event = ToDeviceEvent( @@ -272,7 +272,7 @@ void main() { 'm.megolm_backup.v1' ]) { final secret = await handle.getStored(type); - await client.database.clearSSSSCache(client.id); + await client.encryption.ssss.clearCache(); client.encryption.ssss.pendingShareRequests.clear(); await client.encryption.ssss.request(type, [key]); event = ToDeviceEvent( @@ -294,7 +294,7 @@ void main() { // test different fail scenarios // not encrypted - await client.database.clearSSSSCache(client.id); + await client.encryption.ssss.clearCache(); client.encryption.ssss.pendingShareRequests.clear(); await client.encryption.ssss.request('best animal', [key]); event = ToDeviceEvent( @@ -309,7 +309,7 @@ void main() { expect(await client.encryption.ssss.getCached('best animal'), null); // unknown request id - await client.database.clearSSSSCache(client.id); + await client.encryption.ssss.clearCache(); client.encryption.ssss.pendingShareRequests.clear(); await client.encryption.ssss.request('best animal', [key]); event = ToDeviceEvent( @@ -327,7 +327,7 @@ void main() { expect(await client.encryption.ssss.getCached('best animal'), null); // not from a device we sent the request to - await client.database.clearSSSSCache(client.id); + await client.encryption.ssss.clearCache(); client.encryption.ssss.pendingShareRequests.clear(); await client.encryption.ssss.request('best animal', [key]); event = ToDeviceEvent( @@ -345,7 +345,7 @@ void main() { expect(await client.encryption.ssss.getCached('best animal'), null); // secret not a string - await client.database.clearSSSSCache(client.id); + await client.encryption.ssss.clearCache(); client.encryption.ssss.pendingShareRequests.clear(); await client.encryption.ssss.request('best animal', [key]); event = ToDeviceEvent( @@ -363,7 +363,7 @@ void main() { expect(await client.encryption.ssss.getCached('best animal'), null); // validator doesn't check out - await client.database.clearSSSSCache(client.id); + await client.encryption.ssss.clearCache(); client.encryption.ssss.pendingShareRequests.clear(); await client.encryption.ssss.request('m.megolm_backup.v1', [key]); event = ToDeviceEvent( @@ -386,7 +386,7 @@ void main() { final key = client.userDeviceKeys[client.userID].deviceKeys['OTHERDEVICE']; key.setDirectVerified(true); - await client.database.clearSSSSCache(client.id); + await client.encryption.ssss.clearCache(); client.encryption.ssss.pendingShareRequests.clear(); await client.encryption.ssss.maybeRequestAll([key]); expect(client.encryption.ssss.pendingShareRequests.length, 3); diff --git a/test/user_test.dart b/test/user_test.dart index 0e7de24..ea2d7fe 100644 --- a/test/user_test.dart +++ b/test/user_test.dart @@ -132,6 +132,8 @@ void main() { await client.checkServer('https://fakeserver.notexisting'); expect(user1.canChangePowerLevel, false); }); - client.dispose(); + test('dispose client', () async { + await client.dispose(); + }); }); } From aa9940fdbc25619e594dc984efe1a4a533950af1 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Thu, 10 Sep 2020 09:37:56 +0200 Subject: [PATCH 58/90] fix: Room.requestUser sometimes throws an error --- lib/src/room.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/room.dart b/lib/src/room.dart index 8f528e5..3e2168c 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -1117,7 +1117,7 @@ class Room { final profile = await client.requestProfile(mxID); resp = { 'displayname': profile.displayname, - 'avatar_url': profile.avatarUrl, + 'avatar_url': profile.avatarUrl.toString(), }; } catch (exception) { _requestingMatrixIds.remove(mxID); From 5d45c224a31e6142fff0c6c10ae82a96cd037f32 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Thu, 10 Sep 2020 09:50:02 +0200 Subject: [PATCH 59/90] fix: Mark pending events as failed on startup --- lib/src/database/database.dart | 1 + lib/src/database/database.g.dart | 9 +++++++++ lib/src/database/database.moor | 1 + 3 files changed, 11 insertions(+) diff --git a/lib/src/database/database.dart b/lib/src/database/database.dart index 81dff83..4da4f2d 100644 --- a/lib/src/database/database.dart +++ b/lib/src/database/database.dart @@ -143,6 +143,7 @@ class Database extends _$Database { Future getClient(String name) async { final res = await dbGetClient(name).get(); if (res.isEmpty) return null; + await markPendingEventsAsError(res.first.clientId); return res.first; } diff --git a/lib/src/database/database.g.dart b/lib/src/database/database.g.dart index 283679e..486d2a0 100644 --- a/lib/src/database/database.g.dart +++ b/lib/src/database/database.g.dart @@ -6514,6 +6514,15 @@ abstract class _$Database extends GeneratedDatabase { readsFrom: {files}).map(_rowToDbFile); } + Future markPendingEventsAsError(int client_id) { + return customUpdate( + 'UPDATE events SET status = -1 WHERE client_id = :client_id AND status = 0', + variables: [Variable.withInt(client_id)], + updates: {events}, + updateKind: UpdateKind.update, + ); + } + @override Iterable get allTables => allSchemaEntities.whereType(); @override diff --git a/lib/src/database/database.moor b/lib/src/database/database.moor index 7ce4425..9f00534 100644 --- a/lib/src/database/database.moor +++ b/lib/src/database/database.moor @@ -233,3 +233,4 @@ removeRoom: DELETE FROM rooms WHERE client_id = :client_id AND room_id = :room_i removeRoomEvents: DELETE FROM events WHERE client_id = :client_id AND room_id = :room_id; storeFile: INSERT OR REPLACE INTO files (mxc_uri, bytes, saved_at) VALUES (:mxc_uri, :bytes, :time); dbGetFile: SELECT * FROM files WHERE mxc_uri = :mxc_uri; +markPendingEventsAsError: UPDATE events SET status = -1 WHERE client_id = :client_id AND status = 0; From e08f35b5d018d50622ab5bef481df2c89e0d28d9 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Thu, 10 Sep 2020 11:10:38 +0200 Subject: [PATCH 60/90] fix: Don't trust the info block of events --- lib/src/event.dart | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/src/event.dart b/lib/src/event.dart index 29dfb7b..d369534 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -406,7 +406,7 @@ class Event extends MatrixEvent { // Is this file storeable? final infoMap = getThumbnail ? content['info']['thumbnail_info'] : content['info']; - final storeable = room.client.database != null && + var storeable = room.client.database != null && infoMap is Map && infoMap['size'] is int && infoMap['size'] <= room.client.database.maxFileSize; @@ -422,6 +422,8 @@ class Event extends MatrixEvent { }; uint8list = await downloadCallback(mxContent.getDownloadLink(room.client)); + storeable = storeable && + uint8list.lengthInBytes < room.client.database.maxFileSize; if (storeable) { await room.client.database .storeFile(mxContent.toString(), uint8list, DateTime.now()); From 64b8e014444982c3091340d989d5d9bf92c314cb Mon Sep 17 00:00:00 2001 From: Sorunome Date: Thu, 10 Sep 2020 11:24:37 +0200 Subject: [PATCH 61/90] fix: Handle duplicate indexes properly --- lib/encryption/encryption.dart | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/encryption/encryption.dart b/lib/encryption/encryption.dart index f6f3a3c..b56dd1e 100644 --- a/lib/encryption/encryption.dart +++ b/lib/encryption/encryption.dart @@ -148,6 +148,13 @@ class Encryption { // TODO: maybe clear outbound session, if it is ours throw (DecryptError.CHANNEL_CORRUPTED); } + final existingIndex = inboundGroupSession.indexes.entries.firstWhere( + (e) => e.value == decryptResult.message_index, + orElse: () => null); + if (existingIndex != null && existingIndex.key != messageIndexKey) { + // TODO: maybe clear outbound session, if it is ours + throw (DecryptError.CHANNEL_CORRUPTED); + } inboundGroupSession.indexes[messageIndexKey] = decryptResult.message_index; if (!haveIndex) { From bbc1b63695b63e3316314c0a962d6b62903a5c6d Mon Sep 17 00:00:00 2001 From: Sorunome Date: Thu, 10 Sep 2020 12:56:50 +0200 Subject: [PATCH 62/90] feat: Auto-verify own master key, if there is a valid signature chain within the same account --- lib/encryption/encryption.dart | 16 ++++++++++++++++ lib/src/utils/device_keys_list.dart | 23 +++++++++++++++++++---- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/lib/encryption/encryption.dart b/lib/encryption/encryption.dart index b56dd1e..8675974 100644 --- a/lib/encryption/encryption.dart +++ b/lib/encryption/encryption.dart @@ -316,6 +316,20 @@ class Encryption { return await olmManager.encryptToDeviceMessage(deviceKeys, type, payload); } + Future autovalidateMasterOwnKey() async { + // check if we can set our own master key as verified, if it isn't yet + if (client.database != null && + client.userDeviceKeys.containsKey(client.userID)) { + final masterKey = client.userDeviceKeys[client.userID].masterKey; + if (masterKey != null && + !masterKey.directVerified && + masterKey + .hasValidSignatureChain(onlyValidateUserIds: {client.userID})) { + await masterKey.setVerified(true); + } + } + } + // this method is responsible for all background tasks, such as uploading online key backups bool _backgroundTasksRunning = true; void _backgroundTasks() { @@ -325,6 +339,8 @@ class Encryption { keyManager.backgroundTasks(); + autovalidateMasterOwnKey(); + if (_backgroundTasksRunning) { Timer(Duration(seconds: 10), _backgroundTasks); } diff --git a/lib/src/utils/device_keys_list.dart b/lib/src/utils/device_keys_list.dart index cc8b79c..9fb8f8b 100644 --- a/lib/src/utils/device_keys_list.dart +++ b/lib/src/utils/device_keys_list.dart @@ -157,14 +157,20 @@ abstract class SignableKey extends MatrixSignableKey { return valid; } - bool hasValidSignatureChain({bool verifiedOnly = true, Set visited}) { + bool hasValidSignatureChain( + {bool verifiedOnly = true, + Set visited, + Set onlyValidateUserIds}) { if (!client.encryptionEnabled) { return false; } visited ??= {}; + onlyValidateUserIds ??= {}; final setKey = '${userId};${identifier}'; - if (visited.contains(setKey)) { - return false; // prevent recursion + if (visited.contains(setKey) || + (onlyValidateUserIds.isNotEmpty && + !onlyValidateUserIds.contains(userId))) { + return false; // prevent recursion & validate hasValidSignatureChain } visited.add(setKey); for (final signatureEntries in signatures.entries) { @@ -189,6 +195,13 @@ abstract class SignableKey extends MatrixSignableKey { } else { continue; } + + if (onlyValidateUserIds.isNotEmpty && + !onlyValidateUserIds.contains(key.userId)) { + // we don't want to verify keys from this user + continue; + } + if (key.blocked) { continue; // we can't be bothered about this keys signatures } @@ -228,7 +241,9 @@ abstract class SignableKey extends MatrixSignableKey { } // or else we just recurse into that key and chack if it works out final haveChain = key.hasValidSignatureChain( - verifiedOnly: verifiedOnly, visited: visited); + verifiedOnly: verifiedOnly, + visited: visited, + onlyValidateUserIds: onlyValidateUserIds); if (haveChain) { return true; } From b5ac5001367736dc5b8e42e57a946e8a4525f3be Mon Sep 17 00:00:00 2001 From: Sorunome Date: Thu, 10 Sep 2020 12:11:13 +0200 Subject: [PATCH 63/90] fix: Handle failed to send messages in low network significantly better --- lib/src/database/database.dart | 21 ++++++++++---- lib/src/database/database.g.dart | 4 +-- lib/src/database/database.moor | 2 +- lib/src/timeline.dart | 11 +++++++- test/timeline_test.dart | 48 ++++++++++++++++++++++++++++---- 5 files changed, 70 insertions(+), 16 deletions(-) diff --git a/lib/src/database/database.dart b/lib/src/database/database.dart index 4da4f2d..3998af9 100644 --- a/lib/src/database/database.dart +++ b/lib/src/database/database.dart @@ -399,7 +399,7 @@ class Database extends _$Database { // Is the timeline limited? Then all previous messages should be // removed from the database! if (roomUpdate.limitedTimeline) { - await removeRoomEvents(clientId, roomUpdate.id); + await removeSuccessfulRoomEvents(clientId, roomUpdate.id); await updateRoomSortOrder(0.0, 0.0, clientId, roomUpdate.id); await setRoomPrevBatch(roomUpdate.prev_batch, clientId, roomUpdate.id); } @@ -432,9 +432,10 @@ class Database extends _$Database { status = eventContent['unsigned'][MessageSendingStatusKey]; } if (eventContent['status'] is num) status = eventContent['status']; - if ((status == 1 || status == -1) && + var storeNewEvent = !((status == 1 || status == -1) && eventContent['unsigned'] is Map && - eventContent['unsigned']['transaction_id'] is String) { + eventContent['unsigned']['transaction_id'] is String); + if (!storeNewEvent) { final allOldEvents = await getEvent(clientId, eventContent['event_id'], chatId).get(); if (allOldEvents.isNotEmpty) { @@ -452,8 +453,15 @@ class Database extends _$Database { } else { // status changed and we have an old transaction id --> update event id and stuffs try { - await updateEventStatus(status, eventContent['event_id'], clientId, - eventContent['unsigned']['transaction_id'], chatId); + final updated = await updateEventStatus( + status, + eventContent['event_id'], + clientId, + eventContent['unsigned']['transaction_id'], + chatId); + if (updated == 0) { + storeNewEvent = true; + } } catch (err) { // we could not update the transaction id to the event id....so it already exists // as we just tried to fetch the event previously this is a race condition if the event comes down sync in the mean time @@ -461,7 +469,8 @@ class Database extends _$Database { // than our status. So, we just ignore this error } } - } else { + } + if (storeNewEvent) { DbEvent oldEvent; if (type == 'history') { final allOldEvents = diff --git a/lib/src/database/database.g.dart b/lib/src/database/database.g.dart index 486d2a0..103bd86 100644 --- a/lib/src/database/database.g.dart +++ b/lib/src/database/database.g.dart @@ -6479,9 +6479,9 @@ abstract class _$Database extends GeneratedDatabase { ); } - Future removeRoomEvents(int client_id, String room_id) { + Future removeSuccessfulRoomEvents(int client_id, String room_id) { return customUpdate( - 'DELETE FROM events WHERE client_id = :client_id AND room_id = :room_id', + 'DELETE FROM events WHERE client_id = :client_id AND room_id = :room_id AND status <> -1 AND status <> 0', variables: [Variable.withInt(client_id), Variable.withString(room_id)], updates: {events}, updateKind: UpdateKind.delete, diff --git a/lib/src/database/database.moor b/lib/src/database/database.moor index 9f00534..17221d4 100644 --- a/lib/src/database/database.moor +++ b/lib/src/database/database.moor @@ -230,7 +230,7 @@ getRoom: SELECT * FROM rooms WHERE client_id = :client_id AND room_id = :room_id getEvent: SELECT * FROM events WHERE client_id = :client_id AND event_id = :event_id AND room_id = :room_id; removeEvent: DELETE FROM events WHERE client_id = :client_id AND event_id = :event_id AND room_id = :room_id; removeRoom: DELETE FROM rooms WHERE client_id = :client_id AND room_id = :room_id; -removeRoomEvents: DELETE FROM events WHERE client_id = :client_id AND room_id = :room_id; +removeSuccessfulRoomEvents: DELETE FROM events WHERE client_id = :client_id AND room_id = :room_id AND status <> -1 AND status <> 0; storeFile: INSERT OR REPLACE INTO files (mxc_uri, bytes, saved_at) VALUES (:mxc_uri, :bytes, :time); dbGetFile: SELECT * FROM files WHERE mxc_uri = :mxc_uri; markPendingEventsAsError: UPDATE events SET status = -1 WHERE client_id = :client_id AND status = 0; diff --git a/lib/src/timeline.dart b/lib/src/timeline.dart index df7e091..1ec96aa 100644 --- a/lib/src/timeline.dart +++ b/lib/src/timeline.dart @@ -99,6 +99,7 @@ class Timeline { for (final e in events) { addAggregatedEvent(e); } + _sort(); } /// Don't forget to call this before you dismiss this object! @@ -274,7 +275,15 @@ class Timeline { void _sort() { if (_sortLock || events.length < 2) return; _sortLock = true; - events?.sort((a, b) => b.sortOrder - a.sortOrder > 0 ? 1 : -1); + events?.sort((a, b) { + if (b.status == -1 && a.status != -1) { + return 1; + } + if (a.status == -1 && b.status != -1) { + return -1; + } + return b.sortOrder - a.sortOrder > 0 ? 1 : -1; + }); _sortLock = false; } } diff --git a/test/timeline_test.dart b/test/timeline_test.dart index 5b308f0..3ce1a09 100644 --- a/test/timeline_test.dart +++ b/test/timeline_test.dart @@ -218,6 +218,7 @@ void main() { }); test('Resend message', () async { + timeline.events.clear(); client.onEvent.add(EventUpdate( type: 'timeline', roomID: roomID, @@ -233,6 +234,7 @@ void main() { }, sortOrder: room.newSortOrder)); await Future.delayed(Duration(milliseconds: 50)); + expect(timeline.events[0].status, -1); await timeline.events[0].sendAgain(); await Future.delayed(Duration(milliseconds: 50)); @@ -240,22 +242,23 @@ void main() { expect(updateCount, 17); expect(insertList, [0, 0, 0, 0, 0, 0, 0, 0]); - expect(timeline.events.length, 7); + expect(timeline.events.length, 1); expect(timeline.events[0].status, 1); }); test('Request history', () async { + timeline.events.clear(); await room.requestHistory(); await Future.delayed(Duration(milliseconds: 50)); expect(updateCount, 20); - expect(timeline.events.length, 10); - expect(timeline.events[7].eventId, '3143273582443PhrSn:example.org'); - expect(timeline.events[8].eventId, '2143273582443PhrSn:example.org'); - expect(timeline.events[9].eventId, '1143273582443PhrSn:example.org'); + expect(timeline.events.length, 3); + expect(timeline.events[0].eventId, '3143273582443PhrSn:example.org'); + expect(timeline.events[1].eventId, '2143273582443PhrSn:example.org'); + expect(timeline.events[2].eventId, '1143273582443PhrSn:example.org'); expect(room.prev_batch, 't47409-4357353_219380_26003_2265'); - await timeline.events[9].redact(reason: 'test', txid: '1234'); + await timeline.events[2].redact(reason: 'test', txid: '1234'); }); test('Clear cache on limited timeline', () async { @@ -271,6 +274,39 @@ void main() { expect(timeline.events.isEmpty, true); }); + test('sort errors on top', () async { + timeline.events.clear(); + client.onEvent.add(EventUpdate( + type: 'timeline', + roomID: roomID, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'content': {'msgtype': 'm.text', 'body': 'Testcase'}, + 'sender': '@alice:example.com', + 'status': -1, + 'event_id': 'abc', + 'origin_server_ts': testTimeStamp + }, + sortOrder: room.newSortOrder)); + client.onEvent.add(EventUpdate( + type: 'timeline', + roomID: roomID, + eventType: 'm.room.message', + content: { + 'type': 'm.room.message', + 'content': {'msgtype': 'm.text', 'body': 'Testcase'}, + 'sender': '@alice:example.com', + 'status': 2, + 'event_id': 'def', + 'origin_server_ts': testTimeStamp + 5 + }, + sortOrder: room.newSortOrder)); + await Future.delayed(Duration(milliseconds: 50)); + expect(timeline.events[0].status, -1); + expect(timeline.events[1].status, 2); + }); + test('sending event to failed update', () async { timeline.events.clear(); client.onEvent.add(EventUpdate( From 2c7ae759f8c83ae19e5ab739e532c6ee1d0e42fc Mon Sep 17 00:00:00 2001 From: Sorunome Date: Thu, 10 Sep 2020 12:17:38 +0200 Subject: [PATCH 64/90] fix: Remove potential race conditions and database issues with OTK upload --- lib/encryption/encryption.dart | 2 +- lib/encryption/olm_manager.dart | 113 ++++++++++++++++++-------------- 2 files changed, 63 insertions(+), 52 deletions(-) diff --git a/lib/encryption/encryption.dart b/lib/encryption/encryption.dart index 8675974..97f4188 100644 --- a/lib/encryption/encryption.dart +++ b/lib/encryption/encryption.dart @@ -68,7 +68,7 @@ class Encryption { } void handleDeviceOneTimeKeysCount(Map countJson) { - olmManager.handleDeviceOneTimeKeysCount(countJson); + Zone.root.run(() => olmManager.handleDeviceOneTimeKeysCount(countJson)); } void onSync() { diff --git a/lib/encryption/olm_manager.dart b/lib/encryption/olm_manager.dart index a0078ec..1c2cac7 100644 --- a/lib/encryption/olm_manager.dart +++ b/lib/encryption/olm_manager.dart @@ -130,6 +130,8 @@ class OlmManager { return isValid; } + bool _uploadKeysLock = false; + /// Generates new one time keys, signs everything and upload it to the server. Future uploadKeys( {bool uploadDeviceKeys = false, int oldKeyCount = 0}) async { @@ -137,62 +139,71 @@ class OlmManager { return true; } - // generate one-time keys - // we generate 2/3rds of max, so that other keys people may still have can - // still be used - final oneTimeKeysCount = - (_olmAccount.max_number_of_one_time_keys() * 2 / 3).floor() - - oldKeyCount; - _olmAccount.generate_one_time_keys(oneTimeKeysCount); - final Map oneTimeKeys = - json.decode(_olmAccount.one_time_keys()); - - // now sign all the one-time keys - final signedOneTimeKeys = {}; - for (final entry in oneTimeKeys['curve25519'].entries) { - final key = entry.key; - final value = entry.value; - signedOneTimeKeys['signed_curve25519:$key'] = {}; - signedOneTimeKeys['signed_curve25519:$key'] = signJson({ - 'key': value, - }); + if (_uploadKeysLock) { + return false; } + _uploadKeysLock = true; - // and now generate the payload to upload - final keysContent = { - if (uploadDeviceKeys) - 'device_keys': { - 'user_id': client.userID, - 'device_id': client.deviceID, - 'algorithms': [ - 'm.olm.v1.curve25519-aes-sha2', - 'm.megolm.v1.aes-sha2' - ], - 'keys': {}, - }, - }; - if (uploadDeviceKeys) { - final Map keys = - json.decode(_olmAccount.identity_keys()); - for (final entry in keys.entries) { - final algorithm = entry.key; + try { + // generate one-time keys + // we generate 2/3rds of max, so that other keys people may still have can + // still be used + final oneTimeKeysCount = + (_olmAccount.max_number_of_one_time_keys() * 2 / 3).floor() - + oldKeyCount; + _olmAccount.generate_one_time_keys(oneTimeKeysCount); + final Map oneTimeKeys = + json.decode(_olmAccount.one_time_keys()); + + // now sign all the one-time keys + final signedOneTimeKeys = {}; + for (final entry in oneTimeKeys['curve25519'].entries) { + final key = entry.key; final value = entry.value; - keysContent['device_keys']['keys']['$algorithm:${client.deviceID}'] = - value; + signedOneTimeKeys['signed_curve25519:$key'] = {}; + signedOneTimeKeys['signed_curve25519:$key'] = signJson({ + 'key': value, + }); } - keysContent['device_keys'] = - signJson(keysContent['device_keys'] as Map); - } - final response = await client.uploadDeviceKeys( - deviceKeys: uploadDeviceKeys - ? MatrixDeviceKeys.fromJson(keysContent['device_keys']) - : null, - oneTimeKeys: signedOneTimeKeys, - ); - _olmAccount.mark_keys_as_published(); - await client.database?.updateClientKeys(pickledOlmAccount, client.id); - return response['signed_curve25519'] == oneTimeKeysCount; + // and now generate the payload to upload + final keysContent = { + if (uploadDeviceKeys) + 'device_keys': { + 'user_id': client.userID, + 'device_id': client.deviceID, + 'algorithms': [ + 'm.olm.v1.curve25519-aes-sha2', + 'm.megolm.v1.aes-sha2' + ], + 'keys': {}, + }, + }; + if (uploadDeviceKeys) { + final Map keys = + json.decode(_olmAccount.identity_keys()); + for (final entry in keys.entries) { + final algorithm = entry.key; + final value = entry.value; + keysContent['device_keys']['keys']['$algorithm:${client.deviceID}'] = + value; + } + keysContent['device_keys'] = + signJson(keysContent['device_keys'] as Map); + } + + final response = await client.uploadDeviceKeys( + deviceKeys: uploadDeviceKeys + ? MatrixDeviceKeys.fromJson(keysContent['device_keys']) + : null, + oneTimeKeys: signedOneTimeKeys, + ); + _olmAccount.mark_keys_as_published(); + await client.database?.updateClientKeys(pickledOlmAccount, client.id); + return response['signed_curve25519'] == oneTimeKeysCount; + } finally { + _uploadKeysLock = false; + } } void handleDeviceOneTimeKeysCount(Map countJson) { From cb1ec86b32260d5ce0365771ed14911996b7a4fe Mon Sep 17 00:00:00 2001 From: Sorunome Date: Thu, 10 Sep 2020 13:19:05 +0200 Subject: [PATCH 65/90] feat: Periodically fetch ssss secrets from other devices --- lib/encryption/encryption.dart | 9 +++++ lib/encryption/key_manager.dart | 13 ++++--- lib/encryption/ssss.dart | 40 +++++++++++++++++++--- test/encryption/key_verification_test.dart | 2 +- test/encryption/ssss_test.dart | 25 ++++++++++++++ 5 files changed, 79 insertions(+), 10 deletions(-) diff --git a/lib/encryption/encryption.dart b/lib/encryption/encryption.dart index 97f4188..9775fb1 100644 --- a/lib/encryption/encryption.dart +++ b/lib/encryption/encryption.dart @@ -96,6 +96,10 @@ class Encryption { // some ssss thing. We can do this in the background unawaited(Zone.root.run(() => ssss.handleToDeviceEvent(event))); } + if (event.sender == client.userID) { + // maybe we need to re-try SSSS secrets + unawaited(Zone.root.run(() => ssss.periodicallyRequestMissingCache())); + } } Future handleEventUpdate(EventUpdate update) async { @@ -111,6 +115,11 @@ class Encryption { unawaited(Zone.root .run(() => keyVerificationManager.handleEventUpdate(update))); } + if (update.content['sender'] == client.userID && + !update.content['unsigned'].containsKey('transaction_id')) { + // maybe we need to re-try SSSS secrets + unawaited(Zone.root.run(() => ssss.periodicallyRequestMissingCache())); + } } Future decryptToDeviceEvent(ToDeviceEvent event) async { diff --git a/lib/encryption/key_manager.dart b/lib/encryption/key_manager.dart index d032704..edf908e 100644 --- a/lib/encryption/key_manager.dart +++ b/lib/encryption/key_manager.dart @@ -451,10 +451,15 @@ class KeyManager { try { await loadSingleKey(room.id, sessionId); } catch (err, stacktrace) { - Logs.error( - '[KeyManager] Failed to access online key backup: ' + - err.toString(), - stacktrace); + if (err is MatrixException && err.errcode == 'M_NOT_FOUND') { + Logs.info( + '[KeyManager] Key not in online key backup, requesting it from other devices...'); + } else { + Logs.error( + '[KeyManager] Failed to access online key backup: ' + + err.toString(), + stacktrace); + } } if (!hadPreviously && getInboundGroupSession(room.id, sessionId, senderKey) != null) { diff --git a/lib/encryption/ssss.dart b/lib/encryption/ssss.dart index f6be44d..4670dbb 100644 --- a/lib/encryption/ssss.dart +++ b/lib/encryption/ssss.dart @@ -258,18 +258,27 @@ class SSSS { } } - Future maybeRequestAll(List devices) async { + Future maybeRequestAll([List devices]) async { for (final type in CACHE_TYPES) { - final secret = await getCached(type); - if (secret == null) { - await request(type, devices); + if (keyIdsFromType(type) != null) { + final secret = await getCached(type); + if (secret == null) { + await request(type, devices); + } } } } - Future request(String type, List devices) async { + Future request(String type, [List devices]) async { // only send to own, verified devices Logs.info('[SSSS] Requesting type ${type}...'); + if (devices == null || devices.isEmpty) { + if (!client.userDeviceKeys.containsKey(client.userID)) { + Logs.warning('[SSSS] User does not have any devices'); + return; + } + devices = client.userDeviceKeys[client.userID].deviceKeys.values.toList(); + } devices.removeWhere((DeviceKeys d) => d.userId != client.userID || !d.verified || @@ -294,6 +303,27 @@ class SSSS { }); } + DateTime _lastCacheRequest; + bool _isPeriodicallyRequestingMissingCache = false; + Future periodicallyRequestMissingCache() async { + if (_isPeriodicallyRequestingMissingCache || + (_lastCacheRequest != null && + DateTime.now() + .subtract(Duration(minutes: 15)) + .isBefore(_lastCacheRequest)) || + client.isUnknownSession) { + // we are already requesting right now or we attempted to within the last 15 min + return; + } + _lastCacheRequest = DateTime.now(); + _isPeriodicallyRequestingMissingCache = true; + try { + await maybeRequestAll(); + } finally { + _isPeriodicallyRequestingMissingCache = false; + } + } + Future handleToDeviceEvent(ToDeviceEvent event) async { if (event.type == 'm.secret.request') { // got a request to share a secret diff --git a/test/encryption/key_verification_test.dart b/test/encryption/key_verification_test.dart index 319b8f5..8d7c86b 100644 --- a/test/encryption/key_verification_test.dart +++ b/test/encryption/key_verification_test.dart @@ -32,7 +32,7 @@ class MockSSSS extends SSSS { bool requestedSecrets = false; @override - Future maybeRequestAll(List devices) async { + Future maybeRequestAll([List devices]) async { requestedSecrets = true; final handle = open(); handle.unlock(recoveryKey: SSSS_KEY); diff --git a/test/encryption/ssss_test.dart b/test/encryption/ssss_test.dart index e698e09..a9b2593 100644 --- a/test/encryption/ssss_test.dart +++ b/test/encryption/ssss_test.dart @@ -30,6 +30,19 @@ import 'package:olm/olm.dart' as olm; import '../fake_client.dart'; import '../fake_matrix_api.dart'; +class MockSSSS extends SSSS { + MockSSSS(Encryption encryption) : super(encryption); + + bool requestedSecrets = false; + @override + Future maybeRequestAll([List devices]) async { + requestedSecrets = true; + final handle = open(); + handle.unlock(recoveryKey: SSSS_KEY); + await handle.maybeCacheAll(); + } +} + void main() { group('SSSS', () { var olmEnabled = true; @@ -392,6 +405,18 @@ void main() { expect(client.encryption.ssss.pendingShareRequests.length, 3); }); + test('periodicallyRequestMissingCache', () async { + client.userDeviceKeys[client.userID].masterKey.setDirectVerified(true); + client.encryption.ssss = MockSSSS(client.encryption); + (client.encryption.ssss as MockSSSS).requestedSecrets = false; + await client.encryption.ssss.periodicallyRequestMissingCache(); + expect((client.encryption.ssss as MockSSSS).requestedSecrets, true); + // it should only retry once every 15 min + (client.encryption.ssss as MockSSSS).requestedSecrets = false; + await client.encryption.ssss.periodicallyRequestMissingCache(); + expect((client.encryption.ssss as MockSSSS).requestedSecrets, false); + }); + test('dispose client', () async { await client.dispose(closeDatabase: true); }); From c90e18b55de81acf14623639254fd14d741f540e Mon Sep 17 00:00:00 2001 From: Sorunome Date: Tue, 15 Sep 2020 12:26:49 +0200 Subject: [PATCH 66/90] fix: Handle domains with port or ipv6 addresses correctly --- lib/src/utils/matrix_id_string_extension.dart | 16 ++++++++++++---- test/matrix_id_string_extension_test.dart | 2 ++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/src/utils/matrix_id_string_extension.dart b/lib/src/utils/matrix_id_string_extension.dart index 685e12e..8e91d2c 100644 --- a/lib/src/utils/matrix_id_string_extension.dart +++ b/lib/src/utils/matrix_id_string_extension.dart @@ -3,6 +3,15 @@ extension MatrixIdExtension on String { static const int MAX_LENGTH = 255; + List _getParts() { + final s = substring(1); + final ix = s.indexOf(':'); + if (ix == -1) { + return [substring(1)]; + } + return [s.substring(0, ix), s.substring(ix + 1)]; + } + bool get isValidMatrixId { if (isEmpty ?? true) return false; if (length > MAX_LENGTH) return false; @@ -14,7 +23,7 @@ extension MatrixIdExtension on String { return true; } // all other matrix IDs have to have a domain - final parts = substring(1).split(':'); + final parts = _getParts(); // the localpart can be an empty string, e.g. for aliases if (parts.length != 2 || parts[1].isEmpty) { return false; @@ -24,10 +33,9 @@ extension MatrixIdExtension on String { String get sigil => isValidMatrixId ? substring(0, 1) : null; - String get localpart => - isValidMatrixId ? substring(1).split(':').first : null; + String get localpart => isValidMatrixId ? _getParts().first : null; - String get domain => isValidMatrixId ? substring(1).split(':')[1] : null; + String get domain => isValidMatrixId ? _getParts().last : null; bool equals(String other) => toLowerCase() == other?.toLowerCase(); } diff --git a/test/matrix_id_string_extension_test.dart b/test/matrix_id_string_extension_test.dart index 0ee37c3..01d59c8 100644 --- a/test/matrix_id_string_extension_test.dart +++ b/test/matrix_id_string_extension_test.dart @@ -43,6 +43,8 @@ void main() { expect(mxId.domain, 'example.com'); expect(mxId.equals('@Test:example.com'), true); expect(mxId.equals('@test:example.org'), false); + expect('@user:domain:8448'.localpart, 'user'); + expect('@user:domain:8448'.domain, 'domain:8448'); }); }); } From d9c4472cac54d67f4dc3605bae2a1886eea654b5 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Tue, 15 Sep 2020 19:03:55 +0200 Subject: [PATCH 67/90] feat: Add emote helpers --- lib/src/event.dart | 45 ++++++++++++++++++++ test/event_test.dart | 98 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+) diff --git a/lib/src/event.dart b/lib/src/event.dart index d369534..7807f88 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -762,4 +762,49 @@ class Event extends MatrixEvent { } return this; } + + /// returns if a message is a rich message + bool get isRichMessage => + content['format'] == 'org.matrix.custom.html' && + content['formatted_body'] is String; + + // regexes to fetch the number of emotes, including emoji, and if the message consists of only those + // to match an emoji we can use the following regex: + // \x{00a9}|\x{00ae}|[\x{2000}-\x{3300}]|\x{d83c}[\x{d000}-\x{dfff}|\x{d83d}[\x{d000}-\x{dfff}]|\x{d83e}[\x{d000}-\x{dfff}] + // we need to replace \x{0000} with \u0000, the comment is left in the other format to be able to paste into regex101.com + // to see if there is a custom emote, we use the following regex: ]+data-mx-(?:emote|emoticon)(?==|>|\s)[^>]*> + // now we combind the two to have four regexes: + // 1. are there only emoji, or whitespace + // 2. are there only emoji, emotes, or whitespace + // 3. count number of emoji + // 4- count number of emoji or emotes + static final RegExp _onlyEmojiRegex = RegExp( + r'^(\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff]|\s)*$', + caseSensitive: false, + multiLine: true); + static final RegExp _onlyEmojiEmoteRegex = RegExp( + r'^(\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff]|]+data-mx-(?:emote|emoticon)(?==|>|\s)[^>]*>|\s)*$', + caseSensitive: false, + multiLine: true); + static final RegExp _countEmojiRegex = RegExp( + r'(\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff])', + caseSensitive: false, + multiLine: true); + static final RegExp _countEmojiEmoteRegex = RegExp( + r'(\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff]|]+data-mx-(?:emote|emoticon)(?==|>|\s)[^>]*>)', + caseSensitive: false, + multiLine: true); + + /// Returns if a given event only has emotes, emojis or whitespace as content. + /// This is useful to determine if stand-alone emotes should be displayed bigger. + bool get onlyEmotes => isRichMessage + ? _onlyEmojiEmoteRegex.hasMatch(content['formatted_body']) + : _onlyEmojiRegex.hasMatch(content['body'] ?? ''); + + /// Gets the number of emotes in a given message. This is useful to determine if + /// emotes should be displayed bigger. WARNING: This does **not** test if there are + /// only emotes. Use `event.onlyEmotes` for that! + int get numberEmotes => isRichMessage + ? _countEmojiEmoteRegex.allMatches(content['formatted_body']).length + : _countEmojiRegex.allMatches(content['body'] ?? '').length; } diff --git a/test/event_test.dart b/test/event_test.dart index 9649b7c..3dabdd4 100644 --- a/test/event_test.dart +++ b/test/event_test.dart @@ -1159,5 +1159,103 @@ void main() { await room.client.dispose(closeDatabase: true); }); + test('emote detection', () async { + var event = Event.fromJson({ + 'type': EventTypes.Message, + 'content': { + 'msgtype': 'm.text', + 'body': 'normal message', + }, + 'event_id': '\$edit2', + 'sender': '@alice:example.org', + }, null); + expect(event.onlyEmotes, false); + expect(event.numberEmotes, 0); + event = Event.fromJson({ + 'type': EventTypes.Message, + 'content': { + 'msgtype': 'm.text', + 'body': 'normal message with emoji 🦊', + }, + 'event_id': '\$edit2', + 'sender': '@alice:example.org', + }, null); + expect(event.onlyEmotes, false); + expect(event.numberEmotes, 1); + event = Event.fromJson({ + 'type': EventTypes.Message, + 'content': { + 'msgtype': 'm.text', + 'body': '🦊', + }, + 'event_id': '\$edit2', + 'sender': '@alice:example.org', + }, null); + expect(event.onlyEmotes, true); + expect(event.numberEmotes, 1); + event = Event.fromJson({ + 'type': EventTypes.Message, + 'content': { + 'msgtype': 'm.text', + 'body': '🦊🦊 🦊\n🦊🦊', + }, + 'event_id': '\$edit2', + 'sender': '@alice:example.org', + }, null); + expect(event.onlyEmotes, true); + expect(event.numberEmotes, 5); + event = Event.fromJson({ + 'type': EventTypes.Message, + 'content': { + 'msgtype': 'm.text', + 'body': 'rich message', + 'format': 'org.matrix.custom.html', + 'formatted_body': 'rich message' + }, + 'event_id': '\$edit2', + 'sender': '@alice:example.org', + }, null); + expect(event.onlyEmotes, false); + expect(event.numberEmotes, 0); + event = Event.fromJson({ + 'type': EventTypes.Message, + 'content': { + 'msgtype': 'm.text', + 'body': '🦊', + 'format': 'org.matrix.custom.html', + 'formatted_body': '🦊' + }, + 'event_id': '\$edit2', + 'sender': '@alice:example.org', + }, null); + expect(event.onlyEmotes, true); + expect(event.numberEmotes, 1); + event = Event.fromJson({ + 'type': EventTypes.Message, + 'content': { + 'msgtype': 'm.text', + 'body': ':blah:', + 'format': 'org.matrix.custom.html', + 'formatted_body': '' + }, + 'event_id': '\$edit2', + 'sender': '@alice:example.org', + }, null); + expect(event.onlyEmotes, true); + expect(event.numberEmotes, 1); + event = Event.fromJson({ + 'type': EventTypes.Message, + 'content': { + 'msgtype': 'm.text', + 'body': '🦊 :blah:', + 'format': 'org.matrix.custom.html', + 'formatted_body': '🦊 ' + }, + 'event_id': '\$edit2', + 'sender': '@alice:example.org', + }, null); + expect(event.onlyEmotes, true); + expect(event.numberEmotes, 2); + }); }); } From b05e4da34fd6fb54b2541184ef0b40681d2d49a5 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Wed, 16 Sep 2020 10:18:13 +0200 Subject: [PATCH 68/90] fix: Last event calculation --- analysis_options.yaml | 3 +- lib/src/client.dart | 97 ++++++++++++++++++++++++----------------- lib/src/room.dart | 8 +++- lib/src/utils/logs.dart | 4 ++ 4 files changed, 68 insertions(+), 44 deletions(-) diff --git a/analysis_options.yaml b/analysis_options.yaml index a3efac9..fefe961 100644 --- a/analysis_options.yaml +++ b/analysis_options.yaml @@ -9,5 +9,4 @@ analyzer: errors: todo: ignore exclude: - - example/main.dart - - lib/src/utils/logs.dart \ No newline at end of file + - example/main.dart \ No newline at end of file diff --git a/lib/src/client.dart b/lib/src/client.dart index 8e67d42..661291c 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -59,15 +59,16 @@ class Client extends MatrixApi { Set importantStateEvents; + Set roomPreviewLastEvents; + /// Create a client - /// clientName = unique identifier of this client - /// debug: Print debug output? - /// database: The database instance to use - /// enableE2eeRecovery: Enable additional logic to try to recover from bad e2ee sessions - /// verificationMethods: A set of all the verification methods this client can handle. Includes: + /// [clientName] = unique identifier of this client + /// [database]: The database instance to use + /// [enableE2eeRecovery]: Enable additional logic to try to recover from bad e2ee sessions + /// [verificationMethods]: A set of all the verification methods this client can handle. Includes: /// KeyVerificationMethod.numbers: Compare numbers. Most basic, should be supported /// KeyVerificationMethod.emoji: Compare emojis - /// importantStateEvents: A set of all the important state events to load when the client connects. + /// [importantStateEvents]: A set of all the important state events to load when the client connects. /// To speed up performance only a set of state events is loaded on startup, those that are /// needed to display a room list. All the remaining state events are automatically post-loaded /// when opening the timeline of a room or manually by calling `room.postLoad()`. @@ -80,6 +81,8 @@ class Client extends MatrixApi { /// - m.room.canonical_alias /// - m.room.tombstone /// - *some* m.room.member events, where needed + /// [roomPreviewLastEvents]: The event types that should be used to calculate the last event + /// in a room for the room list. Client( this.clientName, { this.database, @@ -87,11 +90,12 @@ class Client extends MatrixApi { this.verificationMethods, http.Client httpClient, this.importantStateEvents, + this.roomPreviewLastEvents, this.pinUnreadRooms = false, @deprecated bool debug, }) { verificationMethods ??= {}; - importantStateEvents ??= {}; + importantStateEvents ??= {}; importantStateEvents.addAll([ EventTypes.RoomName, EventTypes.RoomAvatar, @@ -101,6 +105,12 @@ class Client extends MatrixApi { EventTypes.RoomCanonicalAlias, EventTypes.RoomTombstone, ]); + roomPreviewLastEvents ??= {}; + roomPreviewLastEvents.addAll([ + EventTypes.Message, + EventTypes.Encrypted, + EventTypes.Sticker, + ]); this.httpClient = httpClient ?? http.Client(); } @@ -1099,44 +1109,49 @@ class Client extends MatrixApi { void _updateRoomsByEventUpdate(EventUpdate eventUpdate) { if (eventUpdate.type == 'history') return; - // Search the room in the rooms - num j = 0; - for (j = 0; j < rooms.length; j++) { - if (rooms[j].id == eventUpdate.roomID) break; - } - final found = (j < rooms.length && rooms[j].id == eventUpdate.roomID); - if (!found) return; - if (eventUpdate.type == 'timeline' || - eventUpdate.type == 'state' || - eventUpdate.type == 'invite_state') { - var stateEvent = - Event.fromJson(eventUpdate.content, rooms[j], eventUpdate.sortOrder); - if (stateEvent.type == EventTypes.Redaction) { - final String redacts = eventUpdate.content['redacts']; - rooms[j].states.states.forEach( - (String key, Map states) => states.forEach( - (String key, Event state) { - if (state.eventId == redacts) { - state.setRedactionEvent(stateEvent); - } - }, - ), - ); - } else { - var prevState = rooms[j].getState(stateEvent.type, stateEvent.stateKey); + + final room = getRoomById(eventUpdate.roomID); + if (room == null) return; + + switch (eventUpdate.type) { + case 'timeline': + case 'state': + case 'invite_state': + var stateEvent = + Event.fromJson(eventUpdate.content, room, eventUpdate.sortOrder); + var prevState = room.getState(stateEvent.type, stateEvent.stateKey); if (prevState != null && prevState.sortOrder > stateEvent.sortOrder) { + Logs.warning(''' +A new ${eventUpdate.type} event of the type ${stateEvent.type} has arrived with a previews +sort order ${stateEvent.sortOrder} than the current ${stateEvent.type} event with a +sort order of ${prevState.sortOrder}. This should never happen...'''); return; } - rooms[j].setState(stateEvent); - } - } else if (eventUpdate.type == 'account_data') { - rooms[j].roomAccountData[eventUpdate.eventType] = - BasicRoomEvent.fromJson(eventUpdate.content); - } else if (eventUpdate.type == 'ephemeral') { - rooms[j].ephemerals[eventUpdate.eventType] = - BasicRoomEvent.fromJson(eventUpdate.content); + if (stateEvent.type == EventTypes.Redaction) { + final String redacts = eventUpdate.content['redacts']; + room.states.states.forEach( + (String key, Map states) => states.forEach( + (String key, Event state) { + if (state.eventId == redacts) { + state.setRedactionEvent(stateEvent); + } + }, + ), + ); + } else { + room.setState(stateEvent); + } + break; + case 'account_data': + room.roomAccountData[eventUpdate.eventType] = + BasicRoomEvent.fromJson(eventUpdate.content); + break; + case 'ephemeral': + room.ephemerals[eventUpdate.eventType] = + BasicRoomEvent.fromJson(eventUpdate.content); + break; } - if (rooms[j].onUpdate != null) rooms[j].onUpdate.add(rooms[j].id); + room.onUpdate.add(room.id); if (['timeline', 'account_data'].contains(eventUpdate.type)) _sortRooms(); } diff --git a/lib/src/room.dart b/lib/src/room.dart index 3e2168c..f46b8a8 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -264,7 +264,13 @@ class Room { // perfect, it is only used for the room preview in the room list and sorting // said room list, so it should be good enough. var lastTime = DateTime.fromMillisecondsSinceEpoch(0); - var lastEvent = getState(EventTypes.Message); + final lastEvents = [ + for (var type in client.roomPreviewLastEvents) getState(type) + ]..removeWhere((e) => e == null); + + var lastEvent = lastEvents.isEmpty + ? null + : lastEvents.reduce((a, b) => a.sortOrder > b.sortOrder ? a : b); if (lastEvent == null) { states.forEach((final String key, final entry) { if (!entry.containsKey('')) return; diff --git a/lib/src/utils/logs.dart b/lib/src/utils/logs.dart index fa2609a..42c76bd 100644 --- a/lib/src/utils/logs.dart +++ b/lib/src/utils/logs.dart @@ -26,20 +26,24 @@ abstract class Logs { static const String _prefixText = '[Famedly Matrix SDK] '; + // ignore: avoid_print static void info(dynamic info) => print( _prefixText + _infoPen(info.toString()), ); + // ignore: avoid_print static void success(dynamic obj, [dynamic stackTrace]) => print( _prefixText + _successPen(obj.toString()), ); + // ignore: avoid_print static void warning(dynamic warning, [dynamic stackTrace]) => print( _prefixText + _warningPen(warning.toString()) + (stackTrace != null ? '\n${stackTrace.toString()}' : ''), ); + // ignore: avoid_print static void error(dynamic obj, [dynamic stackTrace]) => print( _prefixText + _errorPen(obj.toString()) + From df2cfb3faf87930fd16943ec0d86ebc326346d27 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Wed, 16 Sep 2020 13:42:05 +0200 Subject: [PATCH 69/90] fix: Ask only own devices on automated key requests --- lib/encryption/key_manager.dart | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/encryption/key_manager.dart b/lib/encryption/key_manager.dart index edf908e..37221a5 100644 --- a/lib/encryption/key_manager.dart +++ b/lib/encryption/key_manager.dart @@ -207,7 +207,7 @@ class KeyManager { !_requestedSessionIds.contains(requestIdent)) { // do e2ee recovery _requestedSessionIds.add(requestIdent); - unawaited(request(room, sessionId, senderKey)); + unawaited(request(room, sessionId, senderKey, askOnlyOwnDevices: true)); } return null; } @@ -442,8 +442,13 @@ class KeyManager { } /// Request a certain key from another device - Future request(Room room, String sessionId, String senderKey, - {bool tryOnlineBackup = true}) async { + Future request( + Room room, + String sessionId, + String senderKey, { + bool tryOnlineBackup = true, + bool askOnlyOwnDevices = false, + }) async { if (tryOnlineBackup) { // let's first check our online key backup store thingy... var hadPreviously = @@ -470,6 +475,9 @@ class KeyManager { // while we just send the to-device event to '*', we still need to save the // devices themself to know where to send the cancel to after receiving a reply final devices = await room.getUserDeviceKeys(); + if (askOnlyOwnDevices) { + devices.removeWhere((d) => d.userId != client.userID); + } final requestId = client.generateUniqueTransactionId(); final request = KeyManagerKeyShareRequest( requestId: requestId, From 0871e218d18027db700a0af84e31d8bcd60ee131 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Wed, 16 Sep 2020 10:39:04 +0200 Subject: [PATCH 70/90] refactor: Json signature check --- lib/encryption/olm_manager.dart | 8 +++-- .../utils/json_signature_check_extension.dart | 29 +++++++++++++++++++ test/encryption/olm_manager_test.dart | 9 ++---- 3 files changed, 37 insertions(+), 9 deletions(-) create mode 100644 lib/encryption/utils/json_signature_check_extension.dart diff --git a/lib/encryption/olm_manager.dart b/lib/encryption/olm_manager.dart index 1c2cac7..9635939 100644 --- a/lib/encryption/olm_manager.dart +++ b/lib/encryption/olm_manager.dart @@ -24,6 +24,7 @@ import 'package:famedlysdk/matrix_api.dart'; import 'package:olm/olm.dart' as olm; import 'package:pedantic/pedantic.dart'; +import '../encryption/utils/json_signature_check_extension.dart'; import '../src/utils/logs.dart'; import 'encryption.dart'; import 'utils/olm_session.dart'; @@ -75,7 +76,8 @@ class OlmManager { } } - /// Adds a signature to this json from this olm account. + /// Adds a signature to this json from this olm account and returns the signed + /// json. Map signJson(Map payload) { if (!enabled) throw ('Encryption is disabled'); final Map unsigned = payload['unsigned']; @@ -105,6 +107,7 @@ class OlmManager { } /// Checks the signature of a signed json object. + @deprecated bool checkJsonSignature(String key, Map signedJson, String userId, String deviceId) { if (!enabled) throw ('Encryption is disabled'); @@ -406,8 +409,7 @@ class OlmManager { final identityKey = client.userDeviceKeys[userId].deviceKeys[deviceId].curve25519Key; for (Map deviceKey in deviceKeysEntry.value.values) { - if (!checkJsonSignature( - fingerprintKey, deviceKey, userId, deviceId)) { + if (!deviceKey.checkJsonSignature(fingerprintKey, userId, deviceId)) { continue; } var session = olm.Session(); diff --git a/lib/encryption/utils/json_signature_check_extension.dart b/lib/encryption/utils/json_signature_check_extension.dart new file mode 100644 index 0000000..8a2401f --- /dev/null +++ b/lib/encryption/utils/json_signature_check_extension.dart @@ -0,0 +1,29 @@ +import 'package:canonical_json/canonical_json.dart'; +import 'package:famedlysdk/src/utils/logs.dart'; +import 'package:olm/olm.dart' as olm; + +extension JsonSignatureCheckExtension on Map { + /// Checks the signature of a signed json object. + bool checkJsonSignature(String key, String userId, String deviceId) { + final Map signatures = this['signatures']; + if (signatures == null || !signatures.containsKey(userId)) return false; + remove('unsigned'); + remove('signatures'); + if (!signatures[userId].containsKey('ed25519:$deviceId')) return false; + final String signature = signatures[userId]['ed25519:$deviceId']; + final canonical = canonicalJson.encode(this); + final message = String.fromCharCodes(canonical); + var isValid = false; + final olmutil = olm.Utility(); + try { + olmutil.ed25519_verify(key, message, signature); + isValid = true; + } catch (e, s) { + isValid = false; + Logs.error('[LibOlm] Signature check failed: ' + e.toString(), s); + } finally { + olmutil.free(); + } + return isValid; + } +} diff --git a/test/encryption/olm_manager_test.dart b/test/encryption/olm_manager_test.dart index 78e7068..883171c 100644 --- a/test/encryption/olm_manager_test.dart +++ b/test/encryption/olm_manager_test.dart @@ -21,6 +21,7 @@ import 'package:famedlysdk/famedlysdk.dart'; import 'package:famedlysdk/src/utils/logs.dart'; import 'package:test/test.dart'; import 'package:olm/olm.dart' as olm; +import 'package:famedlysdk/encryption/utils/json_signature_check_extension.dart'; import '../fake_client.dart'; import '../fake_matrix_api.dart'; @@ -51,13 +52,9 @@ void main() { }; final signedPayload = client.encryption.olmManager.signJson(payload); expect( - client.encryption.olmManager.checkJsonSignature(client.fingerprintKey, - signedPayload, client.userID, client.deviceID), + signedPayload.checkJsonSignature( + client.fingerprintKey, client.userID, client.deviceID), true); - expect( - client.encryption.olmManager.checkJsonSignature( - client.fingerprintKey, payload, client.userID, client.deviceID), - false); }); test('uploadKeys', () async { From bc8fef4a94eff2e463fef866ae9a13e7e08b15c2 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Thu, 3 Sep 2020 07:38:13 +0000 Subject: [PATCH 71/90] fix: Remove pubspec.lock from repo --- .gitignore | 1 + lib/src/database/database.dart | 2 +- pubspec.lock | 677 --------------------------------- 3 files changed, 2 insertions(+), 678 deletions(-) delete mode 100644 pubspec.lock diff --git a/.gitignore b/.gitignore index 7826f7f..0ab205e 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ coverage_badge.svg .pub-cache/ .pub/ build/ +pubspec.lock # Android related **/android/**/gradle-wrapper.jar diff --git a/lib/src/database/database.dart b/lib/src/database/database.dart index 3998af9..d1b3d2b 100644 --- a/lib/src/database/database.dart +++ b/lib/src/database/database.dart @@ -99,7 +99,7 @@ class Database extends _$Database { await m.createTableIfNotExists(userCrossSigningKeys); await m.createTableIfNotExists(ssssCache); // mark all keys as outdated so that the cross signing keys will be fetched - await m.issueCustomQuery( + await customStatement( 'UPDATE user_device_keys SET outdated = true'); from++; } diff --git a/pubspec.lock b/pubspec.lock deleted file mode 100644 index ebc2d2f..0000000 --- a/pubspec.lock +++ /dev/null @@ -1,677 +0,0 @@ -# Generated by pub -# See https://dart.dev/tools/pub/glossary#lockfile -packages: - _fe_analyzer_shared: - dependency: transitive - description: - name: _fe_analyzer_shared - url: "https://pub.dartlang.org" - source: hosted - version: "3.0.0" - analyzer: - dependency: transitive - description: - name: analyzer - url: "https://pub.dartlang.org" - source: hosted - version: "0.39.8" - analyzer_plugin_fork: - dependency: transitive - description: - name: analyzer_plugin_fork - url: "https://pub.dartlang.org" - source: hosted - version: "0.2.2" - ansicolor: - dependency: "direct main" - description: - name: ansicolor - url: "https://pub.dartlang.org" - source: hosted - version: "1.0.2" - args: - dependency: transitive - description: - name: args - url: "https://pub.dartlang.org" - source: hosted - version: "1.6.0" - asn1lib: - dependency: transitive - description: - name: asn1lib - url: "https://pub.dartlang.org" - source: hosted - version: "0.6.4" - async: - dependency: transitive - description: - name: async - url: "https://pub.dartlang.org" - source: hosted - version: "2.4.1" - base58check: - dependency: "direct main" - description: - name: base58check - url: "https://pub.dartlang.org" - source: hosted - version: "1.0.1" - boolean_selector: - dependency: transitive - description: - name: boolean_selector - url: "https://pub.dartlang.org" - source: hosted - version: "2.0.0" - build: - dependency: transitive - description: - name: build - url: "https://pub.dartlang.org" - source: hosted - version: "1.3.0" - build_config: - dependency: transitive - description: - name: build_config - url: "https://pub.dartlang.org" - source: hosted - version: "0.4.2" - build_daemon: - dependency: transitive - description: - name: build_daemon - url: "https://pub.dartlang.org" - source: hosted - version: "2.1.4" - build_resolvers: - dependency: transitive - description: - name: build_resolvers - url: "https://pub.dartlang.org" - source: hosted - version: "1.3.9" - build_runner: - dependency: "direct dev" - description: - name: build_runner - url: "https://pub.dartlang.org" - source: hosted - version: "1.10.0" - build_runner_core: - dependency: transitive - description: - name: build_runner_core - url: "https://pub.dartlang.org" - source: hosted - version: "5.2.0" - built_collection: - dependency: transitive - description: - name: built_collection - url: "https://pub.dartlang.org" - source: hosted - version: "4.3.2" - built_value: - dependency: transitive - description: - name: built_value - url: "https://pub.dartlang.org" - source: hosted - version: "7.1.0" - canonical_json: - dependency: "direct main" - description: - name: canonical_json - url: "https://pub.dartlang.org" - source: hosted - version: "1.0.0" - charcode: - dependency: transitive - description: - name: charcode - url: "https://pub.dartlang.org" - source: hosted - version: "1.1.3" - checked_yaml: - dependency: transitive - description: - name: checked_yaml - url: "https://pub.dartlang.org" - source: hosted - version: "1.0.2" - cli_util: - dependency: transitive - description: - name: cli_util - url: "https://pub.dartlang.org" - source: hosted - version: "0.1.4" - clock: - dependency: transitive - description: - name: clock - url: "https://pub.dartlang.org" - source: hosted - version: "1.0.1" - code_builder: - dependency: transitive - description: - name: code_builder - url: "https://pub.dartlang.org" - source: hosted - version: "3.2.1" - collection: - dependency: transitive - description: - name: collection - url: "https://pub.dartlang.org" - source: hosted - version: "1.14.12" - convert: - dependency: transitive - description: - name: convert - url: "https://pub.dartlang.org" - source: hosted - version: "2.1.1" - coverage: - dependency: transitive - description: - name: coverage - url: "https://pub.dartlang.org" - source: hosted - version: "0.13.9" - crypto: - dependency: "direct main" - description: - name: crypto - url: "https://pub.dartlang.org" - source: hosted - version: "2.1.4" - csslib: - dependency: transitive - description: - name: csslib - url: "https://pub.dartlang.org" - source: hosted - version: "0.16.1" - dart_style: - dependency: transitive - description: - name: dart_style - url: "https://pub.dartlang.org" - source: hosted - version: "1.3.6" - encrypt: - dependency: "direct main" - description: - name: encrypt - url: "https://pub.dartlang.org" - source: hosted - version: "4.0.2" - ffi: - dependency: transitive - description: - name: ffi - url: "https://pub.dartlang.org" - source: hosted - version: "0.1.3" - fixnum: - dependency: transitive - description: - name: fixnum - url: "https://pub.dartlang.org" - source: hosted - version: "0.10.11" - glob: - dependency: transitive - description: - name: glob - url: "https://pub.dartlang.org" - source: hosted - version: "1.2.0" - graphs: - dependency: transitive - description: - name: graphs - url: "https://pub.dartlang.org" - source: hosted - version: "0.2.0" - html: - dependency: transitive - description: - name: html - url: "https://pub.dartlang.org" - source: hosted - version: "0.14.0+3" - html_unescape: - dependency: "direct main" - description: - name: html_unescape - url: "https://pub.dartlang.org" - source: hosted - version: "1.0.1+3" - http: - dependency: "direct main" - description: - name: http - url: "https://pub.dartlang.org" - source: hosted - version: "0.12.1" - http_multi_server: - dependency: transitive - description: - name: http_multi_server - url: "https://pub.dartlang.org" - source: hosted - version: "2.2.0" - http_parser: - dependency: transitive - description: - name: http_parser - url: "https://pub.dartlang.org" - source: hosted - version: "3.1.4" - io: - dependency: transitive - description: - name: io - url: "https://pub.dartlang.org" - source: hosted - version: "0.3.4" - isolate: - dependency: "direct main" - description: - name: isolate - url: "https://pub.dartlang.org" - source: hosted - version: "2.0.3" - js: - dependency: transitive - description: - name: js - url: "https://pub.dartlang.org" - source: hosted - version: "0.6.1+1" - json_annotation: - dependency: transitive - description: - name: json_annotation - url: "https://pub.dartlang.org" - source: hosted - version: "3.0.1" - lcov: - dependency: transitive - description: - name: lcov - url: "https://pub.dartlang.org" - source: hosted - version: "5.7.0" - logging: - dependency: transitive - description: - name: logging - url: "https://pub.dartlang.org" - source: hosted - version: "0.11.4" - markdown: - dependency: "direct main" - description: - name: markdown - url: "https://pub.dartlang.org" - source: hosted - version: "2.1.3" - matcher: - dependency: transitive - description: - name: matcher - url: "https://pub.dartlang.org" - source: hosted - version: "0.12.6" - matrix_file_e2ee: - dependency: "direct main" - description: - name: matrix_file_e2ee - url: "https://pub.dartlang.org" - source: hosted - version: "1.0.4" - meta: - dependency: transitive - description: - name: meta - url: "https://pub.dartlang.org" - source: hosted - version: "1.1.8" - mime: - dependency: "direct main" - description: - name: mime - url: "https://pub.dartlang.org" - source: hosted - version: "0.9.6+3" - moor: - dependency: "direct main" - description: - name: moor - url: "https://pub.dartlang.org" - source: hosted - version: "3.0.2" - moor_ffi: - dependency: "direct dev" - description: - name: moor_ffi - url: "https://pub.dartlang.org" - source: hosted - version: "0.5.0" - moor_generator: - dependency: "direct dev" - description: - name: moor_generator - url: "https://pub.dartlang.org" - source: hosted - version: "3.0.0" - multi_server_socket: - dependency: transitive - description: - name: multi_server_socket - url: "https://pub.dartlang.org" - source: hosted - version: "1.0.2" - node_interop: - dependency: transitive - description: - name: node_interop - url: "https://pub.dartlang.org" - source: hosted - version: "1.1.1" - node_io: - dependency: transitive - description: - name: node_io - url: "https://pub.dartlang.org" - source: hosted - version: "1.1.1" - node_preamble: - dependency: transitive - description: - name: node_preamble - url: "https://pub.dartlang.org" - source: hosted - version: "1.4.8" - olm: - dependency: "direct main" - description: - name: olm - url: "https://pub.dartlang.org" - source: hosted - version: "1.2.1" - package_config: - dependency: transitive - description: - name: package_config - url: "https://pub.dartlang.org" - source: hosted - version: "1.9.3" - password_hash: - dependency: "direct main" - description: - name: password_hash - url: "https://pub.dartlang.org" - source: hosted - version: "2.0.0" - path: - dependency: transitive - description: - name: path - url: "https://pub.dartlang.org" - source: hosted - version: "1.7.0" - pedantic: - dependency: "direct dev" - description: - name: pedantic - url: "https://pub.dartlang.org" - source: hosted - version: "1.9.0" - pointycastle: - dependency: transitive - description: - name: pointycastle - url: "https://pub.dartlang.org" - source: hosted - version: "1.0.2" - pool: - dependency: transitive - description: - name: pool - url: "https://pub.dartlang.org" - source: hosted - version: "1.4.0" - pub_semver: - dependency: transitive - description: - name: pub_semver - url: "https://pub.dartlang.org" - source: hosted - version: "1.4.4" - pubspec_parse: - dependency: transitive - description: - name: pubspec_parse - url: "https://pub.dartlang.org" - source: hosted - version: "0.1.5" - quiver: - dependency: transitive - description: - name: quiver - url: "https://pub.dartlang.org" - source: hosted - version: "2.1.3" - random_string: - dependency: "direct main" - description: - name: random_string - url: "https://pub.dartlang.org" - source: hosted - version: "2.0.1" - recase: - dependency: transitive - description: - name: recase - url: "https://pub.dartlang.org" - source: hosted - version: "3.0.0" - shelf: - dependency: transitive - description: - name: shelf - url: "https://pub.dartlang.org" - source: hosted - version: "0.7.5" - shelf_packages_handler: - dependency: transitive - description: - name: shelf_packages_handler - url: "https://pub.dartlang.org" - source: hosted - version: "2.0.0" - shelf_static: - dependency: transitive - description: - name: shelf_static - url: "https://pub.dartlang.org" - source: hosted - version: "0.2.8" - shelf_web_socket: - dependency: transitive - description: - name: shelf_web_socket - url: "https://pub.dartlang.org" - source: hosted - version: "0.2.3" - source_gen: - dependency: transitive - description: - name: source_gen - url: "https://pub.dartlang.org" - source: hosted - version: "0.9.5" - source_map_stack_trace: - dependency: transitive - description: - name: source_map_stack_trace - url: "https://pub.dartlang.org" - source: hosted - version: "2.0.0" - source_maps: - dependency: transitive - description: - name: source_maps - url: "https://pub.dartlang.org" - source: hosted - version: "0.10.9" - source_span: - dependency: transitive - description: - name: source_span - url: "https://pub.dartlang.org" - source: hosted - version: "1.7.0" - sqlparser: - dependency: transitive - description: - name: sqlparser - url: "https://pub.dartlang.org" - source: hosted - version: "0.8.1" - stack_trace: - dependency: transitive - description: - name: stack_trace - url: "https://pub.dartlang.org" - source: hosted - version: "1.9.3" - stream_channel: - dependency: transitive - description: - name: stream_channel - url: "https://pub.dartlang.org" - source: hosted - version: "2.0.0" - stream_transform: - dependency: transitive - description: - name: stream_transform - url: "https://pub.dartlang.org" - source: hosted - version: "1.2.0" - string_scanner: - dependency: transitive - description: - name: string_scanner - url: "https://pub.dartlang.org" - source: hosted - version: "1.0.5" - synchronized: - dependency: transitive - description: - name: synchronized - url: "https://pub.dartlang.org" - source: hosted - version: "2.2.0" - term_glyph: - dependency: transitive - description: - name: term_glyph - url: "https://pub.dartlang.org" - source: hosted - version: "1.1.0" - test: - dependency: "direct dev" - description: - name: test - url: "https://pub.dartlang.org" - source: hosted - version: "1.14.3" - test_api: - dependency: transitive - description: - name: test_api - url: "https://pub.dartlang.org" - source: hosted - version: "0.2.15" - test_core: - dependency: transitive - description: - name: test_core - url: "https://pub.dartlang.org" - source: hosted - version: "0.3.4" - test_coverage: - dependency: "direct dev" - description: - name: test_coverage - url: "https://pub.dartlang.org" - source: hosted - version: "0.4.3" - timing: - dependency: transitive - description: - name: timing - url: "https://pub.dartlang.org" - source: hosted - version: "0.1.1+2" - typed_data: - dependency: transitive - description: - name: typed_data - url: "https://pub.dartlang.org" - source: hosted - version: "1.1.6" - unorm_dart: - dependency: transitive - description: - name: unorm_dart - url: "https://pub.dartlang.org" - source: hosted - version: "0.1.2" - vm_service: - dependency: transitive - description: - name: vm_service - url: "https://pub.dartlang.org" - source: hosted - version: "4.0.4" - watcher: - dependency: transitive - description: - name: watcher - url: "https://pub.dartlang.org" - source: hosted - version: "0.9.7+15" - web_socket_channel: - dependency: transitive - description: - name: web_socket_channel - url: "https://pub.dartlang.org" - source: hosted - version: "1.1.0" - webkit_inspection_protocol: - dependency: transitive - description: - name: webkit_inspection_protocol - url: "https://pub.dartlang.org" - source: hosted - version: "0.5.4" - yaml: - dependency: transitive - description: - name: yaml - url: "https://pub.dartlang.org" - source: hosted - version: "2.2.1" -sdks: - dart: ">=2.7.0 <3.0.0" From 5d5c7fa8b4c834f8787d8cf0cfe4bc25c7868f65 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Thu, 17 Sep 2020 12:53:18 +0200 Subject: [PATCH 72/90] fix: Catch all root zone exceptions --- lib/encryption/encryption.dart | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/lib/encryption/encryption.dart b/lib/encryption/encryption.dart index 9775fb1..8c9ee7b 100644 --- a/lib/encryption/encryption.dart +++ b/lib/encryption/encryption.dart @@ -23,12 +23,24 @@ import 'package:pedantic/pedantic.dart'; import '../famedlysdk.dart'; import '../matrix_api.dart'; +import '../src/utils/logs.dart'; import 'cross_signing.dart'; import 'key_manager.dart'; import 'key_verification_manager.dart'; import 'olm_manager.dart'; import 'ssss.dart'; +Future _runInRoot(FutureOr Function() fn) async { + return await Zone.root.run(() async { + try { + return await fn(); + } catch (e, s) { + Logs.error('Error thrown in root zone: ' + e.toString(), s); + } + return null; + }); +} + class Encryption { final Client client; final bool debug; @@ -68,7 +80,7 @@ class Encryption { } void handleDeviceOneTimeKeysCount(Map countJson) { - Zone.root.run(() => olmManager.handleDeviceOneTimeKeysCount(countJson)); + _runInRoot(() => olmManager.handleDeviceOneTimeKeysCount(countJson)); } void onSync() { @@ -84,21 +96,21 @@ class Encryption { if (['m.room_key_request', 'm.forwarded_room_key'].contains(event.type)) { // "just" room key request things. We don't need these asap, so we handle // them in the background - unawaited(Zone.root.run(() => keyManager.handleToDeviceEvent(event))); + unawaited(_runInRoot(() => keyManager.handleToDeviceEvent(event))); } if (event.type.startsWith('m.key.verification.')) { // some key verification event. No need to handle it now, we can easily // do this in the background - unawaited(Zone.root - .run(() => keyVerificationManager.handleToDeviceEvent(event))); + unawaited( + _runInRoot(() => keyVerificationManager.handleToDeviceEvent(event))); } if (event.type.startsWith('m.secret.')) { // some ssss thing. We can do this in the background - unawaited(Zone.root.run(() => ssss.handleToDeviceEvent(event))); + unawaited(_runInRoot(() => ssss.handleToDeviceEvent(event))); } if (event.sender == client.userID) { // maybe we need to re-try SSSS secrets - unawaited(Zone.root.run(() => ssss.periodicallyRequestMissingCache())); + unawaited(_runInRoot(() => ssss.periodicallyRequestMissingCache())); } } @@ -112,13 +124,13 @@ class Encryption { update.content['content']['msgtype'] .startsWith('m.key.verification.'))) { // "just" key verification, no need to do this in sync - unawaited(Zone.root - .run(() => keyVerificationManager.handleEventUpdate(update))); + unawaited( + _runInRoot(() => keyVerificationManager.handleEventUpdate(update))); } if (update.content['sender'] == client.userID && !update.content['unsigned'].containsKey('transaction_id')) { // maybe we need to re-try SSSS secrets - unawaited(Zone.root.run(() => ssss.periodicallyRequestMissingCache())); + unawaited(_runInRoot(() => ssss.periodicallyRequestMissingCache())); } } From 0fa2046c41fe3561cfc8d89269de3229716001dc Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Thu, 17 Sep 2020 16:52:55 +0200 Subject: [PATCH 73/90] fix: Missing null check --- lib/matrix_api/model/sync_update.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/matrix_api/model/sync_update.dart b/lib/matrix_api/model/sync_update.dart index c21ff06..e94a126 100644 --- a/lib/matrix_api/model/sync_update.dart +++ b/lib/matrix_api/model/sync_update.dart @@ -315,8 +315,8 @@ class DeviceListsUpdate { List changed; List left; DeviceListsUpdate.fromJson(Map json) { - changed = List.from(json['changed']); - left = List.from(json['left']); + changed = List.from(json['changed'] ?? []); + left = List.from(json['left'] ?? []); } Map toJson() { final data = {}; From 024a27bfc276547f4665254e7dec614262518169 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Fri, 18 Sep 2020 10:17:08 +0200 Subject: [PATCH 74/90] fix: Back off of failed key queries --- lib/matrix_api/model/keys_query_response.dart | 4 +++- lib/src/client.dart | 18 +++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/lib/matrix_api/model/keys_query_response.dart b/lib/matrix_api/model/keys_query_response.dart index 57bc16e..b368b4f 100644 --- a/lib/matrix_api/model/keys_query_response.dart +++ b/lib/matrix_api/model/keys_query_response.dart @@ -26,7 +26,9 @@ class KeysQueryResponse { Map userSigningKeys; KeysQueryResponse.fromJson(Map json) { - failures = Map.from(json['failures']); + failures = json['failures'] != null + ? Map.from(json['failures']) + : null; deviceKeys = json['device_keys'] != null ? (json['device_keys'] as Map).map( (k, v) => MapEntry( diff --git a/lib/src/client.dart b/lib/src/client.dart index 661291c..0f337b7 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -1122,8 +1122,8 @@ class Client extends MatrixApi { var prevState = room.getState(stateEvent.type, stateEvent.stateKey); if (prevState != null && prevState.sortOrder > stateEvent.sortOrder) { Logs.warning(''' -A new ${eventUpdate.type} event of the type ${stateEvent.type} has arrived with a previews -sort order ${stateEvent.sortOrder} than the current ${stateEvent.type} event with a +A new ${eventUpdate.type} event of the type ${stateEvent.type} has arrived with a previews +sort order ${stateEvent.sortOrder} than the current ${stateEvent.type} event with a sort order of ${prevState.sortOrder}. This should never happen...'''); return; } @@ -1213,6 +1213,7 @@ sort order of ${prevState.sortOrder}. This should never happen...'''); return userIds; } + final Map _keyQueryFailures = {}; Future _updateUserDeviceKeys() async { try { if (!isLogged()) return; @@ -1231,7 +1232,11 @@ sort order of ${prevState.sortOrder}. This should never happen...'''); _userDeviceKeys[userId] = DeviceKeysList(userId, this); } var deviceKeysList = userDeviceKeys[userId]; - if (deviceKeysList.outdated) { + if (deviceKeysList.outdated && + (!_keyQueryFailures.containsKey(userId.domain) || + DateTime.now() + .subtract(Duration(minutes: 5)) + .isAfter(_keyQueryFailures[userId.domain]))) { outdatedLists[userId] = []; } } @@ -1373,6 +1378,13 @@ sort order of ${prevState.sortOrder}. This should never happen...'''); } } } + + // now process all the failures + if (response.failures != null) { + for (final failureDomain in response.failures.keys) { + _keyQueryFailures[failureDomain] = DateTime.now(); + } + } } if (dbActions.isNotEmpty) { From f6259efa591d1c7b7cc28ba03f4219165b57dc4f Mon Sep 17 00:00:00 2001 From: Sorunome Date: Fri, 18 Sep 2020 09:42:17 +0200 Subject: [PATCH 75/90] fix: Better handle online key backup --- lib/encryption/key_manager.dart | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/lib/encryption/key_manager.dart b/lib/encryption/key_manager.dart index 37221a5..4da6c14 100644 --- a/lib/encryption/key_manager.dart +++ b/lib/encryption/key_manager.dart @@ -46,7 +46,7 @@ class KeyManager { encryption.ssss.setValidator(MEGOLM_KEY, (String secret) async { final keyObj = olm.PkDecryption(); try { - final info = await client.getRoomKeysBackup(); + final info = await getRoomKeysBackupInfo(false); if (info.algorithm != RoomKeysAlgorithmType.v1Curve25519AesSha2) { return false; } @@ -204,7 +204,8 @@ class KeyManager { final requestIdent = '$roomId|$sessionId|$senderKey'; if (client.enableE2eeRecovery && room != null && - !_requestedSessionIds.contains(requestIdent)) { + !_requestedSessionIds.contains(requestIdent) && + !client.isUnknownSession) { // do e2ee recovery _requestedSessionIds.add(requestIdent); unawaited(request(room, sessionId, senderKey, askOnlyOwnDevices: true)); @@ -367,6 +368,23 @@ class KeyManager { return (await encryption.ssss.getCached(MEGOLM_KEY)) != null; } + RoomKeysVersionResponse _roomKeysVersionCache; + DateTime _roomKeysVersionCacheDate; + Future getRoomKeysBackupInfo( + [bool useCache = true]) async { + if (_roomKeysVersionCache != null && + _roomKeysVersionCacheDate != null && + useCache && + DateTime.now() + .subtract(Duration(minutes: 5)) + .isBefore(_roomKeysVersionCacheDate)) { + return _roomKeysVersionCache; + } + _roomKeysVersionCache = await client.getRoomKeysBackup(); + _roomKeysVersionCacheDate = DateTime.now(); + return _roomKeysVersionCache; + } + Future loadFromResponse(RoomKeys keys) async { if (!(await isCached())) { return; @@ -374,7 +392,7 @@ class KeyManager { final privateKey = base64.decode(await encryption.ssss.getCached(MEGOLM_KEY)); final decryption = olm.PkDecryption(); - final info = await client.getRoomKeysBackup(); + final info = await getRoomKeysBackupInfo(); String backupPubKey; try { backupPubKey = decryption.init_with_private_key(privateKey); @@ -426,7 +444,7 @@ class KeyManager { } Future loadSingleKey(String roomId, String sessionId) async { - final info = await client.getRoomKeysBackup(); + final info = await getRoomKeysBackupInfo(); final ret = await client.getRoomKeysSingleKey(roomId, sessionId, info.version); final keys = RoomKeys.fromJson({ @@ -449,7 +467,7 @@ class KeyManager { bool tryOnlineBackup = true, bool askOnlyOwnDevices = false, }) async { - if (tryOnlineBackup) { + if (tryOnlineBackup && await isCached()) { // let's first check our online key backup store thingy... var hadPreviously = getInboundGroupSession(room.id, sessionId, senderKey) != null; @@ -530,7 +548,7 @@ class KeyManager { base64.decode(await encryption.ssss.getCached(MEGOLM_KEY)); // decryption is needed to calculate the public key and thus see if the claimed information is in fact valid final decryption = olm.PkDecryption(); - final info = await client.getRoomKeysBackup(); + final info = await getRoomKeysBackupInfo(false); String backupPubKey; try { backupPubKey = decryption.init_with_private_key(privateKey); From 3187275ed762e15c9b049dbad4c3329e46f9612f Mon Sep 17 00:00:00 2001 From: Sorunome Date: Fri, 18 Sep 2020 10:01:03 +0200 Subject: [PATCH 76/90] fix: Don't query /members over and over --- lib/src/room.dart | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/src/room.dart b/lib/src/room.dart index 4305218..f30a534 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -1044,6 +1044,8 @@ class Room { return userList; } + bool _requestedParticipants = false; + /// Request the full list of participants from the server. The local list /// from the store is not complete if the client uses lazy loading. Future> requestParticipants() async { @@ -1054,13 +1056,16 @@ class Room { setState(user); } } - if (participantListComplete) return getParticipants(); + if (_requestedParticipants || participantListComplete) { + return getParticipants(); + } final matrixEvents = await client.requestMembers(id); final users = matrixEvents.map((e) => Event.fromMatrixEvent(e, this).asUser).toList(); for (final user in users) { setState(user); // at *least* cache this in-memory } + _requestedParticipants = true; users.removeWhere( (u) => [Membership.leave, Membership.ban].contains(u.membership)); return users; From a77e776479c7f2b6c654c32ce3ae0a3976e5cc45 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Sat, 19 Sep 2020 12:39:19 +0200 Subject: [PATCH 77/90] feat: Implement ignore list --- lib/src/client.dart | 46 +++++++++++++++++++++++++++++++++++++++ test/client_test.dart | 12 ++++++++++ test/fake_matrix_api.dart | 2 ++ 3 files changed, 60 insertions(+) diff --git a/lib/src/client.dart b/lib/src/client.dart index 0f337b7..ff8cbaf 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -1519,6 +1519,52 @@ sort order of ${prevState.sortOrder}. This should never happen...'''); } } + /// Clear all local cached messages and perform a new clean sync. + Future clearLocalCachedMessages() async { + prevBatch = null; + rooms.forEach((r) => r.prev_batch = null); + await database?.clearCache(id); + } + + /// A list of mxids of users who are ignored. + List get ignoredUsers => + accountData.containsKey('m.ignored_user_list') + ? accountData['m.ignored_user_list'].content['ignored_users'] + : []; + + /// Ignore another user. This will clear the local cached messages to + /// hide all previous messages from this user. + Future ignoreUser(String userId) async { + if (!userId.isValidMatrixId) { + throw Exception('$userId is not a valid mxid!'); + } + await setAccountData( + userID, + 'm.ignored_user_list', + {'ignored_users': ignoredUsers..add(userId)}, + ); + await clearLocalCachedMessages(); + return; + } + + /// Unignore a user. This will clear the local cached messages and request + /// them again from the server to avoid gaps in the timeline. + Future unignoreUser(String userId) async { + if (!userId.isValidMatrixId) { + throw Exception('$userId is not a valid mxid!'); + } + if (!ignoredUsers.contains(userId)) { + throw Exception('$userId is not in the ignore list!'); + } + await setAccountData( + userID, + 'm.ignored_user_list', + {'ignored_users': ignoredUsers..remove(userId)}, + ); + await clearLocalCachedMessages(); + return; + } + bool _disposed = false; Future _currentTransaction = Future.sync(() => {}); diff --git a/test/client_test.dart b/test/client_test.dart index 2fa4178..ce67a9d 100644 --- a/test/client_test.dart +++ b/test/client_test.dart @@ -421,6 +421,18 @@ void main() { test('changePassword', () async { await matrix.changePassword('1234', oldPassword: '123456'); }); + test('ignoredUsers', () async { + expect(matrix.ignoredUsers, []); + matrix.accountData['m.ignored_user_list'] = + BasicEvent(type: 'm.ignored_user_list', content: { + 'ignored_users': ['@charley:stupid.abc'] + }); + expect(matrix.ignoredUsers, ['@charley:stupid.abc']); + }); + test('ignoredUsers', () async { + await matrix.ignoreUser('@charley2:stupid.abc'); + await matrix.unignoreUser('@charley:stupid.abc'); + }); test('dispose', () async { await matrix.dispose(closeDatabase: true); diff --git a/test/fake_matrix_api.dart b/test/fake_matrix_api.dart index 2400de2..156d86a 100644 --- a/test/fake_matrix_api.dart +++ b/test/fake_matrix_api.dart @@ -1982,6 +1982,8 @@ class FakeMatrixApi extends MockClient { '/client/unstable/room_keys/version': (var reqI) => {'version': '5'}, }, 'PUT': { + '/client/r0/user/%40test%3AfakeServer.notExisting/account_data/m.ignored_user_list': + (var req) => {}, '/client/r0/presence/${Uri.encodeComponent('@alice:example.com')}/status': (var req) => {}, '/client/r0/pushrules/global/content/nocake/enabled': (var req) => {}, From 510de0530434e392a281f197f77a9c49349e7fc2 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Sat, 19 Sep 2020 15:05:43 +0200 Subject: [PATCH 78/90] fix: ignore list --- lib/src/client.dart | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/src/client.dart b/lib/src/client.dart index ff8cbaf..ebd85a6 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -1527,10 +1527,12 @@ sort order of ${prevState.sortOrder}. This should never happen...'''); } /// A list of mxids of users who are ignored. - List get ignoredUsers => - accountData.containsKey('m.ignored_user_list') - ? accountData['m.ignored_user_list'].content['ignored_users'] - : []; + List get ignoredUsers => (accountData + .containsKey('m.ignored_user_list') && + accountData['m.ignored_user_list'].content['ignored_users'] is List) + ? List.from( + accountData['m.ignored_user_list'].content['ignored_users']) + : []; /// Ignore another user. This will clear the local cached messages to /// hide all previous messages from this user. From 864cbfa9068f5bb285336ca73ab2551204e62044 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Sun, 20 Sep 2020 10:35:25 +0200 Subject: [PATCH 79/90] fix: Hotfix ignored user list --- lib/src/client.dart | 22 ++++++++++------------ test/client_test.dart | 4 +++- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/src/client.dart b/lib/src/client.dart index ebd85a6..58f3492 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -1529,9 +1529,9 @@ sort order of ${prevState.sortOrder}. This should never happen...'''); /// A list of mxids of users who are ignored. List get ignoredUsers => (accountData .containsKey('m.ignored_user_list') && - accountData['m.ignored_user_list'].content['ignored_users'] is List) + accountData['m.ignored_user_list'].content['ignored_users'] is Map) ? List.from( - accountData['m.ignored_user_list'].content['ignored_users']) + accountData['m.ignored_user_list'].content['ignored_users'].keys) : []; /// Ignore another user. This will clear the local cached messages to @@ -1540,11 +1540,10 @@ sort order of ${prevState.sortOrder}. This should never happen...'''); if (!userId.isValidMatrixId) { throw Exception('$userId is not a valid mxid!'); } - await setAccountData( - userID, - 'm.ignored_user_list', - {'ignored_users': ignoredUsers..add(userId)}, - ); + await setAccountData(userID, 'm.ignored_user_list', { + 'ignored_users': Map.fromEntries( + (ignoredUsers..add(userId)).map((key) => MapEntry(key, {}))), + }); await clearLocalCachedMessages(); return; } @@ -1558,11 +1557,10 @@ sort order of ${prevState.sortOrder}. This should never happen...'''); if (!ignoredUsers.contains(userId)) { throw Exception('$userId is not in the ignore list!'); } - await setAccountData( - userID, - 'm.ignored_user_list', - {'ignored_users': ignoredUsers..remove(userId)}, - ); + await setAccountData(userID, 'm.ignored_user_list', { + 'ignored_users': Map.fromEntries( + (ignoredUsers..remove(userId)).map((key) => MapEntry(key, {}))), + }); await clearLocalCachedMessages(); return; } diff --git a/test/client_test.dart b/test/client_test.dart index ce67a9d..f1a13f4 100644 --- a/test/client_test.dart +++ b/test/client_test.dart @@ -425,7 +425,9 @@ void main() { expect(matrix.ignoredUsers, []); matrix.accountData['m.ignored_user_list'] = BasicEvent(type: 'm.ignored_user_list', content: { - 'ignored_users': ['@charley:stupid.abc'] + 'ignored_users': { + '@charley:stupid.abc': {}, + }, }); expect(matrix.ignoredUsers, ['@charley:stupid.abc']); }); From ba7a01ddea430a6eec2661e2f9b7714d3719c652 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Sun, 20 Sep 2020 11:24:56 +0200 Subject: [PATCH 80/90] fix: emoji regex typo --- lib/src/event.dart | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/src/event.dart b/lib/src/event.dart index 7807f88..740fe0b 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -770,7 +770,7 @@ class Event extends MatrixEvent { // regexes to fetch the number of emotes, including emoji, and if the message consists of only those // to match an emoji we can use the following regex: - // \x{00a9}|\x{00ae}|[\x{2000}-\x{3300}]|\x{d83c}[\x{d000}-\x{dfff}|\x{d83d}[\x{d000}-\x{dfff}]|\x{d83e}[\x{d000}-\x{dfff}] + // \x{00a9}|\x{00ae}|[\x{2000}-\x{3300}]|\x{d83c}[\x{d000}-\x{dfff}]|\x{d83d}[\x{d000}-\x{dfff}]|\x{d83e}[\x{d000}-\x{dfff}] // we need to replace \x{0000} with \u0000, the comment is left in the other format to be able to paste into regex101.com // to see if there is a custom emote, we use the following regex: ]+data-mx-(?:emote|emoticon)(?==|>|\s)[^>]*> // now we combind the two to have four regexes: @@ -779,19 +779,19 @@ class Event extends MatrixEvent { // 3. count number of emoji // 4- count number of emoji or emotes static final RegExp _onlyEmojiRegex = RegExp( - r'^(\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff]|\s)*$', + r'^(\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff]|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff]|\s)*$', caseSensitive: false, multiLine: true); static final RegExp _onlyEmojiEmoteRegex = RegExp( - r'^(\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff]|]+data-mx-(?:emote|emoticon)(?==|>|\s)[^>]*>|\s)*$', + r'^(\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff]|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff]|]+data-mx-(?:emote|emoticon)(?==|>|\s)[^>]*>|\s)*$', caseSensitive: false, multiLine: true); static final RegExp _countEmojiRegex = RegExp( - r'(\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff])', + r'(\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff]|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff])', caseSensitive: false, multiLine: true); static final RegExp _countEmojiEmoteRegex = RegExp( - r'(\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff]|]+data-mx-(?:emote|emoticon)(?==|>|\s)[^>]*>)', + r'(\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff]|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff]|]+data-mx-(?:emote|emoticon)(?==|>|\s)[^>]*>)', caseSensitive: false, multiLine: true); From d42979da12ad6749a3c47c67dc4fb0f0f57420b4 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Sun, 20 Sep 2020 19:09:32 +0200 Subject: [PATCH 81/90] fix: Emoji regex incorrectly using multiline --- lib/src/event.dart | 8 ++++---- test/event_test.dart | 11 +++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/src/event.dart b/lib/src/event.dart index 740fe0b..a92bbd2 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -781,19 +781,19 @@ class Event extends MatrixEvent { static final RegExp _onlyEmojiRegex = RegExp( r'^(\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff]|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff]|\s)*$', caseSensitive: false, - multiLine: true); + multiLine: false); static final RegExp _onlyEmojiEmoteRegex = RegExp( r'^(\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff]|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff]|]+data-mx-(?:emote|emoticon)(?==|>|\s)[^>]*>|\s)*$', caseSensitive: false, - multiLine: true); + multiLine: false); static final RegExp _countEmojiRegex = RegExp( r'(\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff]|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff])', caseSensitive: false, - multiLine: true); + multiLine: false); static final RegExp _countEmojiEmoteRegex = RegExp( r'(\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff]|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff]|]+data-mx-(?:emote|emoticon)(?==|>|\s)[^>]*>)', caseSensitive: false, - multiLine: true); + multiLine: false); /// Returns if a given event only has emotes, emojis or whitespace as content. /// This is useful to determine if stand-alone emotes should be displayed bigger. diff --git a/test/event_test.dart b/test/event_test.dart index 3dabdd4..aaa4aff 100644 --- a/test/event_test.dart +++ b/test/event_test.dart @@ -1171,6 +1171,17 @@ void main() { }, null); expect(event.onlyEmotes, false); expect(event.numberEmotes, 0); + event = Event.fromJson({ + 'type': EventTypes.Message, + 'content': { + 'msgtype': 'm.text', + 'body': 'normal message\n\nvery normal', + }, + 'event_id': '\$edit2', + 'sender': '@alice:example.org', + }, null); + expect(event.onlyEmotes, false); + expect(event.numberEmotes, 0); event = Event.fromJson({ 'type': EventTypes.Message, 'content': { From 0ff971faa99ca946acf116a51fc31ef7fe87a745 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Mon, 21 Sep 2020 08:43:56 +0200 Subject: [PATCH 82/90] fix: Obay variant selectors for emoji regex --- lib/src/event.dart | 10 +++++----- test/event_test.dart | 12 ++++++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/lib/src/event.dart b/lib/src/event.dart index a92bbd2..bb6c428 100644 --- a/lib/src/event.dart +++ b/lib/src/event.dart @@ -770,7 +770,7 @@ class Event extends MatrixEvent { // regexes to fetch the number of emotes, including emoji, and if the message consists of only those // to match an emoji we can use the following regex: - // \x{00a9}|\x{00ae}|[\x{2000}-\x{3300}]|\x{d83c}[\x{d000}-\x{dfff}]|\x{d83d}[\x{d000}-\x{dfff}]|\x{d83e}[\x{d000}-\x{dfff}] + // (?:\x{00a9}|\x{00ae}|[\x{2000}-\x{3300}]|\x{d83c}[\x{d000}-\x{dfff}]|\x{d83d}[\x{d000}-\x{dfff}]|\x{d83e}[\x{d000}-\x{dfff}])[\x{fe00}-\x{fe0f}]? // we need to replace \x{0000} with \u0000, the comment is left in the other format to be able to paste into regex101.com // to see if there is a custom emote, we use the following regex: ]+data-mx-(?:emote|emoticon)(?==|>|\s)[^>]*> // now we combind the two to have four regexes: @@ -779,19 +779,19 @@ class Event extends MatrixEvent { // 3. count number of emoji // 4- count number of emoji or emotes static final RegExp _onlyEmojiRegex = RegExp( - r'^(\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff]|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff]|\s)*$', + r'^((?:\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff]|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff])[\ufe00-\ufe0f]?|\s)*$', caseSensitive: false, multiLine: false); static final RegExp _onlyEmojiEmoteRegex = RegExp( - r'^(\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff]|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff]|]+data-mx-(?:emote|emoticon)(?==|>|\s)[^>]*>|\s)*$', + r'^((?:\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff]|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff])[\ufe00-\ufe0f]?|]+data-mx-(?:emote|emoticon)(?==|>|\s)[^>]*>|\s)*$', caseSensitive: false, multiLine: false); static final RegExp _countEmojiRegex = RegExp( - r'(\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff]|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff])', + r'((?:\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff]|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff])[\ufe00-\ufe0f]?)', caseSensitive: false, multiLine: false); static final RegExp _countEmojiEmoteRegex = RegExp( - r'(\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff]|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff]|]+data-mx-(?:emote|emoticon)(?==|>|\s)[^>]*>)', + r'((?:\u00a9|\u00ae|[\u2000-\u3300]|\ud83c[\ud000-\udfff]|\ud83d[\ud000-\udfff]|\ud83e[\ud000-\udfff])[\ufe00-\ufe0f]?|]+data-mx-(?:emote|emoticon)(?==|>|\s)[^>]*>)', caseSensitive: false, multiLine: false); diff --git a/test/event_test.dart b/test/event_test.dart index aaa4aff..74d7c6a 100644 --- a/test/event_test.dart +++ b/test/event_test.dart @@ -1267,6 +1267,18 @@ void main() { }, null); expect(event.onlyEmotes, true); expect(event.numberEmotes, 2); + // with variant selector + event = Event.fromJson({ + 'type': EventTypes.Message, + 'content': { + 'msgtype': 'm.text', + 'body': '❤️', + }, + 'event_id': '\$edit2', + 'sender': '@alice:example.org', + }, null); + expect(event.onlyEmotes, true); + expect(event.numberEmotes, 1); }); }); } From 70939a7c9caa41e4c45ff301a86777848fd1d7e0 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Mon, 21 Sep 2020 10:24:15 +0200 Subject: [PATCH 83/90] fix: Message index replay attack check --- lib/encryption/encryption.dart | 18 ++++++------------ lib/encryption/utils/session_key.dart | 13 +++++++++---- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/lib/encryption/encryption.dart b/lib/encryption/encryption.dart index 8c9ee7b..db56cb5 100644 --- a/lib/encryption/encryption.dart +++ b/lib/encryption/encryption.dart @@ -160,24 +160,18 @@ class Encryption { final decryptResult = inboundGroupSession.inboundGroupSession .decrypt(event.content['ciphertext']); canRequestSession = false; - final messageIndexKey = event.eventId + + // we can't have the key be an int, else json-serializing will fail, thus we need it to be a string + final messageIndexKey = 'key-' + decryptResult.message_index.toString(); + final messageIndexValue = event.eventId + + '|' + event.originServerTs.millisecondsSinceEpoch.toString(); var haveIndex = inboundGroupSession.indexes.containsKey(messageIndexKey); if (haveIndex && - inboundGroupSession.indexes[messageIndexKey] != - decryptResult.message_index) { + inboundGroupSession.indexes[messageIndexKey] != messageIndexValue) { // TODO: maybe clear outbound session, if it is ours throw (DecryptError.CHANNEL_CORRUPTED); } - final existingIndex = inboundGroupSession.indexes.entries.firstWhere( - (e) => e.value == decryptResult.message_index, - orElse: () => null); - if (existingIndex != null && existingIndex.key != messageIndexKey) { - // TODO: maybe clear outbound session, if it is ours - throw (DecryptError.CHANNEL_CORRUPTED); - } - inboundGroupSession.indexes[messageIndexKey] = - decryptResult.message_index; + inboundGroupSession.indexes[messageIndexKey] = messageIndexValue; if (!haveIndex) { // now we persist the udpated indexes into the database. // the entry should always exist. In the case it doesn't, the following diff --git a/lib/encryption/utils/session_key.dart b/lib/encryption/utils/session_key.dart index b54ca32..471c1f4 100644 --- a/lib/encryption/utils/session_key.dart +++ b/lib/encryption/utils/session_key.dart @@ -26,7 +26,7 @@ import '../../src/utils/logs.dart'; class SessionKey { Map content; - Map indexes; + Map indexes; olm.InboundGroupSession inboundGroupSession; final String key; List get forwardingCurve25519KeyChain => @@ -60,9 +60,14 @@ class SessionKey { Event.getMapFromPayload(dbEntry.senderClaimedKeys); content = parsedContent != null ? Map.from(parsedContent) : null; - indexes = parsedIndexes != null - ? Map.from(parsedIndexes) - : {}; + // we need to try...catch as the map used to be and that will throw an error. + try { + indexes = parsedIndexes != null + ? Map.from(parsedIndexes) + : {}; + } catch (e) { + indexes = {}; + } roomId = dbEntry.roomId; sessionId = dbEntry.sessionId; _setSenderKey(dbEntry.senderKey); From 86a4f90a5aecbfa3bdd3932771df6d611141d211 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Mon, 21 Sep 2020 18:10:46 +0200 Subject: [PATCH 84/90] fix: Run automated key requests in root zone --- lib/encryption/encryption.dart | 27 ++++++++------------------- lib/encryption/key_manager.dart | 4 +++- lib/src/utils/run_in_root.dart | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 20 deletions(-) create mode 100644 lib/src/utils/run_in_root.dart diff --git a/lib/encryption/encryption.dart b/lib/encryption/encryption.dart index db56cb5..aa627a9 100644 --- a/lib/encryption/encryption.dart +++ b/lib/encryption/encryption.dart @@ -23,24 +23,13 @@ import 'package:pedantic/pedantic.dart'; import '../famedlysdk.dart'; import '../matrix_api.dart'; -import '../src/utils/logs.dart'; +import '../src/utils/run_in_root.dart'; import 'cross_signing.dart'; import 'key_manager.dart'; import 'key_verification_manager.dart'; import 'olm_manager.dart'; import 'ssss.dart'; -Future _runInRoot(FutureOr Function() fn) async { - return await Zone.root.run(() async { - try { - return await fn(); - } catch (e, s) { - Logs.error('Error thrown in root zone: ' + e.toString(), s); - } - return null; - }); -} - class Encryption { final Client client; final bool debug; @@ -80,7 +69,7 @@ class Encryption { } void handleDeviceOneTimeKeysCount(Map countJson) { - _runInRoot(() => olmManager.handleDeviceOneTimeKeysCount(countJson)); + runInRoot(() => olmManager.handleDeviceOneTimeKeysCount(countJson)); } void onSync() { @@ -96,21 +85,21 @@ class Encryption { if (['m.room_key_request', 'm.forwarded_room_key'].contains(event.type)) { // "just" room key request things. We don't need these asap, so we handle // them in the background - unawaited(_runInRoot(() => keyManager.handleToDeviceEvent(event))); + unawaited(runInRoot(() => keyManager.handleToDeviceEvent(event))); } if (event.type.startsWith('m.key.verification.')) { // some key verification event. No need to handle it now, we can easily // do this in the background unawaited( - _runInRoot(() => keyVerificationManager.handleToDeviceEvent(event))); + runInRoot(() => keyVerificationManager.handleToDeviceEvent(event))); } if (event.type.startsWith('m.secret.')) { // some ssss thing. We can do this in the background - unawaited(_runInRoot(() => ssss.handleToDeviceEvent(event))); + unawaited(runInRoot(() => ssss.handleToDeviceEvent(event))); } if (event.sender == client.userID) { // maybe we need to re-try SSSS secrets - unawaited(_runInRoot(() => ssss.periodicallyRequestMissingCache())); + unawaited(runInRoot(() => ssss.periodicallyRequestMissingCache())); } } @@ -125,12 +114,12 @@ class Encryption { .startsWith('m.key.verification.'))) { // "just" key verification, no need to do this in sync unawaited( - _runInRoot(() => keyVerificationManager.handleEventUpdate(update))); + runInRoot(() => keyVerificationManager.handleEventUpdate(update))); } if (update.content['sender'] == client.userID && !update.content['unsigned'].containsKey('transaction_id')) { // maybe we need to re-try SSSS secrets - unawaited(_runInRoot(() => ssss.periodicallyRequestMissingCache())); + unawaited(runInRoot(() => ssss.periodicallyRequestMissingCache())); } } diff --git a/lib/encryption/key_manager.dart b/lib/encryption/key_manager.dart index 4da6c14..3c4ce5b 100644 --- a/lib/encryption/key_manager.dart +++ b/lib/encryption/key_manager.dart @@ -29,6 +29,7 @@ import '../matrix_api.dart'; import '../src/database/database.dart'; import '../src/utils/logs.dart'; import '../src/utils/run_in_background.dart'; +import '../src/utils/run_in_root.dart'; const MEGOLM_KEY = 'm.megolm_backup.v1'; @@ -208,7 +209,8 @@ class KeyManager { !client.isUnknownSession) { // do e2ee recovery _requestedSessionIds.add(requestIdent); - unawaited(request(room, sessionId, senderKey, askOnlyOwnDevices: true)); + unawaited(runInRoot(() => + request(room, sessionId, senderKey, askOnlyOwnDevices: true))); } return null; } diff --git a/lib/src/utils/run_in_root.dart b/lib/src/utils/run_in_root.dart new file mode 100644 index 0000000..b898dde --- /dev/null +++ b/lib/src/utils/run_in_root.dart @@ -0,0 +1,32 @@ +/* + * Famedly Matrix SDK + * Copyright (C) 2020 Famedly GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import 'dart:async'; + +import 'logs.dart'; + +Future runInRoot(FutureOr Function() fn) async { + return await Zone.root.run(() async { + try { + return await fn(); + } catch (e, s) { + Logs.error('Error thrown in root zone: ' + e.toString(), s); + } + return null; + }); +} From adb907bbc443f8180e491003ffc99bb78fcab5b3 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Mon, 21 Sep 2020 17:37:03 +0200 Subject: [PATCH 85/90] fix: Clear on logout --- lib/src/client.dart | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/src/client.dart b/lib/src/client.dart index 58f3492..5d8cb94 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -758,6 +758,11 @@ class Client extends MatrixApi { Logs.error('Error during processing events: ' + e.toString(), s); onSyncError.add(SdkError( exception: e is Exception ? e : Exception(e), stackTrace: s)); + if (e is MatrixException && + e.errcode == MatrixError.M_UNKNOWN_TOKEN.toString().split('.').last) { + Logs.warning('The user has been logged out!'); + clear(); + } } } From 5019ebfeb56f0789ab4cc8d27ccda663156b5d68 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Mon, 21 Sep 2020 12:28:13 +0200 Subject: [PATCH 86/90] feat: Auto retry send events --- lib/src/client.dart | 3 +++ lib/src/room.dart | 54 +++++++++++++++++++++++++---------------- test/timeline_test.dart | 3 ++- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/lib/src/client.dart b/lib/src/client.dart index 5d8cb94..71f7504 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -61,6 +61,8 @@ class Client extends MatrixApi { Set roomPreviewLastEvents; + int sendMessageTimeoutSeconds; + /// Create a client /// [clientName] = unique identifier of this client /// [database]: The database instance to use @@ -92,6 +94,7 @@ class Client extends MatrixApi { this.importantStateEvents, this.roomPreviewLastEvents, this.pinUnreadRooms = false, + this.sendMessageTimeoutSeconds = 60, @deprecated bool debug, }) { verificationMethods ??= {}; diff --git a/lib/src/room.dart b/lib/src/room.dart index f30a534..19fa640 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -722,7 +722,7 @@ class Room { content['formatted_body'] = '* ' + content['formatted_body']; } } - + final sentDate = DateTime.now(); final syncUpdate = SyncUpdate() ..rooms = (RoomsUpdate() ..join = ({}..[id] = (JoinedRoomUpdate() @@ -733,7 +733,7 @@ class Room { ..type = type ..eventId = messageID ..senderId = client.userID - ..originServerTs = DateTime.now() + ..originServerTs = sentDate ..unsigned = { MessageSendingStatusKey: 0, 'transaction_id': messageID, @@ -742,26 +742,38 @@ class Room { await _handleFakeSync(syncUpdate); // Send the text and on success, store and display a *sent* event. - try { - final res = await _sendContent( - type, - content, - txid: messageID, - ); - syncUpdate.rooms.join.values.first.timeline.events.first - .unsigned[MessageSendingStatusKey] = 1; - syncUpdate.rooms.join.values.first.timeline.events.first.eventId = res; - await _handleFakeSync(syncUpdate); - - return res; - } catch (e, s) { - Logs.warning( - '[Client] Problem while sending message: ' + e.toString(), s); - syncUpdate.rooms.join.values.first.timeline.events.first - .unsigned[MessageSendingStatusKey] = -1; - await _handleFakeSync(syncUpdate); + String res; + while (res == null) { + try { + res = await _sendContent( + type, + content, + txid: messageID, + ); + } catch (e, s) { + if ((DateTime.now().millisecondsSinceEpoch - + sentDate.millisecondsSinceEpoch) < + (1000 * client.sendMessageTimeoutSeconds)) { + Logs.warning('[Client] Problem while sending message because of "' + + e.toString() + + '". Try again in 1 seconds...'); + await Future.delayed(Duration(seconds: 1)); + } else { + Logs.warning( + '[Client] Problem while sending message: ' + e.toString(), s); + syncUpdate.rooms.join.values.first.timeline.events.first + .unsigned[MessageSendingStatusKey] = -1; + await _handleFakeSync(syncUpdate); + return null; + } + } } - return null; + syncUpdate.rooms.join.values.first.timeline.events.first + .unsigned[MessageSendingStatusKey] = 1; + syncUpdate.rooms.join.values.first.timeline.events.first.eventId = res; + await _handleFakeSync(syncUpdate); + + return res; } /// Call the Matrix API to join this room if the user is not already a member. diff --git a/test/timeline_test.dart b/test/timeline_test.dart index 3ce1a09..8717e75 100644 --- a/test/timeline_test.dart +++ b/test/timeline_test.dart @@ -33,7 +33,8 @@ void main() { var updateCount = 0; var insertList = []; - var client = Client('testclient', httpClient: FakeMatrixApi()); + var client = Client('testclient', + httpClient: FakeMatrixApi(), sendMessageTimeoutSeconds: 5); var room = Room( id: roomID, client: client, prev_batch: '1234', roomAccountData: {}); From b6754fbc4666f7eaf51da78cc1595fcde349a4d2 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Thu, 24 Sep 2020 15:57:58 +0200 Subject: [PATCH 87/90] chore: update emote stuff --- lib/src/room.dart | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/lib/src/room.dart b/lib/src/room.dart index 19fa640..7b46b88 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -446,7 +446,7 @@ class Room { final allMxcs = {}; // for easy dedupint final addEmotePack = (String packName, Map content, [String packNameOverride]) { - if (!(content['short'] is Map)) { + if (!(content['emoticons'] is Map) && !(content['short'] is Map)) { return; } if (content['pack'] is Map && content['pack']['name'] is String) { @@ -459,13 +459,26 @@ class Room { if (!packs.containsKey(packName)) { packs[packName] = {}; } - content['short'].forEach((key, value) { - if (key is String && value is String && value.startsWith('mxc://')) { - if (allMxcs.add(value)) { - packs[packName][key] = value; + if (content['emoticons'] is Map) { + content['emoticons'].forEach((key, value) { + if (key is String && + value is Map && + value['url'] is String && + value['url'].startsWith('mxc://')) { + if (allMxcs.add(value['url'])) { + packs[packName][key] = value['url']; + } } - } - }); + }); + } else { + content['short'].forEach((key, value) { + if (key is String && value is String && value.startsWith('mxc://')) { + if (allMxcs.add(value)) { + packs[packName][key] = value; + } + } + }); + } }; // first add all the user emotes final userEmotes = client.accountData['im.ponies.user_emotes']; @@ -477,9 +490,6 @@ class Room { if (emoteRooms != null && emoteRooms.content['rooms'] is Map) { for (final roomEntry in emoteRooms.content['rooms'].entries) { final roomId = roomEntry.key; - if (roomId == id) { - continue; - } final room = client.getRoomById(roomId); if (room != null && roomEntry.value is Map) { for (final stateKeyEntry in roomEntry.value.entries) { From 877ff9963cc7bd7d9a20685b708f7681c8da104e Mon Sep 17 00:00:00 2001 From: Sorunome Date: Sun, 27 Sep 2020 10:54:54 +0200 Subject: [PATCH 88/90] fix: Don't sort rooms too often --- lib/src/client.dart | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/src/client.dart b/lib/src/client.dart index 71f7504..ce3dc43 100644 --- a/lib/src/client.dart +++ b/lib/src/client.dart @@ -787,6 +787,7 @@ class Client extends MatrixApi { await _handleRooms(sync.rooms.leave, Membership.leave, sortAtTheEnd: sortAtTheEnd); } + _sortRooms(); } if (sync.presence != null) { for (final newPresence in sync.presence) { @@ -1112,7 +1113,6 @@ class Client extends MatrixApi { } if (rooms[j].onUpdate != null) rooms[j].onUpdate.add(rooms[j].id); } - _sortRooms(); } void _updateRoomsByEventUpdate(EventUpdate eventUpdate) { @@ -1160,7 +1160,6 @@ sort order of ${prevState.sortOrder}. This should never happen...'''); break; } room.onUpdate.add(room.id); - if (['timeline', 'account_data'].contains(eventUpdate.type)) _sortRooms(); } bool _sortLock = false; From ab97c596ac13f1d13f059954f0c33a135c9d9b19 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Mon, 28 Sep 2020 10:58:24 +0200 Subject: [PATCH 89/90] chore: Add better debug logging for corrupt sessions --- lib/encryption/encryption.dart | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/encryption/encryption.dart b/lib/encryption/encryption.dart index aa627a9..1a3557a 100644 --- a/lib/encryption/encryption.dart +++ b/lib/encryption/encryption.dart @@ -24,6 +24,7 @@ import 'package:pedantic/pedantic.dart'; import '../famedlysdk.dart'; import '../matrix_api.dart'; import '../src/utils/run_in_root.dart'; +import '../src/utils/logs.dart'; import 'cross_signing.dart'; import 'key_manager.dart'; import 'key_verification_manager.dart'; @@ -158,6 +159,16 @@ class Encryption { if (haveIndex && inboundGroupSession.indexes[messageIndexKey] != messageIndexValue) { // TODO: maybe clear outbound session, if it is ours + // TODO: Make it so that we can't re-request the session keys, this is just for debugging + Logs.error('[Decrypt] Could not decrypt due to a corrupted session.'); + Logs.error('[Decrypt] Want session: $roomId $sessionId $senderKey'); + Logs.error( + '[Decrypt] Have sessoin: ${inboundGroupSession.roomId} ${inboundGroupSession.sessionId} ${inboundGroupSession.senderKey}'); + Logs.error( + '[Decrypt] Want indexes: $messageIndexKey $messageIndexValue'); + Logs.error( + '[Decrypt] Have indexes: $messageIndexKey ${inboundGroupSession.indexes[messageIndexKey]}'); + canRequestSession = true; throw (DecryptError.CHANNEL_CORRUPTED); } inboundGroupSession.indexes[messageIndexKey] = messageIndexValue; From 84cc925b08e97098d00c54fff9c1244f91055de3 Mon Sep 17 00:00:00 2001 From: Christian Pauly Date: Mon, 28 Sep 2020 12:43:23 +0200 Subject: [PATCH 90/90] fix: Mimetype null --- lib/src/utils/matrix_file.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/src/utils/matrix_file.dart b/lib/src/utils/matrix_file.dart index 3c728f2..bfc0f3d 100644 --- a/lib/src/utils/matrix_file.dart +++ b/lib/src/utils/matrix_file.dart @@ -19,7 +19,8 @@ class MatrixFile { } MatrixFile({this.bytes, this.name, this.mimeType}) { - mimeType ??= lookupMimeType(name, headerBytes: bytes); + mimeType ??= + lookupMimeType(name, headerBytes: bytes) ?? 'application/octet-stream'; name = name.split('/').last.toLowerCase(); }