diff --git a/lib/matrix_api/model/keys_query_response.dart b/lib/matrix_api/model/keys_query_response.dart
index 57bc16e..b368b4f 100644
--- a/lib/matrix_api/model/keys_query_response.dart
+++ b/lib/matrix_api/model/keys_query_response.dart
@@ -26,7 +26,9 @@ class KeysQueryResponse {
   Map<String, MatrixCrossSigningKey> userSigningKeys;
 
   KeysQueryResponse.fromJson(Map<String, dynamic> json) {
-    failures = Map<String, dynamic>.from(json['failures']);
+    failures = json['failures'] != null
+        ? Map<String, dynamic>.from(json['failures'])
+        : null;
     deviceKeys = json['device_keys'] != null
         ? (json['device_keys'] as Map).map(
             (k, v) => MapEntry(
diff --git a/lib/src/client.dart b/lib/src/client.dart
index 661291c..0f337b7 100644
--- a/lib/src/client.dart
+++ b/lib/src/client.dart
@@ -1122,8 +1122,8 @@ class Client extends MatrixApi {
         var prevState = room.getState(stateEvent.type, stateEvent.stateKey);
         if (prevState != null && prevState.sortOrder > stateEvent.sortOrder) {
           Logs.warning('''
-A new ${eventUpdate.type} event of the type ${stateEvent.type} has arrived with a previews 
-sort order ${stateEvent.sortOrder} than the current ${stateEvent.type} event with a 
+A new ${eventUpdate.type} event of the type ${stateEvent.type} has arrived with a previews
+sort order ${stateEvent.sortOrder} than the current ${stateEvent.type} event with a
 sort order of ${prevState.sortOrder}. This should never happen...''');
           return;
         }
@@ -1213,6 +1213,7 @@ sort order of ${prevState.sortOrder}. This should never happen...''');
     return userIds;
   }
 
+  final Map<String, DateTime> _keyQueryFailures = {};
   Future<void> _updateUserDeviceKeys() async {
     try {
       if (!isLogged()) return;
@@ -1231,7 +1232,11 @@ sort order of ${prevState.sortOrder}. This should never happen...''');
           _userDeviceKeys[userId] = DeviceKeysList(userId, this);
         }
         var deviceKeysList = userDeviceKeys[userId];
-        if (deviceKeysList.outdated) {
+        if (deviceKeysList.outdated &&
+            (!_keyQueryFailures.containsKey(userId.domain) ||
+                DateTime.now()
+                    .subtract(Duration(minutes: 5))
+                    .isAfter(_keyQueryFailures[userId.domain]))) {
           outdatedLists[userId] = [];
         }
       }
@@ -1373,6 +1378,13 @@ sort order of ${prevState.sortOrder}. This should never happen...''');
             }
           }
         }
+
+        // now process all the failures
+        if (response.failures != null) {
+          for (final failureDomain in response.failures.keys) {
+            _keyQueryFailures[failureDomain] = DateTime.now();
+          }
+        }
       }
 
       if (dbActions.isNotEmpty) {