diff --git a/lib/encryption/key_manager.dart b/lib/encryption/key_manager.dart
index 9621112..d9b2a88 100644
--- a/lib/encryption/key_manager.dart
+++ b/lib/encryption/key_manager.dart
@@ -44,11 +44,11 @@ class KeyManager {
       final keyObj = olm.PkDecryption();
       try {
         final info = await client.api.getRoomKeysBackup();
-        if (!(info.authData is RoomKeysAuthDataV1Curve25519AesSha2)) {
+        if (info.algorithm != RoomKeysAlgorithmType.v1Curve25519AesSha2) {
           return false;
         }
         if (keyObj.init_with_private_key(base64.decode(secret)) ==
-            (info.authData as RoomKeysAuthDataV1Curve25519AesSha2).publicKey) {
+            info.authData['public_key']) {
           _requestedSessionIds.clear();
           return true;
         }
@@ -340,9 +340,8 @@ class KeyManager {
       backupPubKey = decryption.init_with_private_key(privateKey);
 
       if (backupPubKey == null ||
-          !(info.authData is RoomKeysAuthDataV1Curve25519AesSha2) ||
-          (info.authData as RoomKeysAuthDataV1Curve25519AesSha2).publicKey !=
-              backupPubKey) {
+          info.algorithm != RoomKeysAlgorithmType.v1Curve25519AesSha2 ||
+          info.authData['public_key'] != backupPubKey) {
         return;
       }
       for (final roomEntry in keys.rooms.entries) {
diff --git a/lib/matrix_api/matrix_api.dart b/lib/matrix_api/matrix_api.dart
index f09c59b..d939fa7 100644
--- a/lib/matrix_api/matrix_api.dart
+++ b/lib/matrix_api/matrix_api.dart
@@ -2042,13 +2042,13 @@ class MatrixApi {
   /// Create room keys backup
   /// https://matrix.org/docs/spec/client_server/unstable#post-matrix-client-r0-room-keys-version
   Future<String> createRoomKeysBackup(
-      RoomKeysAlgorithmType algorithm, RoomKeysAuthData authData) async {
+      RoomKeysAlgorithmType algorithm, Map<String, dynamic> authData) async {
     final ret = await request(
       RequestType.POST,
       '/client/unstable/room_keys/version',
       data: {
         'algorithm': algorithm.algorithmString,
-        'auth_data': authData.toJson(),
+        'auth_data': authData,
       },
     );
     return ret['version'];
@@ -2071,13 +2071,13 @@ class MatrixApi {
   /// Updates a room key backup
   /// https://matrix.org/docs/spec/client_server/unstable#put-matrix-client-r0-room-keys-version-version
   Future<void> updateRoomKeysBackup(String version,
-      RoomKeysAlgorithmType algorithm, RoomKeysAuthData authData) async {
+      RoomKeysAlgorithmType algorithm, Map<String, dynamic> authData) async {
     await request(
       RequestType.PUT,
       '/client/unstable/room_keys/version/${Uri.encodeComponent(version)}',
       data: {
         'algorithm': algorithm.algorithmString,
-        'auth_data': authData.toJson(),
+        'auth_data': authData,
         'version': version,
       },
     );
diff --git a/lib/matrix_api/model/room_keys_info.dart b/lib/matrix_api/model/room_keys_info.dart
index 42e2398..f7a2cfe 100644
--- a/lib/matrix_api/model/room_keys_info.dart
+++ b/lib/matrix_api/model/room_keys_info.dart
@@ -38,46 +38,9 @@ extension RoomKeysAlgorithmTypeExtension on RoomKeysAlgorithmType {
   }
 }
 
-abstract class RoomKeysAuthData {
-  // This object is used for signing so we need the raw json too
-  Map<String, dynamic> _json;
-
-  RoomKeysAuthData.fromJson(Map<String, dynamic> json) {
-    _json = json;
-  }
-
-  Map<String, dynamic> toJson() {
-    return _json;
-  }
-}
-
-class RoomKeysAuthDataV1Curve25519AesSha2 extends RoomKeysAuthData {
-  String publicKey;
-  Map<String, Map<String, String>> signatures;
-
-  RoomKeysAuthDataV1Curve25519AesSha2.fromJson(Map<String, dynamic> json)
-      : super.fromJson(json) {
-    publicKey = json['public_key'];
-    signatures = json['signatures'] is Map
-        ? Map<String, Map<String, String>>.from((json['signatures'] as Map)
-            .map((k, v) => MapEntry(k, Map<String, String>.from(v))))
-        : null;
-  }
-
-  @override
-  Map<String, dynamic> toJson() {
-    final data = super.toJson();
-    data['public_key'] = publicKey;
-    if (signatures != null) {
-      data['signatures'] = signatures;
-    }
-    return data;
-  }
-}
-
 class RoomKeysVersionResponse {
   RoomKeysAlgorithmType algorithm;
-  RoomKeysAuthData authData;
+  Map<String, dynamic> authData;
   int count;
   String etag;
   String version;
@@ -85,14 +48,7 @@ class RoomKeysVersionResponse {
   RoomKeysVersionResponse.fromJson(Map<String, dynamic> json) {
     algorithm =
         RoomKeysAlgorithmTypeExtension.fromAlgorithmString(json['algorithm']);
-    switch (algorithm) {
-      case RoomKeysAlgorithmType.v1Curve25519AesSha2:
-        authData =
-            RoomKeysAuthDataV1Curve25519AesSha2.fromJson(json['auth_data']);
-        break;
-      default:
-        authData = null;
-    }
+    authData = json['auth_data'];
     count = json['count'];
     etag =
         json['etag'].toString(); // synapse replies an int but docs say string?
@@ -102,7 +58,7 @@ class RoomKeysVersionResponse {
   Map<String, dynamic> toJson() {
     final data = <String, dynamic>{};
     data['algorithm'] = algorithm?.algorithmString;
-    data['auth_data'] = authData?.toJson();
+    data['auth_data'] = authData;
     data['count'] = count;
     data['etag'] = etag;
     data['version'] = version;
diff --git a/test/matrix_api_test.dart b/test/matrix_api_test.dart
index d585b36..a165628 100644
--- a/test/matrix_api_test.dart
+++ b/test/matrix_api_test.dart
@@ -1596,10 +1596,10 @@ void main() {
       matrixApi.accessToken = '1234';
 
       final algorithm = RoomKeysAlgorithmType.v1Curve25519AesSha2;
-      final authData = RoomKeysAuthDataV1Curve25519AesSha2.fromJson({
+      final authData = <String, dynamic>{
         'public_key': 'GXYaxqhNhUK28zUdxOmEsFRguz+PzBsDlTLlF0O0RkM',
         'signatures': {},
-      });
+      };
       final ret = await matrixApi.createRoomKeysBackup(algorithm, authData);
       expect(
           FakeMatrixApi.api['POST']
@@ -1619,10 +1619,10 @@ void main() {
       matrixApi.accessToken = '1234';
 
       final algorithm = RoomKeysAlgorithmType.v1Curve25519AesSha2;
-      final authData = RoomKeysAuthDataV1Curve25519AesSha2.fromJson({
+      final authData = <String, dynamic>{
         'public_key': 'GXYaxqhNhUK28zUdxOmEsFRguz+PzBsDlTLlF0O0RkM',
         'signatures': {},
-      });
+      };
       await matrixApi.updateRoomKeysBackup('5', algorithm, authData);
     });
     test('deleteRoomKeysBackup', () async {