From 5d4f0d58eb2dae1c7a1dead087058b930f213ae9 Mon Sep 17 00:00:00 2001 From: yostyle Date: Fri, 7 Feb 2025 14:43:13 +0100 Subject: [PATCH] Restore data migration from realm --- .../android/sdk/internal/crypto/OlmMachine.kt | 17 ++++ .../sdk/internal/crypto/RustCryptoService.kt | 9 ++- .../internal/crypto/network/RequestSender.kt | 81 ++++++++++--------- .../crypto/store/IMXCommonCryptoStore.kt | 24 ++++-- .../internal/crypto/store/RustCryptoStore.kt | 40 ++++++--- .../crypto/store/db/RealmCryptoStoreModule.kt | 2 + .../store/db/migration/MigrateCryptoTo024.kt | 27 +++---- 7 files changed, 129 insertions(+), 71 deletions(-) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/OlmMachine.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/OlmMachine.kt index a3c68c2230..37c19d2e59 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/OlmMachine.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/OlmMachine.kt @@ -53,6 +53,7 @@ import org.matrix.android.sdk.api.util.toOptional import org.matrix.android.sdk.internal.coroutines.builder.safeInvokeOnClose import org.matrix.android.sdk.internal.crypto.keysbackup.model.rest.DefaultKeysAlgorithmAndData import org.matrix.android.sdk.internal.crypto.keysbackup.model.rest.KeysAlgorithmAndData +import org.matrix.android.sdk.internal.crypto.model.MXInboundMegolmSessionWrapper import org.matrix.android.sdk.internal.crypto.network.RequestSender import org.matrix.android.sdk.internal.crypto.verification.SasVerification import org.matrix.android.sdk.internal.crypto.verification.VerificationRequest @@ -319,6 +320,22 @@ internal class OlmMachine @Inject constructor( inner.receiveVerificationEvent(serializedEvent, roomId) } + /** + * Used for lazy migration of inboundGroupSession from EA to ER. + */ + suspend fun importRoomKey(inbound: MXInboundMegolmSessionWrapper): Result { + Timber.v("Migration:: Tentative lazy migration") + return withContext(coroutineDispatchers.io) { + val export = inbound.exportKeys() + ?: return@withContext Result.failure(Exception("Failed to export key")) + val result = importDecryptedKeys(listOf(export), null).also { + Timber.v("Migration:: Tentative lazy migration result: ${it.totalNumberOfKeys}") + } + if (result.totalNumberOfKeys == 1) return@withContext Result.success(Unit) + return@withContext Result.failure(Exception("Import failed")) + } + } + /** * Mark the given list of users to be tracked, triggering a key query request for them. * diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/RustCryptoService.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/RustCryptoService.kt index e8e0414f05..82b19f9775 100755 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/RustCryptoService.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/RustCryptoService.kt @@ -505,8 +505,15 @@ internal class RustCryptoService @Inject constructor( val content = event.content?.toModel() ?: throw mxCryptoError val roomId = event.roomId val sessionId = content.sessionId + val senderKey = content.senderKey if (roomId != null && sessionId != null) { - perSessionBackupQueryRateLimiter.tryFromBackupIfPossible(sessionId, roomId) + // try to perform a lazy migration from legacy store + val legacy = tryOrNull("Failed to access legacy crypto store") { + cryptoStore.getInboundGroupSession(sessionId, senderKey.orEmpty()) + } + if (legacy == null || olmMachine.importRoomKey(legacy).isFailure) { + perSessionBackupQueryRateLimiter.tryFromBackupIfPossible(sessionId, roomId) + } } } throw mxCryptoError diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/network/RequestSender.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/network/RequestSender.kt index 22ec4ee417..5b9e2463ae 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/network/RequestSender.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/network/RequestSender.kt @@ -22,17 +22,23 @@ import dagger.Lazy import kotlinx.coroutines.CoroutineName import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch import org.matrix.android.sdk.api.auth.UserInteractiveAuthInterceptor +import org.matrix.android.sdk.api.extensions.tryOrNull import org.matrix.android.sdk.api.failure.Failure import org.matrix.android.sdk.api.failure.MatrixError import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysBackupLastVersionResult import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysVersion import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysVersionResult +import org.matrix.android.sdk.api.session.crypto.model.GossipingToDeviceObject import org.matrix.android.sdk.api.session.crypto.model.MXUsersDevicesMap import org.matrix.android.sdk.api.session.events.model.Content import org.matrix.android.sdk.api.session.events.model.Event +import org.matrix.android.sdk.api.session.events.model.EventType import org.matrix.android.sdk.api.session.uia.UiaResult import org.matrix.android.sdk.internal.auth.registration.handleUIA +import org.matrix.android.sdk.internal.crypto.OlmMachine +import org.matrix.android.sdk.internal.crypto.PerSessionBackupQueryRateLimiter import org.matrix.android.sdk.internal.crypto.keysbackup.model.rest.BackupKeysResult import org.matrix.android.sdk.internal.crypto.keysbackup.model.rest.CreateKeysBackupVersionBody import org.matrix.android.sdk.internal.crypto.keysbackup.model.rest.KeysBackupData @@ -53,6 +59,7 @@ import org.matrix.android.sdk.internal.crypto.model.rest.KeysUploadBody import org.matrix.android.sdk.internal.crypto.model.rest.KeysUploadResponse import org.matrix.android.sdk.internal.crypto.model.rest.RestKeyInfo import org.matrix.android.sdk.internal.crypto.model.rest.SignatureUploadResponse +import org.matrix.android.sdk.internal.crypto.store.IMXCommonCryptoStore import org.matrix.android.sdk.internal.crypto.tasks.ClaimOneTimeKeysForUsersDeviceTask import org.matrix.android.sdk.internal.crypto.tasks.DefaultSendVerificationMessageTask import org.matrix.android.sdk.internal.crypto.tasks.DownloadKeysForUsersTask @@ -95,10 +102,10 @@ internal class RequestSender @Inject constructor( private val getRoomSessionDataTask: GetRoomSessionDataTask, private val moshi: Moshi, cryptoCoroutineScope: CoroutineScope, -// private val rateLimiter: PerSessionBackupQueryRateLimiter, -// private val cryptoStore: IMXCommonCryptoStore, + private val rateLimiter: PerSessionBackupQueryRateLimiter, + private val cryptoStore: IMXCommonCryptoStore, private val localEchoRepository: LocalEchoRepository, -// private val olmMachine: Lazy + private val olmMachine: Lazy ) { private val scope = CoroutineScope( @@ -242,40 +249,40 @@ internal class RequestSender @Inject constructor( .adapter>>(Map::class.java) val jsonBody = adapter.fromJson(body)!! -// if (eventType == EventType.ROOM_KEY_REQUEST) { -// scope.launch { -// Timber.v("Intercepting key request, try backup") -// /** -// * It's a bit hacky, check how this can be better integrated with rust? -// */ -// try { -// jsonBody.forEach { (_, deviceToContent) -> -// deviceToContent.forEach { (_, content) -> -// val hashMap = content as? Map<*, *> -// val action = hashMap?.get("action")?.toString() -// if (GossipingToDeviceObject.ACTION_SHARE_REQUEST == action) { -// val requestBody = hashMap["body"] as? Map<*, *> -// val roomId = requestBody?.get("room_id") as? String -// val sessionId = requestBody?.get("session_id") as? String -// val senderKey = requestBody?.get("sender_key") as? String -// if (roomId != null && sessionId != null) { -// // try to perform a lazy migration from legacy store -// val legacy = tryOrNull("Failed to access legacy crypto store") { -// cryptoStore.getInboundGroupSession(sessionId, senderKey.orEmpty()) -// } -// if (legacy == null || olmMachine.get().importRoomKey(legacy).isFailure) { -// rateLimiter.tryFromBackupIfPossible(sessionId, roomId) -// } -// } -// } -// } -// } -// Timber.v("Intercepting key request, try backup") -// } catch (failure: Throwable) { -// Timber.v(failure, "Failed to use backup") -// } -// } -// } + if (eventType == EventType.ROOM_KEY_REQUEST) { + scope.launch { + Timber.v("Intercepting key request, try backup") + /** + * It's a bit hacky, check how this can be better integrated with rust? + */ + try { + jsonBody.forEach { (_, deviceToContent) -> + deviceToContent.forEach { (_, content) -> + val hashMap = content as? Map<*, *> + val action = hashMap?.get("action")?.toString() + if (GossipingToDeviceObject.ACTION_SHARE_REQUEST == action) { + val requestBody = hashMap["body"] as? Map<*, *> + val roomId = requestBody?.get("room_id") as? String + val sessionId = requestBody?.get("session_id") as? String + val senderKey = requestBody?.get("sender_key") as? String + if (roomId != null && sessionId != null) { + // try to perform a lazy migration from legacy store + val legacy = tryOrNull("Failed to access legacy crypto store") { + cryptoStore.getInboundGroupSession(sessionId, senderKey.orEmpty()) + } + if (legacy == null || olmMachine.get().importRoomKey(legacy).isFailure) { + rateLimiter.tryFromBackupIfPossible(sessionId, roomId) + } + } + } + } + } + Timber.v("Intercepting key request, try backup") + } catch (failure: Throwable) { + Timber.v(failure, "Failed to use backup") + } + } + } val userMap = MXUsersDevicesMap() userMap.join(jsonBody) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/IMXCommonCryptoStore.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/IMXCommonCryptoStore.kt index eb58de74c0..ba6197e5e1 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/IMXCommonCryptoStore.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/IMXCommonCryptoStore.kt @@ -35,13 +35,6 @@ import org.matrix.android.sdk.internal.crypto.store.db.CryptoStoreAggregator */ interface IMXCommonCryptoStore { - /** - * Retrieve the known inbound group sessions. - * - * @return the list of all known group sessions, to export them. - */ - fun getInboundGroupSessions(): List - /** * Provides the algorithm used in a dedicated room. * @@ -150,4 +143,21 @@ interface IMXCommonCryptoStore { * @return the device or null if not found */ fun deviceWithIdentityKey(userId: String, identityKey: String): CryptoDeviceInfo? + + /** + * Retrieve an inbound group session. + * Used in rust for lazy migration + * + * @param sessionId the session identifier. + * @param senderKey the base64-encoded curve25519 key of the sender. + * @return an inbound group session. + */ + fun getInboundGroupSession(sessionId: String, senderKey: String): MXInboundMegolmSessionWrapper? + + /** + * Retrieve the known inbound group sessions. + * + * @return the list of all known group sessions, to export them. + */ + fun getInboundGroupSessions(): List } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/RustCryptoStore.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/RustCryptoStore.kt index f3d8545a31..5479e62b7f 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/RustCryptoStore.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/RustCryptoStore.kt @@ -48,6 +48,8 @@ import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoRoomEntityFie import org.matrix.android.sdk.internal.crypto.store.db.model.MyDeviceLastSeenInfoEntity import org.matrix.android.sdk.internal.crypto.store.db.model.MyDeviceLastSeenInfoEntityFields import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity +import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntityFields +import org.matrix.android.sdk.internal.crypto.store.db.model.createPrimaryKey import org.matrix.android.sdk.internal.crypto.store.db.query.getById import org.matrix.android.sdk.internal.crypto.store.db.query.getOrCreate import org.matrix.android.sdk.internal.di.CryptoDatabase @@ -126,6 +128,32 @@ internal class RustCryptoStore @Inject constructor( } } + /** + * Note: the result will be only use to export all the keys and not to use the OlmInboundGroupSessionWrapper2, + * so there is no need to use or update `inboundGroupSessionToRelease` for native memory management. + */ + override fun getInboundGroupSessions(): List { + return doWithRealm(realmConfiguration) { realm -> + realm.where() + .findAll() + .mapNotNull { it.toModel() } + } + } + + /** + * Needed for lazy migration of sessions from the legacy store. + */ + override fun getInboundGroupSession(sessionId: String, senderKey: String): MXInboundMegolmSessionWrapper? { + val key = OlmInboundGroupSessionEntity.createPrimaryKey(sessionId, senderKey) + + return doWithRealm(realmConfiguration) { realm -> + realm.where() + .equalTo(OlmInboundGroupSessionEntityFields.PRIMARY_KEY, key) + .findFirst() + ?.toModel() + } + } + // ================================================ // Things that should be migrated to another store than realm // ================================================ @@ -152,18 +180,6 @@ internal class RustCryptoStore @Inject constructor( } } - /** - * Note: the result will be only use to export all the keys and not to use the OlmInboundGroupSessionWrapper2, - * so there is no need to use or update `inboundGroupSessionToRelease` for native memory management. - */ - override fun getInboundGroupSessions(): List { - return doWithRealm(realmConfiguration) { realm -> - realm.where() - .findAll() - .mapNotNull { it.toModel() } - } - } - override fun getRoomAlgorithm(roomId: String): String? { return doWithRealm(realmConfiguration) { CryptoRoomEntity.getById(it, roomId)?.algorithm diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreModule.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreModule.kt index 8b94baacde..06fe27b539 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreModule.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreModule.kt @@ -20,6 +20,7 @@ import io.realm.annotations.RealmModule import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoRoomEntity import org.matrix.android.sdk.internal.crypto.store.db.model.MyDeviceLastSeenInfoEntity +import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity /** * Realm module for Crypto store classes. @@ -29,6 +30,7 @@ import org.matrix.android.sdk.internal.crypto.store.db.model.MyDeviceLastSeenInf classes = [ CryptoMetadataEntity::class, CryptoRoomEntity::class, + OlmInboundGroupSessionEntity::class, MyDeviceLastSeenInfoEntity::class, ] ) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo024.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo024.kt index e69df4e4fe..3bee29929f 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo024.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo024.kt @@ -17,7 +17,6 @@ package org.matrix.android.sdk.internal.crypto.store.db.migration import io.realm.DynamicRealm -import org.matrix.android.sdk.internal.extensions.safeRemove import org.matrix.android.sdk.internal.util.database.RealmMigrator internal class MigrateCryptoTo024(realm: DynamicRealm) : RealmMigrator(realm, 24) { @@ -33,20 +32,20 @@ internal class MigrateCryptoTo024(realm: DynamicRealm) : RealmMigrator(realm, 24 get("CryptoRoomEntity")?.removeField("outboundSessionInfo") // Warning: order is important, first remove classes that depends on others. - safeRemove("UserEntity") - safeRemove("DeviceInfoEntity") - safeRemove("CrossSigningInfoEntity") - safeRemove("KeyInfoEntity") - safeRemove("TrustLevelEntity") - safeRemove("KeysBackupDataEntity") +// safeRemove("UserEntity") +// safeRemove("DeviceInfoEntity") +// safeRemove("CrossSigningInfoEntity") +// safeRemove("KeyInfoEntity") +// safeRemove("TrustLevelEntity") +// safeRemove("KeysBackupDataEntity") // safeRemove("OlmInboundGroupSessionEntity") - safeRemove("OlmSessionEntity") - safeRemove("AuditTrailEntity") - safeRemove("OutgoingKeyRequestEntity") - safeRemove("KeyRequestReplyEntity") - safeRemove("WithHeldSessionEntity") - safeRemove("SharedSessionEntity") - safeRemove("OutboundGroupSessionInfoEntity") +// safeRemove("OlmSessionEntity") +// safeRemove("AuditTrailEntity") +// safeRemove("OutgoingKeyRequestEntity") +// safeRemove("KeyRequestReplyEntity") +// safeRemove("WithHeldSessionEntity") +// safeRemove("SharedSessionEntity") +// safeRemove("OutboundGroupSessionInfoEntity") } } }