From f65e8359a74e1248c0cba88d1e03b73f3858c76c Mon Sep 17 00:00:00 2001 From: Prateek Sunal Date: Thu, 18 Apr 2024 22:38:10 +0530 Subject: [PATCH] fix: use random path, add date based fields, use collection id to encrypt file key --- mobile/lib/db/upload_locks_db.dart | 92 +++++++++++++------ .../lib/module/upload/service/multipart.dart | 46 +++++++++- mobile/lib/utils/file_uploader.dart | 20 ++-- 3 files changed, 116 insertions(+), 42 deletions(-) diff --git a/mobile/lib/db/upload_locks_db.dart b/mobile/lib/db/upload_locks_db.dart index 746ed0bb6..d22498820 100644 --- a/mobile/lib/db/upload_locks_db.dart +++ b/mobile/lib/db/upload_locks_db.dart @@ -3,10 +3,8 @@ import 'dart:io'; import 'package:path/path.dart'; import 'package:path_provider/path_provider.dart'; -import "package:photos/models/encryption_result.dart"; import "package:photos/module/upload/model/multipart.dart"; import "package:photos/module/upload/service/multipart.dart"; -import "package:photos/utils/crypto_util.dart"; import 'package:sqflite/sqflite.dart'; import "package:sqflite_migration/sqflite_migration.dart"; @@ -25,14 +23,18 @@ class UploadLocksDB { columnID: "id", columnLocalID: "local_id", columnFileHash: "file_hash", + columnCollectionID: "collection_id", columnEncryptedFilePath: "encrypted_file_path", columnEncryptedFileSize: "encrypted_file_size", - columnFileKey: "file_key", - columnFileNonce: "file_nonce", + columnEncryptedFileKey: "encrypted_file_key", + columnFileEncryptionNonce: "file_encryption_nonce", + columnKeyEncryptionNonce: "key_encryption_nonce", columnObjectKey: "object_key", columnCompleteUrl: "complete_url", columnStatus: "status", columnPartSize: "part_size", + columnLastAttemptedAt: "last_attempted_at", + columnCreatedAt: "created_at", ); static const _partsTable = ( @@ -93,14 +95,18 @@ class UploadLocksDB { ${_trackUploadTable.columnID} INTEGER PRIMARY KEY, ${_trackUploadTable.columnLocalID} TEXT NOT NULL, ${_trackUploadTable.columnFileHash} TEXT NOT NULL, + ${_trackUploadTable.columnCollectionID} INTEGER NOT NULL, ${_trackUploadTable.columnEncryptedFilePath} TEXT NOT NULL, ${_trackUploadTable.columnEncryptedFileSize} INTEGER NOT NULL, - ${_trackUploadTable.columnFileKey} TEXT NOT NULL, - ${_trackUploadTable.columnFileNonce} TEXT NOT NULL, + ${_trackUploadTable.columnEncryptedFileKey} TEXT NOT NULL, + ${_trackUploadTable.columnFileEncryptionNonce} TEXT NOT NULL, + ${_trackUploadTable.columnKeyEncryptionNonce} TEXT NOT NULL, ${_trackUploadTable.columnObjectKey} TEXT NOT NULL, ${_trackUploadTable.columnCompleteUrl} TEXT NOT NULL, ${_trackUploadTable.columnStatus} TEXT DEFAULT '${MultipartStatus.pending.name}' NOT NULL, - ${_trackUploadTable.columnPartSize} INTEGER NOT NULL + ${_trackUploadTable.columnPartSize} INTEGER NOT NULL, + ${_trackUploadTable.columnLastAttemptedAt} INTEGER, + ${_trackUploadTable.columnCreatedAt} INTEGER DEFAULT CURRENT_TIMESTAMP NOT NULL, ) ''', ''' @@ -177,29 +183,33 @@ class UploadLocksDB { } // For multipart download tracking - Future doesExists(String localId, String hash) async { + Future doesExists(String localId, String hash, int collectionID) async { final db = await instance.database; final rows = await db.query( _trackUploadTable.table, - where: - '${_trackUploadTable.columnLocalID} = ? AND ${_trackUploadTable.columnFileHash} = ?', - whereArgs: [localId, hash], + where: '${_trackUploadTable.columnLocalID} = ?' + ' AND ${_trackUploadTable.columnFileHash} = ?' + ' AND ${_trackUploadTable.columnCollectionID} = ?', + whereArgs: [localId, hash, collectionID], ); return rows.isNotEmpty; } - Future getFileEncryptionData( + Future<({String encryptedFileKey, String fileNonce, String keyNonce})> + getFileEncryptionData( String localId, String fileHash, + int collectionID, ) async { final db = await instance.database; final rows = await db.query( _trackUploadTable.table, - where: - '${_trackUploadTable.columnLocalID} = ? AND ${_trackUploadTable.columnFileHash} = ?', - whereArgs: [localId, fileHash], + where: '${_trackUploadTable.columnLocalID} = ?' + ' AND ${_trackUploadTable.columnFileHash} = ?' + ' AND ${_trackUploadTable.columnCollectionID} = ?', + whereArgs: [localId, fileHash, collectionID], ); if (rows.isEmpty) { @@ -207,25 +217,25 @@ class UploadLocksDB { } final row = rows.first; - return EncryptionResult( - key: - CryptoUtil.base642bin(row[_trackUploadTable.columnFileKey] as String), - header: CryptoUtil.base642bin( - row[_trackUploadTable.columnFileNonce] as String, - ), + return ( + encryptedFileKey: row[_trackUploadTable.columnEncryptedFileKey] as String, + fileNonce: row[_trackUploadTable.columnFileEncryptionNonce] as String, + keyNonce: row[_trackUploadTable.columnKeyEncryptionNonce] as String, ); } Future getCachedLinks( String localId, String fileHash, + int collectionID, ) async { final db = await instance.database; final rows = await db.query( _trackUploadTable.table, - where: - '${_trackUploadTable.columnLocalID} = ? AND ${_trackUploadTable.columnFileHash} = ?', - whereArgs: [localId, fileHash], + where: '${_trackUploadTable.columnLocalID} = ?' + ' AND ${_trackUploadTable.columnFileHash} = ?' + ' AND ${_trackUploadTable.columnCollectionID} = ?', + whereArgs: [localId, fileHash, collectionID], ); if (rows.isEmpty) { throw Exception("No cached links found for $localId and $fileHash"); @@ -274,11 +284,13 @@ class UploadLocksDB { Future createTrackUploadsEntry( String localId, String fileHash, + int collectionID, MultipartUploadURLs urls, String encryptedFilePath, int fileSize, String fileKey, String fileNonce, + String keyNonce, ) async { final db = await UploadLocksDB.instance.database; final objectKey = urls.objectKey; @@ -288,13 +300,16 @@ class UploadLocksDB { { _trackUploadTable.columnLocalID: localId, _trackUploadTable.columnFileHash: fileHash, + _trackUploadTable.columnCollectionID: collectionID, _trackUploadTable.columnObjectKey: objectKey, _trackUploadTable.columnCompleteUrl: urls.completeURL, _trackUploadTable.columnEncryptedFilePath: encryptedFilePath, _trackUploadTable.columnEncryptedFileSize: fileSize, - _trackUploadTable.columnFileKey: fileKey, - _trackUploadTable.columnFileNonce: fileNonce, - _trackUploadTable.columnPartSize: MultiPartUploader.multipartPartSizeForUpload, + _trackUploadTable.columnEncryptedFileKey: fileKey, + _trackUploadTable.columnFileEncryptionNonce: fileNonce, + _trackUploadTable.columnKeyEncryptionNonce: keyNonce, + _trackUploadTable.columnPartSize: + MultiPartUploader.multipartPartSizeForUpload, }, ); @@ -357,4 +372,27 @@ class UploadLocksDB { whereArgs: [localId], ); } + + Future isEncryptedPathSafeToDelete(String encryptedPath) { + // If lastAttemptedAt exceeds 3 days or createdAt exceeds 7 days + final db = instance.database; + return db.then((db) async { + final rows = await db.query( + _trackUploadTable.table, + where: '${_trackUploadTable.columnEncryptedFilePath} = ?', + whereArgs: [encryptedPath], + ); + if (rows.isEmpty) { + return true; + } + final row = rows.first; + final lastAttemptedAt = + row[_trackUploadTable.columnLastAttemptedAt] as int?; + final createdAt = row[_trackUploadTable.columnCreatedAt] as int; + final now = DateTime.now().millisecondsSinceEpoch; + return (lastAttemptedAt == null || + now - lastAttemptedAt > 3 * 24 * 60 * 60 * 1000) && + now - createdAt > 7 * 24 * 60 * 60 * 1000; + }); + } } diff --git a/mobile/lib/module/upload/service/multipart.dart b/mobile/lib/module/upload/service/multipart.dart index c775a25e5..450335cf2 100644 --- a/mobile/lib/module/upload/service/multipart.dart +++ b/mobile/lib/module/upload/service/multipart.dart @@ -8,6 +8,7 @@ import "package:photos/db/upload_locks_db.dart"; import "package:photos/models/encryption_result.dart"; import "package:photos/module/upload/model/multipart.dart"; import "package:photos/module/upload/model/xml.dart"; +import "package:photos/services/collections_service.dart"; import "package:photos/services/feature_flag_service.dart"; import "package:photos/utils/crypto_util.dart"; @@ -28,8 +29,25 @@ class MultiPartUploader { Future getEncryptionResult( String localId, String fileHash, - ) { - return _db.getFileEncryptionData(localId, fileHash); + int collectionID, + ) async { + final collection = + CollectionsService.instance.getCollectionByID(collectionID); + if (collection == null) { + throw Exception("Collection not found"); + } + final result = + await _db.getFileEncryptionData(localId, fileHash, collectionID); + final encryptedFileKey = CryptoUtil.base642bin(result.encryptedFileKey); + final fileNonce = CryptoUtil.base642bin(result.fileNonce); + + final key = CryptoUtil.base642bin(collection.encryptedKey); + final encryptKeyNonce = CryptoUtil.base642bin(result.keyNonce); + + return EncryptionResult( + key: CryptoUtil.decryptSync(encryptedFileKey, key, encryptKeyNonce), + nonce: fileNonce, + ); } static int get multipartPartSizeForUpload { @@ -40,6 +58,10 @@ class MultiPartUploader { } Future calculatePartCount(int fileSize) async { + // Multipart upload is only enabled for internal users + // and debug builds till it's battle tested. + if (!FeatureFlagService.instance.isInternalUserOrDebugBuild()) return 1; + final partCount = (fileSize / multipartPartSizeForUpload).ceil(); return partCount; } @@ -67,20 +89,34 @@ class MultiPartUploader { Future createTableEntry( String localId, String fileHash, + int collectionID, MultipartUploadURLs urls, String encryptedFilePath, int fileSize, Uint8List fileKey, Uint8List fileNonce, ) async { + final collection = + CollectionsService.instance.getCollectionByID(collectionID); + if (collection == null) { + throw Exception("Collection not found"); + } + + final encryptedResult = CryptoUtil.encryptSync( + fileKey, + CryptoUtil.base642bin(collection.encryptedKey), + ); + await _db.createTrackUploadsEntry( localId, fileHash, + collectionID, urls, encryptedFilePath, fileSize, - CryptoUtil.bin2base64(fileKey), + CryptoUtil.bin2base64(encryptedResult.key!), CryptoUtil.bin2base64(fileNonce), + CryptoUtil.bin2base64(encryptedResult.nonce!), ); } @@ -88,8 +124,10 @@ class MultiPartUploader { File encryptedFile, String localId, String fileHash, + int collectionID, ) async { - final multipartInfo = await _db.getCachedLinks(localId, fileHash); + final multipartInfo = + await _db.getCachedLinks(localId, fileHash, collectionID); Map etags = multipartInfo.partETags ?? {}; diff --git a/mobile/lib/utils/file_uploader.dart b/mobile/lib/utils/file_uploader.dart index f2f79088f..dadfe75a1 100644 --- a/mobile/lib/utils/file_uploader.dart +++ b/mobile/lib/utils/file_uploader.dart @@ -41,6 +41,7 @@ import 'package:photos/utils/file_uploader_util.dart'; import "package:photos/utils/file_util.dart"; import 'package:shared_preferences/shared_preferences.dart'; import 'package:tuple/tuple.dart'; +import "package:uuid/uuid.dart"; class FileUploader { static const kMaximumConcurrentUploads = 4; @@ -426,11 +427,7 @@ class FileUploader { MediaUploadData? mediaUploadData; mediaUploadData = await getUploadDataFromEnteFile(file); - final String uniqueID = lockKey + - "_" + - mediaUploadData.hashData!.fileHash! - .replaceAll('+', '') - .replaceAll('/', ''); + final String uniqueID = const Uuid().v4().toString(); final encryptedFilePath = '$tempDirectory$kUploadTempPrefix${uniqueID}_file.encrypted'; @@ -453,6 +450,7 @@ class FileUploader { await _uploadLocks.doesExists( lockKey, mediaUploadData.hashData!.fileHash!, + collectionID, ); Uint8List? key; @@ -464,6 +462,7 @@ class FileUploader { ? await _multiPartUploader.getEncryptionResult( lockKey, mediaUploadData.hashData!.fileHash!, + collectionID, ) : null; key = multipartEncryptionResult?.key; @@ -534,13 +533,10 @@ class FileUploader { final String thumbnailObjectKey = await _putFile(thumbnailUploadURL, encryptedThumbnailFile); - // Calculate the number of parts for the file. Multiple part upload - // is only enabled for internal users and debug builds till it's battle tested. - final count = FeatureFlagService.instance.isInternalUserOrDebugBuild() - ? await _multiPartUploader.calculatePartCount( + // Calculate the number of parts for the file. + final count = await _multiPartUploader.calculatePartCount( await encryptedFile.length(), - ) - : 1; + ); late String fileObjectKey; @@ -553,6 +549,7 @@ class FileUploader { encryptedFile, lockKey, mediaUploadData.hashData!.fileHash!, + collectionID, ); } else { final fileUploadURLs = @@ -560,6 +557,7 @@ class FileUploader { await _multiPartUploader.createTableEntry( lockKey, mediaUploadData.hashData!.fileHash!, + collectionID, fileUploadURLs, encryptedFilePath, await encryptedFile.length(),