beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

fix: use random path, add date based fields, use collection id to encrypt file key

Browse source
This commit is contained in:
Prateek Sunal 2024-04-18 22:38:10 +05:30
parent 901e50b69b
commit f65e8359a7
3 changed files with 116 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

92
mobile/lib/db/upload_locks_db.dart
View file

@ -3,10 +3,8 @@ import 'dart:io';
import 'package:path/path.dart';
import 'package:path_provider/path_provider.dart';
import "package:photos/models/encryption_result.dart";
import "package:photos/module/upload/model/multipart.dart";
import "package:photos/module/upload/service/multipart.dart";
import "package:photos/utils/crypto_util.dart";
import 'package:sqflite/sqflite.dart';
import "package:sqflite_migration/sqflite_migration.dart";
@ -25,14 +23,18 @@ class UploadLocksDB {
columnID: "id",
columnLocalID: "local_id",
columnFileHash: "file_hash",
columnCollectionID: "collection_id",
columnEncryptedFilePath: "encrypted_file_path",
columnEncryptedFileSize: "encrypted_file_size",
columnFileKey: "file_key",
columnFileNonce: "file_nonce",
columnEncryptedFileKey: "encrypted_file_key",
columnFileEncryptionNonce: "file_encryption_nonce",
columnKeyEncryptionNonce: "key_encryption_nonce",
columnObjectKey: "object_key",
columnCompleteUrl: "complete_url",
columnStatus: "status",
columnPartSize: "part_size",
columnLastAttemptedAt: "last_attempted_at",
columnCreatedAt: "created_at",
);
static const _partsTable = (
@ -93,14 +95,18 @@ class UploadLocksDB {
${_trackUploadTable.columnID} INTEGER PRIMARY KEY,
${_trackUploadTable.columnLocalID} TEXT NOT NULL,
${_trackUploadTable.columnFileHash} TEXT NOT NULL,
${_trackUploadTable.columnCollectionID} INTEGER NOT NULL,
${_trackUploadTable.columnEncryptedFilePath} TEXT NOT NULL,
${_trackUploadTable.columnEncryptedFileSize} INTEGER NOT NULL,
${_trackUploadTable.columnFileKey} TEXT NOT NULL,
${_trackUploadTable.columnFileNonce} TEXT NOT NULL,
${_trackUploadTable.columnEncryptedFileKey} TEXT NOT NULL,
${_trackUploadTable.columnFileEncryptionNonce} TEXT NOT NULL,
${_trackUploadTable.columnKeyEncryptionNonce} TEXT NOT NULL,
${_trackUploadTable.columnObjectKey} TEXT NOT NULL,
${_trackUploadTable.columnCompleteUrl} TEXT NOT NULL,
${_trackUploadTable.columnStatus} TEXT DEFAULT '${MultipartStatus.pending.name}' NOT NULL,
${_trackUploadTable.columnPartSize} INTEGER NOT NULL
${_trackUploadTable.columnPartSize} INTEGER NOT NULL,
${_trackUploadTable.columnLastAttemptedAt} INTEGER,
${_trackUploadTable.columnCreatedAt} INTEGER DEFAULT CURRENT_TIMESTAMP NOT NULL,
)
''',
'''
@ -177,29 +183,33 @@ class UploadLocksDB {
}
// For multipart download tracking
Future<bool> doesExists(String localId, String hash) async {
Future<bool> doesExists(String localId, String hash, int collectionID) async {
final db = await instance.database;
final rows = await db.query(
_trackUploadTable.table,
where:
'${_trackUploadTable.columnLocalID} = ? AND ${_trackUploadTable.columnFileHash} = ?',
whereArgs: [localId, hash],
where: '${_trackUploadTable.columnLocalID} = ?'
' AND ${_trackUploadTable.columnFileHash} = ?'
' AND ${_trackUploadTable.columnCollectionID} = ?',
whereArgs: [localId, hash, collectionID],
);
return rows.isNotEmpty;
}
Future<EncryptionResult> getFileEncryptionData(
Future<({String encryptedFileKey, String fileNonce, String keyNonce})>
getFileEncryptionData(
String localId,
String fileHash,
int collectionID,
) async {
final db = await instance.database;
final rows = await db.query(
_trackUploadTable.table,
where:
'${_trackUploadTable.columnLocalID} = ? AND ${_trackUploadTable.columnFileHash} = ?',
whereArgs: [localId, fileHash],
where: '${_trackUploadTable.columnLocalID} = ?'
' AND ${_trackUploadTable.columnFileHash} = ?'
' AND ${_trackUploadTable.columnCollectionID} = ?',
whereArgs: [localId, fileHash, collectionID],
);
if (rows.isEmpty) {
@ -207,25 +217,25 @@ class UploadLocksDB {
}
final row = rows.first;
return EncryptionResult(
key:
CryptoUtil.base642bin(row[_trackUploadTable.columnFileKey] as String),
header: CryptoUtil.base642bin(
row[_trackUploadTable.columnFileNonce] as String,
),
return (
encryptedFileKey: row[_trackUploadTable.columnEncryptedFileKey] as String,
fileNonce: row[_trackUploadTable.columnFileEncryptionNonce] as String,
keyNonce: row[_trackUploadTable.columnKeyEncryptionNonce] as String,
);
}
Future<MultipartInfo> getCachedLinks(
String localId,
String fileHash,
int collectionID,
) async {
final db = await instance.database;
final rows = await db.query(
_trackUploadTable.table,
where:
'${_trackUploadTable.columnLocalID} = ? AND ${_trackUploadTable.columnFileHash} = ?',
whereArgs: [localId, fileHash],
where: '${_trackUploadTable.columnLocalID} = ?'
' AND ${_trackUploadTable.columnFileHash} = ?'
' AND ${_trackUploadTable.columnCollectionID} = ?',
whereArgs: [localId, fileHash, collectionID],
);
if (rows.isEmpty) {
throw Exception("No cached links found for $localId and $fileHash");
@ -274,11 +284,13 @@ class UploadLocksDB {
Future<void> createTrackUploadsEntry(
String localId,
String fileHash,
int collectionID,
MultipartUploadURLs urls,
String encryptedFilePath,
int fileSize,
String fileKey,
String fileNonce,
String keyNonce,
) async {
final db = await UploadLocksDB.instance.database;
final objectKey = urls.objectKey;
@ -288,13 +300,16 @@ class UploadLocksDB {
{
_trackUploadTable.columnLocalID: localId,
_trackUploadTable.columnFileHash: fileHash,
_trackUploadTable.columnCollectionID: collectionID,
_trackUploadTable.columnObjectKey: objectKey,
_trackUploadTable.columnCompleteUrl: urls.completeURL,
_trackUploadTable.columnEncryptedFilePath: encryptedFilePath,
_trackUploadTable.columnEncryptedFileSize: fileSize,
_trackUploadTable.columnFileKey: fileKey,
_trackUploadTable.columnFileNonce: fileNonce,
_trackUploadTable.columnPartSize: MultiPartUploader.multipartPartSizeForUpload,
_trackUploadTable.columnEncryptedFileKey: fileKey,
_trackUploadTable.columnFileEncryptionNonce: fileNonce,
_trackUploadTable.columnKeyEncryptionNonce: keyNonce,
_trackUploadTable.columnPartSize:
MultiPartUploader.multipartPartSizeForUpload,
},
);
@ -357,4 +372,27 @@ class UploadLocksDB {
whereArgs: [localId],
);
}
Future<bool> isEncryptedPathSafeToDelete(String encryptedPath) {
// If lastAttemptedAt exceeds 3 days or createdAt exceeds 7 days
final db = instance.database;
return db.then((db) async {
final rows = await db.query(
_trackUploadTable.table,
where: '${_trackUploadTable.columnEncryptedFilePath} = ?',
whereArgs: [encryptedPath],
);
if (rows.isEmpty) {
return true;
}
final row = rows.first;
final lastAttemptedAt =
row[_trackUploadTable.columnLastAttemptedAt] as int?;
final createdAt = row[_trackUploadTable.columnCreatedAt] as int;
final now = DateTime.now().millisecondsSinceEpoch;
return (lastAttemptedAt == null ||
now - lastAttemptedAt > 3 * 24 * 60 * 60 * 1000) &&
now - createdAt > 7 * 24 * 60 * 60 * 1000;
});
}
}

46
mobile/lib/module/upload/service/multipart.dart
View file

@ -8,6 +8,7 @@ import "package:photos/db/upload_locks_db.dart";
import "package:photos/models/encryption_result.dart";
import "package:photos/module/upload/model/multipart.dart";
import "package:photos/module/upload/model/xml.dart";
import "package:photos/services/collections_service.dart";
import "package:photos/services/feature_flag_service.dart";
import "package:photos/utils/crypto_util.dart";
@ -28,8 +29,25 @@ class MultiPartUploader {
Future<EncryptionResult> getEncryptionResult(
String localId,
String fileHash,
) {
return _db.getFileEncryptionData(localId, fileHash);
int collectionID,
) async {
final collection =
CollectionsService.instance.getCollectionByID(collectionID);
if (collection == null) {
throw Exception("Collection not found");
}
final result =
await _db.getFileEncryptionData(localId, fileHash, collectionID);
final encryptedFileKey = CryptoUtil.base642bin(result.encryptedFileKey);
final fileNonce = CryptoUtil.base642bin(result.fileNonce);
final key = CryptoUtil.base642bin(collection.encryptedKey);
final encryptKeyNonce = CryptoUtil.base642bin(result.keyNonce);
return EncryptionResult(
key: CryptoUtil.decryptSync(encryptedFileKey, key, encryptKeyNonce),
nonce: fileNonce,
);
}
static int get multipartPartSizeForUpload {
@ -40,6 +58,10 @@ class MultiPartUploader {
}
Future<int> calculatePartCount(int fileSize) async {
// Multipart upload is only enabled for internal users
// and debug builds till it's battle tested.
if (!FeatureFlagService.instance.isInternalUserOrDebugBuild()) return 1;
final partCount = (fileSize / multipartPartSizeForUpload).ceil();
return partCount;
}
@ -67,20 +89,34 @@ class MultiPartUploader {
Future<void> createTableEntry(
String localId,
String fileHash,
int collectionID,
MultipartUploadURLs urls,
String encryptedFilePath,
int fileSize,
Uint8List fileKey,
Uint8List fileNonce,
) async {
final collection =
CollectionsService.instance.getCollectionByID(collectionID);
if (collection == null) {
throw Exception("Collection not found");
}
final encryptedResult = CryptoUtil.encryptSync(
fileKey,
CryptoUtil.base642bin(collection.encryptedKey),
);
await _db.createTrackUploadsEntry(
localId,
fileHash,
collectionID,
urls,
encryptedFilePath,
fileSize,
CryptoUtil.bin2base64(fileKey),
CryptoUtil.bin2base64(encryptedResult.key!),
CryptoUtil.bin2base64(fileNonce),
CryptoUtil.bin2base64(encryptedResult.nonce!),
);
}
@ -88,8 +124,10 @@ class MultiPartUploader {
File encryptedFile,
String localId,
String fileHash,
int collectionID,
) async {
final multipartInfo = await _db.getCachedLinks(localId, fileHash);
final multipartInfo =
await _db.getCachedLinks(localId, fileHash, collectionID);
Map<int, String> etags = multipartInfo.partETags ?? {};

20
mobile/lib/utils/file_uploader.dart
View file

@ -41,6 +41,7 @@ import 'package:photos/utils/file_uploader_util.dart';
import "package:photos/utils/file_util.dart";
import 'package:shared_preferences/shared_preferences.dart';
import 'package:tuple/tuple.dart';
import "package:uuid/uuid.dart";
class FileUploader {
static const kMaximumConcurrentUploads = 4;
@ -426,11 +427,7 @@ class FileUploader {
MediaUploadData? mediaUploadData;
mediaUploadData = await getUploadDataFromEnteFile(file);
final String uniqueID = lockKey +
"_" +
mediaUploadData.hashData!.fileHash!
.replaceAll('+', '')
.replaceAll('/', '');
final String uniqueID = const Uuid().v4().toString();
final encryptedFilePath =
'$tempDirectory$kUploadTempPrefix${uniqueID}_file.encrypted';
@ -453,6 +450,7 @@ class FileUploader {
await _uploadLocks.doesExists(
lockKey,
mediaUploadData.hashData!.fileHash!,
collectionID,
);
Uint8List? key;
@ -464,6 +462,7 @@ class FileUploader {
? await _multiPartUploader.getEncryptionResult(
lockKey,
mediaUploadData.hashData!.fileHash!,
collectionID,
)
: null;
key = multipartEncryptionResult?.key;
@ -534,13 +533,10 @@ class FileUploader {
final String thumbnailObjectKey =
await _putFile(thumbnailUploadURL, encryptedThumbnailFile);
// Calculate the number of parts for the file. Multiple part upload
// is only enabled for internal users and debug builds till it's battle tested.
final count = FeatureFlagService.instance.isInternalUserOrDebugBuild()
? await _multiPartUploader.calculatePartCount(
// Calculate the number of parts for the file.
final count = await _multiPartUploader.calculatePartCount(
await encryptedFile.length(),
)
: 1;
);
late String fileObjectKey;
@ -553,6 +549,7 @@ class FileUploader {
encryptedFile,
lockKey,
mediaUploadData.hashData!.fileHash!,
collectionID,
);
} else {
final fileUploadURLs =
@ -560,6 +557,7 @@ class FileUploader {
await _multiPartUploader.createTableEntry(
lockKey,
mediaUploadData.hashData!.fileHash!,
collectionID,
fileUploadURLs,
encryptedFilePath,
await encryptedFile.length(),