beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Enhance re-processing of live photos (#1570)

Browse source 
* Improve live photos change check

* Ignore files that are already marked for reupload
This commit is contained in:
Neeraj Gupta 2023-12-06 12:56:26 +05:30 committed by GitHub
parent 15c3ae9cdd
commit 034d9e9914
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 112 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
lib/db/file_updation_db.dart
View file

@ -14,7 +14,7 @@ class FileUpdationDB {
static const tableName = 're_upload_tracker';
static const columnLocalID = 'local_id';
static const columnReason = 'reason';
static const livePhotoSize = 'livePhotoSize';
static const livePhotoCheck = 'livePhotoCheck';
static const modificationTimeUpdated = 'modificationTimeUpdated';

10
lib/db/files_db.dart
View file

@ -1449,20 +1449,16 @@ class FilesDB {
return result;
}
// For a given userID, return unique uploadedFileId for the given userID
Future<List<String>> getLivePhotosWithBadSize(
int userId,
int sizeInBytes,
) async {
// For a given userID, return unique localID for all uploaded live photos
Future<List<String>> getLivePhotosForUser(int userId) async {
final db = await instance.database;
final rows = await db.query(
filesTable,
columns: [columnLocalID],
distinct: true,
where: '$columnOwnerID = ? AND '
'($columnFileSize IS NULL OR $columnFileSize = ?) AND '
'$columnFileType = ? AND $columnLocalID IS NOT NULL',
whereArgs: [userId, sizeInBytes, getInt(FileType.livePhoto)],
whereArgs: [userId, getInt(FileType.livePhoto)],
);
final result = <String>[];
for (final row in rows) {

115
lib/services/local_file_update_service.dart
View file

@ -7,6 +7,7 @@ import "package:photos/core/configuration.dart";
import 'package:photos/core/errors.dart';
import 'package:photos/db/file_updation_db.dart';
import 'package:photos/db/files_db.dart';
import "package:photos/extensions/list.dart";
import "package:photos/extensions/stop_watch.dart";
import 'package:photos/models/file/file.dart';
import 'package:photos/models/file/file_type.dart';
@ -21,9 +22,9 @@ class LocalFileUpdateService {
late FileUpdationDB _fileUpdationDB;
late SharedPreferences _prefs;
late Logger _logger;
final String _iosLivePhotoSizeMigrationDone = 'fm_ios_live_photo_size';
final String _doneLivePhotoImport = 'fm_import_ios_live_photo_size';
static int fourMBWithChunkSize = 4194338;
final String _iosLivePhotoSizeMigrationDone = 'fm_ios_live_photo_check';
final String _doneLivePhotoImport = 'fm_import_ios_live_photo_check';
static int twoHundredKb = 200 * 1024;
final List<String> _oldMigrationKeys = [
'fm_badCreationTime',
'fm_badCreationTimeCompleted',
@ -31,6 +32,8 @@ class LocalFileUpdateService {
'fm_missingLocationV2MigrationDone',
'fm_badLocationImportDone',
'fm_badLocationMigrationDone',
'fm_ios_live_photo_size',
'fm_import_ios_live_photo_size',
];
Completer<void>? _existingMigration;
@ -55,9 +58,8 @@ class LocalFileUpdateService {
_existingMigration = Completer<void>();
try {
await _markFilesWhichAreActuallyUpdated();
if (Platform.isAndroid) {
_cleanUpOlderMigration().ignore();
} else {
_cleanUpOlderMigration().ignore();
if (!Platform.isAndroid) {
await _handleLivePhotosSizedCheck();
}
} catch (e, s) {
@ -78,15 +80,16 @@ class LocalFileUpdateService {
}
}
if (hasOldMigrationKey) {
for (var element in _oldMigrationKeys) {
_prefs.remove(element);
}
await _fileUpdationDB.deleteByReasons([
'missing_location',
'badCreationTime',
'missingLocationV2',
'badLocationCord',
'livePhotoSize',
]);
for (var element in _oldMigrationKeys) {
await _prefs.remove(element);
}
}
}
@ -212,28 +215,32 @@ class LocalFileUpdateService {
return;
}
await _importLivePhotoReUploadCandidates();
final sTime = DateTime.now().microsecondsSinceEpoch;
// singleRunLimit indicates number of files to check during single
// invocation of this method. The limit act as a crude way to limit the
// resource consumed by the method
const int singleRunLimit = 50;
const int singleRunLimit = 500;
final localIDsToProcess =
await _fileUpdationDB.getLocalIDsForPotentialReUpload(
singleRunLimit,
FileUpdationDB.livePhotoSize,
FileUpdationDB.livePhotoCheck,
);
if (localIDsToProcess.isNotEmpty) {
await _checkLivePhotoWithLowOrUnknownSize(
localIDsToProcess,
);
final eTime = DateTime.now().microsecondsSinceEpoch;
final d = Duration(microseconds: eTime - sTime);
_logger.info(
'Performed hashCheck for ${localIDsToProcess.length} livePhoto files '
'completed in ${d.inSeconds.toString()} secs',
);
final chunks = localIDsToProcess.chunks(10);
for (final chunk in chunks) {
final sTime = DateTime.now().microsecondsSinceEpoch;
await _checkLivePhotoWithLowOrUnknownSize(
chunk,
);
final eTime = DateTime.now().microsecondsSinceEpoch;
final d = Duration(microseconds: eTime - sTime);
_logger.info(
'Performed hashCheck for ${chunk.length} livePhoto files '
'completed in ${d.inSeconds.toString()} secs',
);
}
} else {
_prefs.setBool(_iosLivePhotoSizeMigrationDone, true);
await _prefs.setBool(_iosLivePhotoSizeMigrationDone, true);
}
} catch (e, s) {
_logger.severe('error while checking livePhotoSize check', e, s);
@ -249,6 +256,7 @@ class LocalFileUpdateService {
final List<EnteFile> localFilesForUser = [];
final Set<String> localIDsWithFile = {};
final Set<int> missingSizeIDs = {};
final Set<String> processedIDs = {};
for (EnteFile file in result) {
if (file.ownerID == null || file.ownerID == userID) {
localFilesForUser.add(file);
@ -256,6 +264,10 @@ class LocalFileUpdateService {
if (file.isUploaded && file.fileSize == null) {
missingSizeIDs.add(file.uploadedFileID!);
}
if (file.isUploaded && file.updationTime == null) {
// file already queued for re-upload
processedIDs.add(file.localID!);
}
}
}
if (missingSizeIDs.isNotEmpty) {
@ -265,7 +277,6 @@ class LocalFileUpdateService {
return;
}
final Set<String> processedIDs = {};
// if a file for localID doesn't exist, then mark it as processed
// otherwise the app will be stuck in retrying same set of ids
@ -283,9 +294,6 @@ class LocalFileUpdateService {
continue;
} else if (file.fileType != FileType.livePhoto) {
_logger.severe('fileType is not livePhoto, skip this file');
continue;
} else if (file.fileSize! != fourMBWithChunkSize) {
// back-filled size is not of our interest
processedIDs.add(file.localID!);
continue;
}
@ -293,20 +301,24 @@ class LocalFileUpdateService {
continue;
}
try {
final MediaUploadData uploadData = await getUploadData(file);
_logger.info(
'Found livePhoto on local with hash ${uploadData.hashData?.fileHash ?? "null"} and existing hash ${file.hash ?? "null"}',
);
await clearCache(file);
await FilesDB.instance.markFilesForReUpload(
userID,
file.localID!,
file.title,
file.location,
file.creationTime!,
file.modificationTime!,
file.fileType,
);
late MediaUploadData uploadData;
late int mediaUploadSize;
(uploadData, mediaUploadSize) = await getUploadDataWithSizeSize(file);
if ((file.fileSize! - mediaUploadSize).abs() > twoHundredKb) {
_logger.info(
'Re-upload livePhoto localHash ${uploadData.hashData?.fileHash ?? "null"} & localSize: $mediaUploadSize'
' and remoteHash ${file.hash ?? "null"} & removeSize: ${file.fileSize!}',
);
await FilesDB.instance.markFilesForReUpload(
userID,
file.localID!,
file.title,
file.location,
file.creationTime!,
file.modificationTime!,
file.fileType,
);
}
processedIDs.add(file.localID!);
} on InvalidFileError catch (e) {
if (e.reason == InvalidReason.livePhotoToImageTypeChanged ||
@ -324,9 +336,10 @@ class LocalFileUpdateService {
_logger.severe("livePhoto check failed", e);
} finally {}
}
_logger.info('completed check for ${localIDsToProcess.length} files');
await _fileUpdationDB.deleteByLocalIDs(
processedIDs.toList(),
FileUpdationDB.livePhotoSize,
FileUpdationDB.livePhotoCheck,
);
}
@ -337,11 +350,11 @@ class LocalFileUpdateService {
_logger.info('_importLivePhotoReUploadCandidates');
final EnteWatch watch = EnteWatch("_importLivePhotoReUploadCandidates");
final int ownerID = Configuration.instance.getUserID()!;
final List<String> localIDs = await FilesDB.instance
.getLivePhotosWithBadSize(ownerID, fourMBWithChunkSize);
final List<String> localIDs =
await FilesDB.instance.getLivePhotosForUser(ownerID);
await _fileUpdationDB.insertMultiple(
localIDs,
FileUpdationDB.livePhotoSize,
FileUpdationDB.livePhotoCheck,
);
watch.log("imported ${localIDs.length} files");
await _prefs.setBool(_doneLivePhotoImport, true);
@ -357,4 +370,18 @@ class LocalFileUpdateService {
}
return mediaUploadData;
}
Future<(MediaUploadData, int)> getUploadDataWithSizeSize(
EnteFile file,
) async {
final mediaUploadData = await getUploadDataFromEnteFile(file);
final int size = await mediaUploadData.sourceFile!.length();
// delete the file from app's internal cache if it was copied to app
// for upload. Shared Media should only be cleared when the upload
// succeeds.
if (Platform.isIOS && mediaUploadData.sourceFile != null) {
await mediaUploadData.sourceFile?.delete();
}
return (mediaUploadData, size);
}
}

44
lib/utils/file_uploader_util.dart
View file

@ -5,6 +5,7 @@ import 'dart:typed_data';
import 'dart:ui' as ui;
import "package:archive/archive_io.dart";
import "package:computer/computer.dart";
import 'package:logging/logging.dart';
import "package:motion_photos/motion_photos.dart";
import 'package:motionphoto/motionphoto.dart';
@ -21,6 +22,7 @@ import "package:photos/models/metadata/file_magic.dart";
import "package:photos/services/file_magic_service.dart";
import 'package:photos/utils/crypto_util.dart';
import 'package:photos/utils/file_util.dart';
import "package:uuid/uuid.dart";
import 'package:video_thumbnail/video_thumbnail.dart';
final _logger = Logger("FileUtil");
@ -125,13 +127,14 @@ Future<MediaUploadData> _getMediaUploadDataFromAssetFile(EnteFile file) async {
fileHash = '$fileHash$kLivePhotoHashSeparator$livePhotoVideoHash';
final tempPath = Configuration.instance.getTempDirectory();
// .elp -> ente live photo
final livePhotoPath = tempPath + file.generatedID.toString() + ".elp";
_logger.fine("Uploading zipped live photo from " + livePhotoPath);
final encoder = ZipFileEncoder();
encoder.create(livePhotoPath);
await encoder.addFile(videoUrl, "video" + extension(videoUrl.path));
await encoder.addFile(sourceFile, "image" + extension(sourceFile.path));
encoder.close();
final uniqueId = const Uuid().v4().toString();
final livePhotoPath = tempPath + uniqueId + "_${file.generatedID}.elp";
_logger.fine("Creating zip for live photo from " + livePhotoPath);
await zip(
zipPath: livePhotoPath,
imagePath: sourceFile.path,
videoPath: videoUrl.path,
);
// delete the temporary video and image copy (only in IOS)
if (Platform.isIOS) {
await sourceFile.delete();
@ -168,6 +171,33 @@ Future<MediaUploadData> _getMediaUploadDataFromAssetFile(EnteFile file) async {
);
}
Future<void> _computeZip(Map<String, dynamic> args) async {
final String zipPath = args['zipPath'];
final String imagePath = args['imagePath'];
final String videoPath = args['videoPath'];
final encoder = ZipFileEncoder();
encoder.create(zipPath);
await encoder.addFile(File(imagePath), "image" + extension(imagePath));
await encoder.addFile(File(videoPath), "video" + extension(videoPath));
encoder.close();
}
Future<void> zip({
required String zipPath,
required String imagePath,
required String videoPath,
}) {
return Computer.shared().compute(
_computeZip,
param: {
'zipPath': zipPath,
'imagePath': imagePath,
'videoPath': videoPath,
},
taskName: 'zip',
);
}
Future<Uint8List?> _getThumbnailForUpload(
AssetEntity asset,
EnteFile file,