beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

feat: add status, fix xml parsing

Browse source
This commit is contained in:
Prateek Sunal 2024-04-13 13:47:13 +05:30
parent 7495a0e388
commit ea38997ff9
4 changed files with 60 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
mobile/lib/db/upload_locks_db.dart
View file

@ -28,22 +28,23 @@ class UploadLocksDB {
columnFileKey: "file_key",
columnObjectKey: "object_key",
columnCompleteUrl: "complete_url",
columnCompletionStatus: "completion_status",
columnStatus: "status",
columnPartSize: "part_size",
);
static const _trackStatus = (
pending: "pending",
completed: "completed",
);
static const _partsTable = (
table: "upload_parts",
columnObjectKey: "object_key",
columnPartNumber: "part_number",
columnPartUrl: "part_url",
columnPartETag: "part_etag",
columnPartStatus: "part_status",
);
static const trackStatus = (
pending: "pending",
uploaded: "uploaded",
completed: "completed",
);
static const _partStatus = (
pending: "pending",
uploaded: "uploaded",
@ -98,10 +99,6 @@ class UploadLocksDB {
return;
}
// drop
// await db.execute('DROP TABLE IF EXISTS ${_trackUploadTable.table}');
// await db.execute('DROP TABLE IF EXISTS ${_partsTable.table}');
await db.execute(
'''
CREATE TABLE ${_trackUploadTable.table} (
@ -113,7 +110,7 @@ class UploadLocksDB {
${_trackUploadTable.columnFileKey} TEXT NOT NULL,
${_trackUploadTable.columnObjectKey} TEXT NOT NULL,
${_trackUploadTable.columnCompleteUrl} TEXT NOT NULL,
${_trackUploadTable.columnCompletionStatus} TEXT NOT NULL,
${_trackUploadTable.columnStatus} TEXT DEFAULT '${trackStatus.pending}' NOT NULL,
${_trackUploadTable.columnPartSize} INTEGER NOT NULL
)
''',
@ -124,6 +121,7 @@ class UploadLocksDB {
${_partsTable.columnObjectKey} TEXT NOT NULL REFERENCES ${_trackUploadTable.table}(${_trackUploadTable.columnObjectKey}) ON DELETE CASCADE,
${_partsTable.columnPartNumber} INTEGER NOT NULL,
${_partsTable.columnPartUrl} TEXT NOT NULL,
${_partsTable.columnPartETag} TEXT,
${_partsTable.columnPartStatus} TEXT NOT NULL,
PRIMARY KEY (${_partsTable.columnObjectKey}, ${_partsTable.columnPartNumber})
)
@ -204,7 +202,7 @@ class UploadLocksDB {
return rows.isNotEmpty;
}
Future<MultipartUploadURLs> getCachedLinks(
Future<(MultipartUploadURLs, String)> getCachedLinks(
String localId,
String fileHash,
) async {
@ -231,12 +229,16 @@ class UploadLocksDB {
partsStatus.length,
(index) => "",
);
final Map<int, String> partETags = {};
for (final part in partsStatus) {
final partNumber = part[_partsTable.columnPartNumber] as int;
final partUrl = part[_partsTable.columnPartUrl] as String;
final partStatus = part[_partsTable.columnPartStatus] as String;
partsURLs[partNumber] = partUrl;
if (part[_partsTable.columnPartETag] != null) {
partETags[partNumber] = part[_partsTable.columnPartETag] as String;
}
partUploadStatus.add(partStatus == "uploaded");
}
final urls = MultipartUploadURLs(
@ -244,9 +246,10 @@ class UploadLocksDB {
completeURL: row[_trackUploadTable.columnCompleteUrl] as String,
partsURLs: partsURLs,
partUploadStatus: partUploadStatus,
partETags: partETags,
);
return urls;
return (urls, row[_trackUploadTable.columnStatus] as String);
}
Future<void> createTrackUploadsEntry(
@ -270,7 +273,6 @@ class UploadLocksDB {
_trackUploadTable.columnEncryptedFilePath: encryptedFilePath,
_trackUploadTable.columnEncryptedFileSize: fileSize,
_trackUploadTable.columnFileKey: fileKey,
_trackUploadTable.columnCompletionStatus: _trackStatus.pending,
_trackUploadTable.columnPartSize: multipartPartSize,
},
);
@ -289,29 +291,19 @@ class UploadLocksDB {
},
);
}
// print all database entries
final trackUploads = await db.query(_trackUploadTable.table);
final parts = await db.query(_partsTable.table);
print("Track Uploads:");
for (final trackUpload in trackUploads) {
print(trackUpload);
}
print("Parts:");
for (final part in parts) {
print(part);
}
}
Future<void> updatePartStatus(
String objectKey,
int partNumber,
String etag,
) async {
final db = await instance.database;
await db.update(
_partsTable.table,
{
_partsTable.columnPartStatus: _partStatus.uploaded,
_partsTable.columnPartETag: etag,
},
where:
'${_partsTable.columnObjectKey} = ? AND ${_partsTable.columnPartNumber} = ?',
@ -319,17 +311,29 @@ class UploadLocksDB {
);
}
Future<void> updateCompletionStatus(
Future<void> updateTrackUploadStatus(
String objectKey,
String status,
) async {
final db = await instance.database;
await db.update(
_trackUploadTable.table,
{
_trackUploadTable.columnCompletionStatus: _trackStatus.completed,
_trackUploadTable.columnStatus: status,
},
where: '${_trackUploadTable.columnObjectKey} = ?',
whereArgs: [objectKey],
);
}
Future<int> deleteCompletedRecord(
String localId,
) async {
final db = await instance.database;
return await db.delete(
_trackUploadTable.table,
where: '${_trackUploadTable.columnLocalID} = ?',
whereArgs: [localId],
);
}
}

2
mobile/lib/utils/file_uploader.dart
View file

@ -612,6 +612,8 @@ class FileUploader {
}
await FilesDB.instance.update(remoteFile);
}
await UploadLocksDB.instance.deleteCompletedRecord(lockKey);
if (!_isBackground) {
Bus.instance.fire(
LocalPhotosUpdatedEvent(

34
mobile/lib/utils/multipart_upload_util.dart
View file

@ -13,6 +13,7 @@ import "package:photos/utils/xml_parser_util.dart";
final _enteDio = NetworkClient.instance.enteDio;
final _dio = NetworkClient.instance.getDio();
final _uploadLocksDb = UploadLocksDB.instance;
class PartETag extends XmlParsableObject {
final int partNumber;
@ -37,12 +38,14 @@ class MultipartUploadURLs {
final List<String> partsURLs;
final String completeURL;
final List<bool>? partUploadStatus;
final Map<int, String>? partETags;
MultipartUploadURLs({
required this.objectKey,
required this.partsURLs,
required this.completeURL,
this.partUploadStatus,
this.partETags,
});
factory MultipartUploadURLs.fromMap(Map<String, dynamic> map) {
@ -83,7 +86,7 @@ Future<void> createTableEntry(
int fileSize,
Uint8List fileKey,
) async {
await UploadLocksDB.instance.createTrackUploadsEntry(
await _uploadLocksDb.createTrackUploadsEntry(
localId,
fileHash,
urls,
@ -98,13 +101,19 @@ Future<String> putExistingMultipartFile(
String localId,
String fileHash,
) async {
final urls = await UploadLocksDB.instance.getCachedLinks(localId, fileHash);
final (urls, status) = await _uploadLocksDb.getCachedLinks(localId, fileHash);
// upload individual parts and get their etags
final etags = await uploadParts(urls, encryptedFile);
Map<int, String> etags = urls.partETags ?? {};
// complete the multipart upload
await completeMultipartUpload(urls.objectKey, etags, urls.completeURL);
if (status == UploadLocksDB.trackStatus.pending) {
// upload individual parts and get their etags
etags = await uploadParts(urls, encryptedFile);
}
if (status != UploadLocksDB.trackStatus.completed) {
// complete the multipart upload
await completeMultipartUpload(urls.objectKey, etags, urls.completeURL);
}
return urls.objectKey;
}
@ -129,7 +138,7 @@ Future<Map<int, String>> uploadParts(
final partsURLs = url.partsURLs;
final partUploadStatus = url.partUploadStatus;
final partsLength = partsURLs.length;
final etags = <int, String>{};
final etags = url.partETags ?? <int, String>{};
for (int i = 0; i < partsLength; i++) {
if (i < (partUploadStatus?.length ?? 0) &&
@ -163,8 +172,12 @@ Future<Map<int, String>> uploadParts(
etags[i] = eTag!;
await UploadLocksDB.instance.updatePartStatus(url.objectKey, i);
await _uploadLocksDb.updatePartStatus(url.objectKey, i, eTag);
}
await _uploadLocksDb.updateTrackUploadStatus(
url.objectKey,
UploadLocksDB.trackStatus.uploaded,
);
return etags;
}
@ -193,7 +206,10 @@ Future<void> completeMultipartUpload(
contentType: "text/xml",
),
);
await UploadLocksDB.instance.updateCompletionStatus(objectKey);
await _uploadLocksDb.updateTrackUploadStatus(
objectKey,
UploadLocksDB.trackStatus.completed,
);
} catch (e) {
Logger("MultipartUpload").severe(e);
rethrow;

4
mobile/lib/utils/xml_parser_util.dart
View file

@ -1,6 +1,5 @@
// ignore_for_file: implementation_imports
import "package:xml/src/xml/entities/named_entities.dart";
import "package:xml/xml.dart";
// used for classes that can be converted to xml
@ -16,7 +15,6 @@ String convertJs2Xml(Map<String, dynamic> json) {
return builder.buildDocument().toXmlString(
pretty: true,
indent: ' ',
entityMapping: defaultMyEntityMapping,
);
}
@ -38,6 +36,6 @@ void buildXml(XmlBuilder builder, dynamic node) {
},
);
} else {
builder.text(node.toString());
builder.text(node is String ? node : node.toString());
}
}