feat: add multipart upload support

mobile/lib/core/constants.dart

@@ -1,3 +1,5 @@
+import "package:photos/utils/crypto_util.dart";
+
 const int thumbnailSmallSize = 256;
 const int thumbnailQuality = 50;
 const int thumbnailLargeSize = 512;
@@ -45,6 +47,14 @@ class FFDefault {
   static const bool enablePasskey = false;
 }
 
+// this is the chunk size of the un-encrypted file which is read and encrypted before uploading it as a single part.
+const multipartPartSize = 20 * 1024 * 1024;
+
+const fileReaderChunkSize = encryptionChunkSize;
+
+final fileChunksCombinedForAUploadPart =
+    (multipartPartSize / fileReaderChunkSize).floor();
+
 const kDefaultProductionEndpoint = 'https://api.ente.io';
 
 const int intMaxValue = 9223372036854775807;

mobile/lib/utils/file_uploader.dart

@@ -37,6 +37,7 @@ import 'package:photos/utils/crypto_util.dart';
 import 'package:photos/utils/file_download_util.dart';
 import 'package:photos/utils/file_uploader_util.dart';
 import "package:photos/utils/file_util.dart";
+import "package:photos/utils/multipart_upload_util.dart";
 import 'package:shared_preferences/shared_preferences.dart';
 import 'package:tuple/tuple.dart';
 import "package:uuid/uuid.dart";
@@ -492,9 +493,17 @@ class FileUploader {
       final String thumbnailObjectKey =
           await _putFile(thumbnailUploadURL, encryptedThumbnailFile);
 
-      final fileUploadURL = await _getUploadURL();
+      final count = await calculatePartCount(
-      final String fileObjectKey = await _putFile(fileUploadURL, encryptedFile);
+        await encryptedFile.length(),
+      );
 
+      final fileUploadURLs = await getMultipartUploadURLs(count);
+      final fileObjectKey = fileUploadURLs.objectKey;
+
+      await putMultipartFile(fileUploadURLs, encryptedFile);
+
+      // final fileUploadURL = await _getUploadURL();
+      // fileObjectKey = await _putFile(fileUploadURL, encryptedFile);
       final metadata = await file.getMetadataForUpload(mediaUploadData);
       final encryptedMetadataResult = await CryptoUtil.encryptChaCha(
         utf8.encode(jsonEncode(metadata)) as Uint8List,

mobile/lib/utils/multipart_upload_util.dart

@@ -0,0 +1,255 @@
+// ignore_for_file: implementation_imports
+
+import "dart:io";
+
+import "package:dio/dio.dart";
+import "package:logging/logging.dart";
+import "package:photos/core/constants.dart";
+import "package:photos/core/network/network.dart";
+import "package:xml/src/xml/entities/named_entities.dart";
+import "package:xml/xml.dart";
+
+final _enteDio = NetworkClient.instance.enteDio;
+final _dio = NetworkClient.instance.getDio();
+
+class MultipartUploadURLs {
+  final String objectKey;
+  final List<String> partsURLs;
+  final String completeURL;
+
+  MultipartUploadURLs({
+    required this.objectKey,
+    required this.partsURLs,
+    required this.completeURL,
+  });
+
+  factory MultipartUploadURLs.fromMap(Map<String, dynamic> map) {
+    return MultipartUploadURLs(
+      objectKey: map["urls"]["objectKey"],
+      partsURLs: (map["urls"]["partURLs"] as List).cast<String>(),
+      completeURL: map["urls"]["completeURL"],
+    );
+  }
+}
+
+Future<int> calculatePartCount(int fileSize) async {
+  final partCount = (fileSize / multipartPartSize).ceil();
+  return partCount;
+}
+
+Future<MultipartUploadURLs> getMultipartUploadURLs(int count) async {
+  try {
+    final response = await _enteDio.get(
+      "/files/multipart-upload-urls",
+      queryParameters: {
+        "count": count,
+      },
+    );
+
+    return MultipartUploadURLs.fromMap(response.data);
+  } on Exception catch (e) {
+    Logger("MultipartUploadURL").severe(e);
+    rethrow;
+  }
+}
+
+Future<void> putMultipartFile(
+  MultipartUploadURLs urls,
+  File encryptedFile,
+) async {
+  // upload individual parts and get their etags
+  final etags = await uploadParts(urls.partsURLs, encryptedFile);
+
+  print(etags);
+
+  // complete the multipart upload
+  await completeMultipartUpload(etags, urls.completeURL);
+}
+
+Future<Map<int, String>> uploadParts(
+  List<String> partsURLs,
+  File encryptedFile,
+) async {
+  final partsLength = partsURLs.length;
+  final etags = <int, String>{};
+
+  for (int i = 0; i < partsLength; i++) {
+    final partURL = partsURLs[i];
+    final isLastPart = i == partsLength - 1;
+    final fileSize = isLastPart
+        ? encryptedFile.lengthSync() % multipartPartSize
+        : multipartPartSize;
+
+    final response = await _dio.put(
+      partURL,
+      data: encryptedFile.openRead(
+        i * multipartPartSize,
+        isLastPart ? null : multipartPartSize,
+      ),
+      options: Options(
+        headers: {
+          Headers.contentLengthHeader: fileSize,
+        },
+      ),
+    );
+
+    final eTag = response.headers.value("etag");
+
+    if (eTag?.isEmpty ?? true) {
+      throw Exception('ETAG_MISSING');
+    }
+
+    etags[i] = eTag!;
+  }
+
+  return etags;
+}
+
+Future<void> completeMultipartUpload(
+  Map<int, String> partEtags,
+  String completeURL,
+) async {
+  final body = convertJs2Xml({
+    'CompleteMultipartUpload': partEtags.entries.toList(),
+  });
+
+  print(body);
+
+  try {
+    await _dio.post(
+      completeURL,
+      data: body,
+      options: Options(
+        contentType: "text/xml",
+      ),
+    );
+  } catch (e) {
+    Logger("MultipartUpload").severe(e);
+    rethrow;
+  }
+}
+
+// for converting the response to xml
+String convertJs2Xml(Map<String, dynamic> json) {
+  final builder = XmlBuilder();
+  buildXml(builder, json);
+  return builder.buildDocument().toXmlString(
+        pretty: true,
+        indent: '    ',
+        entityMapping: defaultMyEntityMapping,
+      );
+}
+
+void buildXml(XmlBuilder builder, dynamic node) {
+  if (node is Map<String, dynamic>) {
+    node.forEach((key, value) {
+      builder.element(key, nest: () => buildXml(builder, value));
+    });
+  } else if (node is List<dynamic>) {
+    for (var item in node) {
+      buildXml(builder, item);
+    }
+  } else {
+    builder.element(
+      "Part",
+      nest: () {
+        builder.attribute(
+          "PartNumber",
+          (node as MapEntry<int, String>).key + 1,
+        );
+        print(node.value);
+        builder.attribute("ETag", node.value);
+      },
+    );
+  }
+}
+
+XmlEntityMapping defaultMyEntityMapping = MyXmlDefaultEntityMapping.xml();
+
+class MyXmlDefaultEntityMapping extends XmlDefaultEntityMapping {
+  MyXmlDefaultEntityMapping.xml() : this(xmlEntities);
+  MyXmlDefaultEntityMapping.html() : this(htmlEntities);
+  MyXmlDefaultEntityMapping.html5() : this(html5Entities);
+  MyXmlDefaultEntityMapping(super.entities);
+
+  @override
+  String encodeText(String input) =>
+      input.replaceAllMapped(_textPattern, _textReplace);
+
+  @override
+  String encodeAttributeValue(String input, XmlAttributeType type) {
+    switch (type) {
+      case XmlAttributeType.SINGLE_QUOTE:
+        return input.replaceAllMapped(
+          _singeQuoteAttributePattern,
+          _singeQuoteAttributeReplace,
+        );
+      case XmlAttributeType.DOUBLE_QUOTE:
+        return input.replaceAllMapped(
+          _doubleQuoteAttributePattern,
+          _doubleQuoteAttributeReplace,
+        );
+    }
+  }
+}
+
+final _textPattern = RegExp(r'[&<>' + _highlyDiscouragedCharClass + r']');
+
+String _textReplace(Match match) {
+  final toEscape = match.group(0)!;
+  switch (toEscape) {
+    case '<':
+      return '&lt;';
+    case '&':
+      return '&amp;';
+    case '>':
+      return '&gt;';
+    default:
+      return _asNumericCharacterReferences(toEscape);
+  }
+}
+
+final _singeQuoteAttributePattern =
+    RegExp(r"['&<>\n\r\t" + _highlyDiscouragedCharClass + r']');
+
+String _singeQuoteAttributeReplace(Match match) {
+  final toEscape = match.group(0)!;
+  switch (toEscape) {
+    case "'":
+      return '';
+    case '&':
+      return '&amp;';
+    case '<':
+      return '&lt;';
+    case '>':
+      return '&gt;';
+    default:
+      return _asNumericCharacterReferences(toEscape);
+  }
+}
+
+final _doubleQuoteAttributePattern =
+    RegExp(r'["&<>\n\r\t' + _highlyDiscouragedCharClass + r']');
+
+String _doubleQuoteAttributeReplace(Match match) {
+  final toEscape = match.group(0)!;
+  switch (toEscape) {
+    case '"':
+      return '';
+    case '&':
+      return '&amp;';
+    case '<':
+      return '&lt;';
+    case '>':
+      return '&gt;';
+    default:
+      return _asNumericCharacterReferences(toEscape);
+  }
+}
+
+const _highlyDiscouragedCharClass =
+    r'\u0001-\u0008\u000b\u000c\u000e-\u001f\u007f-\u0084\u0086-\u009f';
+
+String _asNumericCharacterReferences(String toEscape) => toEscape.runes
+    .map((rune) => '&#x${rune.toRadixString(16).toUpperCase()};')
+    .join();

mobile/pubspec.lock

@@ -2505,7 +2505,7 @@ packages:
     source: hosted
     version: "0.2.0+3"
   xml:
-    dependency: transitive
+    dependency: "direct main"
     description:
       name: xml
       sha256: "5bc72e1e45e941d825fd7468b9b4cc3b9327942649aeb6fc5cdbf135f0a86e84"

mobile/pubspec.yaml

@@ -172,6 +172,7 @@ dependencies:
   wallpaper_manager_flutter: ^0.0.2
   wechat_assets_picker: ^8.6.3
   widgets_to_image: ^0.0.2
+  xml: ^6.3.0
 
 dependency_overrides:
   # current fork of tfite_flutter_helper depends on ffi: ^1.x.x