From 22ca572532ba04d7c4f03a293026a5f6948fe9fd Mon Sep 17 00:00:00 2001 From: Vishnu Mohandas Date: Sat, 28 Mar 2020 19:26:06 +0530 Subject: [PATCH] Update diff handling --- lib/db/db_helper.dart | 83 ++++++++++----------- lib/main.dart | 3 +- lib/models/photo.dart | 31 ++++++-- lib/photo_sync_manager.dart | 140 +++++++++++++++++++++--------------- pubspec.lock | 2 +- pubspec.yaml | 1 + 6 files changed, 149 insertions(+), 111 deletions(-) diff --git a/lib/db/db_helper.dart b/lib/db/db_helper.dart index 94eba33b5..e1f327f7c 100644 --- a/lib/db/db_helper.dart +++ b/lib/db/db_helper.dart @@ -11,7 +11,6 @@ class DatabaseHelper { static final table = 'photos'; - static final columnId = 'photo_id'; static final columnLocalPath = 'local_path'; static final columnUrl = 'url'; static final columnHash = 'hash'; @@ -42,11 +41,10 @@ class DatabaseHelper { Future _onCreate(Database db, int version) async { await db.execute(''' CREATE TABLE $table ( - $columnId VARCHAR(255) PRIMARY KEY, $columnLocalPath TEXT NOT NULL, - $columnUrl TEXT NOT NULL, + $columnUrl TEXT, $columnHash TEXT NOT NULL, - $columnSyncTimestamp TEXT NOT NULL + $columnSyncTimestamp TEXT ) '''); } @@ -54,7 +52,6 @@ class DatabaseHelper { Future insertPhoto(Photo photo) async { Database db = await instance.database; var row = new Map(); - row[columnId] = photo.photoID; row[columnLocalPath] = photo.localPath; row[columnUrl] = photo.url; row[columnHash] = photo.hash; @@ -65,49 +62,47 @@ class DatabaseHelper { Future> getAllPhotos() async { Database db = await instance.database; var results = await db.query(table); + return _convertToPhotos(results); + } + + Future> getPhotosToBeUploaded() async { + Database db = await instance.database; + var results = await db.query(table, where: '$columnUrl IS NULL'); + return _convertToPhotos(results); + } + + // We are assuming here that the hash column in the map is set. The other + // column values will be used to update the row. + Future updateUrlAndTimestamp( + String hash, String url, String timestamp) async { + Database db = await instance.database; + var row = new Map(); + row[columnUrl] = url; + row[columnSyncTimestamp] = timestamp; + return await db + .update(table, row, where: '$columnHash = ?', whereArgs: [hash]); + } + + Future containsPath(String path) async { + Database db = await instance.database; + return (await db + .query(table, where: '$columnLocalPath =?', whereArgs: [path])) + .length > + 0; + } + + Future containsPhotoHash(String hash) async { + Database db = await instance.database; + return (await db.query(table, where: '$columnHash =?', whereArgs: [hash])) + .length > + 0; + } + + List _convertToPhotos(List> results) { var photos = List(); for (var result in results) { photos.add(Photo.fromRow(result)); } return photos; } - - // Helper methods - - // Inserts a row in the database where each key in the Map is a column name - // and the value is the column value. The return value is the id of the - // inserted row. - Future insert(Map row) async { - Database db = await instance.database; - return await db.insert(table, row); - } - - // All of the rows are returned as a list of maps, where each map is - // a key-value list of columns. - Future>> queryAllRows() async { - Database db = await instance.database; - return await db.query(table); - } - - // We are assuming here that the id column in the map is set. The other - // column values will be used to update the row. - Future update(Map row) async { - Database db = await instance.database; - int id = row[columnId]; - return await db.update(table, row, where: '$columnId = ?', whereArgs: [id]); - } - - // Deletes the row specified by the id. The number of affected rows is - // returned. This should be 1 as long as the row exists. - Future delete(int id) async { - Database db = await instance.database; - return await db.delete(table, where: '$columnId = ?', whereArgs: [id]); - } - - Future containsPath(String path) async { - Database db = await instance.database; - return (await db.query(table, where: '$columnLocalPath =?', whereArgs: [path])) - .length > - 0; - } } diff --git a/lib/main.dart b/lib/main.dart index 6cad888d6..88ec3a325 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -18,7 +18,8 @@ void main() async { WidgetsFlutterBinding.ensureInitialized(); await provider.refreshGalleryList(); var assets = await provider.list[0].assetList; - PhotoSyncManager(assets); + var photoSyncManager = PhotoSyncManager(assets); + await photoSyncManager.init(); runApp(MyApp2()); } diff --git a/lib/models/photo.dart b/lib/models/photo.dart index 8d69e12bb..a0ec197cb 100644 --- a/lib/models/photo.dart +++ b/lib/models/photo.dart @@ -1,19 +1,38 @@ +import 'dart:io'; + +import 'package:crypto/crypto.dart'; +import 'package:photo_manager/photo_manager.dart'; + class Photo { - String photoID; String url; String localPath; String hash; int syncTimestamp; + Photo(); + Photo.fromJson(Map json) - : photoID = json["photoID"], - url = json["url"], + : url = json["url"], + hash = json["hash"], syncTimestamp = json["syncTimestamp"]; Photo.fromRow(Map row) - : photoID = row["photo_id"], - localPath = row["local_path"], + : localPath = row["local_path"], url = row["url"], hash = row["hash"], - syncTimestamp = int.parse(row["sync_timestamp"]); + syncTimestamp = row["sync_timestamp"] == null + ? -1 + : int.parse(row["sync_timestamp"]); + + static Future fromAsset(AssetEntity asset) async { + Photo photo = Photo(); + var file = (await asset.originFile); + photo.localPath = file.path; + photo.hash = getHash(file); + return photo; + } + + static String getHash(File file) { + return sha256.convert(file.readAsBytesSync()).toString(); + } } diff --git a/lib/photo_sync_manager.dart b/lib/photo_sync_manager.dart index b3a72219b..9ef37e642 100644 --- a/lib/photo_sync_manager.dart +++ b/lib/photo_sync_manager.dart @@ -9,99 +9,121 @@ import 'package:dio/dio.dart'; import 'package:myapp/models/photo.dart'; class PhotoSyncManager { - final logger = Logger(); - final dio = Dio(); - final endpoint = "http://172.20.10.6:8080"; - final user = "umbu"; - static final lastSyncTimestampKey = "last_sync_timestamp_0"; + final _logger = Logger(); + final _dio = Dio(); + final _endpoint = "http://192.168.0.106:8080"; + final _user = "umbu"; + final List _assets; + static final _lastSyncTimestampKey = "last_sync_timestamp_0"; + static final _lastDBUpdateTimestampKey = "last_db_update_timestamp"; - PhotoSyncManager(List assets) { - logger.i("PhotoSyncManager init"); - _syncPhotos(assets); + PhotoSyncManager(this._assets) { + _logger.i("PhotoSyncManager init"); + _assets.sort((first, second) => second + .modifiedDateTime.millisecondsSinceEpoch + .compareTo(first.modifiedDateTime.millisecondsSinceEpoch)); } - _syncPhotos(List assets) async { + Future init() async { + await _updateDatabase(); + await _syncPhotos(); + } + + Future _updateDatabase() async { SharedPreferences prefs = await SharedPreferences.getInstance(); - var lastSyncTimestamp = prefs.getInt(lastSyncTimestampKey); + var lastDBUpdateTimestamp = prefs.getInt(_lastDBUpdateTimestampKey); + if (lastDBUpdateTimestamp == null) { + lastDBUpdateTimestamp = 0; + } + for (AssetEntity asset in _assets) { + if (asset.createDateTime.millisecondsSinceEpoch > lastDBUpdateTimestamp) { + await DatabaseHelper.instance.insertPhoto(await Photo.fromAsset(asset)); + } + } + return await prefs.setInt( + _lastDBUpdateTimestampKey, DateTime.now().millisecondsSinceEpoch); + } + + _syncPhotos() async { + SharedPreferences prefs = await SharedPreferences.getInstance(); + var lastSyncTimestamp = prefs.getInt(_lastSyncTimestampKey); if (lastSyncTimestamp == null) { lastSyncTimestamp = 0; } - logger.i("Last sync timestamp: " + lastSyncTimestamp.toString()); + _logger.i("Last sync timestamp: " + lastSyncTimestamp.toString()); - await _downloadDiff(lastSyncTimestamp, prefs); + List diff = await _getDiff(lastSyncTimestamp); + await _downloadDiff(diff, prefs); - await _uploadDiff(assets, prefs); + await _uploadDiff(prefs); // TODO: Fix race conditions triggered due to concurrent syncs. // Add device_id/last_sync_timestamp to the upload request? } - Future _uploadDiff(List assets, SharedPreferences prefs) async { - assets.sort((first, second) => second - .modifiedDateTime.millisecondsSinceEpoch - .compareTo(first.modifiedDateTime.millisecondsSinceEpoch)); - var uploadedAssetCount = 0; - for (AssetEntity asset in assets) { + Future _uploadDiff(SharedPreferences prefs) async { + var uploadedCount = 0; + List photosToBeUploaded = + await DatabaseHelper.instance.getPhotosToBeUploaded(); + for (Photo photo in photosToBeUploaded) { // TODO: Fix me - if (uploadedAssetCount == 100) { + if (uploadedCount == 100) { return; } - var containsPath = await DatabaseHelper.instance - .containsPath((await asset.originFile).path); - if (!containsPath) { - var response = await _uploadFile(asset); - prefs.setInt(lastSyncTimestampKey, response.syncTimestamp); - uploadedAssetCount++; - } + var uploadedPhoto = await _uploadFile(photo.localPath, photo.hash); + await DatabaseHelper.instance.updateUrlAndTimestamp(photo.hash, + uploadedPhoto.url, uploadedPhoto.syncTimestamp.toString()); + prefs.setInt(_lastSyncTimestampKey, uploadedPhoto.syncTimestamp); + uploadedCount++; } } - Future _downloadDiff(int lastSyncTimestamp, SharedPreferences prefs) async { - Response response = await dio.get(endpoint + "/diff", queryParameters: { - "user": user, + Future _downloadDiff(List diff, SharedPreferences prefs) async { + var externalPath = (await getApplicationDocumentsDirectory()).path; + _logger.i("External path: " + externalPath); + var path = externalPath + "/photos/"; + for (Photo photo in diff) { + if (await DatabaseHelper.instance.containsPhotoHash(photo.hash)) { + await DatabaseHelper.instance.updateUrlAndTimestamp( + photo.hash, photo.url, photo.syncTimestamp.toString()); + continue; + } else { + var localPath = path + basename(photo.url); + await _dio.download(_endpoint + photo.url, localPath); + photo.localPath = localPath; + await insertPhotoToDB(photo); + } + await prefs.setInt(_lastSyncTimestampKey, photo.syncTimestamp); + } + } + + Future> _getDiff(int lastSyncTimestamp) async { + Response response = await _dio.get(_endpoint + "/diff", queryParameters: { + "user": _user, "lastSyncTimestamp": lastSyncTimestamp }); - logger.i(response.toString()); - var externalPath = (await getApplicationDocumentsDirectory()).path; - logger.i("External path: " + externalPath); - var path = externalPath + "/photos/"; - - List photos = (response.data["diff"] as List) + _logger.i(response.toString()); + return (response.data["diff"] as List) .map((photo) => new Photo.fromJson(photo)) .toList(); - for (Photo photo in photos) { - await dio.download(endpoint + photo.url, path + basename(photo.url)); - photo.hash = _getHash(photo); - photo.localPath = path + basename(photo.url); - insertPhotoToDB(photo); - prefs.setInt(lastSyncTimestampKey, photo.syncTimestamp); - logger.i("Downloaded " + photo.url + " to " + path); - } } - Future _uploadFile(AssetEntity entity) async { - logger.i("Uploading: " + entity.id); - var path = (await entity.originFile).path; + Future _uploadFile(String path, String hash) async { var formData = FormData.fromMap({ - "file": await MultipartFile.fromFile(path, filename: entity.title), - "user": user, + "file": await MultipartFile.fromFile(path, filename: basename(path)), + "user": _user, }); - var response = await dio.post(endpoint + "/upload", data: formData); - logger.i(response.toString()); + var response = await _dio.post(_endpoint + "/upload", data: formData); + _logger.i(response.toString()); var photo = Photo.fromJson(response.data); - photo.hash = _getHash(photo); + _logger.i("Locally computed hash for " + path + ": " + hash); + _logger.i("Server computed hash for " + path + ": " + photo.hash); photo.localPath = path; - insertPhotoToDB(photo); return photo; } - String _getHash(Photo photo) { - // TODO: Compute hash - return "hash"; - } - Future insertPhotoToDB(Photo photo) async { - logger.i("Inserting to DB"); + _logger.i("Inserting to DB"); await DatabaseHelper.instance.insertPhoto(photo); PhotoLoader.instance.reloadPhotos(); } diff --git a/pubspec.lock b/pubspec.lock index 8e982d21b..3356f6930 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -51,7 +51,7 @@ packages: source: hosted version: "2.1.1" crypto: - dependency: transitive + dependency: "direct main" description: name: crypto url: "https://pub.dartlang.org" diff --git a/pubspec.yaml b/pubspec.yaml index 1ba40d43d..dd966ebcb 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -31,6 +31,7 @@ dependencies: logger: ^0.8.3 dio: ^3.0.9 local_image_provider: ^1.0.0 + crypto: ^2.1.3 dev_dependencies: flutter_test: