Update diff handling
This commit is contained in:
parent
22d3f2c9ed
commit
22ca572532
|
@ -11,7 +11,6 @@ class DatabaseHelper {
|
||||||
|
|
||||||
static final table = 'photos';
|
static final table = 'photos';
|
||||||
|
|
||||||
static final columnId = 'photo_id';
|
|
||||||
static final columnLocalPath = 'local_path';
|
static final columnLocalPath = 'local_path';
|
||||||
static final columnUrl = 'url';
|
static final columnUrl = 'url';
|
||||||
static final columnHash = 'hash';
|
static final columnHash = 'hash';
|
||||||
|
@ -42,11 +41,10 @@ class DatabaseHelper {
|
||||||
Future _onCreate(Database db, int version) async {
|
Future _onCreate(Database db, int version) async {
|
||||||
await db.execute('''
|
await db.execute('''
|
||||||
CREATE TABLE $table (
|
CREATE TABLE $table (
|
||||||
$columnId VARCHAR(255) PRIMARY KEY,
|
|
||||||
$columnLocalPath TEXT NOT NULL,
|
$columnLocalPath TEXT NOT NULL,
|
||||||
$columnUrl TEXT NOT NULL,
|
$columnUrl TEXT,
|
||||||
$columnHash TEXT NOT NULL,
|
$columnHash TEXT NOT NULL,
|
||||||
$columnSyncTimestamp TEXT NOT NULL
|
$columnSyncTimestamp TEXT
|
||||||
)
|
)
|
||||||
''');
|
''');
|
||||||
}
|
}
|
||||||
|
@ -54,7 +52,6 @@ class DatabaseHelper {
|
||||||
Future<int> insertPhoto(Photo photo) async {
|
Future<int> insertPhoto(Photo photo) async {
|
||||||
Database db = await instance.database;
|
Database db = await instance.database;
|
||||||
var row = new Map<String, dynamic>();
|
var row = new Map<String, dynamic>();
|
||||||
row[columnId] = photo.photoID;
|
|
||||||
row[columnLocalPath] = photo.localPath;
|
row[columnLocalPath] = photo.localPath;
|
||||||
row[columnUrl] = photo.url;
|
row[columnUrl] = photo.url;
|
||||||
row[columnHash] = photo.hash;
|
row[columnHash] = photo.hash;
|
||||||
|
@ -65,49 +62,47 @@ class DatabaseHelper {
|
||||||
Future<List<Photo>> getAllPhotos() async {
|
Future<List<Photo>> getAllPhotos() async {
|
||||||
Database db = await instance.database;
|
Database db = await instance.database;
|
||||||
var results = await db.query(table);
|
var results = await db.query(table);
|
||||||
|
return _convertToPhotos(results);
|
||||||
|
}
|
||||||
|
|
||||||
|
Future<List<Photo>> getPhotosToBeUploaded() async {
|
||||||
|
Database db = await instance.database;
|
||||||
|
var results = await db.query(table, where: '$columnUrl IS NULL');
|
||||||
|
return _convertToPhotos(results);
|
||||||
|
}
|
||||||
|
|
||||||
|
// We are assuming here that the hash column in the map is set. The other
|
||||||
|
// column values will be used to update the row.
|
||||||
|
Future<int> updateUrlAndTimestamp(
|
||||||
|
String hash, String url, String timestamp) async {
|
||||||
|
Database db = await instance.database;
|
||||||
|
var row = new Map<String, dynamic>();
|
||||||
|
row[columnUrl] = url;
|
||||||
|
row[columnSyncTimestamp] = timestamp;
|
||||||
|
return await db
|
||||||
|
.update(table, row, where: '$columnHash = ?', whereArgs: [hash]);
|
||||||
|
}
|
||||||
|
|
||||||
|
Future<bool> containsPath(String path) async {
|
||||||
|
Database db = await instance.database;
|
||||||
|
return (await db
|
||||||
|
.query(table, where: '$columnLocalPath =?', whereArgs: [path]))
|
||||||
|
.length >
|
||||||
|
0;
|
||||||
|
}
|
||||||
|
|
||||||
|
Future<bool> containsPhotoHash(String hash) async {
|
||||||
|
Database db = await instance.database;
|
||||||
|
return (await db.query(table, where: '$columnHash =?', whereArgs: [hash]))
|
||||||
|
.length >
|
||||||
|
0;
|
||||||
|
}
|
||||||
|
|
||||||
|
List<Photo> _convertToPhotos(List<Map<String, dynamic>> results) {
|
||||||
var photos = List<Photo>();
|
var photos = List<Photo>();
|
||||||
for (var result in results) {
|
for (var result in results) {
|
||||||
photos.add(Photo.fromRow(result));
|
photos.add(Photo.fromRow(result));
|
||||||
}
|
}
|
||||||
return photos;
|
return photos;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Helper methods
|
|
||||||
|
|
||||||
// Inserts a row in the database where each key in the Map is a column name
|
|
||||||
// and the value is the column value. The return value is the id of the
|
|
||||||
// inserted row.
|
|
||||||
Future<int> insert(Map<String, dynamic> row) async {
|
|
||||||
Database db = await instance.database;
|
|
||||||
return await db.insert(table, row);
|
|
||||||
}
|
|
||||||
|
|
||||||
// All of the rows are returned as a list of maps, where each map is
|
|
||||||
// a key-value list of columns.
|
|
||||||
Future<List<Map<String, dynamic>>> queryAllRows() async {
|
|
||||||
Database db = await instance.database;
|
|
||||||
return await db.query(table);
|
|
||||||
}
|
|
||||||
|
|
||||||
// We are assuming here that the id column in the map is set. The other
|
|
||||||
// column values will be used to update the row.
|
|
||||||
Future<int> update(Map<String, dynamic> row) async {
|
|
||||||
Database db = await instance.database;
|
|
||||||
int id = row[columnId];
|
|
||||||
return await db.update(table, row, where: '$columnId = ?', whereArgs: [id]);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Deletes the row specified by the id. The number of affected rows is
|
|
||||||
// returned. This should be 1 as long as the row exists.
|
|
||||||
Future<int> delete(int id) async {
|
|
||||||
Database db = await instance.database;
|
|
||||||
return await db.delete(table, where: '$columnId = ?', whereArgs: [id]);
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<bool> containsPath(String path) async {
|
|
||||||
Database db = await instance.database;
|
|
||||||
return (await db.query(table, where: '$columnLocalPath =?', whereArgs: [path]))
|
|
||||||
.length >
|
|
||||||
0;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,7 +18,8 @@ void main() async {
|
||||||
WidgetsFlutterBinding.ensureInitialized();
|
WidgetsFlutterBinding.ensureInitialized();
|
||||||
await provider.refreshGalleryList();
|
await provider.refreshGalleryList();
|
||||||
var assets = await provider.list[0].assetList;
|
var assets = await provider.list[0].assetList;
|
||||||
PhotoSyncManager(assets);
|
var photoSyncManager = PhotoSyncManager(assets);
|
||||||
|
await photoSyncManager.init();
|
||||||
runApp(MyApp2());
|
runApp(MyApp2());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,19 +1,38 @@
|
||||||
|
import 'dart:io';
|
||||||
|
|
||||||
|
import 'package:crypto/crypto.dart';
|
||||||
|
import 'package:photo_manager/photo_manager.dart';
|
||||||
|
|
||||||
class Photo {
|
class Photo {
|
||||||
String photoID;
|
|
||||||
String url;
|
String url;
|
||||||
String localPath;
|
String localPath;
|
||||||
String hash;
|
String hash;
|
||||||
int syncTimestamp;
|
int syncTimestamp;
|
||||||
|
|
||||||
|
Photo();
|
||||||
|
|
||||||
Photo.fromJson(Map<String, dynamic> json)
|
Photo.fromJson(Map<String, dynamic> json)
|
||||||
: photoID = json["photoID"],
|
: url = json["url"],
|
||||||
url = json["url"],
|
hash = json["hash"],
|
||||||
syncTimestamp = json["syncTimestamp"];
|
syncTimestamp = json["syncTimestamp"];
|
||||||
|
|
||||||
Photo.fromRow(Map<String, dynamic> row)
|
Photo.fromRow(Map<String, dynamic> row)
|
||||||
: photoID = row["photo_id"],
|
: localPath = row["local_path"],
|
||||||
localPath = row["local_path"],
|
|
||||||
url = row["url"],
|
url = row["url"],
|
||||||
hash = row["hash"],
|
hash = row["hash"],
|
||||||
syncTimestamp = int.parse(row["sync_timestamp"]);
|
syncTimestamp = row["sync_timestamp"] == null
|
||||||
|
? -1
|
||||||
|
: int.parse(row["sync_timestamp"]);
|
||||||
|
|
||||||
|
static Future<Photo> fromAsset(AssetEntity asset) async {
|
||||||
|
Photo photo = Photo();
|
||||||
|
var file = (await asset.originFile);
|
||||||
|
photo.localPath = file.path;
|
||||||
|
photo.hash = getHash(file);
|
||||||
|
return photo;
|
||||||
|
}
|
||||||
|
|
||||||
|
static String getHash(File file) {
|
||||||
|
return sha256.convert(file.readAsBytesSync()).toString();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -9,99 +9,121 @@ import 'package:dio/dio.dart';
|
||||||
import 'package:myapp/models/photo.dart';
|
import 'package:myapp/models/photo.dart';
|
||||||
|
|
||||||
class PhotoSyncManager {
|
class PhotoSyncManager {
|
||||||
final logger = Logger();
|
final _logger = Logger();
|
||||||
final dio = Dio();
|
final _dio = Dio();
|
||||||
final endpoint = "http://172.20.10.6:8080";
|
final _endpoint = "http://192.168.0.106:8080";
|
||||||
final user = "umbu";
|
final _user = "umbu";
|
||||||
static final lastSyncTimestampKey = "last_sync_timestamp_0";
|
final List<AssetEntity> _assets;
|
||||||
|
static final _lastSyncTimestampKey = "last_sync_timestamp_0";
|
||||||
|
static final _lastDBUpdateTimestampKey = "last_db_update_timestamp";
|
||||||
|
|
||||||
PhotoSyncManager(List<AssetEntity> assets) {
|
PhotoSyncManager(this._assets) {
|
||||||
logger.i("PhotoSyncManager init");
|
_logger.i("PhotoSyncManager init");
|
||||||
_syncPhotos(assets);
|
_assets.sort((first, second) => second
|
||||||
|
.modifiedDateTime.millisecondsSinceEpoch
|
||||||
|
.compareTo(first.modifiedDateTime.millisecondsSinceEpoch));
|
||||||
}
|
}
|
||||||
|
|
||||||
_syncPhotos(List<AssetEntity> assets) async {
|
Future<void> init() async {
|
||||||
|
await _updateDatabase();
|
||||||
|
await _syncPhotos();
|
||||||
|
}
|
||||||
|
|
||||||
|
Future<bool> _updateDatabase() async {
|
||||||
SharedPreferences prefs = await SharedPreferences.getInstance();
|
SharedPreferences prefs = await SharedPreferences.getInstance();
|
||||||
var lastSyncTimestamp = prefs.getInt(lastSyncTimestampKey);
|
var lastDBUpdateTimestamp = prefs.getInt(_lastDBUpdateTimestampKey);
|
||||||
|
if (lastDBUpdateTimestamp == null) {
|
||||||
|
lastDBUpdateTimestamp = 0;
|
||||||
|
}
|
||||||
|
for (AssetEntity asset in _assets) {
|
||||||
|
if (asset.createDateTime.millisecondsSinceEpoch > lastDBUpdateTimestamp) {
|
||||||
|
await DatabaseHelper.instance.insertPhoto(await Photo.fromAsset(asset));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return await prefs.setInt(
|
||||||
|
_lastDBUpdateTimestampKey, DateTime.now().millisecondsSinceEpoch);
|
||||||
|
}
|
||||||
|
|
||||||
|
_syncPhotos() async {
|
||||||
|
SharedPreferences prefs = await SharedPreferences.getInstance();
|
||||||
|
var lastSyncTimestamp = prefs.getInt(_lastSyncTimestampKey);
|
||||||
if (lastSyncTimestamp == null) {
|
if (lastSyncTimestamp == null) {
|
||||||
lastSyncTimestamp = 0;
|
lastSyncTimestamp = 0;
|
||||||
}
|
}
|
||||||
logger.i("Last sync timestamp: " + lastSyncTimestamp.toString());
|
_logger.i("Last sync timestamp: " + lastSyncTimestamp.toString());
|
||||||
|
|
||||||
await _downloadDiff(lastSyncTimestamp, prefs);
|
List<Photo> diff = await _getDiff(lastSyncTimestamp);
|
||||||
|
await _downloadDiff(diff, prefs);
|
||||||
|
|
||||||
await _uploadDiff(assets, prefs);
|
await _uploadDiff(prefs);
|
||||||
|
|
||||||
// TODO: Fix race conditions triggered due to concurrent syncs.
|
// TODO: Fix race conditions triggered due to concurrent syncs.
|
||||||
// Add device_id/last_sync_timestamp to the upload request?
|
// Add device_id/last_sync_timestamp to the upload request?
|
||||||
}
|
}
|
||||||
|
|
||||||
Future _uploadDiff(List<AssetEntity> assets, SharedPreferences prefs) async {
|
Future _uploadDiff(SharedPreferences prefs) async {
|
||||||
assets.sort((first, second) => second
|
var uploadedCount = 0;
|
||||||
.modifiedDateTime.millisecondsSinceEpoch
|
List<Photo> photosToBeUploaded =
|
||||||
.compareTo(first.modifiedDateTime.millisecondsSinceEpoch));
|
await DatabaseHelper.instance.getPhotosToBeUploaded();
|
||||||
var uploadedAssetCount = 0;
|
for (Photo photo in photosToBeUploaded) {
|
||||||
for (AssetEntity asset in assets) {
|
|
||||||
// TODO: Fix me
|
// TODO: Fix me
|
||||||
if (uploadedAssetCount == 100) {
|
if (uploadedCount == 100) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
var containsPath = await DatabaseHelper.instance
|
var uploadedPhoto = await _uploadFile(photo.localPath, photo.hash);
|
||||||
.containsPath((await asset.originFile).path);
|
await DatabaseHelper.instance.updateUrlAndTimestamp(photo.hash,
|
||||||
if (!containsPath) {
|
uploadedPhoto.url, uploadedPhoto.syncTimestamp.toString());
|
||||||
var response = await _uploadFile(asset);
|
prefs.setInt(_lastSyncTimestampKey, uploadedPhoto.syncTimestamp);
|
||||||
prefs.setInt(lastSyncTimestampKey, response.syncTimestamp);
|
uploadedCount++;
|
||||||
uploadedAssetCount++;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Future _downloadDiff(int lastSyncTimestamp, SharedPreferences prefs) async {
|
Future _downloadDiff(List<Photo> diff, SharedPreferences prefs) async {
|
||||||
Response response = await dio.get(endpoint + "/diff", queryParameters: {
|
var externalPath = (await getApplicationDocumentsDirectory()).path;
|
||||||
"user": user,
|
_logger.i("External path: " + externalPath);
|
||||||
|
var path = externalPath + "/photos/";
|
||||||
|
for (Photo photo in diff) {
|
||||||
|
if (await DatabaseHelper.instance.containsPhotoHash(photo.hash)) {
|
||||||
|
await DatabaseHelper.instance.updateUrlAndTimestamp(
|
||||||
|
photo.hash, photo.url, photo.syncTimestamp.toString());
|
||||||
|
continue;
|
||||||
|
} else {
|
||||||
|
var localPath = path + basename(photo.url);
|
||||||
|
await _dio.download(_endpoint + photo.url, localPath);
|
||||||
|
photo.localPath = localPath;
|
||||||
|
await insertPhotoToDB(photo);
|
||||||
|
}
|
||||||
|
await prefs.setInt(_lastSyncTimestampKey, photo.syncTimestamp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Future<List<Photo>> _getDiff(int lastSyncTimestamp) async {
|
||||||
|
Response response = await _dio.get(_endpoint + "/diff", queryParameters: {
|
||||||
|
"user": _user,
|
||||||
"lastSyncTimestamp": lastSyncTimestamp
|
"lastSyncTimestamp": lastSyncTimestamp
|
||||||
});
|
});
|
||||||
logger.i(response.toString());
|
_logger.i(response.toString());
|
||||||
var externalPath = (await getApplicationDocumentsDirectory()).path;
|
return (response.data["diff"] as List)
|
||||||
logger.i("External path: " + externalPath);
|
|
||||||
var path = externalPath + "/photos/";
|
|
||||||
|
|
||||||
List<Photo> photos = (response.data["diff"] as List)
|
|
||||||
.map((photo) => new Photo.fromJson(photo))
|
.map((photo) => new Photo.fromJson(photo))
|
||||||
.toList();
|
.toList();
|
||||||
for (Photo photo in photos) {
|
|
||||||
await dio.download(endpoint + photo.url, path + basename(photo.url));
|
|
||||||
photo.hash = _getHash(photo);
|
|
||||||
photo.localPath = path + basename(photo.url);
|
|
||||||
insertPhotoToDB(photo);
|
|
||||||
prefs.setInt(lastSyncTimestampKey, photo.syncTimestamp);
|
|
||||||
logger.i("Downloaded " + photo.url + " to " + path);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Future<Photo> _uploadFile(AssetEntity entity) async {
|
Future<Photo> _uploadFile(String path, String hash) async {
|
||||||
logger.i("Uploading: " + entity.id);
|
|
||||||
var path = (await entity.originFile).path;
|
|
||||||
var formData = FormData.fromMap({
|
var formData = FormData.fromMap({
|
||||||
"file": await MultipartFile.fromFile(path, filename: entity.title),
|
"file": await MultipartFile.fromFile(path, filename: basename(path)),
|
||||||
"user": user,
|
"user": _user,
|
||||||
});
|
});
|
||||||
var response = await dio.post(endpoint + "/upload", data: formData);
|
var response = await _dio.post(_endpoint + "/upload", data: formData);
|
||||||
logger.i(response.toString());
|
_logger.i(response.toString());
|
||||||
var photo = Photo.fromJson(response.data);
|
var photo = Photo.fromJson(response.data);
|
||||||
photo.hash = _getHash(photo);
|
_logger.i("Locally computed hash for " + path + ": " + hash);
|
||||||
|
_logger.i("Server computed hash for " + path + ": " + photo.hash);
|
||||||
photo.localPath = path;
|
photo.localPath = path;
|
||||||
insertPhotoToDB(photo);
|
|
||||||
return photo;
|
return photo;
|
||||||
}
|
}
|
||||||
|
|
||||||
String _getHash(Photo photo) {
|
|
||||||
// TODO: Compute hash
|
|
||||||
return "hash";
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<void> insertPhotoToDB(Photo photo) async {
|
Future<void> insertPhotoToDB(Photo photo) async {
|
||||||
logger.i("Inserting to DB");
|
_logger.i("Inserting to DB");
|
||||||
await DatabaseHelper.instance.insertPhoto(photo);
|
await DatabaseHelper.instance.insertPhoto(photo);
|
||||||
PhotoLoader.instance.reloadPhotos();
|
PhotoLoader.instance.reloadPhotos();
|
||||||
}
|
}
|
||||||
|
|
|
@ -51,7 +51,7 @@ packages:
|
||||||
source: hosted
|
source: hosted
|
||||||
version: "2.1.1"
|
version: "2.1.1"
|
||||||
crypto:
|
crypto:
|
||||||
dependency: transitive
|
dependency: "direct main"
|
||||||
description:
|
description:
|
||||||
name: crypto
|
name: crypto
|
||||||
url: "https://pub.dartlang.org"
|
url: "https://pub.dartlang.org"
|
||||||
|
|
|
@ -31,6 +31,7 @@ dependencies:
|
||||||
logger: ^0.8.3
|
logger: ^0.8.3
|
||||||
dio: ^3.0.9
|
dio: ^3.0.9
|
||||||
local_image_provider: ^1.0.0
|
local_image_provider: ^1.0.0
|
||||||
|
crypto: ^2.1.3
|
||||||
|
|
||||||
dev_dependencies:
|
dev_dependencies:
|
||||||
flutter_test:
|
flutter_test:
|
||||||
|
|
Loading…
Reference in a new issue