made chnages to better handle file import

This commit is contained in:
Abhinav-grd 2021-08-13 23:51:14 +05:30
parent 96e36c8bf0
commit 5d965da21e
3 changed files with 47 additions and 25 deletions

View file

@ -17,6 +17,7 @@ import UploadManager, {
UPLOAD_STAGES, UPLOAD_STAGES,
} from 'services/upload/uploadManager'; } from 'services/upload/uploadManager';
import uploadManager from 'services/upload/uploadManager'; import uploadManager from 'services/upload/uploadManager';
import { METADATA_FOLDER_NAME } from 'services/exportService';
interface Props { interface Props {
syncWithRemote: (force?: boolean, silent?: boolean) => Promise<void>; syncWithRemote: (force?: boolean, silent?: boolean) => Promise<void>;
@ -169,8 +170,12 @@ export default function Upload(props: Props) {
function getCollectionWiseFiles() { function getCollectionWiseFiles() {
const collectionWiseFiles = new Map<string, globalThis.File[]>(); const collectionWiseFiles = new Map<string, globalThis.File[]>();
for (const file of props.acceptedFiles) { for (const file of props.acceptedFiles) {
const filePath = file['path']; const filePath = file['path'] as string;
const folderPath = filePath.substr(0, filePath.lastIndexOf('/'));
let folderPath = filePath.substr(0, filePath.lastIndexOf('/'));
if (folderPath.endsWith(METADATA_FOLDER_NAME)) {
folderPath = folderPath.substr(0, folderPath.lastIndexOf('/'));
}
const folderName = folderPath.substr( const folderName = folderPath.substr(
folderPath.lastIndexOf('/') + 1 folderPath.lastIndexOf('/') + 1
); );

View file

@ -10,7 +10,11 @@ import {
import { retryAsyncFunction } from 'utils/network'; import { retryAsyncFunction } from 'utils/network';
import { logError } from 'utils/sentry'; import { logError } from 'utils/sentry';
import { getData, LS_KEYS } from 'utils/storage/localStorage'; import { getData, LS_KEYS } from 'utils/storage/localStorage';
import { Collection, getLocalCollections } from './collectionService'; import {
Collection,
getLocalCollections,
getNonEmptyCollections,
} from './collectionService';
import downloadManager from './downloadManager'; import downloadManager from './downloadManager';
import { File, FILE_TYPE, getLocalFiles } from './fileService'; import { File, FILE_TYPE, getLocalFiles } from './fileService';
import { decodeMotionPhoto } from './motionPhotoService'; import { decodeMotionPhoto } from './motionPhotoService';
@ -63,8 +67,8 @@ export enum ExportType {
RETRY_FAILED, RETRY_FAILED,
} }
const ExportRecordFileName = 'export_status.json'; const EXPORT_RECORD_FILE_NAME = 'export_status.json';
const MetadataFolderName = 'metadata'; export const METADATA_FOLDER_NAME = 'metadata';
class ExportService { class ExportService {
ElectronAPIs: any; ElectronAPIs: any;
@ -103,6 +107,10 @@ class ExportService {
let filesToExport: File[]; let filesToExport: File[];
const allFiles = await getLocalFiles(); const allFiles = await getLocalFiles();
const collections = await getLocalCollections(); const collections = await getLocalCollections();
const nonEmptyCollections = getNonEmptyCollections(
collections,
allFiles
);
const exportRecord = await this.getExportRecord(exportDir); const exportRecord = await this.getExportRecord(exportDir);
if (exportType === ExportType.NEW) { if (exportType === ExportType.NEW) {
@ -117,7 +125,7 @@ class ExportService {
} }
this.exportInProgress = this.fileExporter( this.exportInProgress = this.fileExporter(
filesToExport, filesToExport,
collections, nonEmptyCollections,
updateProgress, updateProgress,
exportDir exportDir
); );
@ -162,7 +170,7 @@ class ExportService {
collectionFolderPath collectionFolderPath
); );
await this.ElectronAPIs.checkExistsAndCreateCollectionDir( await this.ElectronAPIs.checkExistsAndCreateCollectionDir(
`${collectionFolderPath}/${MetadataFolderName}` `${collectionFolderPath}/${METADATA_FOLDER_NAME}`
); );
collectionIDMap.set(collection.id, collectionFolderPath); collectionIDMap.set(collection.id, collectionFolderPath);
} }
@ -267,7 +275,7 @@ class ExportService {
const exportRecord = await this.getExportRecord(folder); const exportRecord = await this.getExportRecord(folder);
const newRecord = { ...exportRecord, ...newData }; const newRecord = { ...exportRecord, ...newData };
await this.ElectronAPIs.setExportRecord( await this.ElectronAPIs.setExportRecord(
`${folder}/${ExportRecordFileName}`, `${folder}/${EXPORT_RECORD_FILE_NAME}`,
JSON.stringify(newRecord, null, 2) JSON.stringify(newRecord, null, 2)
); );
} catch (e) { } catch (e) {
@ -283,7 +291,7 @@ class ExportService {
folder = getData(LS_KEYS.EXPORT)?.folder; folder = getData(LS_KEYS.EXPORT)?.folder;
} }
const recordFile = await this.ElectronAPIs.getExportRecord( const recordFile = await this.ElectronAPIs.getExportRecord(
`${folder}/${ExportRecordFileName}` `${folder}/${EXPORT_RECORD_FILE_NAME}`
); );
if (recordFile) { if (recordFile) {
return JSON.parse(recordFile); return JSON.parse(recordFile);
@ -303,15 +311,9 @@ class ExportService {
if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
this.exportMotionPhoto(fileStream, file, collectionPath); this.exportMotionPhoto(fileStream, file, collectionPath);
} else { } else {
this.ElectronAPIs.saveStreamToDisk( this.saveMediaFile(collectionPath, uid, fileStream);
`${collectionPath}/${uid}`, this.saveMetadataFile(collectionPath, uid, file.metadata);
fileStream
);
} }
this.ElectronAPIs.saveFileToDisk(
`${collectionPath}/${MetadataFolderName}/${uid}.json`,
getGoogleLikeMetadataFile(uid, file.metadata)
);
} }
private async exportMotionPhoto( private async exportMotionPhoto(
@ -324,12 +326,27 @@ class ExportService {
const motionPhoto = await decodeMotionPhoto(fileBlob, originalName); const motionPhoto = await decodeMotionPhoto(fileBlob, originalName);
const imageStream = generateStreamFromArrayBuffer(motionPhoto.image); const imageStream = generateStreamFromArrayBuffer(motionPhoto.image);
const imageSavePath = `${collectionPath}/${file.id}_${motionPhoto.imageNameTitle}`; const imageUID = `${file.id}_${motionPhoto.imageNameTitle}`;
this.ElectronAPIs.saveStreamToDisk(imageSavePath, imageStream); this.saveMediaFile(collectionPath, imageUID, imageStream);
this.saveMetadataFile(collectionPath, imageUID, file.metadata);
const videoStream = generateStreamFromArrayBuffer(motionPhoto.video); const videoStream = generateStreamFromArrayBuffer(motionPhoto.video);
const videoSavePath = `${collectionPath}/${file.id}_${motionPhoto.videoNameTitle}`; const videoUID = `${file.id}_${motionPhoto.videoNameTitle}`;
this.ElectronAPIs.saveStreamToDisk(videoSavePath, videoStream); this.saveMediaFile(collectionPath, videoUID, videoStream);
this.saveMetadataFile(collectionPath, videoUID, file.metadata);
}
private saveMediaFile(collectionPath, uid, fileStream) {
this.ElectronAPIs.saveStreamToDisk(
`${collectionPath}/${uid}`,
fileStream
);
}
private saveMetadataFile(collectionPath, uid, metadata) {
this.ElectronAPIs.saveFileToDisk(
`${collectionPath}/${METADATA_FOLDER_NAME}/${uid}.json`,
getGoogleLikeMetadataFile(uid, metadata)
);
} }
private sanitizeName(name) { private sanitizeName(name) {

View file

@ -55,18 +55,18 @@ export const getGoogleLikeMetadataFile = (
uid: string, uid: string,
metadata: MetadataObject metadata: MetadataObject
) => { ) => {
const creationTime = metadata.creationTime / 1000000; const creationTime = Math.floor(metadata.creationTime / 1000000);
const modificationTime = metadata.modificationTime / 1000000; const modificationTime = Math.floor(metadata.modificationTime / 1000000);
return JSON.stringify( return JSON.stringify(
{ {
title: uid, title: uid,
creationTime: { creationTime: {
timestamp: creationTime, timestamp: creationTime,
formatted: formatDate(creationTime), formatted: formatDate(creationTime * 1000),
}, },
modificationTime: { modificationTime: {
timestamp: modificationTime, timestamp: modificationTime,
formatted: formatDate(modificationTime), formatted: formatDate(modificationTime * 1000),
}, },
geoData: { geoData: {
latitude: metadata.latitude, latitude: metadata.latitude,