diff --git a/package.json b/package.json
index 83f50f9d7..1674865f5 100644
--- a/package.json
+++ b/package.json
@@ -41,6 +41,7 @@
"next": "^11.1.3",
"node-forge": "^0.10.0",
"photoswipe": "file:./thirdparty/photoswipe",
+ "piexifjs": "^1.0.6",
"react": "^17.0.2",
"react-bootstrap": "^1.3.0",
"react-burger-menu": "^3.0.4",
diff --git a/src/components/ExportFinished.tsx b/src/components/ExportFinished.tsx
index dedae4d34..5fc263d88 100644
--- a/src/components/ExportFinished.tsx
+++ b/src/components/ExportFinished.tsx
@@ -29,8 +29,8 @@ export default function ExportFinished(props: Props) {
padding: '0 5%',
}}>
-
-
+
+
{formatDateTime(props.lastExportTime)}
@@ -38,7 +38,7 @@ export default function ExportFinished(props: Props) {
-
+
{props.exportStats.success} / {totalFiles}
diff --git a/src/components/ExportModal.tsx b/src/components/ExportModal.tsx
index a03d4b722..f7c3c6c80 100644
--- a/src/components/ExportModal.tsx
+++ b/src/components/ExportModal.tsx
@@ -8,9 +8,11 @@ import exportService, {
ExportType,
} from 'services/exportService';
import { getLocalFiles } from 'services/fileService';
+import { User } from 'services/userService';
import styled from 'styled-components';
import { sleep } from 'utils/common';
import { getExportRecordFileUID } from 'utils/export';
+import { logError } from 'utils/sentry';
import { getData, LS_KEYS, setData } from 'utils/storage/localStorage';
import constants from 'utils/strings/constants';
import { Label, Row, Value } from './Container';
@@ -105,30 +107,41 @@ export default function ExportModal(props: Props) {
return;
}
const main = async () => {
+ const user: User = getData(LS_KEYS.USER);
if (exportStage === ExportStage.FINISHED) {
- const localFiles = await getLocalFiles();
- const exportRecord = await exportService.getExportRecord();
- const exportedFileCnt = exportRecord.exportedFiles.length;
- const failedFilesCnt = exportRecord.failedFiles.length;
- const syncedFilesCnt = localFiles.length;
- if (syncedFilesCnt > exportedFileCnt + failedFilesCnt) {
- updateExportProgress({
- current: exportedFileCnt + failedFilesCnt,
- total: syncedFilesCnt,
- });
- const exportFileUIDs = new Set([
- ...exportRecord.exportedFiles,
- ...exportRecord.failedFiles,
- ]);
- const unExportedFiles = localFiles.filter(
- (file) =>
- !exportFileUIDs.has(getExportRecordFileUID(file))
+ try {
+ const localFiles = await getLocalFiles();
+ const userPersonalFiles = localFiles.filter(
+ (file) => file.ownerID === user?.id
);
- exportService.addFilesQueuedRecord(
- exportFolder,
- unExportedFiles
- );
- updateExportStage(ExportStage.PAUSED);
+ const exportRecord = await exportService.getExportRecord();
+ const exportedFileCnt = exportRecord.exportedFiles?.length;
+ const failedFilesCnt = exportRecord.failedFiles?.length;
+ const syncedFilesCnt = userPersonalFiles.length;
+ if (syncedFilesCnt > exportedFileCnt + failedFilesCnt) {
+ updateExportProgress({
+ current: exportedFileCnt + failedFilesCnt,
+ total: syncedFilesCnt,
+ });
+ const exportFileUIDs = new Set([
+ ...exportRecord.exportedFiles,
+ ...exportRecord.failedFiles,
+ ]);
+ const unExportedFiles = userPersonalFiles.filter(
+ (file) =>
+ !exportFileUIDs.has(
+ getExportRecordFileUID(file)
+ )
+ );
+ exportService.addFilesQueuedRecord(
+ exportFolder,
+ unExportedFiles
+ );
+ updateExportStage(ExportStage.PAUSED);
+ }
+ } catch (e) {
+ setExportStage(ExportStage.INIT);
+ logError(e, 'error while updating exportModal on reopen');
}
}
};
@@ -154,7 +167,7 @@ export default function ExportModal(props: Props) {
const updateExportTime = (newTime: number) => {
setLastExportTime(newTime);
- exportService.updateExportRecord({ time: newTime });
+ exportService.updateExportRecord({ lastAttemptTimestamp: newTime });
};
const updateExportProgress = (newProgress: ExportProgress) => {
@@ -178,8 +191,8 @@ export default function ExportModal(props: Props) {
updateExportStage(ExportStage.INPROGRESS);
await sleep(100);
};
- const postExportRun = async (paused: Boolean) => {
- if (!paused) {
+ const postExportRun = async (exportResult?: { paused?: boolean }) => {
+ if (!exportResult?.paused) {
updateExportStage(ExportStage.FINISHED);
await sleep(100);
updateExportTime(Date.now());
@@ -189,22 +202,22 @@ export default function ExportModal(props: Props) {
const startExport = async () => {
await preExportRun();
updateExportProgress({ current: 0, total: 0 });
- const { paused } = await exportService.exportFiles(
+ const exportResult = await exportService.exportFiles(
updateExportProgress,
ExportType.NEW
);
- await postExportRun(paused);
+ await postExportRun(exportResult);
};
const stopExport = async () => {
exportService.stopRunningExport();
- postExportRun(false);
+ postExportRun();
};
const pauseExport = () => {
updateExportStage(ExportStage.PAUSED);
exportService.pauseRunningExport();
- postExportRun(true);
+ postExportRun({ paused: true });
};
const resumeExport = async () => {
@@ -219,23 +232,23 @@ export default function ExportModal(props: Props) {
current: pausedStageProgress.current + progress.current,
total: pausedStageProgress.current + progress.total,
});
- const { paused } = await exportService.exportFiles(
+ const exportResult = await exportService.exportFiles(
updateExportStatsWithOffset,
ExportType.PENDING
);
- await postExportRun(paused);
+ await postExportRun(exportResult);
};
const retryFailedExport = async () => {
await preExportRun();
updateExportProgress({ current: 0, total: exportStats.failed });
- const { paused } = await exportService.exportFiles(
+ const exportResult = await exportService.exportFiles(
updateExportProgress,
ExportType.RETRY_FAILED
);
- await postExportRun(paused);
+ await postExportRun(exportResult);
};
const syncExportStatsWithReport = async () => {
@@ -327,11 +340,9 @@ export default function ExportModal(props: Props) {
) : (
<>
- {/* */}
{exportFolder}
- {/* */}
{(exportStage === ExportStage.FINISHED ||
exportStage === ExportStage.INIT) && (
;
export interface ExportProgress {
current: number;
total: number;
}
+export interface ExportedCollectionPaths {
+ [collectionID: number]: string;
+}
export interface ExportStats {
failed: number;
success: number;
}
+const LATEST_EXPORT_VERSION = 1;
+
export interface ExportRecord {
- stage: ExportStage;
- lastAttemptTimestamp: number;
- progress: ExportProgress;
- queuedFiles: string[];
- exportedFiles: string[];
- failedFiles: string[];
+ version?: number;
+ stage?: ExportStage;
+ lastAttemptTimestamp?: number;
+ progress?: ExportProgress;
+ queuedFiles?: string[];
+ exportedFiles?: string[];
+ failedFiles?: string[];
+ exportedCollectionPaths?: ExportedCollectionPaths;
}
export enum ExportStage {
INIT,
@@ -74,12 +102,14 @@ class ExportService {
ElectronAPIs: any;
private exportInProgress: Promise<{ paused: boolean }> = null;
- private recordUpdateInProgress = Promise.resolve();
+ private exportRecordUpdater = new QueueProcessor(1);
private stopExport: boolean = false;
private pauseExport: boolean = false;
+ private allElectronAPIsExist: boolean = false;
constructor() {
this.ElectronAPIs = runningInBrowser() && window['ElectronAPIs'];
+ this.allElectronAPIsExist = !!this.ElectronAPIs.exists;
}
async selectExportDirectory() {
return await this.ElectronAPIs.selectRootDirectory();
@@ -94,53 +124,117 @@ class ExportService {
updateProgress: (progress: ExportProgress) => void,
exportType: ExportType
) {
- if (this.exportInProgress) {
- this.ElectronAPIs.sendNotification(ExportNotification.IN_PROGRESS);
- return this.exportInProgress;
- }
- this.ElectronAPIs.showOnTray('starting export');
- const exportDir = getData(LS_KEYS.EXPORT)?.folder;
- if (!exportDir) {
- // no-export folder set
- return;
- }
- let filesToExport: File[];
- const allFiles = await getLocalFiles();
- const collections = await getLocalCollections();
- const nonEmptyCollections = getNonEmptyCollections(
- collections,
- allFiles
- );
- const exportRecord = await this.getExportRecord(exportDir);
+ try {
+ if (this.exportInProgress) {
+ this.ElectronAPIs.sendNotification(
+ ExportNotification.IN_PROGRESS
+ );
+ return await this.exportInProgress;
+ }
+ this.ElectronAPIs.showOnTray('starting export');
+ const exportDir = getData(LS_KEYS.EXPORT)?.folder;
+ if (!exportDir) {
+ // no-export folder set
+ return;
+ }
+ const user: User = getData(LS_KEYS.USER);
- if (exportType === ExportType.NEW) {
- filesToExport = await getFilesUploadedAfterLastExport(
- allFiles,
+ let filesToExport: File[];
+ const localFiles = await getLocalFiles();
+ const userPersonalFiles = localFiles
+ .filter((file) => file.ownerID === user?.id)
+ .sort((fileA, fileB) => fileA.id - fileB.id);
+
+ const collections = await getLocalCollections();
+ const nonEmptyCollections = getNonEmptyCollections(
+ collections,
+ userPersonalFiles
+ );
+ const userCollections = nonEmptyCollections
+ .filter((collection) => collection.owner.id === user?.id)
+ .sort(
+ (collectionA, collectionB) =>
+ collectionA.id - collectionB.id
+ );
+ if (this.checkAllElectronAPIsExists()) {
+ await this.migrateExport(
+ exportDir,
+ collections,
+ userPersonalFiles
+ );
+ }
+ const exportRecord = await this.getExportRecord(exportDir);
+
+ if (exportType === ExportType.NEW) {
+ filesToExport = getFilesUploadedAfterLastExport(
+ userPersonalFiles,
+ exportRecord
+ );
+ } else if (exportType === ExportType.RETRY_FAILED) {
+ filesToExport = getExportFailedFiles(
+ userPersonalFiles,
+ exportRecord
+ );
+ } else {
+ filesToExport = getExportQueuedFiles(
+ userPersonalFiles,
+ exportRecord
+ );
+ }
+ const collectionIDPathMap: CollectionIDPathMap =
+ getCollectionIDPathMapFromExportRecord(exportRecord);
+ const newCollections = getCollectionsCreatedAfterLastExport(
+ userCollections,
exportRecord
);
- } else if (exportType === ExportType.RETRY_FAILED) {
- filesToExport = await getExportFailedFiles(allFiles, exportRecord);
- } else {
- filesToExport = await getExportPendingFiles(allFiles, exportRecord);
+
+ const renamedCollections = getCollectionsRenamedAfterLastExport(
+ userCollections,
+ exportRecord
+ );
+ this.exportInProgress = this.fileExporter(
+ filesToExport,
+ newCollections,
+ renamedCollections,
+ collectionIDPathMap,
+ updateProgress,
+ exportDir
+ );
+ const resp = await this.exportInProgress;
+ this.exportInProgress = null;
+ return resp;
+ } catch (e) {
+ logError(e, 'exportFiles failed');
+ return { paused: false };
}
- this.exportInProgress = this.fileExporter(
- filesToExport,
- nonEmptyCollections,
- updateProgress,
- exportDir
- );
- const resp = await this.exportInProgress;
- this.exportInProgress = null;
- return resp;
}
async fileExporter(
files: File[],
- collections: Collection[],
+ newCollections: Collection[],
+ renamedCollections: Collection[],
+ collectionIDPathMap: CollectionIDPathMap,
updateProgress: (progress: ExportProgress) => void,
- dir: string
+ exportDir: string
): Promise<{ paused: boolean }> {
try {
+ if (newCollections?.length) {
+ await this.createNewCollectionFolders(
+ newCollections,
+ exportDir,
+ collectionIDPathMap
+ );
+ }
+ if (
+ renamedCollections?.length &&
+ this.checkAllElectronAPIsExists()
+ ) {
+ await this.renameCollectionFolders(
+ renamedCollections,
+ exportDir,
+ collectionIDPathMap
+ );
+ }
if (!files?.length) {
this.ElectronAPIs.sendNotification(
ExportNotification.UP_TO_DATE
@@ -149,7 +243,7 @@ class ExportService {
}
this.stopExport = false;
this.pauseExport = false;
- this.addFilesQueuedRecord(dir, files);
+ this.addFilesQueuedRecord(exportDir, files);
const failedFileCount = 0;
this.ElectronAPIs.showOnTray({
@@ -161,19 +255,6 @@ class ExportService {
});
this.ElectronAPIs.sendNotification(ExportNotification.START);
- const collectionIDMap = new Map();
- for (const collection of collections) {
- const collectionFolderPath = `${dir}/${
- collection.id
- }_${this.sanitizeName(collection.name)}`;
- await this.ElectronAPIs.checkExistsAndCreateCollectionDir(
- collectionFolderPath
- );
- await this.ElectronAPIs.checkExistsAndCreateCollectionDir(
- `${collectionFolderPath}/${METADATA_FOLDER_NAME}`
- );
- collectionIDMap.set(collection.id, collectionFolderPath);
- }
for (const [index, file] of files.entries()) {
if (this.stopExport || this.pauseExport) {
if (this.pauseExport) {
@@ -184,20 +265,26 @@ class ExportService {
}
break;
}
- const collectionPath = collectionIDMap.get(file.collectionID);
+ const collectionPath = collectionIDPathMap.get(
+ file.collectionID
+ );
try {
await this.downloadAndSave(file, collectionPath);
- await this.addFileExportRecord(
- dir,
+ await this.addFileExportedRecord(
+ exportDir,
file,
RecordType.SUCCESS
);
} catch (e) {
- await this.addFileExportRecord(
- dir,
+ await this.addFileExportedRecord(
+ exportDir,
file,
RecordType.FAILED
);
+ console.log(
+ `export failed for fileID:${file.id}, reason:`,
+ e
+ );
logError(
e,
'download and save failed for file during export'
@@ -227,7 +314,8 @@ class ExportService {
}
return { paused: false };
} catch (e) {
- logError(e, 'export failed ');
+ logError(e, 'fileExporter failed');
+ throw e;
}
}
async addFilesQueuedRecord(folder: string, files: File[]) {
@@ -236,7 +324,7 @@ class ExportService {
await this.updateExportRecord(exportRecord, folder);
}
- async addFileExportRecord(folder: string, file: File, type: RecordType) {
+ async addFileExportedRecord(folder: string, file: File, type: RecordType) {
const fileUID = getExportRecordFileUID(file);
const exportRecord = await this.getExportRecord(folder);
exportRecord.queuedFiles = exportRecord.queuedFiles.filter(
@@ -265,28 +353,47 @@ class ExportService {
await this.updateExportRecord(exportRecord, folder);
}
- async updateExportRecord(newData: Record, folder?: string) {
- await this.recordUpdateInProgress;
- this.recordUpdateInProgress = (async () => {
- try {
- if (!folder) {
- folder = getData(LS_KEYS.EXPORT)?.folder;
- }
- const exportRecord = await this.getExportRecord(folder);
- const newRecord = { ...exportRecord, ...newData };
- await this.ElectronAPIs.setExportRecord(
- `${folder}/${EXPORT_RECORD_FILE_NAME}`,
- JSON.stringify(newRecord, null, 2)
- );
- } catch (e) {
- logError(e, 'error updating Export Record');
+ async addCollectionExportedRecord(
+ folder: string,
+ collection: Collection,
+ collectionFolderPath: string
+ ) {
+ const exportRecord = await this.getExportRecord(folder);
+ if (!exportRecord?.exportedCollectionPaths) {
+ exportRecord.exportedCollectionPaths = {};
+ }
+ exportRecord.exportedCollectionPaths = {
+ ...exportRecord.exportedCollectionPaths,
+ [collection.id]: collectionFolderPath,
+ };
+
+ await this.updateExportRecord(exportRecord, folder);
+ }
+
+ async updateExportRecord(newData: ExportRecord, folder?: string) {
+ const response = this.exportRecordUpdater.queueUpRequest(() =>
+ this.updateExportRecordHelper(folder, newData)
+ );
+ await response.promise;
+ }
+ async updateExportRecordHelper(folder: string, newData: ExportRecord) {
+ try {
+ if (!folder) {
+ folder = getData(LS_KEYS.EXPORT)?.folder;
}
- })();
+ const exportRecord = await this.getExportRecord(folder);
+ const newRecord = { ...exportRecord, ...newData };
+ await this.ElectronAPIs.setExportRecord(
+ `${folder}/${EXPORT_RECORD_FILE_NAME}`,
+ JSON.stringify(newRecord, null, 2)
+ );
+ } catch (e) {
+ logError(e, 'error updating Export Record');
+ }
}
async getExportRecord(folder?: string): Promise {
try {
- await this.recordUpdateInProgress;
if (!folder) {
folder = getData(LS_KEYS.EXPORT)?.folder;
}
@@ -303,16 +410,89 @@ class ExportService {
}
}
+ async createNewCollectionFolders(
+ newCollections: Collection[],
+ exportFolder: string,
+ collectionIDPathMap: CollectionIDPathMap
+ ) {
+ for (const collection of newCollections) {
+ const collectionFolderPath = getUniqueCollectionFolderPath(
+ exportFolder,
+ collection
+ );
+ await this.ElectronAPIs.checkExistsAndCreateCollectionDir(
+ collectionFolderPath
+ );
+ await this.ElectronAPIs.checkExistsAndCreateCollectionDir(
+ getMetadataFolderPath(collectionFolderPath)
+ );
+ await this.addCollectionExportedRecord(
+ exportFolder,
+ collection,
+ collectionFolderPath
+ );
+ collectionIDPathMap.set(collection.id, collectionFolderPath);
+ }
+ }
+ async renameCollectionFolders(
+ renamedCollections: Collection[],
+ exportFolder: string,
+ collectionIDPathMap: CollectionIDPathMap
+ ) {
+ for (const collection of renamedCollections) {
+ const oldCollectionFolderPath = collectionIDPathMap.get(
+ collection.id
+ );
+
+ const newCollectionFolderPath = getUniqueCollectionFolderPath(
+ exportFolder,
+ collection
+ );
+ await this.ElectronAPIs.checkExistsAndRename(
+ oldCollectionFolderPath,
+ newCollectionFolderPath
+ );
+
+ await this.addCollectionExportedRecord(
+ exportFolder,
+ collection,
+ newCollectionFolderPath
+ );
+ collectionIDPathMap.set(collection.id, newCollectionFolderPath);
+ }
+ }
+
async downloadAndSave(file: File, collectionPath: string) {
- const uid = `${file.id}_${this.sanitizeName(file.metadata.title)}`;
- const fileStream = await retryAsyncFunction(() =>
+ file.metadata = mergeMetadata([file])[0].metadata;
+ const fileSaveName = getUniqueFileSaveName(
+ collectionPath,
+ file.metadata.title,
+ file.id
+ );
+ let fileStream = await retryAsyncFunction(() =>
downloadManager.downloadFile(file)
);
+ const fileType = getFileExtension(file.metadata.title);
+ if (
+ file.pubMagicMetadata?.data.editedTime &&
+ (fileType === TYPE_JPEG || fileType === TYPE_JPG)
+ ) {
+ const fileBlob = await new Response(fileStream).blob();
+ const updatedFileBlob = await updateFileCreationDateInEXIF(
+ fileBlob,
+ new Date(file.pubMagicMetadata.data.editedTime / 1000)
+ );
+ fileStream = updatedFileBlob.stream();
+ }
if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
- this.exportMotionPhoto(fileStream, file, collectionPath);
+ await this.exportMotionPhoto(fileStream, file, collectionPath);
} else {
- this.saveMediaFile(collectionPath, uid, fileStream);
- this.saveMetadataFile(collectionPath, uid, file.metadata);
+ this.saveMediaFile(collectionPath, fileSaveName, fileStream);
+ await this.saveMetadataFile(
+ collectionPath,
+ fileSaveName,
+ file.metadata
+ );
}
}
@@ -324,37 +504,173 @@ class ExportService {
const fileBlob = await new Response(fileStream).blob();
const originalName = fileNameWithoutExtension(file.metadata.title);
const motionPhoto = await decodeMotionPhoto(fileBlob, originalName);
-
const imageStream = generateStreamFromArrayBuffer(motionPhoto.image);
- const imageUID = `${file.id}_${motionPhoto.imageNameTitle}`;
- this.saveMediaFile(collectionPath, imageUID, imageStream);
- this.saveMetadataFile(collectionPath, imageUID, file.metadata);
+ const imageSaveName = getUniqueFileSaveName(
+ collectionPath,
+ motionPhoto.imageNameTitle,
+ file.id
+ );
+ this.saveMediaFile(collectionPath, imageSaveName, imageStream);
+ await this.saveMetadataFile(
+ collectionPath,
+ imageSaveName,
+ file.metadata
+ );
const videoStream = generateStreamFromArrayBuffer(motionPhoto.video);
- const videoUID = `${file.id}_${motionPhoto.videoNameTitle}`;
- this.saveMediaFile(collectionPath, videoUID, videoStream);
- this.saveMetadataFile(collectionPath, videoUID, file.metadata);
+ const videoSaveName = getUniqueFileSaveName(
+ collectionPath,
+ motionPhoto.videoNameTitle,
+ file.id
+ );
+ this.saveMediaFile(collectionPath, videoSaveName, videoStream);
+ await this.saveMetadataFile(
+ collectionPath,
+ videoSaveName,
+ file.metadata
+ );
}
- private saveMediaFile(collectionPath, uid, fileStream) {
+ private saveMediaFile(
+ collectionFolderPath: string,
+ fileSaveName: string,
+ fileStream: ReadableStream
+ ) {
this.ElectronAPIs.saveStreamToDisk(
- `${collectionPath}/${uid}`,
+ getFileSavePath(collectionFolderPath, fileSaveName),
fileStream
);
}
- private saveMetadataFile(collectionPath, uid, metadata) {
- this.ElectronAPIs.saveFileToDisk(
- `${collectionPath}/${METADATA_FOLDER_NAME}/${uid}.json`,
- getGoogleLikeMetadataFile(uid, metadata)
+ private async saveMetadataFile(
+ collectionFolderPath: string,
+ fileSaveName: string,
+ metadata: MetadataObject
+ ) {
+ await this.ElectronAPIs.saveFileToDisk(
+ getFileMetadataSavePath(collectionFolderPath, fileSaveName),
+ getGoogleLikeMetadataFile(fileSaveName, metadata)
);
}
- private sanitizeName(name) {
- return name.replaceAll('/', '_').replaceAll(' ', '_');
- }
-
isExportInProgress = () => {
return this.exportInProgress !== null;
};
+
+ exists = (path: string) => {
+ return this.ElectronAPIs.exists(path);
+ };
+
+ checkAllElectronAPIsExists = () => this.allElectronAPIsExist;
+
+ /*
+ this function migrates the exportRecord file to apply any schema changes.
+ currently we apply only a single migration to update file and collection name to newer format
+ so there is just a if condition check,
+ later this will be converted to a loop which applies the migration one by one
+ till the files reaches the latest version
+ */
+ private async migrateExport(
+ exportDir: string,
+ collections: Collection[],
+ allFiles: File[]
+ ) {
+ const exportRecord = await this.getExportRecord(exportDir);
+ const currentVersion = exportRecord?.version ?? 0;
+ if (currentVersion === 0) {
+ const collectionIDPathMap = new Map();
+
+ await this.migrateCollectionFolders(
+ collections,
+ exportDir,
+ collectionIDPathMap
+ );
+ await this.migrateFiles(
+ getExportedFiles(allFiles, exportRecord),
+ collectionIDPathMap
+ );
+
+ await this.updateExportRecord({
+ version: LATEST_EXPORT_VERSION,
+ });
+ }
+ }
+
+ /*
+ This updates the folder name of already exported folders from the earlier format of
+ `collectionID_collectionName` to newer `collectionName(numbered)` format
+ */
+ private async migrateCollectionFolders(
+ collections: Collection[],
+ exportDir: string,
+ collectionIDPathMap: CollectionIDPathMap
+ ) {
+ for (const collection of collections) {
+ const oldCollectionFolderPath = getOldCollectionFolderPath(
+ exportDir,
+ collection
+ );
+ const newCollectionFolderPath = getUniqueCollectionFolderPath(
+ exportDir,
+ collection
+ );
+ collectionIDPathMap.set(collection.id, newCollectionFolderPath);
+ if (this.ElectronAPIs.exists(oldCollectionFolderPath)) {
+ await this.ElectronAPIs.checkExistsAndRename(
+ oldCollectionFolderPath,
+ newCollectionFolderPath
+ );
+ await this.addCollectionExportedRecord(
+ exportDir,
+ collection,
+ newCollectionFolderPath
+ );
+ }
+ }
+ }
+
+ /*
+ This updates the file name of already exported files from the earlier format of
+ `fileID_fileName` to newer `fileName(numbered)` format
+ */
+ private async migrateFiles(
+ files: File[],
+ collectionIDPathMap: Map
+ ) {
+ for (let file of files) {
+ const oldFileSavePath = getOldFileSavePath(
+ collectionIDPathMap.get(file.collectionID),
+ file
+ );
+ const oldFileMetadataSavePath = getOldFileMetadataSavePath(
+ collectionIDPathMap.get(file.collectionID),
+ file
+ );
+ file = mergeMetadata([file])[0];
+ const newFileSaveName = getUniqueFileSaveName(
+ collectionIDPathMap.get(file.collectionID),
+ file.metadata.title,
+ file.id
+ );
+
+ const newFileSavePath = getFileSavePath(
+ collectionIDPathMap.get(file.collectionID),
+ newFileSaveName
+ );
+
+ const newFileMetadataSavePath = getFileMetadataSavePath(
+ collectionIDPathMap.get(file.collectionID),
+ newFileSaveName
+ );
+ await this.ElectronAPIs.checkExistsAndRename(
+ oldFileSavePath,
+ newFileSavePath
+ );
+ console.log(oldFileMetadataSavePath, newFileMetadataSavePath);
+ await this.ElectronAPIs.checkExistsAndRename(
+ oldFileMetadataSavePath,
+ newFileMetadataSavePath
+ );
+ }
+ }
}
export default new ExportService();
diff --git a/src/services/motionPhotoService.ts b/src/services/motionPhotoService.ts
index 425a86fcd..27b8440da 100644
--- a/src/services/motionPhotoService.ts
+++ b/src/services/motionPhotoService.ts
@@ -4,8 +4,8 @@ import { fileExtensionWithDot } from 'utils/file';
class MotionPhoto {
image: Uint8Array;
video: Uint8Array;
- imageNameTitle: String;
- videoNameTitle: String;
+ imageNameTitle: string;
+ videoNameTitle: string;
}
export const decodeMotionPhoto = async (
diff --git a/src/services/upload/exifService.ts b/src/services/upload/exifService.ts
index d93a49caf..5792267c8 100644
--- a/src/services/upload/exifService.ts
+++ b/src/services/upload/exifService.ts
@@ -1,6 +1,6 @@
import exifr from 'exifr';
+import piexif from 'piexifjs';
import { logError } from 'utils/sentry';
-
import { NULL_LOCATION, Location } from './metadataService';
import { FileTypeInfo } from './readFileService';
@@ -55,6 +55,65 @@ export async function getExifData(
}
}
+export async function updateFileCreationDateInEXIF(
+ fileBlob: Blob,
+ updatedDate: Date
+) {
+ try {
+ const fileURL = URL.createObjectURL(fileBlob);
+ let imageDataURL = await convertImageToDataURL(fileURL);
+ imageDataURL =
+ 'data:image/jpeg;base64' +
+ imageDataURL.slice(imageDataURL.indexOf(','));
+ const exifObj = piexif.load(imageDataURL);
+ if (!exifObj['Exif']) {
+ exifObj['Exif'] = {};
+ }
+ exifObj['Exif'][piexif.ExifIFD.DateTimeOriginal] =
+ convertToExifDateFormat(updatedDate);
+
+ const exifBytes = piexif.dump(exifObj);
+ const exifInsertedFile = piexif.insert(exifBytes, imageDataURL);
+ return dataURIToBlob(exifInsertedFile);
+ } catch (e) {
+ logError(e, 'updateFileModifyDateInEXIF failed');
+ return fileBlob;
+ }
+}
+
+export async function convertImageToDataURL(url: string) {
+ const blob = await fetch(url).then((r) => r.blob());
+ const dataUrl = await new Promise((resolve) => {
+ const reader = new FileReader();
+ reader.onload = () => resolve(reader.result as string);
+ reader.readAsDataURL(blob);
+ });
+ return dataUrl;
+}
+
+function dataURIToBlob(dataURI) {
+ // convert base64 to raw binary data held in a string
+ // doesn't handle URLEncoded DataURIs - see SO answer #6850276 for code that does this
+ const byteString = atob(dataURI.split(',')[1]);
+
+ // separate out the mime component
+ const mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0];
+
+ // write the bytes of the string to an ArrayBuffer
+ const ab = new ArrayBuffer(byteString.length);
+
+ // create a view into the buffer
+ const ia = new Uint8Array(ab);
+
+ // set the bytes of the buffer to the correct values
+ for (let i = 0; i < byteString.length; i++) {
+ ia[i] = byteString.charCodeAt(i);
+ }
+
+ // write the ArrayBuffer to a blob, and you're done
+ const blob = new Blob([ab], { type: mimeString });
+ return blob;
+}
export async function getRawExif(
receivedFile: File,
fileTypeInfo: FileTypeInfo
@@ -93,3 +152,9 @@ function getEXIFLocation(exifData): Location {
}
return { latitude: exifData.latitude, longitude: exifData.longitude };
}
+
+function convertToExifDateFormat(date: Date) {
+ return `${date.getFullYear()}:${
+ date.getMonth() + 1
+ }:${date.getDate()} ${date.getHours()}:${date.getMinutes()}:${date.getSeconds()}`;
+}
diff --git a/src/services/upload/readFileService.ts b/src/services/upload/readFileService.ts
index 3342fd572..b7742b67b 100644
--- a/src/services/upload/readFileService.ts
+++ b/src/services/upload/readFileService.ts
@@ -6,6 +6,7 @@ import { logError } from 'utils/sentry';
import { FILE_READER_CHUNK_SIZE, MULTIPART_PART_SIZE } from './uploadService';
import FileType from 'file-type/browser';
import { CustomError } from 'utils/common/errorUtil';
+import { getFileExtension } from 'utils/file';
const TYPE_VIDEO = 'video';
const TYPE_IMAGE = 'image';
@@ -48,7 +49,7 @@ export async function getFileType(
}
return { fileType, exactType: typeParts[1] };
} catch (e) {
- const fileFormat = receivedFile.name.split('.').pop();
+ const fileFormat = getFileExtension(receivedFile.name);
const formatMissedByTypeDetection = FORMAT_MISSED_BY_FILE_TYPE_LIB.find(
(a) => a.exactType === fileFormat
);
diff --git a/src/services/upload/uploadManager.ts b/src/services/upload/uploadManager.ts
index 1c67cbb6d..fe9db1016 100644
--- a/src/services/upload/uploadManager.ts
+++ b/src/services/upload/uploadManager.ts
@@ -125,7 +125,7 @@ class UploadManager {
fileWithCollection.collectionID,
title
),
- parsedMetaDataJSON
+ { ...parsedMetaDataJSON }
);
UIService.increaseFileUploaded();
}
diff --git a/src/utils/export/index.ts b/src/utils/export/index.ts
index 1aca1446b..ffad98a7f 100644
--- a/src/utils/export/index.ts
+++ b/src/utils/export/index.ts
@@ -1,46 +1,120 @@
-import { ExportRecord } from 'services/exportService';
+import { Collection } from 'services/collectionService';
+import exportService, {
+ CollectionIDPathMap,
+ ExportRecord,
+ METADATA_FOLDER_NAME,
+} from 'services/exportService';
import { File } from 'services/fileService';
import { MetadataObject } from 'services/upload/uploadService';
-import { formatDate } from 'utils/file';
+import { formatDate, splitFilenameAndExtension } from 'utils/file';
export const getExportRecordFileUID = (file: File) =>
`${file.id}_${file.collectionID}_${file.updationTime}`;
-export const getExportPendingFiles = async (
+export const getExportQueuedFiles = (
allFiles: File[],
exportRecord: ExportRecord
) => {
const queuedFiles = new Set(exportRecord?.queuedFiles);
const unExportedFiles = allFiles.filter((file) => {
if (queuedFiles.has(getExportRecordFileUID(file))) {
- return file;
+ return true;
}
+ return false;
});
return unExportedFiles;
};
-export const getFilesUploadedAfterLastExport = async (
+export const getCollectionsCreatedAfterLastExport = (
+ collections: Collection[],
+ exportRecord: ExportRecord
+) => {
+ const exportedCollections = new Set(
+ Object.keys(exportRecord?.exportedCollectionPaths ?? {}).map((x) =>
+ Number(x)
+ )
+ );
+ const unExportedCollections = collections.filter((collection) => {
+ if (!exportedCollections.has(collection.id)) {
+ return true;
+ }
+ return false;
+ });
+ return unExportedCollections;
+};
+export const getCollectionIDPathMapFromExportRecord = (
+ exportRecord: ExportRecord
+): CollectionIDPathMap => {
+ return new Map(
+ Object.entries(exportRecord.exportedCollectionPaths ?? {}).map((e) => {
+ return [Number(e[0]), String(e[1])];
+ })
+ );
+};
+
+export const getCollectionsRenamedAfterLastExport = (
+ collections: Collection[],
+ exportRecord: ExportRecord
+) => {
+ const collectionIDPathMap =
+ getCollectionIDPathMapFromExportRecord(exportRecord);
+ const renamedCollections = collections.filter((collection) => {
+ if (collectionIDPathMap.has(collection.id)) {
+ const currentFolderName = collectionIDPathMap.get(collection.id);
+ const startIndex = currentFolderName.lastIndexOf('/');
+ const lastIndex = currentFolderName.lastIndexOf('(');
+ const nameRoot = currentFolderName.slice(
+ startIndex + 1,
+ lastIndex !== -1 ? lastIndex : currentFolderName.length
+ );
+
+ if (nameRoot !== sanitizeName(collection.name)) {
+ return true;
+ }
+ }
+ return false;
+ });
+ return renamedCollections;
+};
+
+export const getFilesUploadedAfterLastExport = (
allFiles: File[],
exportRecord: ExportRecord
) => {
const exportedFiles = new Set(exportRecord?.exportedFiles);
const unExportedFiles = allFiles.filter((file) => {
if (!exportedFiles.has(getExportRecordFileUID(file))) {
- return file;
+ return true;
}
+ return false;
});
return unExportedFiles;
};
-export const getExportFailedFiles = async (
+export const getExportedFiles = (
+ allFiles: File[],
+ exportRecord: ExportRecord
+) => {
+ const exportedFileIds = new Set(exportRecord?.exportedFiles);
+ const exportedFiles = allFiles.filter((file) => {
+ if (exportedFileIds.has(getExportRecordFileUID(file))) {
+ return true;
+ }
+ return false;
+ });
+ return exportedFiles;
+};
+
+export const getExportFailedFiles = (
allFiles: File[],
exportRecord: ExportRecord
) => {
const failedFiles = new Set(exportRecord?.failedFiles);
const filesToExport = allFiles.filter((file) => {
if (failedFiles.has(getExportRecordFileUID(file))) {
- return file;
+ return true;
}
+ return false;
});
return filesToExport;
};
@@ -52,14 +126,16 @@ export const dedupe = (files: any[]) => {
};
export const getGoogleLikeMetadataFile = (
- uid: string,
+ fileSaveName: string,
metadata: MetadataObject
) => {
const creationTime = Math.floor(metadata.creationTime / 1000000);
- const modificationTime = Math.floor(metadata.modificationTime / 1000000);
+ const modificationTime = Math.floor(
+ (metadata.modificationTime ?? metadata.creationTime) / 1000000
+ );
return JSON.stringify(
{
- title: uid,
+ title: fileSaveName,
creationTime: {
timestamp: creationTime,
formatted: formatDate(creationTime * 1000),
@@ -77,3 +153,85 @@ export const getGoogleLikeMetadataFile = (
2
);
};
+
+export const oldSanitizeName = (name: string) =>
+ name.replaceAll('/', '_').replaceAll(' ', '_');
+
+export const sanitizeName = (name: string) =>
+ name.replace(/[^a-z0-9.]/gi, '_').toLowerCase();
+
+export const getUniqueCollectionFolderPath = (
+ dir: string,
+ collection: Collection
+): string => {
+ if (!exportService.checkAllElectronAPIsExists()) {
+ return getOldCollectionFolderPath(dir, collection);
+ }
+ let collectionFolderPath = `${dir}/${sanitizeName(collection.name)}`;
+ let count = 1;
+ while (exportService.exists(collectionFolderPath)) {
+ collectionFolderPath = `${dir}/${sanitizeName(
+ collection.name
+ )}(${count})`;
+ count++;
+ }
+ return collectionFolderPath;
+};
+
+export const getMetadataFolderPath = (collectionFolderPath: string) =>
+ `${collectionFolderPath}/${METADATA_FOLDER_NAME}`;
+
+export const getUniqueFileSaveName = (
+ collectionPath: string,
+ filename: string,
+ fileID: number
+) => {
+ if (!exportService.checkAllElectronAPIsExists()) {
+ return getOldFileSaveName(filename, fileID);
+ }
+ let fileSaveName = sanitizeName(filename);
+ let count = 1;
+ while (
+ exportService.exists(getFileSavePath(collectionPath, fileSaveName))
+ ) {
+ const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
+ if (filenameParts[1]) {
+ fileSaveName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
+ } else {
+ fileSaveName = `${filenameParts[0]}(${count})`;
+ }
+ count++;
+ }
+ return fileSaveName;
+};
+
+export const getOldFileSaveName = (filename: string, fileID: number) =>
+ `${fileID}_${oldSanitizeName(filename)}`;
+
+export const getFileMetadataSavePath = (
+ collectionFolderPath: string,
+ fileSaveName: string
+) => `${collectionFolderPath}/${METADATA_FOLDER_NAME}/${fileSaveName}.json`;
+
+export const getFileSavePath = (
+ collectionFolderPath: string,
+ fileSaveName: string
+) => `${collectionFolderPath}/${fileSaveName}`;
+
+export const getOldCollectionFolderPath = (
+ dir: string,
+ collection: Collection
+) => `${dir}/${collection.id}_${oldSanitizeName(collection.name)}`;
+
+export const getOldFileSavePath = (collectionFolderPath: string, file: File) =>
+ `${collectionFolderPath}/${file.id}_${oldSanitizeName(
+ file.metadata.title
+ )}`;
+
+export const getOldFileMetadataSavePath = (
+ collectionFolderPath: string,
+ file: File
+) =>
+ `${collectionFolderPath}/${METADATA_FOLDER_NAME}/${
+ file.id
+ }_${oldSanitizeName(file.metadata.title)}.json`;
diff --git a/src/utils/file/index.ts b/src/utils/file/index.ts
index 1e7b08f25..ea911e08a 100644
--- a/src/utils/file/index.ts
+++ b/src/utils/file/index.ts
@@ -16,9 +16,12 @@ import { logError } from 'utils/sentry';
import { User } from 'services/userService';
import CryptoWorker from 'utils/crypto';
import { getData, LS_KEYS } from 'utils/storage/localStorage';
+import { updateFileCreationDateInEXIF } from 'services/upload/exifService';
export const TYPE_HEIC = 'heic';
export const TYPE_HEIF = 'heif';
+export const TYPE_JPEG = 'jpeg';
+export const TYPE_JPG = 'jpg';
const UNSUPPORTED_FORMATS = ['flv', 'mkv', '3gp', 'avi', 'wmv'];
export function downloadAsFile(filename: string, content: string) {
@@ -40,13 +43,32 @@ export function downloadAsFile(filename: string, content: string) {
export async function downloadFile(file: File) {
const a = document.createElement('a');
a.style.display = 'none';
- const cachedFileUrl = await DownloadManager.getCachedOriginalFile(file);
- const fileURL =
- cachedFileUrl ??
- URL.createObjectURL(
+ let fileURL = await DownloadManager.getCachedOriginalFile(file);
+ let tempURL;
+ if (!fileURL) {
+ tempURL = URL.createObjectURL(
await new Response(await DownloadManager.downloadFile(file)).blob()
);
+ fileURL = tempURL;
+ }
+ const fileType = getFileExtension(file.metadata.title);
+ let tempEditedFileURL;
+ if (
+ file.pubMagicMetadata?.data.editedTime &&
+ (fileType === TYPE_JPEG || fileType === TYPE_JPG)
+ ) {
+ let fileBlob = await (await fetch(fileURL)).blob();
+
+ fileBlob = await updateFileCreationDateInEXIF(
+ fileBlob,
+ new Date(file.pubMagicMetadata.data.editedTime / 1000)
+ );
+ tempEditedFileURL = URL.createObjectURL(fileBlob);
+ fileURL = tempEditedFileURL;
+ }
+
a.href = fileURL;
+
if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
a.download = fileNameWithoutExtension(file.metadata.title) + '.zip';
} else {
@@ -55,6 +77,8 @@ export async function downloadFile(file: File) {
document.body.appendChild(a);
a.click();
a.remove();
+ tempURL && URL.revokeObjectURL(tempURL);
+ tempEditedFileURL && URL.revokeObjectURL(tempEditedFileURL);
}
export function isFileHEIC(mimeType: string) {
@@ -257,6 +281,10 @@ export function splitFilenameAndExtension(filename): [string, string] {
];
}
+export function getFileExtension(filename) {
+ return splitFilenameAndExtension(filename)[1];
+}
+
export function generateStreamFromArrayBuffer(data: Uint8Array) {
return new ReadableStream({
async start(controller: ReadableStreamDefaultController) {
@@ -273,7 +301,7 @@ export async function convertForPreview(file: File, fileBlob: Blob) {
fileBlob = new Blob([motionPhoto.image]);
}
- const typeFromExtension = file.metadata.title.split('.')[-1];
+ const typeFromExtension = getFileExtension(file.metadata.title);
const worker = await new CryptoWorker();
const mimeType =
diff --git a/src/utils/sentry/index.ts b/src/utils/sentry/index.ts
index dae3f9996..b3d4c87ca 100644
--- a/src/utils/sentry/index.ts
+++ b/src/utils/sentry/index.ts
@@ -9,7 +9,7 @@ export const logError = (
) => {
const err = errorWithContext(error, msg);
if (!process.env.NEXT_PUBLIC_SENTRY_ENV) {
- console.log({ error, msg, info });
+ console.log(error, { msg, info });
}
Sentry.captureException(err, {
level: Sentry.Severity.Info,
diff --git a/yarn.lock b/yarn.lock
index 67a62af1f..e75a5a4d2 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -5055,6 +5055,11 @@ picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.2, picomatch@^2.2.3:
resolved "https://registry.npmjs.org/picomatch/-/picomatch-2.3.0.tgz"
integrity sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw==
+piexifjs@^1.0.6:
+ version "1.0.6"
+ resolved "https://registry.yarnpkg.com/piexifjs/-/piexifjs-1.0.6.tgz#883811d73f447218d0d06e9ed7866d04533e59e0"
+ integrity sha512-0wVyH0cKohzBQ5Gi2V1BuxYpxWfxF3cSqfFXfPIpl5tl9XLS5z4ogqhUCD20AbHi0h9aJkqXNJnkVev6gwh2ag==
+
pify@^2.0.0:
version "2.3.0"
resolved "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz"