commit
1f0ff4862d
|
@ -41,6 +41,7 @@
|
||||||
"next": "^11.1.3",
|
"next": "^11.1.3",
|
||||||
"node-forge": "^0.10.0",
|
"node-forge": "^0.10.0",
|
||||||
"photoswipe": "file:./thirdparty/photoswipe",
|
"photoswipe": "file:./thirdparty/photoswipe",
|
||||||
|
"piexifjs": "^1.0.6",
|
||||||
"react": "^17.0.2",
|
"react": "^17.0.2",
|
||||||
"react-bootstrap": "^1.3.0",
|
"react-bootstrap": "^1.3.0",
|
||||||
"react-burger-menu": "^3.0.4",
|
"react-burger-menu": "^3.0.4",
|
||||||
|
|
|
@ -29,8 +29,8 @@ export default function ExportFinished(props: Props) {
|
||||||
padding: '0 5%',
|
padding: '0 5%',
|
||||||
}}>
|
}}>
|
||||||
<Row>
|
<Row>
|
||||||
<Label width="40%">{constants.LAST_EXPORT_TIME}</Label>
|
<Label width="35%">{constants.LAST_EXPORT_TIME}</Label>
|
||||||
<Value width="60%">
|
<Value width="65%">
|
||||||
{formatDateTime(props.lastExportTime)}
|
{formatDateTime(props.lastExportTime)}
|
||||||
</Value>
|
</Value>
|
||||||
</Row>
|
</Row>
|
||||||
|
@ -38,7 +38,7 @@ export default function ExportFinished(props: Props) {
|
||||||
<Label width="60%">
|
<Label width="60%">
|
||||||
{constants.SUCCESSFULLY_EXPORTED_FILES}
|
{constants.SUCCESSFULLY_EXPORTED_FILES}
|
||||||
</Label>
|
</Label>
|
||||||
<Value width="35%">
|
<Value width="40%">
|
||||||
<ComfySpan>
|
<ComfySpan>
|
||||||
{props.exportStats.success} / {totalFiles}
|
{props.exportStats.success} / {totalFiles}
|
||||||
</ComfySpan>
|
</ComfySpan>
|
||||||
|
|
|
@ -8,9 +8,11 @@ import exportService, {
|
||||||
ExportType,
|
ExportType,
|
||||||
} from 'services/exportService';
|
} from 'services/exportService';
|
||||||
import { getLocalFiles } from 'services/fileService';
|
import { getLocalFiles } from 'services/fileService';
|
||||||
|
import { User } from 'services/userService';
|
||||||
import styled from 'styled-components';
|
import styled from 'styled-components';
|
||||||
import { sleep } from 'utils/common';
|
import { sleep } from 'utils/common';
|
||||||
import { getExportRecordFileUID } from 'utils/export';
|
import { getExportRecordFileUID } from 'utils/export';
|
||||||
|
import { logError } from 'utils/sentry';
|
||||||
import { getData, LS_KEYS, setData } from 'utils/storage/localStorage';
|
import { getData, LS_KEYS, setData } from 'utils/storage/localStorage';
|
||||||
import constants from 'utils/strings/constants';
|
import constants from 'utils/strings/constants';
|
||||||
import { Label, Row, Value } from './Container';
|
import { Label, Row, Value } from './Container';
|
||||||
|
@ -105,12 +107,17 @@ export default function ExportModal(props: Props) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const main = async () => {
|
const main = async () => {
|
||||||
|
const user: User = getData(LS_KEYS.USER);
|
||||||
if (exportStage === ExportStage.FINISHED) {
|
if (exportStage === ExportStage.FINISHED) {
|
||||||
|
try {
|
||||||
const localFiles = await getLocalFiles();
|
const localFiles = await getLocalFiles();
|
||||||
|
const userPersonalFiles = localFiles.filter(
|
||||||
|
(file) => file.ownerID === user?.id
|
||||||
|
);
|
||||||
const exportRecord = await exportService.getExportRecord();
|
const exportRecord = await exportService.getExportRecord();
|
||||||
const exportedFileCnt = exportRecord.exportedFiles.length;
|
const exportedFileCnt = exportRecord.exportedFiles?.length;
|
||||||
const failedFilesCnt = exportRecord.failedFiles.length;
|
const failedFilesCnt = exportRecord.failedFiles?.length;
|
||||||
const syncedFilesCnt = localFiles.length;
|
const syncedFilesCnt = userPersonalFiles.length;
|
||||||
if (syncedFilesCnt > exportedFileCnt + failedFilesCnt) {
|
if (syncedFilesCnt > exportedFileCnt + failedFilesCnt) {
|
||||||
updateExportProgress({
|
updateExportProgress({
|
||||||
current: exportedFileCnt + failedFilesCnt,
|
current: exportedFileCnt + failedFilesCnt,
|
||||||
|
@ -120,9 +127,11 @@ export default function ExportModal(props: Props) {
|
||||||
...exportRecord.exportedFiles,
|
...exportRecord.exportedFiles,
|
||||||
...exportRecord.failedFiles,
|
...exportRecord.failedFiles,
|
||||||
]);
|
]);
|
||||||
const unExportedFiles = localFiles.filter(
|
const unExportedFiles = userPersonalFiles.filter(
|
||||||
(file) =>
|
(file) =>
|
||||||
!exportFileUIDs.has(getExportRecordFileUID(file))
|
!exportFileUIDs.has(
|
||||||
|
getExportRecordFileUID(file)
|
||||||
|
)
|
||||||
);
|
);
|
||||||
exportService.addFilesQueuedRecord(
|
exportService.addFilesQueuedRecord(
|
||||||
exportFolder,
|
exportFolder,
|
||||||
|
@ -130,6 +139,10 @@ export default function ExportModal(props: Props) {
|
||||||
);
|
);
|
||||||
updateExportStage(ExportStage.PAUSED);
|
updateExportStage(ExportStage.PAUSED);
|
||||||
}
|
}
|
||||||
|
} catch (e) {
|
||||||
|
setExportStage(ExportStage.INIT);
|
||||||
|
logError(e, 'error while updating exportModal on reopen');
|
||||||
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
main();
|
main();
|
||||||
|
@ -154,7 +167,7 @@ export default function ExportModal(props: Props) {
|
||||||
|
|
||||||
const updateExportTime = (newTime: number) => {
|
const updateExportTime = (newTime: number) => {
|
||||||
setLastExportTime(newTime);
|
setLastExportTime(newTime);
|
||||||
exportService.updateExportRecord({ time: newTime });
|
exportService.updateExportRecord({ lastAttemptTimestamp: newTime });
|
||||||
};
|
};
|
||||||
|
|
||||||
const updateExportProgress = (newProgress: ExportProgress) => {
|
const updateExportProgress = (newProgress: ExportProgress) => {
|
||||||
|
@ -178,8 +191,8 @@ export default function ExportModal(props: Props) {
|
||||||
updateExportStage(ExportStage.INPROGRESS);
|
updateExportStage(ExportStage.INPROGRESS);
|
||||||
await sleep(100);
|
await sleep(100);
|
||||||
};
|
};
|
||||||
const postExportRun = async (paused: Boolean) => {
|
const postExportRun = async (exportResult?: { paused?: boolean }) => {
|
||||||
if (!paused) {
|
if (!exportResult?.paused) {
|
||||||
updateExportStage(ExportStage.FINISHED);
|
updateExportStage(ExportStage.FINISHED);
|
||||||
await sleep(100);
|
await sleep(100);
|
||||||
updateExportTime(Date.now());
|
updateExportTime(Date.now());
|
||||||
|
@ -189,22 +202,22 @@ export default function ExportModal(props: Props) {
|
||||||
const startExport = async () => {
|
const startExport = async () => {
|
||||||
await preExportRun();
|
await preExportRun();
|
||||||
updateExportProgress({ current: 0, total: 0 });
|
updateExportProgress({ current: 0, total: 0 });
|
||||||
const { paused } = await exportService.exportFiles(
|
const exportResult = await exportService.exportFiles(
|
||||||
updateExportProgress,
|
updateExportProgress,
|
||||||
ExportType.NEW
|
ExportType.NEW
|
||||||
);
|
);
|
||||||
await postExportRun(paused);
|
await postExportRun(exportResult);
|
||||||
};
|
};
|
||||||
|
|
||||||
const stopExport = async () => {
|
const stopExport = async () => {
|
||||||
exportService.stopRunningExport();
|
exportService.stopRunningExport();
|
||||||
postExportRun(false);
|
postExportRun();
|
||||||
};
|
};
|
||||||
|
|
||||||
const pauseExport = () => {
|
const pauseExport = () => {
|
||||||
updateExportStage(ExportStage.PAUSED);
|
updateExportStage(ExportStage.PAUSED);
|
||||||
exportService.pauseRunningExport();
|
exportService.pauseRunningExport();
|
||||||
postExportRun(true);
|
postExportRun({ paused: true });
|
||||||
};
|
};
|
||||||
|
|
||||||
const resumeExport = async () => {
|
const resumeExport = async () => {
|
||||||
|
@ -219,23 +232,23 @@ export default function ExportModal(props: Props) {
|
||||||
current: pausedStageProgress.current + progress.current,
|
current: pausedStageProgress.current + progress.current,
|
||||||
total: pausedStageProgress.current + progress.total,
|
total: pausedStageProgress.current + progress.total,
|
||||||
});
|
});
|
||||||
const { paused } = await exportService.exportFiles(
|
const exportResult = await exportService.exportFiles(
|
||||||
updateExportStatsWithOffset,
|
updateExportStatsWithOffset,
|
||||||
ExportType.PENDING
|
ExportType.PENDING
|
||||||
);
|
);
|
||||||
|
|
||||||
await postExportRun(paused);
|
await postExportRun(exportResult);
|
||||||
};
|
};
|
||||||
|
|
||||||
const retryFailedExport = async () => {
|
const retryFailedExport = async () => {
|
||||||
await preExportRun();
|
await preExportRun();
|
||||||
updateExportProgress({ current: 0, total: exportStats.failed });
|
updateExportProgress({ current: 0, total: exportStats.failed });
|
||||||
|
|
||||||
const { paused } = await exportService.exportFiles(
|
const exportResult = await exportService.exportFiles(
|
||||||
updateExportProgress,
|
updateExportProgress,
|
||||||
ExportType.RETRY_FAILED
|
ExportType.RETRY_FAILED
|
||||||
);
|
);
|
||||||
await postExportRun(paused);
|
await postExportRun(exportResult);
|
||||||
};
|
};
|
||||||
|
|
||||||
const syncExportStatsWithReport = async () => {
|
const syncExportStatsWithReport = async () => {
|
||||||
|
@ -327,11 +340,9 @@ export default function ExportModal(props: Props) {
|
||||||
</Button>
|
</Button>
|
||||||
) : (
|
) : (
|
||||||
<>
|
<>
|
||||||
{/* <span style={{ overflow: 'hidden', direction: 'rtl', height: '1.5rem', width: '90%', whiteSpace: 'nowrap' }}> */}
|
|
||||||
<ExportFolderPathContainer>
|
<ExportFolderPathContainer>
|
||||||
{exportFolder}
|
{exportFolder}
|
||||||
</ExportFolderPathContainer>
|
</ExportFolderPathContainer>
|
||||||
{/* </span> */}
|
|
||||||
{(exportStage === ExportStage.FINISHED ||
|
{(exportStage === ExportStage.FINISHED ||
|
||||||
exportStage === ExportStage.INIT) && (
|
exportStage === ExportStage.INIT) && (
|
||||||
<FolderIconWrapper
|
<FolderIconWrapper
|
||||||
|
|
|
@ -1,11 +1,23 @@
|
||||||
import { runningInBrowser } from 'utils/common';
|
import { runningInBrowser } from 'utils/common';
|
||||||
import {
|
import {
|
||||||
getExportPendingFiles,
|
getExportQueuedFiles,
|
||||||
getExportFailedFiles,
|
getExportFailedFiles,
|
||||||
getFilesUploadedAfterLastExport,
|
getFilesUploadedAfterLastExport,
|
||||||
dedupe,
|
dedupe,
|
||||||
getGoogleLikeMetadataFile,
|
getGoogleLikeMetadataFile,
|
||||||
getExportRecordFileUID,
|
getExportRecordFileUID,
|
||||||
|
getUniqueCollectionFolderPath,
|
||||||
|
getUniqueFileSaveName,
|
||||||
|
getOldFileSavePath,
|
||||||
|
getOldCollectionFolderPath,
|
||||||
|
getFileMetadataSavePath,
|
||||||
|
getFileSavePath,
|
||||||
|
getOldFileMetadataSavePath,
|
||||||
|
getExportedFiles,
|
||||||
|
getMetadataFolderPath,
|
||||||
|
getCollectionsCreatedAfterLastExport,
|
||||||
|
getCollectionsRenamedAfterLastExport,
|
||||||
|
getCollectionIDPathMapFromExportRecord,
|
||||||
} from 'utils/export';
|
} from 'utils/export';
|
||||||
import { retryAsyncFunction } from 'utils/network';
|
import { retryAsyncFunction } from 'utils/network';
|
||||||
import { logError } from 'utils/sentry';
|
import { logError } from 'utils/sentry';
|
||||||
|
@ -21,24 +33,40 @@ import { decodeMotionPhoto } from './motionPhotoService';
|
||||||
import {
|
import {
|
||||||
fileNameWithoutExtension,
|
fileNameWithoutExtension,
|
||||||
generateStreamFromArrayBuffer,
|
generateStreamFromArrayBuffer,
|
||||||
|
getFileExtension,
|
||||||
|
mergeMetadata,
|
||||||
|
TYPE_JPEG,
|
||||||
|
TYPE_JPG,
|
||||||
} from 'utils/file';
|
} from 'utils/file';
|
||||||
|
import { User } from './userService';
|
||||||
|
import { updateFileCreationDateInEXIF } from './upload/exifService';
|
||||||
|
import { MetadataObject } from './upload/uploadService';
|
||||||
|
import QueueProcessor from './upload/queueProcessor';
|
||||||
|
|
||||||
|
export type CollectionIDPathMap = Map<number, string>;
|
||||||
export interface ExportProgress {
|
export interface ExportProgress {
|
||||||
current: number;
|
current: number;
|
||||||
total: number;
|
total: number;
|
||||||
}
|
}
|
||||||
|
export interface ExportedCollectionPaths {
|
||||||
|
[collectionID: number]: string;
|
||||||
|
}
|
||||||
export interface ExportStats {
|
export interface ExportStats {
|
||||||
failed: number;
|
failed: number;
|
||||||
success: number;
|
success: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const LATEST_EXPORT_VERSION = 1;
|
||||||
|
|
||||||
export interface ExportRecord {
|
export interface ExportRecord {
|
||||||
stage: ExportStage;
|
version?: number;
|
||||||
lastAttemptTimestamp: number;
|
stage?: ExportStage;
|
||||||
progress: ExportProgress;
|
lastAttemptTimestamp?: number;
|
||||||
queuedFiles: string[];
|
progress?: ExportProgress;
|
||||||
exportedFiles: string[];
|
queuedFiles?: string[];
|
||||||
failedFiles: string[];
|
exportedFiles?: string[];
|
||||||
|
failedFiles?: string[];
|
||||||
|
exportedCollectionPaths?: ExportedCollectionPaths;
|
||||||
}
|
}
|
||||||
export enum ExportStage {
|
export enum ExportStage {
|
||||||
INIT,
|
INIT,
|
||||||
|
@ -74,12 +102,14 @@ class ExportService {
|
||||||
ElectronAPIs: any;
|
ElectronAPIs: any;
|
||||||
|
|
||||||
private exportInProgress: Promise<{ paused: boolean }> = null;
|
private exportInProgress: Promise<{ paused: boolean }> = null;
|
||||||
private recordUpdateInProgress = Promise.resolve();
|
private exportRecordUpdater = new QueueProcessor<void>(1);
|
||||||
private stopExport: boolean = false;
|
private stopExport: boolean = false;
|
||||||
private pauseExport: boolean = false;
|
private pauseExport: boolean = false;
|
||||||
|
private allElectronAPIsExist: boolean = false;
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
this.ElectronAPIs = runningInBrowser() && window['ElectronAPIs'];
|
this.ElectronAPIs = runningInBrowser() && window['ElectronAPIs'];
|
||||||
|
this.allElectronAPIsExist = !!this.ElectronAPIs.exists;
|
||||||
}
|
}
|
||||||
async selectExportDirectory() {
|
async selectExportDirectory() {
|
||||||
return await this.ElectronAPIs.selectRootDirectory();
|
return await this.ElectronAPIs.selectRootDirectory();
|
||||||
|
@ -94,9 +124,12 @@ class ExportService {
|
||||||
updateProgress: (progress: ExportProgress) => void,
|
updateProgress: (progress: ExportProgress) => void,
|
||||||
exportType: ExportType
|
exportType: ExportType
|
||||||
) {
|
) {
|
||||||
|
try {
|
||||||
if (this.exportInProgress) {
|
if (this.exportInProgress) {
|
||||||
this.ElectronAPIs.sendNotification(ExportNotification.IN_PROGRESS);
|
this.ElectronAPIs.sendNotification(
|
||||||
return this.exportInProgress;
|
ExportNotification.IN_PROGRESS
|
||||||
|
);
|
||||||
|
return await this.exportInProgress;
|
||||||
}
|
}
|
||||||
this.ElectronAPIs.showOnTray('starting export');
|
this.ElectronAPIs.showOnTray('starting export');
|
||||||
const exportDir = getData(LS_KEYS.EXPORT)?.folder;
|
const exportDir = getData(LS_KEYS.EXPORT)?.folder;
|
||||||
|
@ -104,43 +137,104 @@ class ExportService {
|
||||||
// no-export folder set
|
// no-export folder set
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
const user: User = getData(LS_KEYS.USER);
|
||||||
|
|
||||||
let filesToExport: File[];
|
let filesToExport: File[];
|
||||||
const allFiles = await getLocalFiles();
|
const localFiles = await getLocalFiles();
|
||||||
|
const userPersonalFiles = localFiles
|
||||||
|
.filter((file) => file.ownerID === user?.id)
|
||||||
|
.sort((fileA, fileB) => fileA.id - fileB.id);
|
||||||
|
|
||||||
const collections = await getLocalCollections();
|
const collections = await getLocalCollections();
|
||||||
const nonEmptyCollections = getNonEmptyCollections(
|
const nonEmptyCollections = getNonEmptyCollections(
|
||||||
collections,
|
collections,
|
||||||
allFiles
|
userPersonalFiles
|
||||||
);
|
);
|
||||||
|
const userCollections = nonEmptyCollections
|
||||||
|
.filter((collection) => collection.owner.id === user?.id)
|
||||||
|
.sort(
|
||||||
|
(collectionA, collectionB) =>
|
||||||
|
collectionA.id - collectionB.id
|
||||||
|
);
|
||||||
|
if (this.checkAllElectronAPIsExists()) {
|
||||||
|
await this.migrateExport(
|
||||||
|
exportDir,
|
||||||
|
collections,
|
||||||
|
userPersonalFiles
|
||||||
|
);
|
||||||
|
}
|
||||||
const exportRecord = await this.getExportRecord(exportDir);
|
const exportRecord = await this.getExportRecord(exportDir);
|
||||||
|
|
||||||
if (exportType === ExportType.NEW) {
|
if (exportType === ExportType.NEW) {
|
||||||
filesToExport = await getFilesUploadedAfterLastExport(
|
filesToExport = getFilesUploadedAfterLastExport(
|
||||||
allFiles,
|
userPersonalFiles,
|
||||||
exportRecord
|
exportRecord
|
||||||
);
|
);
|
||||||
} else if (exportType === ExportType.RETRY_FAILED) {
|
} else if (exportType === ExportType.RETRY_FAILED) {
|
||||||
filesToExport = await getExportFailedFiles(allFiles, exportRecord);
|
filesToExport = getExportFailedFiles(
|
||||||
|
userPersonalFiles,
|
||||||
|
exportRecord
|
||||||
|
);
|
||||||
} else {
|
} else {
|
||||||
filesToExport = await getExportPendingFiles(allFiles, exportRecord);
|
filesToExport = getExportQueuedFiles(
|
||||||
|
userPersonalFiles,
|
||||||
|
exportRecord
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
const collectionIDPathMap: CollectionIDPathMap =
|
||||||
|
getCollectionIDPathMapFromExportRecord(exportRecord);
|
||||||
|
const newCollections = getCollectionsCreatedAfterLastExport(
|
||||||
|
userCollections,
|
||||||
|
exportRecord
|
||||||
|
);
|
||||||
|
|
||||||
|
const renamedCollections = getCollectionsRenamedAfterLastExport(
|
||||||
|
userCollections,
|
||||||
|
exportRecord
|
||||||
|
);
|
||||||
this.exportInProgress = this.fileExporter(
|
this.exportInProgress = this.fileExporter(
|
||||||
filesToExport,
|
filesToExport,
|
||||||
nonEmptyCollections,
|
newCollections,
|
||||||
|
renamedCollections,
|
||||||
|
collectionIDPathMap,
|
||||||
updateProgress,
|
updateProgress,
|
||||||
exportDir
|
exportDir
|
||||||
);
|
);
|
||||||
const resp = await this.exportInProgress;
|
const resp = await this.exportInProgress;
|
||||||
this.exportInProgress = null;
|
this.exportInProgress = null;
|
||||||
return resp;
|
return resp;
|
||||||
|
} catch (e) {
|
||||||
|
logError(e, 'exportFiles failed');
|
||||||
|
return { paused: false };
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fileExporter(
|
async fileExporter(
|
||||||
files: File[],
|
files: File[],
|
||||||
collections: Collection[],
|
newCollections: Collection[],
|
||||||
|
renamedCollections: Collection[],
|
||||||
|
collectionIDPathMap: CollectionIDPathMap,
|
||||||
updateProgress: (progress: ExportProgress) => void,
|
updateProgress: (progress: ExportProgress) => void,
|
||||||
dir: string
|
exportDir: string
|
||||||
): Promise<{ paused: boolean }> {
|
): Promise<{ paused: boolean }> {
|
||||||
try {
|
try {
|
||||||
|
if (newCollections?.length) {
|
||||||
|
await this.createNewCollectionFolders(
|
||||||
|
newCollections,
|
||||||
|
exportDir,
|
||||||
|
collectionIDPathMap
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
renamedCollections?.length &&
|
||||||
|
this.checkAllElectronAPIsExists()
|
||||||
|
) {
|
||||||
|
await this.renameCollectionFolders(
|
||||||
|
renamedCollections,
|
||||||
|
exportDir,
|
||||||
|
collectionIDPathMap
|
||||||
|
);
|
||||||
|
}
|
||||||
if (!files?.length) {
|
if (!files?.length) {
|
||||||
this.ElectronAPIs.sendNotification(
|
this.ElectronAPIs.sendNotification(
|
||||||
ExportNotification.UP_TO_DATE
|
ExportNotification.UP_TO_DATE
|
||||||
|
@ -149,7 +243,7 @@ class ExportService {
|
||||||
}
|
}
|
||||||
this.stopExport = false;
|
this.stopExport = false;
|
||||||
this.pauseExport = false;
|
this.pauseExport = false;
|
||||||
this.addFilesQueuedRecord(dir, files);
|
this.addFilesQueuedRecord(exportDir, files);
|
||||||
const failedFileCount = 0;
|
const failedFileCount = 0;
|
||||||
|
|
||||||
this.ElectronAPIs.showOnTray({
|
this.ElectronAPIs.showOnTray({
|
||||||
|
@ -161,19 +255,6 @@ class ExportService {
|
||||||
});
|
});
|
||||||
this.ElectronAPIs.sendNotification(ExportNotification.START);
|
this.ElectronAPIs.sendNotification(ExportNotification.START);
|
||||||
|
|
||||||
const collectionIDMap = new Map<number, string>();
|
|
||||||
for (const collection of collections) {
|
|
||||||
const collectionFolderPath = `${dir}/${
|
|
||||||
collection.id
|
|
||||||
}_${this.sanitizeName(collection.name)}`;
|
|
||||||
await this.ElectronAPIs.checkExistsAndCreateCollectionDir(
|
|
||||||
collectionFolderPath
|
|
||||||
);
|
|
||||||
await this.ElectronAPIs.checkExistsAndCreateCollectionDir(
|
|
||||||
`${collectionFolderPath}/${METADATA_FOLDER_NAME}`
|
|
||||||
);
|
|
||||||
collectionIDMap.set(collection.id, collectionFolderPath);
|
|
||||||
}
|
|
||||||
for (const [index, file] of files.entries()) {
|
for (const [index, file] of files.entries()) {
|
||||||
if (this.stopExport || this.pauseExport) {
|
if (this.stopExport || this.pauseExport) {
|
||||||
if (this.pauseExport) {
|
if (this.pauseExport) {
|
||||||
|
@ -184,20 +265,26 @@ class ExportService {
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
const collectionPath = collectionIDMap.get(file.collectionID);
|
const collectionPath = collectionIDPathMap.get(
|
||||||
|
file.collectionID
|
||||||
|
);
|
||||||
try {
|
try {
|
||||||
await this.downloadAndSave(file, collectionPath);
|
await this.downloadAndSave(file, collectionPath);
|
||||||
await this.addFileExportRecord(
|
await this.addFileExportedRecord(
|
||||||
dir,
|
exportDir,
|
||||||
file,
|
file,
|
||||||
RecordType.SUCCESS
|
RecordType.SUCCESS
|
||||||
);
|
);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
await this.addFileExportRecord(
|
await this.addFileExportedRecord(
|
||||||
dir,
|
exportDir,
|
||||||
file,
|
file,
|
||||||
RecordType.FAILED
|
RecordType.FAILED
|
||||||
);
|
);
|
||||||
|
console.log(
|
||||||
|
`export failed for fileID:${file.id}, reason:`,
|
||||||
|
e
|
||||||
|
);
|
||||||
logError(
|
logError(
|
||||||
e,
|
e,
|
||||||
'download and save failed for file during export'
|
'download and save failed for file during export'
|
||||||
|
@ -227,7 +314,8 @@ class ExportService {
|
||||||
}
|
}
|
||||||
return { paused: false };
|
return { paused: false };
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
logError(e, 'export failed ');
|
logError(e, 'fileExporter failed');
|
||||||
|
throw e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
async addFilesQueuedRecord(folder: string, files: File[]) {
|
async addFilesQueuedRecord(folder: string, files: File[]) {
|
||||||
|
@ -236,7 +324,7 @@ class ExportService {
|
||||||
await this.updateExportRecord(exportRecord, folder);
|
await this.updateExportRecord(exportRecord, folder);
|
||||||
}
|
}
|
||||||
|
|
||||||
async addFileExportRecord(folder: string, file: File, type: RecordType) {
|
async addFileExportedRecord(folder: string, file: File, type: RecordType) {
|
||||||
const fileUID = getExportRecordFileUID(file);
|
const fileUID = getExportRecordFileUID(file);
|
||||||
const exportRecord = await this.getExportRecord(folder);
|
const exportRecord = await this.getExportRecord(folder);
|
||||||
exportRecord.queuedFiles = exportRecord.queuedFiles.filter(
|
exportRecord.queuedFiles = exportRecord.queuedFiles.filter(
|
||||||
|
@ -265,9 +353,30 @@ class ExportService {
|
||||||
await this.updateExportRecord(exportRecord, folder);
|
await this.updateExportRecord(exportRecord, folder);
|
||||||
}
|
}
|
||||||
|
|
||||||
async updateExportRecord(newData: Record<string, any>, folder?: string) {
|
async addCollectionExportedRecord(
|
||||||
await this.recordUpdateInProgress;
|
folder: string,
|
||||||
this.recordUpdateInProgress = (async () => {
|
collection: Collection,
|
||||||
|
collectionFolderPath: string
|
||||||
|
) {
|
||||||
|
const exportRecord = await this.getExportRecord(folder);
|
||||||
|
if (!exportRecord?.exportedCollectionPaths) {
|
||||||
|
exportRecord.exportedCollectionPaths = {};
|
||||||
|
}
|
||||||
|
exportRecord.exportedCollectionPaths = {
|
||||||
|
...exportRecord.exportedCollectionPaths,
|
||||||
|
[collection.id]: collectionFolderPath,
|
||||||
|
};
|
||||||
|
|
||||||
|
await this.updateExportRecord(exportRecord, folder);
|
||||||
|
}
|
||||||
|
|
||||||
|
async updateExportRecord(newData: ExportRecord, folder?: string) {
|
||||||
|
const response = this.exportRecordUpdater.queueUpRequest(() =>
|
||||||
|
this.updateExportRecordHelper(folder, newData)
|
||||||
|
);
|
||||||
|
await response.promise;
|
||||||
|
}
|
||||||
|
async updateExportRecordHelper(folder: string, newData: ExportRecord) {
|
||||||
try {
|
try {
|
||||||
if (!folder) {
|
if (!folder) {
|
||||||
folder = getData(LS_KEYS.EXPORT)?.folder;
|
folder = getData(LS_KEYS.EXPORT)?.folder;
|
||||||
|
@ -281,12 +390,10 @@ class ExportService {
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
logError(e, 'error updating Export Record');
|
logError(e, 'error updating Export Record');
|
||||||
}
|
}
|
||||||
})();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async getExportRecord(folder?: string): Promise<ExportRecord> {
|
async getExportRecord(folder?: string): Promise<ExportRecord> {
|
||||||
try {
|
try {
|
||||||
await this.recordUpdateInProgress;
|
|
||||||
if (!folder) {
|
if (!folder) {
|
||||||
folder = getData(LS_KEYS.EXPORT)?.folder;
|
folder = getData(LS_KEYS.EXPORT)?.folder;
|
||||||
}
|
}
|
||||||
|
@ -303,16 +410,89 @@ class ExportService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async createNewCollectionFolders(
|
||||||
|
newCollections: Collection[],
|
||||||
|
exportFolder: string,
|
||||||
|
collectionIDPathMap: CollectionIDPathMap
|
||||||
|
) {
|
||||||
|
for (const collection of newCollections) {
|
||||||
|
const collectionFolderPath = getUniqueCollectionFolderPath(
|
||||||
|
exportFolder,
|
||||||
|
collection
|
||||||
|
);
|
||||||
|
await this.ElectronAPIs.checkExistsAndCreateCollectionDir(
|
||||||
|
collectionFolderPath
|
||||||
|
);
|
||||||
|
await this.ElectronAPIs.checkExistsAndCreateCollectionDir(
|
||||||
|
getMetadataFolderPath(collectionFolderPath)
|
||||||
|
);
|
||||||
|
await this.addCollectionExportedRecord(
|
||||||
|
exportFolder,
|
||||||
|
collection,
|
||||||
|
collectionFolderPath
|
||||||
|
);
|
||||||
|
collectionIDPathMap.set(collection.id, collectionFolderPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
async renameCollectionFolders(
|
||||||
|
renamedCollections: Collection[],
|
||||||
|
exportFolder: string,
|
||||||
|
collectionIDPathMap: CollectionIDPathMap
|
||||||
|
) {
|
||||||
|
for (const collection of renamedCollections) {
|
||||||
|
const oldCollectionFolderPath = collectionIDPathMap.get(
|
||||||
|
collection.id
|
||||||
|
);
|
||||||
|
|
||||||
|
const newCollectionFolderPath = getUniqueCollectionFolderPath(
|
||||||
|
exportFolder,
|
||||||
|
collection
|
||||||
|
);
|
||||||
|
await this.ElectronAPIs.checkExistsAndRename(
|
||||||
|
oldCollectionFolderPath,
|
||||||
|
newCollectionFolderPath
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.addCollectionExportedRecord(
|
||||||
|
exportFolder,
|
||||||
|
collection,
|
||||||
|
newCollectionFolderPath
|
||||||
|
);
|
||||||
|
collectionIDPathMap.set(collection.id, newCollectionFolderPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async downloadAndSave(file: File, collectionPath: string) {
|
async downloadAndSave(file: File, collectionPath: string) {
|
||||||
const uid = `${file.id}_${this.sanitizeName(file.metadata.title)}`;
|
file.metadata = mergeMetadata([file])[0].metadata;
|
||||||
const fileStream = await retryAsyncFunction(() =>
|
const fileSaveName = getUniqueFileSaveName(
|
||||||
|
collectionPath,
|
||||||
|
file.metadata.title,
|
||||||
|
file.id
|
||||||
|
);
|
||||||
|
let fileStream = await retryAsyncFunction(() =>
|
||||||
downloadManager.downloadFile(file)
|
downloadManager.downloadFile(file)
|
||||||
);
|
);
|
||||||
|
const fileType = getFileExtension(file.metadata.title);
|
||||||
|
if (
|
||||||
|
file.pubMagicMetadata?.data.editedTime &&
|
||||||
|
(fileType === TYPE_JPEG || fileType === TYPE_JPG)
|
||||||
|
) {
|
||||||
|
const fileBlob = await new Response(fileStream).blob();
|
||||||
|
const updatedFileBlob = await updateFileCreationDateInEXIF(
|
||||||
|
fileBlob,
|
||||||
|
new Date(file.pubMagicMetadata.data.editedTime / 1000)
|
||||||
|
);
|
||||||
|
fileStream = updatedFileBlob.stream();
|
||||||
|
}
|
||||||
if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
|
if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
|
||||||
this.exportMotionPhoto(fileStream, file, collectionPath);
|
await this.exportMotionPhoto(fileStream, file, collectionPath);
|
||||||
} else {
|
} else {
|
||||||
this.saveMediaFile(collectionPath, uid, fileStream);
|
this.saveMediaFile(collectionPath, fileSaveName, fileStream);
|
||||||
this.saveMetadataFile(collectionPath, uid, file.metadata);
|
await this.saveMetadataFile(
|
||||||
|
collectionPath,
|
||||||
|
fileSaveName,
|
||||||
|
file.metadata
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -324,37 +504,173 @@ class ExportService {
|
||||||
const fileBlob = await new Response(fileStream).blob();
|
const fileBlob = await new Response(fileStream).blob();
|
||||||
const originalName = fileNameWithoutExtension(file.metadata.title);
|
const originalName = fileNameWithoutExtension(file.metadata.title);
|
||||||
const motionPhoto = await decodeMotionPhoto(fileBlob, originalName);
|
const motionPhoto = await decodeMotionPhoto(fileBlob, originalName);
|
||||||
|
|
||||||
const imageStream = generateStreamFromArrayBuffer(motionPhoto.image);
|
const imageStream = generateStreamFromArrayBuffer(motionPhoto.image);
|
||||||
const imageUID = `${file.id}_${motionPhoto.imageNameTitle}`;
|
const imageSaveName = getUniqueFileSaveName(
|
||||||
this.saveMediaFile(collectionPath, imageUID, imageStream);
|
collectionPath,
|
||||||
this.saveMetadataFile(collectionPath, imageUID, file.metadata);
|
motionPhoto.imageNameTitle,
|
||||||
|
file.id
|
||||||
|
);
|
||||||
|
this.saveMediaFile(collectionPath, imageSaveName, imageStream);
|
||||||
|
await this.saveMetadataFile(
|
||||||
|
collectionPath,
|
||||||
|
imageSaveName,
|
||||||
|
file.metadata
|
||||||
|
);
|
||||||
|
|
||||||
const videoStream = generateStreamFromArrayBuffer(motionPhoto.video);
|
const videoStream = generateStreamFromArrayBuffer(motionPhoto.video);
|
||||||
const videoUID = `${file.id}_${motionPhoto.videoNameTitle}`;
|
const videoSaveName = getUniqueFileSaveName(
|
||||||
this.saveMediaFile(collectionPath, videoUID, videoStream);
|
collectionPath,
|
||||||
this.saveMetadataFile(collectionPath, videoUID, file.metadata);
|
motionPhoto.videoNameTitle,
|
||||||
|
file.id
|
||||||
|
);
|
||||||
|
this.saveMediaFile(collectionPath, videoSaveName, videoStream);
|
||||||
|
await this.saveMetadataFile(
|
||||||
|
collectionPath,
|
||||||
|
videoSaveName,
|
||||||
|
file.metadata
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private saveMediaFile(collectionPath, uid, fileStream) {
|
private saveMediaFile(
|
||||||
|
collectionFolderPath: string,
|
||||||
|
fileSaveName: string,
|
||||||
|
fileStream: ReadableStream<any>
|
||||||
|
) {
|
||||||
this.ElectronAPIs.saveStreamToDisk(
|
this.ElectronAPIs.saveStreamToDisk(
|
||||||
`${collectionPath}/${uid}`,
|
getFileSavePath(collectionFolderPath, fileSaveName),
|
||||||
fileStream
|
fileStream
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
private saveMetadataFile(collectionPath, uid, metadata) {
|
private async saveMetadataFile(
|
||||||
this.ElectronAPIs.saveFileToDisk(
|
collectionFolderPath: string,
|
||||||
`${collectionPath}/${METADATA_FOLDER_NAME}/${uid}.json`,
|
fileSaveName: string,
|
||||||
getGoogleLikeMetadataFile(uid, metadata)
|
metadata: MetadataObject
|
||||||
|
) {
|
||||||
|
await this.ElectronAPIs.saveFileToDisk(
|
||||||
|
getFileMetadataSavePath(collectionFolderPath, fileSaveName),
|
||||||
|
getGoogleLikeMetadataFile(fileSaveName, metadata)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private sanitizeName(name) {
|
|
||||||
return name.replaceAll('/', '_').replaceAll(' ', '_');
|
|
||||||
}
|
|
||||||
|
|
||||||
isExportInProgress = () => {
|
isExportInProgress = () => {
|
||||||
return this.exportInProgress !== null;
|
return this.exportInProgress !== null;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
exists = (path: string) => {
|
||||||
|
return this.ElectronAPIs.exists(path);
|
||||||
|
};
|
||||||
|
|
||||||
|
checkAllElectronAPIsExists = () => this.allElectronAPIsExist;
|
||||||
|
|
||||||
|
/*
|
||||||
|
this function migrates the exportRecord file to apply any schema changes.
|
||||||
|
currently we apply only a single migration to update file and collection name to newer format
|
||||||
|
so there is just a if condition check,
|
||||||
|
later this will be converted to a loop which applies the migration one by one
|
||||||
|
till the files reaches the latest version
|
||||||
|
*/
|
||||||
|
private async migrateExport(
|
||||||
|
exportDir: string,
|
||||||
|
collections: Collection[],
|
||||||
|
allFiles: File[]
|
||||||
|
) {
|
||||||
|
const exportRecord = await this.getExportRecord(exportDir);
|
||||||
|
const currentVersion = exportRecord?.version ?? 0;
|
||||||
|
if (currentVersion === 0) {
|
||||||
|
const collectionIDPathMap = new Map<number, string>();
|
||||||
|
|
||||||
|
await this.migrateCollectionFolders(
|
||||||
|
collections,
|
||||||
|
exportDir,
|
||||||
|
collectionIDPathMap
|
||||||
|
);
|
||||||
|
await this.migrateFiles(
|
||||||
|
getExportedFiles(allFiles, exportRecord),
|
||||||
|
collectionIDPathMap
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.updateExportRecord({
|
||||||
|
version: LATEST_EXPORT_VERSION,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
This updates the folder name of already exported folders from the earlier format of
|
||||||
|
`collectionID_collectionName` to newer `collectionName(numbered)` format
|
||||||
|
*/
|
||||||
|
private async migrateCollectionFolders(
|
||||||
|
collections: Collection[],
|
||||||
|
exportDir: string,
|
||||||
|
collectionIDPathMap: CollectionIDPathMap
|
||||||
|
) {
|
||||||
|
for (const collection of collections) {
|
||||||
|
const oldCollectionFolderPath = getOldCollectionFolderPath(
|
||||||
|
exportDir,
|
||||||
|
collection
|
||||||
|
);
|
||||||
|
const newCollectionFolderPath = getUniqueCollectionFolderPath(
|
||||||
|
exportDir,
|
||||||
|
collection
|
||||||
|
);
|
||||||
|
collectionIDPathMap.set(collection.id, newCollectionFolderPath);
|
||||||
|
if (this.ElectronAPIs.exists(oldCollectionFolderPath)) {
|
||||||
|
await this.ElectronAPIs.checkExistsAndRename(
|
||||||
|
oldCollectionFolderPath,
|
||||||
|
newCollectionFolderPath
|
||||||
|
);
|
||||||
|
await this.addCollectionExportedRecord(
|
||||||
|
exportDir,
|
||||||
|
collection,
|
||||||
|
newCollectionFolderPath
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
This updates the file name of already exported files from the earlier format of
|
||||||
|
`fileID_fileName` to newer `fileName(numbered)` format
|
||||||
|
*/
|
||||||
|
private async migrateFiles(
|
||||||
|
files: File[],
|
||||||
|
collectionIDPathMap: Map<number, string>
|
||||||
|
) {
|
||||||
|
for (let file of files) {
|
||||||
|
const oldFileSavePath = getOldFileSavePath(
|
||||||
|
collectionIDPathMap.get(file.collectionID),
|
||||||
|
file
|
||||||
|
);
|
||||||
|
const oldFileMetadataSavePath = getOldFileMetadataSavePath(
|
||||||
|
collectionIDPathMap.get(file.collectionID),
|
||||||
|
file
|
||||||
|
);
|
||||||
|
file = mergeMetadata([file])[0];
|
||||||
|
const newFileSaveName = getUniqueFileSaveName(
|
||||||
|
collectionIDPathMap.get(file.collectionID),
|
||||||
|
file.metadata.title,
|
||||||
|
file.id
|
||||||
|
);
|
||||||
|
|
||||||
|
const newFileSavePath = getFileSavePath(
|
||||||
|
collectionIDPathMap.get(file.collectionID),
|
||||||
|
newFileSaveName
|
||||||
|
);
|
||||||
|
|
||||||
|
const newFileMetadataSavePath = getFileMetadataSavePath(
|
||||||
|
collectionIDPathMap.get(file.collectionID),
|
||||||
|
newFileSaveName
|
||||||
|
);
|
||||||
|
await this.ElectronAPIs.checkExistsAndRename(
|
||||||
|
oldFileSavePath,
|
||||||
|
newFileSavePath
|
||||||
|
);
|
||||||
|
console.log(oldFileMetadataSavePath, newFileMetadataSavePath);
|
||||||
|
await this.ElectronAPIs.checkExistsAndRename(
|
||||||
|
oldFileMetadataSavePath,
|
||||||
|
newFileMetadataSavePath
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
export default new ExportService();
|
export default new ExportService();
|
||||||
|
|
|
@ -4,8 +4,8 @@ import { fileExtensionWithDot } from 'utils/file';
|
||||||
class MotionPhoto {
|
class MotionPhoto {
|
||||||
image: Uint8Array;
|
image: Uint8Array;
|
||||||
video: Uint8Array;
|
video: Uint8Array;
|
||||||
imageNameTitle: String;
|
imageNameTitle: string;
|
||||||
videoNameTitle: String;
|
videoNameTitle: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const decodeMotionPhoto = async (
|
export const decodeMotionPhoto = async (
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
import exifr from 'exifr';
|
import exifr from 'exifr';
|
||||||
|
import piexif from 'piexifjs';
|
||||||
import { logError } from 'utils/sentry';
|
import { logError } from 'utils/sentry';
|
||||||
|
|
||||||
import { NULL_LOCATION, Location } from './metadataService';
|
import { NULL_LOCATION, Location } from './metadataService';
|
||||||
import { FileTypeInfo } from './readFileService';
|
import { FileTypeInfo } from './readFileService';
|
||||||
|
|
||||||
|
@ -55,6 +55,65 @@ export async function getExifData(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function updateFileCreationDateInEXIF(
|
||||||
|
fileBlob: Blob,
|
||||||
|
updatedDate: Date
|
||||||
|
) {
|
||||||
|
try {
|
||||||
|
const fileURL = URL.createObjectURL(fileBlob);
|
||||||
|
let imageDataURL = await convertImageToDataURL(fileURL);
|
||||||
|
imageDataURL =
|
||||||
|
'data:image/jpeg;base64' +
|
||||||
|
imageDataURL.slice(imageDataURL.indexOf(','));
|
||||||
|
const exifObj = piexif.load(imageDataURL);
|
||||||
|
if (!exifObj['Exif']) {
|
||||||
|
exifObj['Exif'] = {};
|
||||||
|
}
|
||||||
|
exifObj['Exif'][piexif.ExifIFD.DateTimeOriginal] =
|
||||||
|
convertToExifDateFormat(updatedDate);
|
||||||
|
|
||||||
|
const exifBytes = piexif.dump(exifObj);
|
||||||
|
const exifInsertedFile = piexif.insert(exifBytes, imageDataURL);
|
||||||
|
return dataURIToBlob(exifInsertedFile);
|
||||||
|
} catch (e) {
|
||||||
|
logError(e, 'updateFileModifyDateInEXIF failed');
|
||||||
|
return fileBlob;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function convertImageToDataURL(url: string) {
|
||||||
|
const blob = await fetch(url).then((r) => r.blob());
|
||||||
|
const dataUrl = await new Promise<string>((resolve) => {
|
||||||
|
const reader = new FileReader();
|
||||||
|
reader.onload = () => resolve(reader.result as string);
|
||||||
|
reader.readAsDataURL(blob);
|
||||||
|
});
|
||||||
|
return dataUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
function dataURIToBlob(dataURI) {
|
||||||
|
// convert base64 to raw binary data held in a string
|
||||||
|
// doesn't handle URLEncoded DataURIs - see SO answer #6850276 for code that does this
|
||||||
|
const byteString = atob(dataURI.split(',')[1]);
|
||||||
|
|
||||||
|
// separate out the mime component
|
||||||
|
const mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0];
|
||||||
|
|
||||||
|
// write the bytes of the string to an ArrayBuffer
|
||||||
|
const ab = new ArrayBuffer(byteString.length);
|
||||||
|
|
||||||
|
// create a view into the buffer
|
||||||
|
const ia = new Uint8Array(ab);
|
||||||
|
|
||||||
|
// set the bytes of the buffer to the correct values
|
||||||
|
for (let i = 0; i < byteString.length; i++) {
|
||||||
|
ia[i] = byteString.charCodeAt(i);
|
||||||
|
}
|
||||||
|
|
||||||
|
// write the ArrayBuffer to a blob, and you're done
|
||||||
|
const blob = new Blob([ab], { type: mimeString });
|
||||||
|
return blob;
|
||||||
|
}
|
||||||
export async function getRawExif(
|
export async function getRawExif(
|
||||||
receivedFile: File,
|
receivedFile: File,
|
||||||
fileTypeInfo: FileTypeInfo
|
fileTypeInfo: FileTypeInfo
|
||||||
|
@ -93,3 +152,9 @@ function getEXIFLocation(exifData): Location {
|
||||||
}
|
}
|
||||||
return { latitude: exifData.latitude, longitude: exifData.longitude };
|
return { latitude: exifData.latitude, longitude: exifData.longitude };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function convertToExifDateFormat(date: Date) {
|
||||||
|
return `${date.getFullYear()}:${
|
||||||
|
date.getMonth() + 1
|
||||||
|
}:${date.getDate()} ${date.getHours()}:${date.getMinutes()}:${date.getSeconds()}`;
|
||||||
|
}
|
||||||
|
|
|
@ -6,6 +6,7 @@ import { logError } from 'utils/sentry';
|
||||||
import { FILE_READER_CHUNK_SIZE, MULTIPART_PART_SIZE } from './uploadService';
|
import { FILE_READER_CHUNK_SIZE, MULTIPART_PART_SIZE } from './uploadService';
|
||||||
import FileType from 'file-type/browser';
|
import FileType from 'file-type/browser';
|
||||||
import { CustomError } from 'utils/common/errorUtil';
|
import { CustomError } from 'utils/common/errorUtil';
|
||||||
|
import { getFileExtension } from 'utils/file';
|
||||||
|
|
||||||
const TYPE_VIDEO = 'video';
|
const TYPE_VIDEO = 'video';
|
||||||
const TYPE_IMAGE = 'image';
|
const TYPE_IMAGE = 'image';
|
||||||
|
@ -48,7 +49,7 @@ export async function getFileType(
|
||||||
}
|
}
|
||||||
return { fileType, exactType: typeParts[1] };
|
return { fileType, exactType: typeParts[1] };
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
const fileFormat = receivedFile.name.split('.').pop();
|
const fileFormat = getFileExtension(receivedFile.name);
|
||||||
const formatMissedByTypeDetection = FORMAT_MISSED_BY_FILE_TYPE_LIB.find(
|
const formatMissedByTypeDetection = FORMAT_MISSED_BY_FILE_TYPE_LIB.find(
|
||||||
(a) => a.exactType === fileFormat
|
(a) => a.exactType === fileFormat
|
||||||
);
|
);
|
||||||
|
|
|
@ -125,7 +125,7 @@ class UploadManager {
|
||||||
fileWithCollection.collectionID,
|
fileWithCollection.collectionID,
|
||||||
title
|
title
|
||||||
),
|
),
|
||||||
parsedMetaDataJSON
|
{ ...parsedMetaDataJSON }
|
||||||
);
|
);
|
||||||
UIService.increaseFileUploaded();
|
UIService.increaseFileUploaded();
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,46 +1,120 @@
|
||||||
import { ExportRecord } from 'services/exportService';
|
import { Collection } from 'services/collectionService';
|
||||||
|
import exportService, {
|
||||||
|
CollectionIDPathMap,
|
||||||
|
ExportRecord,
|
||||||
|
METADATA_FOLDER_NAME,
|
||||||
|
} from 'services/exportService';
|
||||||
import { File } from 'services/fileService';
|
import { File } from 'services/fileService';
|
||||||
import { MetadataObject } from 'services/upload/uploadService';
|
import { MetadataObject } from 'services/upload/uploadService';
|
||||||
import { formatDate } from 'utils/file';
|
import { formatDate, splitFilenameAndExtension } from 'utils/file';
|
||||||
|
|
||||||
export const getExportRecordFileUID = (file: File) =>
|
export const getExportRecordFileUID = (file: File) =>
|
||||||
`${file.id}_${file.collectionID}_${file.updationTime}`;
|
`${file.id}_${file.collectionID}_${file.updationTime}`;
|
||||||
|
|
||||||
export const getExportPendingFiles = async (
|
export const getExportQueuedFiles = (
|
||||||
allFiles: File[],
|
allFiles: File[],
|
||||||
exportRecord: ExportRecord
|
exportRecord: ExportRecord
|
||||||
) => {
|
) => {
|
||||||
const queuedFiles = new Set(exportRecord?.queuedFiles);
|
const queuedFiles = new Set(exportRecord?.queuedFiles);
|
||||||
const unExportedFiles = allFiles.filter((file) => {
|
const unExportedFiles = allFiles.filter((file) => {
|
||||||
if (queuedFiles.has(getExportRecordFileUID(file))) {
|
if (queuedFiles.has(getExportRecordFileUID(file))) {
|
||||||
return file;
|
return true;
|
||||||
}
|
}
|
||||||
|
return false;
|
||||||
});
|
});
|
||||||
return unExportedFiles;
|
return unExportedFiles;
|
||||||
};
|
};
|
||||||
|
|
||||||
export const getFilesUploadedAfterLastExport = async (
|
export const getCollectionsCreatedAfterLastExport = (
|
||||||
|
collections: Collection[],
|
||||||
|
exportRecord: ExportRecord
|
||||||
|
) => {
|
||||||
|
const exportedCollections = new Set(
|
||||||
|
Object.keys(exportRecord?.exportedCollectionPaths ?? {}).map((x) =>
|
||||||
|
Number(x)
|
||||||
|
)
|
||||||
|
);
|
||||||
|
const unExportedCollections = collections.filter((collection) => {
|
||||||
|
if (!exportedCollections.has(collection.id)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
return unExportedCollections;
|
||||||
|
};
|
||||||
|
export const getCollectionIDPathMapFromExportRecord = (
|
||||||
|
exportRecord: ExportRecord
|
||||||
|
): CollectionIDPathMap => {
|
||||||
|
return new Map<number, string>(
|
||||||
|
Object.entries(exportRecord.exportedCollectionPaths ?? {}).map((e) => {
|
||||||
|
return [Number(e[0]), String(e[1])];
|
||||||
|
})
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getCollectionsRenamedAfterLastExport = (
|
||||||
|
collections: Collection[],
|
||||||
|
exportRecord: ExportRecord
|
||||||
|
) => {
|
||||||
|
const collectionIDPathMap =
|
||||||
|
getCollectionIDPathMapFromExportRecord(exportRecord);
|
||||||
|
const renamedCollections = collections.filter((collection) => {
|
||||||
|
if (collectionIDPathMap.has(collection.id)) {
|
||||||
|
const currentFolderName = collectionIDPathMap.get(collection.id);
|
||||||
|
const startIndex = currentFolderName.lastIndexOf('/');
|
||||||
|
const lastIndex = currentFolderName.lastIndexOf('(');
|
||||||
|
const nameRoot = currentFolderName.slice(
|
||||||
|
startIndex + 1,
|
||||||
|
lastIndex !== -1 ? lastIndex : currentFolderName.length
|
||||||
|
);
|
||||||
|
|
||||||
|
if (nameRoot !== sanitizeName(collection.name)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
return renamedCollections;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getFilesUploadedAfterLastExport = (
|
||||||
allFiles: File[],
|
allFiles: File[],
|
||||||
exportRecord: ExportRecord
|
exportRecord: ExportRecord
|
||||||
) => {
|
) => {
|
||||||
const exportedFiles = new Set(exportRecord?.exportedFiles);
|
const exportedFiles = new Set(exportRecord?.exportedFiles);
|
||||||
const unExportedFiles = allFiles.filter((file) => {
|
const unExportedFiles = allFiles.filter((file) => {
|
||||||
if (!exportedFiles.has(getExportRecordFileUID(file))) {
|
if (!exportedFiles.has(getExportRecordFileUID(file))) {
|
||||||
return file;
|
return true;
|
||||||
}
|
}
|
||||||
|
return false;
|
||||||
});
|
});
|
||||||
return unExportedFiles;
|
return unExportedFiles;
|
||||||
};
|
};
|
||||||
|
|
||||||
export const getExportFailedFiles = async (
|
export const getExportedFiles = (
|
||||||
|
allFiles: File[],
|
||||||
|
exportRecord: ExportRecord
|
||||||
|
) => {
|
||||||
|
const exportedFileIds = new Set(exportRecord?.exportedFiles);
|
||||||
|
const exportedFiles = allFiles.filter((file) => {
|
||||||
|
if (exportedFileIds.has(getExportRecordFileUID(file))) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
return exportedFiles;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getExportFailedFiles = (
|
||||||
allFiles: File[],
|
allFiles: File[],
|
||||||
exportRecord: ExportRecord
|
exportRecord: ExportRecord
|
||||||
) => {
|
) => {
|
||||||
const failedFiles = new Set(exportRecord?.failedFiles);
|
const failedFiles = new Set(exportRecord?.failedFiles);
|
||||||
const filesToExport = allFiles.filter((file) => {
|
const filesToExport = allFiles.filter((file) => {
|
||||||
if (failedFiles.has(getExportRecordFileUID(file))) {
|
if (failedFiles.has(getExportRecordFileUID(file))) {
|
||||||
return file;
|
return true;
|
||||||
}
|
}
|
||||||
|
return false;
|
||||||
});
|
});
|
||||||
return filesToExport;
|
return filesToExport;
|
||||||
};
|
};
|
||||||
|
@ -52,14 +126,16 @@ export const dedupe = (files: any[]) => {
|
||||||
};
|
};
|
||||||
|
|
||||||
export const getGoogleLikeMetadataFile = (
|
export const getGoogleLikeMetadataFile = (
|
||||||
uid: string,
|
fileSaveName: string,
|
||||||
metadata: MetadataObject
|
metadata: MetadataObject
|
||||||
) => {
|
) => {
|
||||||
const creationTime = Math.floor(metadata.creationTime / 1000000);
|
const creationTime = Math.floor(metadata.creationTime / 1000000);
|
||||||
const modificationTime = Math.floor(metadata.modificationTime / 1000000);
|
const modificationTime = Math.floor(
|
||||||
|
(metadata.modificationTime ?? metadata.creationTime) / 1000000
|
||||||
|
);
|
||||||
return JSON.stringify(
|
return JSON.stringify(
|
||||||
{
|
{
|
||||||
title: uid,
|
title: fileSaveName,
|
||||||
creationTime: {
|
creationTime: {
|
||||||
timestamp: creationTime,
|
timestamp: creationTime,
|
||||||
formatted: formatDate(creationTime * 1000),
|
formatted: formatDate(creationTime * 1000),
|
||||||
|
@ -77,3 +153,85 @@ export const getGoogleLikeMetadataFile = (
|
||||||
2
|
2
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const oldSanitizeName = (name: string) =>
|
||||||
|
name.replaceAll('/', '_').replaceAll(' ', '_');
|
||||||
|
|
||||||
|
export const sanitizeName = (name: string) =>
|
||||||
|
name.replace(/[^a-z0-9.]/gi, '_').toLowerCase();
|
||||||
|
|
||||||
|
export const getUniqueCollectionFolderPath = (
|
||||||
|
dir: string,
|
||||||
|
collection: Collection
|
||||||
|
): string => {
|
||||||
|
if (!exportService.checkAllElectronAPIsExists()) {
|
||||||
|
return getOldCollectionFolderPath(dir, collection);
|
||||||
|
}
|
||||||
|
let collectionFolderPath = `${dir}/${sanitizeName(collection.name)}`;
|
||||||
|
let count = 1;
|
||||||
|
while (exportService.exists(collectionFolderPath)) {
|
||||||
|
collectionFolderPath = `${dir}/${sanitizeName(
|
||||||
|
collection.name
|
||||||
|
)}(${count})`;
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
return collectionFolderPath;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getMetadataFolderPath = (collectionFolderPath: string) =>
|
||||||
|
`${collectionFolderPath}/${METADATA_FOLDER_NAME}`;
|
||||||
|
|
||||||
|
export const getUniqueFileSaveName = (
|
||||||
|
collectionPath: string,
|
||||||
|
filename: string,
|
||||||
|
fileID: number
|
||||||
|
) => {
|
||||||
|
if (!exportService.checkAllElectronAPIsExists()) {
|
||||||
|
return getOldFileSaveName(filename, fileID);
|
||||||
|
}
|
||||||
|
let fileSaveName = sanitizeName(filename);
|
||||||
|
let count = 1;
|
||||||
|
while (
|
||||||
|
exportService.exists(getFileSavePath(collectionPath, fileSaveName))
|
||||||
|
) {
|
||||||
|
const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
|
||||||
|
if (filenameParts[1]) {
|
||||||
|
fileSaveName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
|
||||||
|
} else {
|
||||||
|
fileSaveName = `${filenameParts[0]}(${count})`;
|
||||||
|
}
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
return fileSaveName;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getOldFileSaveName = (filename: string, fileID: number) =>
|
||||||
|
`${fileID}_${oldSanitizeName(filename)}`;
|
||||||
|
|
||||||
|
export const getFileMetadataSavePath = (
|
||||||
|
collectionFolderPath: string,
|
||||||
|
fileSaveName: string
|
||||||
|
) => `${collectionFolderPath}/${METADATA_FOLDER_NAME}/${fileSaveName}.json`;
|
||||||
|
|
||||||
|
export const getFileSavePath = (
|
||||||
|
collectionFolderPath: string,
|
||||||
|
fileSaveName: string
|
||||||
|
) => `${collectionFolderPath}/${fileSaveName}`;
|
||||||
|
|
||||||
|
export const getOldCollectionFolderPath = (
|
||||||
|
dir: string,
|
||||||
|
collection: Collection
|
||||||
|
) => `${dir}/${collection.id}_${oldSanitizeName(collection.name)}`;
|
||||||
|
|
||||||
|
export const getOldFileSavePath = (collectionFolderPath: string, file: File) =>
|
||||||
|
`${collectionFolderPath}/${file.id}_${oldSanitizeName(
|
||||||
|
file.metadata.title
|
||||||
|
)}`;
|
||||||
|
|
||||||
|
export const getOldFileMetadataSavePath = (
|
||||||
|
collectionFolderPath: string,
|
||||||
|
file: File
|
||||||
|
) =>
|
||||||
|
`${collectionFolderPath}/${METADATA_FOLDER_NAME}/${
|
||||||
|
file.id
|
||||||
|
}_${oldSanitizeName(file.metadata.title)}.json`;
|
||||||
|
|
|
@ -16,9 +16,12 @@ import { logError } from 'utils/sentry';
|
||||||
import { User } from 'services/userService';
|
import { User } from 'services/userService';
|
||||||
import CryptoWorker from 'utils/crypto';
|
import CryptoWorker from 'utils/crypto';
|
||||||
import { getData, LS_KEYS } from 'utils/storage/localStorage';
|
import { getData, LS_KEYS } from 'utils/storage/localStorage';
|
||||||
|
import { updateFileCreationDateInEXIF } from 'services/upload/exifService';
|
||||||
|
|
||||||
export const TYPE_HEIC = 'heic';
|
export const TYPE_HEIC = 'heic';
|
||||||
export const TYPE_HEIF = 'heif';
|
export const TYPE_HEIF = 'heif';
|
||||||
|
export const TYPE_JPEG = 'jpeg';
|
||||||
|
export const TYPE_JPG = 'jpg';
|
||||||
const UNSUPPORTED_FORMATS = ['flv', 'mkv', '3gp', 'avi', 'wmv'];
|
const UNSUPPORTED_FORMATS = ['flv', 'mkv', '3gp', 'avi', 'wmv'];
|
||||||
|
|
||||||
export function downloadAsFile(filename: string, content: string) {
|
export function downloadAsFile(filename: string, content: string) {
|
||||||
|
@ -40,13 +43,32 @@ export function downloadAsFile(filename: string, content: string) {
|
||||||
export async function downloadFile(file: File) {
|
export async function downloadFile(file: File) {
|
||||||
const a = document.createElement('a');
|
const a = document.createElement('a');
|
||||||
a.style.display = 'none';
|
a.style.display = 'none';
|
||||||
const cachedFileUrl = await DownloadManager.getCachedOriginalFile(file);
|
let fileURL = await DownloadManager.getCachedOriginalFile(file);
|
||||||
const fileURL =
|
let tempURL;
|
||||||
cachedFileUrl ??
|
if (!fileURL) {
|
||||||
URL.createObjectURL(
|
tempURL = URL.createObjectURL(
|
||||||
await new Response(await DownloadManager.downloadFile(file)).blob()
|
await new Response(await DownloadManager.downloadFile(file)).blob()
|
||||||
);
|
);
|
||||||
|
fileURL = tempURL;
|
||||||
|
}
|
||||||
|
const fileType = getFileExtension(file.metadata.title);
|
||||||
|
let tempEditedFileURL;
|
||||||
|
if (
|
||||||
|
file.pubMagicMetadata?.data.editedTime &&
|
||||||
|
(fileType === TYPE_JPEG || fileType === TYPE_JPG)
|
||||||
|
) {
|
||||||
|
let fileBlob = await (await fetch(fileURL)).blob();
|
||||||
|
|
||||||
|
fileBlob = await updateFileCreationDateInEXIF(
|
||||||
|
fileBlob,
|
||||||
|
new Date(file.pubMagicMetadata.data.editedTime / 1000)
|
||||||
|
);
|
||||||
|
tempEditedFileURL = URL.createObjectURL(fileBlob);
|
||||||
|
fileURL = tempEditedFileURL;
|
||||||
|
}
|
||||||
|
|
||||||
a.href = fileURL;
|
a.href = fileURL;
|
||||||
|
|
||||||
if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
|
if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
|
||||||
a.download = fileNameWithoutExtension(file.metadata.title) + '.zip';
|
a.download = fileNameWithoutExtension(file.metadata.title) + '.zip';
|
||||||
} else {
|
} else {
|
||||||
|
@ -55,6 +77,8 @@ export async function downloadFile(file: File) {
|
||||||
document.body.appendChild(a);
|
document.body.appendChild(a);
|
||||||
a.click();
|
a.click();
|
||||||
a.remove();
|
a.remove();
|
||||||
|
tempURL && URL.revokeObjectURL(tempURL);
|
||||||
|
tempEditedFileURL && URL.revokeObjectURL(tempEditedFileURL);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function isFileHEIC(mimeType: string) {
|
export function isFileHEIC(mimeType: string) {
|
||||||
|
@ -257,6 +281,10 @@ export function splitFilenameAndExtension(filename): [string, string] {
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function getFileExtension(filename) {
|
||||||
|
return splitFilenameAndExtension(filename)[1];
|
||||||
|
}
|
||||||
|
|
||||||
export function generateStreamFromArrayBuffer(data: Uint8Array) {
|
export function generateStreamFromArrayBuffer(data: Uint8Array) {
|
||||||
return new ReadableStream({
|
return new ReadableStream({
|
||||||
async start(controller: ReadableStreamDefaultController) {
|
async start(controller: ReadableStreamDefaultController) {
|
||||||
|
@ -273,7 +301,7 @@ export async function convertForPreview(file: File, fileBlob: Blob) {
|
||||||
fileBlob = new Blob([motionPhoto.image]);
|
fileBlob = new Blob([motionPhoto.image]);
|
||||||
}
|
}
|
||||||
|
|
||||||
const typeFromExtension = file.metadata.title.split('.')[-1];
|
const typeFromExtension = getFileExtension(file.metadata.title);
|
||||||
const worker = await new CryptoWorker();
|
const worker = await new CryptoWorker();
|
||||||
|
|
||||||
const mimeType =
|
const mimeType =
|
||||||
|
|
|
@ -9,7 +9,7 @@ export const logError = (
|
||||||
) => {
|
) => {
|
||||||
const err = errorWithContext(error, msg);
|
const err = errorWithContext(error, msg);
|
||||||
if (!process.env.NEXT_PUBLIC_SENTRY_ENV) {
|
if (!process.env.NEXT_PUBLIC_SENTRY_ENV) {
|
||||||
console.log({ error, msg, info });
|
console.log(error, { msg, info });
|
||||||
}
|
}
|
||||||
Sentry.captureException(err, {
|
Sentry.captureException(err, {
|
||||||
level: Sentry.Severity.Info,
|
level: Sentry.Severity.Info,
|
||||||
|
|
|
@ -5055,6 +5055,11 @@ picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.2, picomatch@^2.2.3:
|
||||||
resolved "https://registry.npmjs.org/picomatch/-/picomatch-2.3.0.tgz"
|
resolved "https://registry.npmjs.org/picomatch/-/picomatch-2.3.0.tgz"
|
||||||
integrity sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw==
|
integrity sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw==
|
||||||
|
|
||||||
|
piexifjs@^1.0.6:
|
||||||
|
version "1.0.6"
|
||||||
|
resolved "https://registry.yarnpkg.com/piexifjs/-/piexifjs-1.0.6.tgz#883811d73f447218d0d06e9ed7866d04533e59e0"
|
||||||
|
integrity sha512-0wVyH0cKohzBQ5Gi2V1BuxYpxWfxF3cSqfFXfPIpl5tl9XLS5z4ogqhUCD20AbHi0h9aJkqXNJnkVev6gwh2ag==
|
||||||
|
|
||||||
pify@^2.0.0:
|
pify@^2.0.0:
|
||||||
version "2.3.0"
|
version "2.3.0"
|
||||||
resolved "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz"
|
resolved "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz"
|
||||||
|
|
Loading…
Reference in a new issue