update save file name and collection name parenthesied number for same name instead of using ids to make them unique
This commit is contained in:
parent
8c3106f66d
commit
4deba2df42
|
@ -6,6 +6,8 @@ import {
|
||||||
dedupe,
|
dedupe,
|
||||||
getGoogleLikeMetadataFile,
|
getGoogleLikeMetadataFile,
|
||||||
getExportRecordFileUID,
|
getExportRecordFileUID,
|
||||||
|
getUniqueCollectionFolderPath,
|
||||||
|
getUniqueFileSaveName,
|
||||||
} from 'utils/export';
|
} from 'utils/export';
|
||||||
import { retryAsyncFunction } from 'utils/network';
|
import { retryAsyncFunction } from 'utils/network';
|
||||||
import { logError } from 'utils/sentry';
|
import { logError } from 'utils/sentry';
|
||||||
|
@ -79,6 +81,7 @@ class ExportService {
|
||||||
private recordUpdateInProgress = Promise.resolve();
|
private recordUpdateInProgress = Promise.resolve();
|
||||||
private stopExport: boolean = false;
|
private stopExport: boolean = false;
|
||||||
private pauseExport: boolean = false;
|
private pauseExport: boolean = false;
|
||||||
|
private usedFilenames = new Map<number, Set<string>>();
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
this.ElectronAPIs = runningInBrowser() && window['ElectronAPIs'];
|
this.ElectronAPIs = runningInBrowser() && window['ElectronAPIs'];
|
||||||
|
@ -96,48 +99,61 @@ class ExportService {
|
||||||
updateProgress: (progress: ExportProgress) => void,
|
updateProgress: (progress: ExportProgress) => void,
|
||||||
exportType: ExportType
|
exportType: ExportType
|
||||||
) {
|
) {
|
||||||
if (this.exportInProgress) {
|
try {
|
||||||
this.ElectronAPIs.sendNotification(ExportNotification.IN_PROGRESS);
|
if (this.exportInProgress) {
|
||||||
return this.exportInProgress;
|
this.ElectronAPIs.sendNotification(
|
||||||
}
|
ExportNotification.IN_PROGRESS
|
||||||
this.ElectronAPIs.showOnTray('starting export');
|
);
|
||||||
const exportDir = getData(LS_KEYS.EXPORT)?.folder;
|
return this.exportInProgress;
|
||||||
if (!exportDir) {
|
}
|
||||||
// no-export folder set
|
this.ElectronAPIs.showOnTray('starting export');
|
||||||
return;
|
const exportDir = getData(LS_KEYS.EXPORT)?.folder;
|
||||||
}
|
if (!exportDir) {
|
||||||
let filesToExport: File[];
|
// no-export folder set
|
||||||
const allFiles = await getLocalFiles();
|
return;
|
||||||
const collections = await getLocalCollections();
|
}
|
||||||
const nonEmptyCollections = getNonEmptyCollections(
|
let filesToExport: File[];
|
||||||
collections,
|
const allFiles = await getLocalFiles();
|
||||||
allFiles
|
const collections = await getLocalCollections();
|
||||||
);
|
const nonEmptyCollections = getNonEmptyCollections(
|
||||||
const user: User = getData(LS_KEYS.USER);
|
collections,
|
||||||
const userCollections = nonEmptyCollections.filter(
|
allFiles
|
||||||
(collection) => collection.owner.id === user?.id
|
|
||||||
);
|
|
||||||
const exportRecord = await this.getExportRecord(exportDir);
|
|
||||||
|
|
||||||
if (exportType === ExportType.NEW) {
|
|
||||||
filesToExport = await getFilesUploadedAfterLastExport(
|
|
||||||
allFiles,
|
|
||||||
exportRecord
|
|
||||||
);
|
);
|
||||||
} else if (exportType === ExportType.RETRY_FAILED) {
|
const user: User = getData(LS_KEYS.USER);
|
||||||
filesToExport = await getExportFailedFiles(allFiles, exportRecord);
|
const userCollections = nonEmptyCollections.filter(
|
||||||
} else {
|
(collection) => collection.owner.id === user?.id
|
||||||
filesToExport = await getExportPendingFiles(allFiles, exportRecord);
|
);
|
||||||
|
const exportRecord = await this.getExportRecord(exportDir);
|
||||||
|
|
||||||
|
if (exportType === ExportType.NEW) {
|
||||||
|
filesToExport = await getFilesUploadedAfterLastExport(
|
||||||
|
allFiles,
|
||||||
|
exportRecord
|
||||||
|
);
|
||||||
|
} else if (exportType === ExportType.RETRY_FAILED) {
|
||||||
|
filesToExport = await getExportFailedFiles(
|
||||||
|
allFiles,
|
||||||
|
exportRecord
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
filesToExport = await getExportPendingFiles(
|
||||||
|
allFiles,
|
||||||
|
exportRecord
|
||||||
|
);
|
||||||
|
}
|
||||||
|
this.exportInProgress = this.fileExporter(
|
||||||
|
filesToExport,
|
||||||
|
userCollections,
|
||||||
|
updateProgress,
|
||||||
|
exportDir
|
||||||
|
);
|
||||||
|
const resp = await this.exportInProgress;
|
||||||
|
this.exportInProgress = null;
|
||||||
|
return resp;
|
||||||
|
} catch (e) {
|
||||||
|
logError(e, 'exportFiles failed');
|
||||||
|
return { paused: false };
|
||||||
}
|
}
|
||||||
this.exportInProgress = this.fileExporter(
|
|
||||||
filesToExport,
|
|
||||||
userCollections,
|
|
||||||
updateProgress,
|
|
||||||
exportDir
|
|
||||||
);
|
|
||||||
const resp = await this.exportInProgress;
|
|
||||||
this.exportInProgress = null;
|
|
||||||
return resp;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fileExporter(
|
async fileExporter(
|
||||||
|
@ -166,20 +182,27 @@ class ExportService {
|
||||||
total: files.length,
|
total: files.length,
|
||||||
});
|
});
|
||||||
this.ElectronAPIs.sendNotification(ExportNotification.START);
|
this.ElectronAPIs.sendNotification(ExportNotification.START);
|
||||||
|
collections.sort(
|
||||||
const collectionIDMap = new Map<number, string>();
|
(collectionA, collectionB) => collectionA.id - collectionB.id
|
||||||
|
);
|
||||||
|
const collectionIDPathMap = new Map<number, string>();
|
||||||
|
const usedCollectionPaths = new Set<string>();
|
||||||
for (const collection of collections) {
|
for (const collection of collections) {
|
||||||
const collectionFolderPath = `${dir}/${
|
const collectionFolderPath = getUniqueCollectionFolderPath(
|
||||||
collection.id
|
dir,
|
||||||
}_${this.sanitizeName(collection.name)}`;
|
collection.name,
|
||||||
|
usedCollectionPaths
|
||||||
|
);
|
||||||
|
usedCollectionPaths.add(collectionFolderPath);
|
||||||
|
collectionIDPathMap.set(collection.id, collectionFolderPath);
|
||||||
await this.ElectronAPIs.checkExistsAndCreateCollectionDir(
|
await this.ElectronAPIs.checkExistsAndCreateCollectionDir(
|
||||||
collectionFolderPath
|
collectionFolderPath
|
||||||
);
|
);
|
||||||
await this.ElectronAPIs.checkExistsAndCreateCollectionDir(
|
await this.ElectronAPIs.checkExistsAndCreateCollectionDir(
|
||||||
`${collectionFolderPath}/${METADATA_FOLDER_NAME}`
|
`${collectionFolderPath}/${METADATA_FOLDER_NAME}`
|
||||||
);
|
);
|
||||||
collectionIDMap.set(collection.id, collectionFolderPath);
|
|
||||||
}
|
}
|
||||||
|
files.sort((fileA, fileB) => fileA.id - fileB.id);
|
||||||
for (const [index, file] of files.entries()) {
|
for (const [index, file] of files.entries()) {
|
||||||
if (this.stopExport || this.pauseExport) {
|
if (this.stopExport || this.pauseExport) {
|
||||||
if (this.pauseExport) {
|
if (this.pauseExport) {
|
||||||
|
@ -190,7 +213,9 @@ class ExportService {
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
const collectionPath = collectionIDMap.get(file.collectionID);
|
const collectionPath = collectionIDPathMap.get(
|
||||||
|
file.collectionID
|
||||||
|
);
|
||||||
try {
|
try {
|
||||||
await this.downloadAndSave(file, collectionPath);
|
await this.downloadAndSave(file, collectionPath);
|
||||||
await this.addFileExportRecord(
|
await this.addFileExportRecord(
|
||||||
|
@ -233,7 +258,8 @@ class ExportService {
|
||||||
}
|
}
|
||||||
return { paused: false };
|
return { paused: false };
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
logError(e, 'export failed ');
|
logError(e, 'fileExporter failed');
|
||||||
|
throw e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
async addFilesQueuedRecord(folder: string, files: File[]) {
|
async addFilesQueuedRecord(folder: string, files: File[]) {
|
||||||
|
@ -310,17 +336,23 @@ class ExportService {
|
||||||
}
|
}
|
||||||
|
|
||||||
async downloadAndSave(file: File, collectionPath: string) {
|
async downloadAndSave(file: File, collectionPath: string) {
|
||||||
const uid = `${file.id}_${this.sanitizeName(file.metadata.title)}`;
|
const usedFileNamesInCollection = this.usedFilenames.get(
|
||||||
|
file.collectionID
|
||||||
|
);
|
||||||
|
const fileSaveName = getUniqueFileSaveName(
|
||||||
|
file.metadata.title,
|
||||||
|
usedFileNamesInCollection
|
||||||
|
);
|
||||||
const fileStream = await retryAsyncFunction(() =>
|
const fileStream = await retryAsyncFunction(() =>
|
||||||
downloadManager.downloadFile(file)
|
downloadManager.downloadFile(file)
|
||||||
);
|
);
|
||||||
if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
|
if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
|
||||||
this.exportMotionPhoto(fileStream, file, collectionPath);
|
this.exportMotionPhoto(fileStream, file, collectionPath);
|
||||||
} else {
|
} else {
|
||||||
this.saveMediaFile(collectionPath, uid, fileStream);
|
this.saveMediaFile(collectionPath, fileSaveName, fileStream);
|
||||||
this.saveMetadataFile(
|
this.saveMetadataFile(
|
||||||
collectionPath,
|
collectionPath,
|
||||||
uid,
|
fileSaveName,
|
||||||
mergeMetadata([file])[0].metadata
|
mergeMetadata([file])[0].metadata
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
@ -334,14 +366,22 @@ class ExportService {
|
||||||
const fileBlob = await new Response(fileStream).blob();
|
const fileBlob = await new Response(fileStream).blob();
|
||||||
const originalName = fileNameWithoutExtension(file.metadata.title);
|
const originalName = fileNameWithoutExtension(file.metadata.title);
|
||||||
const motionPhoto = await decodeMotionPhoto(fileBlob, originalName);
|
const motionPhoto = await decodeMotionPhoto(fileBlob, originalName);
|
||||||
|
const usedFileNamesInCollection = this.usedFilenames.get(
|
||||||
|
file.collectionID
|
||||||
|
);
|
||||||
const imageStream = generateStreamFromArrayBuffer(motionPhoto.image);
|
const imageStream = generateStreamFromArrayBuffer(motionPhoto.image);
|
||||||
const imageUID = `${file.id}_${motionPhoto.imageNameTitle}`;
|
const imageSaveName = getUniqueFileSaveName(
|
||||||
this.saveMediaFile(collectionPath, imageUID, imageStream);
|
motionPhoto.imageNameTitle,
|
||||||
this.saveMetadataFile(collectionPath, imageUID, file.metadata);
|
usedFileNamesInCollection
|
||||||
|
);
|
||||||
|
this.saveMediaFile(collectionPath, imageSaveName, imageStream);
|
||||||
|
this.saveMetadataFile(collectionPath, imageSaveName, file.metadata);
|
||||||
|
|
||||||
const videoStream = generateStreamFromArrayBuffer(motionPhoto.video);
|
const videoStream = generateStreamFromArrayBuffer(motionPhoto.video);
|
||||||
const videoUID = `${file.id}_${motionPhoto.videoNameTitle}`;
|
const videoUID = getUniqueFileSaveName(
|
||||||
|
motionPhoto.videoNameTitle,
|
||||||
|
usedFileNamesInCollection
|
||||||
|
);
|
||||||
this.saveMediaFile(collectionPath, videoUID, videoStream);
|
this.saveMediaFile(collectionPath, videoUID, videoStream);
|
||||||
this.saveMetadataFile(collectionPath, videoUID, file.metadata);
|
this.saveMetadataFile(collectionPath, videoUID, file.metadata);
|
||||||
}
|
}
|
||||||
|
@ -359,10 +399,6 @@ class ExportService {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private sanitizeName(name) {
|
|
||||||
return name.replaceAll('/', '_').replaceAll(' ', '_');
|
|
||||||
}
|
|
||||||
|
|
||||||
isExportInProgress = () => {
|
isExportInProgress = () => {
|
||||||
return this.exportInProgress !== null;
|
return this.exportInProgress !== null;
|
||||||
};
|
};
|
||||||
|
|
|
@ -4,8 +4,8 @@ import { fileExtensionWithDot } from 'utils/file';
|
||||||
class MotionPhoto {
|
class MotionPhoto {
|
||||||
image: Uint8Array;
|
image: Uint8Array;
|
||||||
video: Uint8Array;
|
video: Uint8Array;
|
||||||
imageNameTitle: String;
|
imageNameTitle: string;
|
||||||
videoNameTitle: String;
|
videoNameTitle: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const decodeMotionPhoto = async (
|
export const decodeMotionPhoto = async (
|
||||||
|
|
|
@ -77,3 +77,41 @@ export const getGoogleLikeMetadataFile = (
|
||||||
2
|
2
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const getUniqueCollectionFolderPath = (
|
||||||
|
dir: string,
|
||||||
|
collectionName: string,
|
||||||
|
usedCollectionPaths: Set<string>
|
||||||
|
): string => {
|
||||||
|
let collectionFolderPath = `${dir}/${sanitizeName(collectionName)}`;
|
||||||
|
let count = 1;
|
||||||
|
while (
|
||||||
|
usedCollectionPaths &&
|
||||||
|
usedCollectionPaths.has(collectionFolderPath)
|
||||||
|
) {
|
||||||
|
collectionFolderPath = `${dir}/${sanitizeName(
|
||||||
|
collectionName
|
||||||
|
)}(${count})`;
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
return collectionFolderPath;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const sanitizeName = (name: string) => {
|
||||||
|
return name.replaceAll('/', '_').replaceAll(' ', '_');
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getUniqueFileSaveName = (
|
||||||
|
filename: string,
|
||||||
|
usedFileNamesInCollection: Set<string>
|
||||||
|
) => {
|
||||||
|
let fileSaveName = filename;
|
||||||
|
const count = 1;
|
||||||
|
while (
|
||||||
|
usedFileNamesInCollection &&
|
||||||
|
usedFileNamesInCollection.has(fileSaveName)
|
||||||
|
) {
|
||||||
|
fileSaveName = filename + `(${count})`;
|
||||||
|
}
|
||||||
|
return fileSaveName;
|
||||||
|
};
|
||||||
|
|
Loading…
Reference in a new issue