update save file name and collection name parenthesied number for same name instead of using ids to make them unique

This commit is contained in:
Abhinav 2021-11-09 22:16:54 +05:30
parent 8c3106f66d
commit 4deba2df42
3 changed files with 136 additions and 62 deletions

View file

@ -6,6 +6,8 @@ import {
dedupe, dedupe,
getGoogleLikeMetadataFile, getGoogleLikeMetadataFile,
getExportRecordFileUID, getExportRecordFileUID,
getUniqueCollectionFolderPath,
getUniqueFileSaveName,
} from 'utils/export'; } from 'utils/export';
import { retryAsyncFunction } from 'utils/network'; import { retryAsyncFunction } from 'utils/network';
import { logError } from 'utils/sentry'; import { logError } from 'utils/sentry';
@ -79,6 +81,7 @@ class ExportService {
private recordUpdateInProgress = Promise.resolve(); private recordUpdateInProgress = Promise.resolve();
private stopExport: boolean = false; private stopExport: boolean = false;
private pauseExport: boolean = false; private pauseExport: boolean = false;
private usedFilenames = new Map<number, Set<string>>();
constructor() { constructor() {
this.ElectronAPIs = runningInBrowser() && window['ElectronAPIs']; this.ElectronAPIs = runningInBrowser() && window['ElectronAPIs'];
@ -96,48 +99,61 @@ class ExportService {
updateProgress: (progress: ExportProgress) => void, updateProgress: (progress: ExportProgress) => void,
exportType: ExportType exportType: ExportType
) { ) {
if (this.exportInProgress) { try {
this.ElectronAPIs.sendNotification(ExportNotification.IN_PROGRESS); if (this.exportInProgress) {
return this.exportInProgress; this.ElectronAPIs.sendNotification(
} ExportNotification.IN_PROGRESS
this.ElectronAPIs.showOnTray('starting export'); );
const exportDir = getData(LS_KEYS.EXPORT)?.folder; return this.exportInProgress;
if (!exportDir) { }
// no-export folder set this.ElectronAPIs.showOnTray('starting export');
return; const exportDir = getData(LS_KEYS.EXPORT)?.folder;
} if (!exportDir) {
let filesToExport: File[]; // no-export folder set
const allFiles = await getLocalFiles(); return;
const collections = await getLocalCollections(); }
const nonEmptyCollections = getNonEmptyCollections( let filesToExport: File[];
collections, const allFiles = await getLocalFiles();
allFiles const collections = await getLocalCollections();
); const nonEmptyCollections = getNonEmptyCollections(
const user: User = getData(LS_KEYS.USER); collections,
const userCollections = nonEmptyCollections.filter( allFiles
(collection) => collection.owner.id === user?.id
);
const exportRecord = await this.getExportRecord(exportDir);
if (exportType === ExportType.NEW) {
filesToExport = await getFilesUploadedAfterLastExport(
allFiles,
exportRecord
); );
} else if (exportType === ExportType.RETRY_FAILED) { const user: User = getData(LS_KEYS.USER);
filesToExport = await getExportFailedFiles(allFiles, exportRecord); const userCollections = nonEmptyCollections.filter(
} else { (collection) => collection.owner.id === user?.id
filesToExport = await getExportPendingFiles(allFiles, exportRecord); );
const exportRecord = await this.getExportRecord(exportDir);
if (exportType === ExportType.NEW) {
filesToExport = await getFilesUploadedAfterLastExport(
allFiles,
exportRecord
);
} else if (exportType === ExportType.RETRY_FAILED) {
filesToExport = await getExportFailedFiles(
allFiles,
exportRecord
);
} else {
filesToExport = await getExportPendingFiles(
allFiles,
exportRecord
);
}
this.exportInProgress = this.fileExporter(
filesToExport,
userCollections,
updateProgress,
exportDir
);
const resp = await this.exportInProgress;
this.exportInProgress = null;
return resp;
} catch (e) {
logError(e, 'exportFiles failed');
return { paused: false };
} }
this.exportInProgress = this.fileExporter(
filesToExport,
userCollections,
updateProgress,
exportDir
);
const resp = await this.exportInProgress;
this.exportInProgress = null;
return resp;
} }
async fileExporter( async fileExporter(
@ -166,20 +182,27 @@ class ExportService {
total: files.length, total: files.length,
}); });
this.ElectronAPIs.sendNotification(ExportNotification.START); this.ElectronAPIs.sendNotification(ExportNotification.START);
collections.sort(
const collectionIDMap = new Map<number, string>(); (collectionA, collectionB) => collectionA.id - collectionB.id
);
const collectionIDPathMap = new Map<number, string>();
const usedCollectionPaths = new Set<string>();
for (const collection of collections) { for (const collection of collections) {
const collectionFolderPath = `${dir}/${ const collectionFolderPath = getUniqueCollectionFolderPath(
collection.id dir,
}_${this.sanitizeName(collection.name)}`; collection.name,
usedCollectionPaths
);
usedCollectionPaths.add(collectionFolderPath);
collectionIDPathMap.set(collection.id, collectionFolderPath);
await this.ElectronAPIs.checkExistsAndCreateCollectionDir( await this.ElectronAPIs.checkExistsAndCreateCollectionDir(
collectionFolderPath collectionFolderPath
); );
await this.ElectronAPIs.checkExistsAndCreateCollectionDir( await this.ElectronAPIs.checkExistsAndCreateCollectionDir(
`${collectionFolderPath}/${METADATA_FOLDER_NAME}` `${collectionFolderPath}/${METADATA_FOLDER_NAME}`
); );
collectionIDMap.set(collection.id, collectionFolderPath);
} }
files.sort((fileA, fileB) => fileA.id - fileB.id);
for (const [index, file] of files.entries()) { for (const [index, file] of files.entries()) {
if (this.stopExport || this.pauseExport) { if (this.stopExport || this.pauseExport) {
if (this.pauseExport) { if (this.pauseExport) {
@ -190,7 +213,9 @@ class ExportService {
} }
break; break;
} }
const collectionPath = collectionIDMap.get(file.collectionID); const collectionPath = collectionIDPathMap.get(
file.collectionID
);
try { try {
await this.downloadAndSave(file, collectionPath); await this.downloadAndSave(file, collectionPath);
await this.addFileExportRecord( await this.addFileExportRecord(
@ -233,7 +258,8 @@ class ExportService {
} }
return { paused: false }; return { paused: false };
} catch (e) { } catch (e) {
logError(e, 'export failed '); logError(e, 'fileExporter failed');
throw e;
} }
} }
async addFilesQueuedRecord(folder: string, files: File[]) { async addFilesQueuedRecord(folder: string, files: File[]) {
@ -310,17 +336,23 @@ class ExportService {
} }
async downloadAndSave(file: File, collectionPath: string) { async downloadAndSave(file: File, collectionPath: string) {
const uid = `${file.id}_${this.sanitizeName(file.metadata.title)}`; const usedFileNamesInCollection = this.usedFilenames.get(
file.collectionID
);
const fileSaveName = getUniqueFileSaveName(
file.metadata.title,
usedFileNamesInCollection
);
const fileStream = await retryAsyncFunction(() => const fileStream = await retryAsyncFunction(() =>
downloadManager.downloadFile(file) downloadManager.downloadFile(file)
); );
if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
this.exportMotionPhoto(fileStream, file, collectionPath); this.exportMotionPhoto(fileStream, file, collectionPath);
} else { } else {
this.saveMediaFile(collectionPath, uid, fileStream); this.saveMediaFile(collectionPath, fileSaveName, fileStream);
this.saveMetadataFile( this.saveMetadataFile(
collectionPath, collectionPath,
uid, fileSaveName,
mergeMetadata([file])[0].metadata mergeMetadata([file])[0].metadata
); );
} }
@ -334,14 +366,22 @@ class ExportService {
const fileBlob = await new Response(fileStream).blob(); const fileBlob = await new Response(fileStream).blob();
const originalName = fileNameWithoutExtension(file.metadata.title); const originalName = fileNameWithoutExtension(file.metadata.title);
const motionPhoto = await decodeMotionPhoto(fileBlob, originalName); const motionPhoto = await decodeMotionPhoto(fileBlob, originalName);
const usedFileNamesInCollection = this.usedFilenames.get(
file.collectionID
);
const imageStream = generateStreamFromArrayBuffer(motionPhoto.image); const imageStream = generateStreamFromArrayBuffer(motionPhoto.image);
const imageUID = `${file.id}_${motionPhoto.imageNameTitle}`; const imageSaveName = getUniqueFileSaveName(
this.saveMediaFile(collectionPath, imageUID, imageStream); motionPhoto.imageNameTitle,
this.saveMetadataFile(collectionPath, imageUID, file.metadata); usedFileNamesInCollection
);
this.saveMediaFile(collectionPath, imageSaveName, imageStream);
this.saveMetadataFile(collectionPath, imageSaveName, file.metadata);
const videoStream = generateStreamFromArrayBuffer(motionPhoto.video); const videoStream = generateStreamFromArrayBuffer(motionPhoto.video);
const videoUID = `${file.id}_${motionPhoto.videoNameTitle}`; const videoUID = getUniqueFileSaveName(
motionPhoto.videoNameTitle,
usedFileNamesInCollection
);
this.saveMediaFile(collectionPath, videoUID, videoStream); this.saveMediaFile(collectionPath, videoUID, videoStream);
this.saveMetadataFile(collectionPath, videoUID, file.metadata); this.saveMetadataFile(collectionPath, videoUID, file.metadata);
} }
@ -359,10 +399,6 @@ class ExportService {
); );
} }
private sanitizeName(name) {
return name.replaceAll('/', '_').replaceAll(' ', '_');
}
isExportInProgress = () => { isExportInProgress = () => {
return this.exportInProgress !== null; return this.exportInProgress !== null;
}; };

View file

@ -4,8 +4,8 @@ import { fileExtensionWithDot } from 'utils/file';
class MotionPhoto { class MotionPhoto {
image: Uint8Array; image: Uint8Array;
video: Uint8Array; video: Uint8Array;
imageNameTitle: String; imageNameTitle: string;
videoNameTitle: String; videoNameTitle: string;
} }
export const decodeMotionPhoto = async ( export const decodeMotionPhoto = async (

View file

@ -77,3 +77,41 @@ export const getGoogleLikeMetadataFile = (
2 2
); );
}; };
export const getUniqueCollectionFolderPath = (
dir: string,
collectionName: string,
usedCollectionPaths: Set<string>
): string => {
let collectionFolderPath = `${dir}/${sanitizeName(collectionName)}`;
let count = 1;
while (
usedCollectionPaths &&
usedCollectionPaths.has(collectionFolderPath)
) {
collectionFolderPath = `${dir}/${sanitizeName(
collectionName
)}(${count})`;
count++;
}
return collectionFolderPath;
};
export const sanitizeName = (name: string) => {
return name.replaceAll('/', '_').replaceAll(' ', '_');
};
export const getUniqueFileSaveName = (
filename: string,
usedFileNamesInCollection: Set<string>
) => {
let fileSaveName = filename;
const count = 1;
while (
usedFileNamesInCollection &&
usedFileNamesInCollection.has(fileSaveName)
) {
fileSaveName = filename + `(${count})`;
}
return fileSaveName;
};