From b651d16ee50c48bb1b9f79fac3714b84527493e7 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 2 Nov 2021 17:31:53 +0530 Subject: [PATCH 01/58] skip shared-collections during export --- src/components/ExportModal.tsx | 9 +++++++-- src/services/exportService.ts | 7 ++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/components/ExportModal.tsx b/src/components/ExportModal.tsx index a03d4b722..de604e0a1 100644 --- a/src/components/ExportModal.tsx +++ b/src/components/ExportModal.tsx @@ -8,6 +8,7 @@ import exportService, { ExportType, } from 'services/exportService'; import { getLocalFiles } from 'services/fileService'; +import { User } from 'services/userService'; import styled from 'styled-components'; import { sleep } from 'utils/common'; import { getExportRecordFileUID } from 'utils/export'; @@ -105,12 +106,16 @@ export default function ExportModal(props: Props) { return; } const main = async () => { + const user: User = getData(LS_KEYS.USER); if (exportStage === ExportStage.FINISHED) { const localFiles = await getLocalFiles(); + const userPersonalFiles = localFiles.filter( + (file) => file.ownerID === user?.id + ); const exportRecord = await exportService.getExportRecord(); const exportedFileCnt = exportRecord.exportedFiles.length; const failedFilesCnt = exportRecord.failedFiles.length; - const syncedFilesCnt = localFiles.length; + const syncedFilesCnt = userPersonalFiles.length; if (syncedFilesCnt > exportedFileCnt + failedFilesCnt) { updateExportProgress({ current: exportedFileCnt + failedFilesCnt, @@ -120,7 +125,7 @@ export default function ExportModal(props: Props) { ...exportRecord.exportedFiles, ...exportRecord.failedFiles, ]); - const unExportedFiles = localFiles.filter( + const unExportedFiles = userPersonalFiles.filter( (file) => !exportFileUIDs.has(getExportRecordFileUID(file)) ); diff --git a/src/services/exportService.ts b/src/services/exportService.ts index 6f0e69eea..eb553b8de 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -22,6 +22,7 @@ import { fileNameWithoutExtension, generateStreamFromArrayBuffer, } from 'utils/file'; +import { User } from './userService'; export interface ExportProgress { current: number; @@ -111,6 +112,10 @@ class ExportService { collections, allFiles ); + const user: User = getData(LS_KEYS.USER); + const userCollections = nonEmptyCollections.filter( + (collection) => collection.owner.id === user?.id + ); const exportRecord = await this.getExportRecord(exportDir); if (exportType === ExportType.NEW) { @@ -125,7 +130,7 @@ class ExportService { } this.exportInProgress = this.fileExporter( filesToExport, - nonEmptyCollections, + userCollections, updateProgress, exportDir ); From 7245af0cc8dbe5edb8a52794d4609ccd62a8dff2 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 9 Nov 2021 17:32:24 +0530 Subject: [PATCH 02/58] honor edited property in metadata --- src/services/exportService.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index eb553b8de..4b683d4df 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -21,6 +21,7 @@ import { decodeMotionPhoto } from './motionPhotoService'; import { fileNameWithoutExtension, generateStreamFromArrayBuffer, + mergeMetadata, } from 'utils/file'; import { User } from './userService'; @@ -317,7 +318,11 @@ class ExportService { this.exportMotionPhoto(fileStream, file, collectionPath); } else { this.saveMediaFile(collectionPath, uid, fileStream); - this.saveMetadataFile(collectionPath, uid, file.metadata); + this.saveMetadataFile( + collectionPath, + uid, + mergeMetadata([file])[0].metadata + ); } } From 8c3106f66d2b78d5c0af429ce6f25a2b4b0dc92c Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 9 Nov 2021 21:04:31 +0530 Subject: [PATCH 03/58] added fail safe for exportModal state update on reopen --- src/components/ExportModal.tsx | 58 +++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/src/components/ExportModal.tsx b/src/components/ExportModal.tsx index de604e0a1..beeffed80 100644 --- a/src/components/ExportModal.tsx +++ b/src/components/ExportModal.tsx @@ -12,6 +12,7 @@ import { User } from 'services/userService'; import styled from 'styled-components'; import { sleep } from 'utils/common'; import { getExportRecordFileUID } from 'utils/export'; +import { logError } from 'utils/sentry'; import { getData, LS_KEYS, setData } from 'utils/storage/localStorage'; import constants from 'utils/strings/constants'; import { Label, Row, Value } from './Container'; @@ -108,32 +109,39 @@ export default function ExportModal(props: Props) { const main = async () => { const user: User = getData(LS_KEYS.USER); if (exportStage === ExportStage.FINISHED) { - const localFiles = await getLocalFiles(); - const userPersonalFiles = localFiles.filter( - (file) => file.ownerID === user?.id - ); - const exportRecord = await exportService.getExportRecord(); - const exportedFileCnt = exportRecord.exportedFiles.length; - const failedFilesCnt = exportRecord.failedFiles.length; - const syncedFilesCnt = userPersonalFiles.length; - if (syncedFilesCnt > exportedFileCnt + failedFilesCnt) { - updateExportProgress({ - current: exportedFileCnt + failedFilesCnt, - total: syncedFilesCnt, - }); - const exportFileUIDs = new Set([ - ...exportRecord.exportedFiles, - ...exportRecord.failedFiles, - ]); - const unExportedFiles = userPersonalFiles.filter( - (file) => - !exportFileUIDs.has(getExportRecordFileUID(file)) + try { + const localFiles = await getLocalFiles(); + const userPersonalFiles = localFiles.filter( + (file) => file.ownerID === user?.id ); - exportService.addFilesQueuedRecord( - exportFolder, - unExportedFiles - ); - updateExportStage(ExportStage.PAUSED); + const exportRecord = await exportService.getExportRecord(); + const exportedFileCnt = exportRecord.exportedFiles.length; + const failedFilesCnt = exportRecord.failedFiles.length; + const syncedFilesCnt = userPersonalFiles.length; + if (syncedFilesCnt > exportedFileCnt + failedFilesCnt) { + updateExportProgress({ + current: exportedFileCnt + failedFilesCnt, + total: syncedFilesCnt, + }); + const exportFileUIDs = new Set([ + ...exportRecord.exportedFiles, + ...exportRecord.failedFiles, + ]); + const unExportedFiles = userPersonalFiles.filter( + (file) => + !exportFileUIDs.has( + getExportRecordFileUID(file) + ) + ); + exportService.addFilesQueuedRecord( + exportFolder, + unExportedFiles + ); + updateExportStage(ExportStage.PAUSED); + } + } catch (e) { + setExportStage(ExportStage.INIT); + logError(e, 'error while updating exportModal on reopen'); } } }; From 4deba2df42fcf6f3c82a25596527003446b3b39f Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 9 Nov 2021 22:16:54 +0530 Subject: [PATCH 04/58] update save file name and collection name parenthesied number for same name instead of using ids to make them unique --- src/services/exportService.ts | 156 ++++++++++++++++++----------- src/services/motionPhotoService.ts | 4 +- src/utils/export/index.ts | 38 +++++++ 3 files changed, 136 insertions(+), 62 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index 4b683d4df..afda0ceb1 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -6,6 +6,8 @@ import { dedupe, getGoogleLikeMetadataFile, getExportRecordFileUID, + getUniqueCollectionFolderPath, + getUniqueFileSaveName, } from 'utils/export'; import { retryAsyncFunction } from 'utils/network'; import { logError } from 'utils/sentry'; @@ -79,6 +81,7 @@ class ExportService { private recordUpdateInProgress = Promise.resolve(); private stopExport: boolean = false; private pauseExport: boolean = false; + private usedFilenames = new Map>(); constructor() { this.ElectronAPIs = runningInBrowser() && window['ElectronAPIs']; @@ -96,48 +99,61 @@ class ExportService { updateProgress: (progress: ExportProgress) => void, exportType: ExportType ) { - if (this.exportInProgress) { - this.ElectronAPIs.sendNotification(ExportNotification.IN_PROGRESS); - return this.exportInProgress; - } - this.ElectronAPIs.showOnTray('starting export'); - const exportDir = getData(LS_KEYS.EXPORT)?.folder; - if (!exportDir) { - // no-export folder set - return; - } - let filesToExport: File[]; - const allFiles = await getLocalFiles(); - const collections = await getLocalCollections(); - const nonEmptyCollections = getNonEmptyCollections( - collections, - allFiles - ); - const user: User = getData(LS_KEYS.USER); - const userCollections = nonEmptyCollections.filter( - (collection) => collection.owner.id === user?.id - ); - const exportRecord = await this.getExportRecord(exportDir); - - if (exportType === ExportType.NEW) { - filesToExport = await getFilesUploadedAfterLastExport( - allFiles, - exportRecord + try { + if (this.exportInProgress) { + this.ElectronAPIs.sendNotification( + ExportNotification.IN_PROGRESS + ); + return this.exportInProgress; + } + this.ElectronAPIs.showOnTray('starting export'); + const exportDir = getData(LS_KEYS.EXPORT)?.folder; + if (!exportDir) { + // no-export folder set + return; + } + let filesToExport: File[]; + const allFiles = await getLocalFiles(); + const collections = await getLocalCollections(); + const nonEmptyCollections = getNonEmptyCollections( + collections, + allFiles ); - } else if (exportType === ExportType.RETRY_FAILED) { - filesToExport = await getExportFailedFiles(allFiles, exportRecord); - } else { - filesToExport = await getExportPendingFiles(allFiles, exportRecord); + const user: User = getData(LS_KEYS.USER); + const userCollections = nonEmptyCollections.filter( + (collection) => collection.owner.id === user?.id + ); + const exportRecord = await this.getExportRecord(exportDir); + + if (exportType === ExportType.NEW) { + filesToExport = await getFilesUploadedAfterLastExport( + allFiles, + exportRecord + ); + } else if (exportType === ExportType.RETRY_FAILED) { + filesToExport = await getExportFailedFiles( + allFiles, + exportRecord + ); + } else { + filesToExport = await getExportPendingFiles( + allFiles, + exportRecord + ); + } + this.exportInProgress = this.fileExporter( + filesToExport, + userCollections, + updateProgress, + exportDir + ); + const resp = await this.exportInProgress; + this.exportInProgress = null; + return resp; + } catch (e) { + logError(e, 'exportFiles failed'); + return { paused: false }; } - this.exportInProgress = this.fileExporter( - filesToExport, - userCollections, - updateProgress, - exportDir - ); - const resp = await this.exportInProgress; - this.exportInProgress = null; - return resp; } async fileExporter( @@ -166,20 +182,27 @@ class ExportService { total: files.length, }); this.ElectronAPIs.sendNotification(ExportNotification.START); - - const collectionIDMap = new Map(); + collections.sort( + (collectionA, collectionB) => collectionA.id - collectionB.id + ); + const collectionIDPathMap = new Map(); + const usedCollectionPaths = new Set(); for (const collection of collections) { - const collectionFolderPath = `${dir}/${ - collection.id - }_${this.sanitizeName(collection.name)}`; + const collectionFolderPath = getUniqueCollectionFolderPath( + dir, + collection.name, + usedCollectionPaths + ); + usedCollectionPaths.add(collectionFolderPath); + collectionIDPathMap.set(collection.id, collectionFolderPath); await this.ElectronAPIs.checkExistsAndCreateCollectionDir( collectionFolderPath ); await this.ElectronAPIs.checkExistsAndCreateCollectionDir( `${collectionFolderPath}/${METADATA_FOLDER_NAME}` ); - collectionIDMap.set(collection.id, collectionFolderPath); } + files.sort((fileA, fileB) => fileA.id - fileB.id); for (const [index, file] of files.entries()) { if (this.stopExport || this.pauseExport) { if (this.pauseExport) { @@ -190,7 +213,9 @@ class ExportService { } break; } - const collectionPath = collectionIDMap.get(file.collectionID); + const collectionPath = collectionIDPathMap.get( + file.collectionID + ); try { await this.downloadAndSave(file, collectionPath); await this.addFileExportRecord( @@ -233,7 +258,8 @@ class ExportService { } return { paused: false }; } catch (e) { - logError(e, 'export failed '); + logError(e, 'fileExporter failed'); + throw e; } } async addFilesQueuedRecord(folder: string, files: File[]) { @@ -310,17 +336,23 @@ class ExportService { } async downloadAndSave(file: File, collectionPath: string) { - const uid = `${file.id}_${this.sanitizeName(file.metadata.title)}`; + const usedFileNamesInCollection = this.usedFilenames.get( + file.collectionID + ); + const fileSaveName = getUniqueFileSaveName( + file.metadata.title, + usedFileNamesInCollection + ); const fileStream = await retryAsyncFunction(() => downloadManager.downloadFile(file) ); if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { this.exportMotionPhoto(fileStream, file, collectionPath); } else { - this.saveMediaFile(collectionPath, uid, fileStream); + this.saveMediaFile(collectionPath, fileSaveName, fileStream); this.saveMetadataFile( collectionPath, - uid, + fileSaveName, mergeMetadata([file])[0].metadata ); } @@ -334,14 +366,22 @@ class ExportService { const fileBlob = await new Response(fileStream).blob(); const originalName = fileNameWithoutExtension(file.metadata.title); const motionPhoto = await decodeMotionPhoto(fileBlob, originalName); - + const usedFileNamesInCollection = this.usedFilenames.get( + file.collectionID + ); const imageStream = generateStreamFromArrayBuffer(motionPhoto.image); - const imageUID = `${file.id}_${motionPhoto.imageNameTitle}`; - this.saveMediaFile(collectionPath, imageUID, imageStream); - this.saveMetadataFile(collectionPath, imageUID, file.metadata); + const imageSaveName = getUniqueFileSaveName( + motionPhoto.imageNameTitle, + usedFileNamesInCollection + ); + this.saveMediaFile(collectionPath, imageSaveName, imageStream); + this.saveMetadataFile(collectionPath, imageSaveName, file.metadata); const videoStream = generateStreamFromArrayBuffer(motionPhoto.video); - const videoUID = `${file.id}_${motionPhoto.videoNameTitle}`; + const videoUID = getUniqueFileSaveName( + motionPhoto.videoNameTitle, + usedFileNamesInCollection + ); this.saveMediaFile(collectionPath, videoUID, videoStream); this.saveMetadataFile(collectionPath, videoUID, file.metadata); } @@ -359,10 +399,6 @@ class ExportService { ); } - private sanitizeName(name) { - return name.replaceAll('/', '_').replaceAll(' ', '_'); - } - isExportInProgress = () => { return this.exportInProgress !== null; }; diff --git a/src/services/motionPhotoService.ts b/src/services/motionPhotoService.ts index 425a86fcd..27b8440da 100644 --- a/src/services/motionPhotoService.ts +++ b/src/services/motionPhotoService.ts @@ -4,8 +4,8 @@ import { fileExtensionWithDot } from 'utils/file'; class MotionPhoto { image: Uint8Array; video: Uint8Array; - imageNameTitle: String; - videoNameTitle: String; + imageNameTitle: string; + videoNameTitle: string; } export const decodeMotionPhoto = async ( diff --git a/src/utils/export/index.ts b/src/utils/export/index.ts index 1aca1446b..7960c7fa2 100644 --- a/src/utils/export/index.ts +++ b/src/utils/export/index.ts @@ -77,3 +77,41 @@ export const getGoogleLikeMetadataFile = ( 2 ); }; + +export const getUniqueCollectionFolderPath = ( + dir: string, + collectionName: string, + usedCollectionPaths: Set +): string => { + let collectionFolderPath = `${dir}/${sanitizeName(collectionName)}`; + let count = 1; + while ( + usedCollectionPaths && + usedCollectionPaths.has(collectionFolderPath) + ) { + collectionFolderPath = `${dir}/${sanitizeName( + collectionName + )}(${count})`; + count++; + } + return collectionFolderPath; +}; + +export const sanitizeName = (name: string) => { + return name.replaceAll('/', '_').replaceAll(' ', '_'); +}; + +export const getUniqueFileSaveName = ( + filename: string, + usedFileNamesInCollection: Set +) => { + let fileSaveName = filename; + const count = 1; + while ( + usedFileNamesInCollection && + usedFileNamesInCollection.has(fileSaveName) + ) { + fileSaveName = filename + `(${count})`; + } + return fileSaveName; +}; From dfefa9f546f01fe2d66c5047f1e8c2616a6c0809 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 9 Nov 2021 22:25:45 +0530 Subject: [PATCH 05/58] merge metadata for file before export --- src/services/exportService.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index afda0ceb1..96d44fef5 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -339,6 +339,7 @@ class ExportService { const usedFileNamesInCollection = this.usedFilenames.get( file.collectionID ); + file.metadata = mergeMetadata([file])[0].metadata; const fileSaveName = getUniqueFileSaveName( file.metadata.title, usedFileNamesInCollection @@ -350,11 +351,7 @@ class ExportService { this.exportMotionPhoto(fileStream, file, collectionPath); } else { this.saveMediaFile(collectionPath, fileSaveName, fileStream); - this.saveMetadataFile( - collectionPath, - fileSaveName, - mergeMetadata([file])[0].metadata - ); + this.saveMetadataFile(collectionPath, fileSaveName, file.metadata); } } From c6318dd80807406294af8029877c96d01e0b7859 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 9 Nov 2021 22:46:04 +0530 Subject: [PATCH 06/58] remove commented code --- src/components/ExportModal.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/components/ExportModal.tsx b/src/components/ExportModal.tsx index beeffed80..029ebb675 100644 --- a/src/components/ExportModal.tsx +++ b/src/components/ExportModal.tsx @@ -340,11 +340,9 @@ export default function ExportModal(props: Props) { ) : ( <> - {/* */} {exportFolder} - {/* */} {(exportStage === ExportStage.FINISHED || exportStage === ExportStage.INIT) && ( Date: Wed, 10 Nov 2021 18:01:32 +0530 Subject: [PATCH 07/58] add package ro edit exif --- package.json | 1 + yarn.lock | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/package.json b/package.json index 0f589b5a6..4884e4230 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "next": "^11.1.2", "node-forge": "^0.10.0", "photoswipe": "file:./thirdparty/photoswipe", + "piexifjs": "^1.0.6", "react": "^17.0.2", "react-bootstrap": "^1.3.0", "react-burger-menu": "^3.0.4", diff --git a/yarn.lock b/yarn.lock index 143b09bb9..d3bd5bf98 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5040,6 +5040,11 @@ picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.2, picomatch@^2.2.3: resolved "https://registry.npmjs.org/picomatch/-/picomatch-2.3.0.tgz" integrity sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw== +piexifjs@^1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/piexifjs/-/piexifjs-1.0.6.tgz#883811d73f447218d0d06e9ed7866d04533e59e0" + integrity sha512-0wVyH0cKohzBQ5Gi2V1BuxYpxWfxF3cSqfFXfPIpl5tl9XLS5z4ogqhUCD20AbHi0h9aJkqXNJnkVev6gwh2ag== + pify@^2.0.0: version "2.3.0" resolved "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz" From 824dc54d7693a65dbc28dc75cf9f46d61aefb50d Mon Sep 17 00:00:00 2001 From: Abhinav Date: Wed, 10 Nov 2021 19:10:02 +0530 Subject: [PATCH 08/58] adds getFileExtension util --- src/services/upload/readFileService.ts | 3 ++- src/utils/file/index.ts | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/services/upload/readFileService.ts b/src/services/upload/readFileService.ts index 3342fd572..b7742b67b 100644 --- a/src/services/upload/readFileService.ts +++ b/src/services/upload/readFileService.ts @@ -6,6 +6,7 @@ import { logError } from 'utils/sentry'; import { FILE_READER_CHUNK_SIZE, MULTIPART_PART_SIZE } from './uploadService'; import FileType from 'file-type/browser'; import { CustomError } from 'utils/common/errorUtil'; +import { getFileExtension } from 'utils/file'; const TYPE_VIDEO = 'video'; const TYPE_IMAGE = 'image'; @@ -48,7 +49,7 @@ export async function getFileType( } return { fileType, exactType: typeParts[1] }; } catch (e) { - const fileFormat = receivedFile.name.split('.').pop(); + const fileFormat = getFileExtension(receivedFile.name); const formatMissedByTypeDetection = FORMAT_MISSED_BY_FILE_TYPE_LIB.find( (a) => a.exactType === fileFormat ); diff --git a/src/utils/file/index.ts b/src/utils/file/index.ts index 5ea88f252..0ec6d4080 100644 --- a/src/utils/file/index.ts +++ b/src/utils/file/index.ts @@ -250,6 +250,10 @@ export function splitFilenameAndExtension(filename): [string, string] { ]; } +export function getFileExtension(filename) { + return splitFilenameAndExtension(filename)[1]; +} + export function generateStreamFromArrayBuffer(data: Uint8Array) { return new ReadableStream({ async start(controller: ReadableStreamDefaultController) { @@ -266,7 +270,7 @@ export async function convertForPreview(file: File, fileBlob: Blob) { fileBlob = new Blob([motionPhoto.image]); } - const typeFromExtension = file.metadata.title.split('.')[-1]; + const typeFromExtension = getFileExtension(file.metadata.title); const worker = await new CryptoWorker(); const mimeType = From e6056f0ea362b4332ef127d8cc40d99a77afda62 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Wed, 10 Nov 2021 19:10:38 +0530 Subject: [PATCH 09/58] adds exif update logic --- src/services/upload/exifService.ts | 53 +++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/src/services/upload/exifService.ts b/src/services/upload/exifService.ts index 004958b6f..3fda007a4 100644 --- a/src/services/upload/exifService.ts +++ b/src/services/upload/exifService.ts @@ -1,5 +1,5 @@ import exifr from 'exifr'; - +import piexif from 'piexifjs'; import { NULL_LOCATION, Location } from './metadataService'; const EXIF_TAGS_NEEDED = [ @@ -30,6 +30,57 @@ export async function getExifData( return parsedEXIFData; } +export async function updateFileModifyDateInEXIF( + fileBlob: Blob, + updatedDate: Date +) { + const fileURL = URL.createObjectURL(fileBlob); + const imageDataURL = await convertImageToDataURL(fileURL); + const exifObj = piexif.load(imageDataURL); + if (!exifObj['0th']) { + exifObj['0th'] = {}; + } + exifObj['0th'][piexif.ImageIFD.DateTime] = updatedDate; + console.log(exifObj); + const exifBytes = piexif.dump(exifObj); + const exifInsertedFile = piexif.insert(exifBytes, imageDataURL); + return dataURIToBlob(exifInsertedFile); +} + +export async function convertImageToDataURL(url: string) { + const blob = await fetch(url).then((r) => r.blob()); + const dataUrl = await new Promise((resolve) => { + const reader = new FileReader(); + reader.onload = () => resolve(reader.result); + reader.readAsDataURL(blob); + }); + return dataUrl; +} + +function dataURIToBlob(dataURI) { + // convert base64 to raw binary data held in a string + // doesn't handle URLEncoded DataURIs - see SO answer #6850276 for code that does this + const byteString = atob(dataURI.split(',')[1]); + + // separate out the mime component + const mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0]; + + // write the bytes of the string to an ArrayBuffer + const ab = new ArrayBuffer(byteString.length); + + // create a view into the buffer + const ia = new Uint8Array(ab); + + // set the bytes of the buffer to the correct values + for (let i = 0; i < byteString.length; i++) { + ia[i] = byteString.charCodeAt(i); + } + + // write the ArrayBuffer to a blob, and you're done + const blob = new Blob([ab], { type: mimeString }); + return blob; +} + function getUNIXTime(exifData: any) { const dateTime: Date = exifData.DateTimeOriginal ?? exifData.CreateDate ?? exifData.ModifyDate; From 2636568724ade2af83396fc7fe350c922d6b3d07 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Thu, 11 Nov 2021 11:28:47 +0530 Subject: [PATCH 10/58] if file has edited Time and is jpeg then update the exif ModifyDate with the edited time --- src/services/exportService.ts | 18 +++++++++++++++++- src/utils/file/index.ts | 2 ++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index 96d44fef5..0e8944d61 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -23,9 +23,13 @@ import { decodeMotionPhoto } from './motionPhotoService'; import { fileNameWithoutExtension, generateStreamFromArrayBuffer, + getFileExtension, mergeMetadata, + TYPE_JPEG, + TYPE_JPG, } from 'utils/file'; import { User } from './userService'; +import { updateFileModifyDateInEXIF } from './upload/exifService'; export interface ExportProgress { current: number; @@ -344,9 +348,21 @@ class ExportService { file.metadata.title, usedFileNamesInCollection ); - const fileStream = await retryAsyncFunction(() => + let fileStream = await retryAsyncFunction(() => downloadManager.downloadFile(file) ); + const fileType = getFileExtension(file.metadata.title); + if ( + file.pubMagicMetadata?.data.editedTime && + (fileType === TYPE_JPEG || fileType === TYPE_JPG) + ) { + const fileBlob = await new Response(fileStream).blob(); + const updatedFileBlob = await updateFileModifyDateInEXIF( + fileBlob, + new Date(file.pubMagicMetadata.data.editedTime / 1000) + ); + fileStream = updatedFileBlob.stream(); + } if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { this.exportMotionPhoto(fileStream, file, collectionPath); } else { diff --git a/src/utils/file/index.ts b/src/utils/file/index.ts index 0ec6d4080..95b642920 100644 --- a/src/utils/file/index.ts +++ b/src/utils/file/index.ts @@ -19,6 +19,8 @@ import { getData, LS_KEYS } from 'utils/storage/localStorage'; export const TYPE_HEIC = 'heic'; export const TYPE_HEIF = 'heif'; +export const TYPE_JPEG = 'jpeg'; +export const TYPE_JPG = 'jpg'; const UNSUPPORTED_FORMATS = ['flv', 'mkv', '3gp', 'avi', 'wmv']; export function downloadAsFile(filename: string, content: string) { From f92fa1c33191c0aadc9e900a64a101d645ce95c9 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Thu, 11 Nov 2021 11:29:39 +0530 Subject: [PATCH 11/58] replace dataURL so that piexif considers file as JPEG --- src/services/upload/exifService.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/services/upload/exifService.ts b/src/services/upload/exifService.ts index 3fda007a4..ec89391b6 100644 --- a/src/services/upload/exifService.ts +++ b/src/services/upload/exifService.ts @@ -35,7 +35,10 @@ export async function updateFileModifyDateInEXIF( updatedDate: Date ) { const fileURL = URL.createObjectURL(fileBlob); - const imageDataURL = await convertImageToDataURL(fileURL); + let imageDataURL = await convertImageToDataURL(fileURL); + imageDataURL = + 'data:image/jpeg;base64' + + imageDataURL.slice(imageDataURL.indexOf(',')); const exifObj = piexif.load(imageDataURL); if (!exifObj['0th']) { exifObj['0th'] = {}; @@ -49,9 +52,9 @@ export async function updateFileModifyDateInEXIF( export async function convertImageToDataURL(url: string) { const blob = await fetch(url).then((r) => r.blob()); - const dataUrl = await new Promise((resolve) => { + const dataUrl = await new Promise((resolve) => { const reader = new FileReader(); - reader.onload = () => resolve(reader.result); + reader.onload = () => resolve(reader.result as string); reader.readAsDataURL(blob); }); return dataUrl; From 220d7ba62e127ef70a0097ddb57c165c0a42ab38 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Thu, 11 Nov 2021 12:14:51 +0530 Subject: [PATCH 12/58] fixed date format set in exif --- src/services/upload/exifService.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/services/upload/exifService.ts b/src/services/upload/exifService.ts index ec89391b6..ec30f7560 100644 --- a/src/services/upload/exifService.ts +++ b/src/services/upload/exifService.ts @@ -43,8 +43,9 @@ export async function updateFileModifyDateInEXIF( if (!exifObj['0th']) { exifObj['0th'] = {}; } - exifObj['0th'][piexif.ImageIFD.DateTime] = updatedDate; - console.log(exifObj); + exifObj['0th'][piexif.ImageIFD.DateTime] = + convertToExifDateFormat(updatedDate); + const exifBytes = piexif.dump(exifObj); const exifInsertedFile = piexif.insert(exifBytes, imageDataURL); return dataURIToBlob(exifInsertedFile); @@ -105,3 +106,9 @@ function getEXIFLocation(exifData): Location { } return { latitude: exifData.latitude, longitude: exifData.longitude }; } + +function convertToExifDateFormat(date: Date) { + return `${date.getFullYear()}:${ + date.getMonth() + 1 + }:${date.getDate()} ${date.getHours()}:${date.getMinutes()}:${date.getSeconds()}`; +} From dd6fbb29dff064a9f30700f7997aba5bac604f1f Mon Sep 17 00:00:00 2001 From: Abhinav Date: Thu, 11 Nov 2021 13:02:20 +0530 Subject: [PATCH 13/58] await on returned promised to handle error itself --- src/services/exportService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index 0e8944d61..bfc6f8855 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -108,7 +108,7 @@ class ExportService { this.ElectronAPIs.sendNotification( ExportNotification.IN_PROGRESS ); - return this.exportInProgress; + return await this.exportInProgress; } this.ElectronAPIs.showOnTray('starting export'); const exportDir = getData(LS_KEYS.EXPORT)?.folder; From 9a3aa75d1efafbcdb4d6632e7702b04502eb89fd Mon Sep 17 00:00:00 2001 From: Abhinav Date: Thu, 11 Nov 2021 13:07:19 +0530 Subject: [PATCH 14/58] better variable names --- src/services/exportService.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index bfc6f8855..1507d2a1e 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -391,12 +391,12 @@ class ExportService { this.saveMetadataFile(collectionPath, imageSaveName, file.metadata); const videoStream = generateStreamFromArrayBuffer(motionPhoto.video); - const videoUID = getUniqueFileSaveName( + const videoSaveName = getUniqueFileSaveName( motionPhoto.videoNameTitle, usedFileNamesInCollection ); - this.saveMediaFile(collectionPath, videoUID, videoStream); - this.saveMetadataFile(collectionPath, videoUID, file.metadata); + this.saveMediaFile(collectionPath, videoSaveName, videoStream); + this.saveMetadataFile(collectionPath, videoSaveName, file.metadata); } private saveMediaFile(collectionPath, uid, fileStream) { From 6a482c937eb0503ce955d08240534fdf38b57fd4 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Thu, 11 Nov 2021 13:11:16 +0530 Subject: [PATCH 15/58] added error handling for updateFileModifyDateInEXIF --- src/services/upload/exifService.ts | 34 ++++++++++++++++++------------ 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/services/upload/exifService.ts b/src/services/upload/exifService.ts index ec30f7560..d14538e7a 100644 --- a/src/services/upload/exifService.ts +++ b/src/services/upload/exifService.ts @@ -1,5 +1,6 @@ import exifr from 'exifr'; import piexif from 'piexifjs'; +import { logError } from 'utils/sentry'; import { NULL_LOCATION, Location } from './metadataService'; const EXIF_TAGS_NEEDED = [ @@ -34,21 +35,26 @@ export async function updateFileModifyDateInEXIF( fileBlob: Blob, updatedDate: Date ) { - const fileURL = URL.createObjectURL(fileBlob); - let imageDataURL = await convertImageToDataURL(fileURL); - imageDataURL = - 'data:image/jpeg;base64' + - imageDataURL.slice(imageDataURL.indexOf(',')); - const exifObj = piexif.load(imageDataURL); - if (!exifObj['0th']) { - exifObj['0th'] = {}; - } - exifObj['0th'][piexif.ImageIFD.DateTime] = - convertToExifDateFormat(updatedDate); + try { + const fileURL = URL.createObjectURL(fileBlob); + let imageDataURL = await convertImageToDataURL(fileURL); + imageDataURL = + 'data:image/jpeg;base64' + + imageDataURL.slice(imageDataURL.indexOf(',')); + const exifObj = piexif.load(imageDataURL); + if (!exifObj['0th']) { + exifObj['0th'] = {}; + } + exifObj['0th'][piexif.ImageIFD.DateTime] = + convertToExifDateFormat(updatedDate); - const exifBytes = piexif.dump(exifObj); - const exifInsertedFile = piexif.insert(exifBytes, imageDataURL); - return dataURIToBlob(exifInsertedFile); + const exifBytes = piexif.dump(exifObj); + const exifInsertedFile = piexif.insert(exifBytes, imageDataURL); + return dataURIToBlob(exifInsertedFile); + } catch (e) { + logError(e, 'updateFileModifyDateInEXIF failed'); + return fileBlob; + } } export async function convertImageToDataURL(url: string) { From f0787b7dfb30074ed9ae91d8501501782cb8e82b Mon Sep 17 00:00:00 2001 From: Abhinav Date: Thu, 11 Nov 2021 13:13:59 +0530 Subject: [PATCH 16/58] refactor code --- src/utils/export/index.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/utils/export/index.ts b/src/utils/export/index.ts index 7960c7fa2..5c73c6664 100644 --- a/src/utils/export/index.ts +++ b/src/utils/export/index.ts @@ -97,10 +97,6 @@ export const getUniqueCollectionFolderPath = ( return collectionFolderPath; }; -export const sanitizeName = (name: string) => { - return name.replaceAll('/', '_').replaceAll(' ', '_'); -}; - export const getUniqueFileSaveName = ( filename: string, usedFileNamesInCollection: Set @@ -115,3 +111,7 @@ export const getUniqueFileSaveName = ( } return fileSaveName; }; + +export const sanitizeName = (name: string) => { + return name.replaceAll('/', '_').replaceAll(' ', '_'); +}; From 3de18c4fbdedf46609856b4712733ee887ed17e0 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 16 Nov 2021 16:42:54 +0530 Subject: [PATCH 17/58] migrate export if previous export version less than current --- src/services/exportService.ts | 176 ++++++++++++++++++++++++++++------ src/utils/export/index.ts | 61 +++++++++--- 2 files changed, 197 insertions(+), 40 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index 1507d2a1e..0df5dbfaa 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -8,6 +8,12 @@ import { getExportRecordFileUID, getUniqueCollectionFolderPath, getUniqueFileSaveName, + getOldFileSavePath, + getOldCollectionFolderPath, + getFileMetadataSavePath, + getFileSavePath, + getOldFileMetadataSavePath, + getExportedFiles, } from 'utils/export'; import { retryAsyncFunction } from 'utils/network'; import { logError } from 'utils/sentry'; @@ -30,6 +36,7 @@ import { } from 'utils/file'; import { User } from './userService'; import { updateFileModifyDateInEXIF } from './upload/exifService'; +import { MetadataObject } from './upload/uploadService'; export interface ExportProgress { current: number; @@ -40,13 +47,16 @@ export interface ExportStats { success: number; } +const LATEST_EXPORT_VERSION = 1; + export interface ExportRecord { - stage: ExportStage; - lastAttemptTimestamp: number; - progress: ExportProgress; - queuedFiles: string[]; - exportedFiles: string[]; - failedFiles: string[]; + version?: number; + stage?: ExportStage; + lastAttemptTimestamp?: number; + progress?: ExportProgress; + queuedFiles?: string[]; + exportedFiles?: string[]; + failedFiles?: string[]; } export enum ExportStage { INIT, @@ -117,33 +127,44 @@ class ExportService { return; } let filesToExport: File[]; - const allFiles = await getLocalFiles(); + const allFiles = (await getLocalFiles()).sort( + (fileA, fileB) => fileA.id - fileB.id + ); const collections = await getLocalCollections(); const nonEmptyCollections = getNonEmptyCollections( collections, allFiles ); const user: User = getData(LS_KEYS.USER); - const userCollections = nonEmptyCollections.filter( - (collection) => collection.owner.id === user?.id - ); + const userCollections = nonEmptyCollections + .filter((collection) => collection.owner.id === user?.id) + .sort( + (collectionA, collectionB) => + collectionA.id - collectionB.id + ); const exportRecord = await this.getExportRecord(exportDir); + if ( + !exportRecord.version || + exportRecord.version < LATEST_EXPORT_VERSION + ) { + const exportedFiles = getExportedFiles(allFiles, exportRecord); + await this.migrateExport( + exportDir, + exportRecord.version ?? 0, + collections, + exportedFiles + ); + } if (exportType === ExportType.NEW) { - filesToExport = await getFilesUploadedAfterLastExport( + filesToExport = getFilesUploadedAfterLastExport( allFiles, exportRecord ); } else if (exportType === ExportType.RETRY_FAILED) { - filesToExport = await getExportFailedFiles( - allFiles, - exportRecord - ); + filesToExport = getExportFailedFiles(allFiles, exportRecord); } else { - filesToExport = await getExportPendingFiles( - allFiles, - exportRecord - ); + filesToExport = getExportPendingFiles(allFiles, exportRecord); } this.exportInProgress = this.fileExporter( filesToExport, @@ -186,9 +207,6 @@ class ExportService { total: files.length, }); this.ElectronAPIs.sendNotification(ExportNotification.START); - collections.sort( - (collectionA, collectionB) => collectionA.id - collectionB.id - ); const collectionIDPathMap = new Map(); const usedCollectionPaths = new Set(); for (const collection of collections) { @@ -206,7 +224,6 @@ class ExportService { `${collectionFolderPath}/${METADATA_FOLDER_NAME}` ); } - files.sort((fileA, fileB) => fileA.id - fileB.id); for (const [index, file] of files.entries()) { if (this.stopExport || this.pauseExport) { if (this.pauseExport) { @@ -301,7 +318,7 @@ class ExportService { await this.updateExportRecord(exportRecord, folder); } - async updateExportRecord(newData: Record, folder?: string) { + async updateExportRecord(newData: ExportRecord, folder?: string) { await this.recordUpdateInProgress; this.recordUpdateInProgress = (async () => { try { @@ -399,21 +416,122 @@ class ExportService { this.saveMetadataFile(collectionPath, videoSaveName, file.metadata); } - private saveMediaFile(collectionPath, uid, fileStream) { + private saveMediaFile( + collectionFolderPath: string, + fileSaveName: string, + fileStream: ReadableStream + ) { this.ElectronAPIs.saveStreamToDisk( - `${collectionPath}/${uid}`, + getFileSavePath(collectionFolderPath, fileSaveName), fileStream ); } - private saveMetadataFile(collectionPath, uid, metadata) { + private saveMetadataFile( + collectionFolderPath: string, + fileSaveName: string, + metadata: MetadataObject + ) { this.ElectronAPIs.saveFileToDisk( - `${collectionPath}/${METADATA_FOLDER_NAME}/${uid}.json`, - getGoogleLikeMetadataFile(uid, metadata) + getFileMetadataSavePath(collectionFolderPath, fileSaveName), + getGoogleLikeMetadataFile(fileSaveName, metadata) ); } isExportInProgress = () => { return this.exportInProgress !== null; }; + + private async migrateExport( + exportDir: string, + currentVersion: number, + collections: Collection[], + files: File[] + ) { + if (currentVersion === 0) { + const collectionIDPathMap = new Map(); + + await this.migrateCollectionFolders( + collections, + exportDir, + collectionIDPathMap + ); + await this.migrateFiles(files, collectionIDPathMap); + + await this.updateExportRecord({ + version: LATEST_EXPORT_VERSION, + }); + } + } + + private async migrateCollectionFolders( + collections: Collection[], + dir: string, + collectionIDPathMap: Map + ) { + const tempUsedCollectionPaths = new Set(); + for (const collection of collections) { + const oldCollectionFolderPath = getOldCollectionFolderPath( + dir, + collection + ); + const newCollectionFolderPath = getUniqueCollectionFolderPath( + dir, + collection.name, + tempUsedCollectionPaths + ); + tempUsedCollectionPaths.add(newCollectionFolderPath); + collectionIDPathMap.set(collection.id, newCollectionFolderPath); + await this.ElectronAPIs.checkExistsAndRename( + oldCollectionFolderPath, + newCollectionFolderPath + ); + await this.ElectronAPIs.checkExistsAndRename( + `${oldCollectionFolderPath}/${METADATA_FOLDER_NAME}/`, + `${newCollectionFolderPath}/${METADATA_FOLDER_NAME}` + ); + } + } + + private async migrateFiles( + files: File[], + collectionIDPathMap: Map + ) { + const tempUsedFileSaveNames = new Set(); + for (let file of files) { + const oldFileSavePath = getOldFileSavePath( + collectionIDPathMap.get(file.collectionID), + file + ); + const oldFileMetadataSavePath = getOldFileMetadataSavePath( + collectionIDPathMap.get(file.collectionID), + file + ); + file = mergeMetadata([file])[0]; + const newFileSaveName = getUniqueFileSaveName( + file.metadata.title, + tempUsedFileSaveNames + ); + tempUsedFileSaveNames.add(newFileSaveName); + + const newFileSavePath = getFileSavePath( + collectionIDPathMap.get(file.collectionID), + newFileSaveName + ); + + const newFileMetadataSavePath = getFileMetadataSavePath( + collectionIDPathMap.get(file.collectionID), + newFileSaveName + ); + await this.ElectronAPIs.checkExistsAndRename( + oldFileSavePath, + newFileSavePath + ); + console.log(oldFileMetadataSavePath, newFileMetadataSavePath); + await this.ElectronAPIs.checkExistsAndRename( + oldFileMetadataSavePath, + newFileMetadataSavePath + ); + } + } } export default new ExportService(); diff --git a/src/utils/export/index.ts b/src/utils/export/index.ts index 5c73c6664..11fdb9a2d 100644 --- a/src/utils/export/index.ts +++ b/src/utils/export/index.ts @@ -1,4 +1,5 @@ -import { ExportRecord } from 'services/exportService'; +import { Collection } from 'services/collectionService'; +import { ExportRecord, METADATA_FOLDER_NAME } from 'services/exportService'; import { File } from 'services/fileService'; import { MetadataObject } from 'services/upload/uploadService'; import { formatDate } from 'utils/file'; @@ -6,7 +7,7 @@ import { formatDate } from 'utils/file'; export const getExportRecordFileUID = (file: File) => `${file.id}_${file.collectionID}_${file.updationTime}`; -export const getExportPendingFiles = async ( +export const getExportPendingFiles = ( allFiles: File[], exportRecord: ExportRecord ) => { @@ -19,7 +20,7 @@ export const getExportPendingFiles = async ( return unExportedFiles; }; -export const getFilesUploadedAfterLastExport = async ( +export const getFilesUploadedAfterLastExport = ( allFiles: File[], exportRecord: ExportRecord ) => { @@ -32,7 +33,20 @@ export const getFilesUploadedAfterLastExport = async ( return unExportedFiles; }; -export const getExportFailedFiles = async ( +export const getExportedFiles = ( + allFiles: File[], + exportRecord: ExportRecord +) => { + const exportedFileIds = new Set(exportRecord?.exportedFiles); + const exportedFiles = allFiles.filter((file) => { + if (exportedFileIds.has(getExportRecordFileUID(file))) { + return file; + } + }); + return exportedFiles; +}; + +export const getExportFailedFiles = ( allFiles: File[], exportRecord: ExportRecord ) => { @@ -52,14 +66,14 @@ export const dedupe = (files: any[]) => { }; export const getGoogleLikeMetadataFile = ( - uid: string, + fileSaveName: string, metadata: MetadataObject ) => { const creationTime = Math.floor(metadata.creationTime / 1000000); const modificationTime = Math.floor(metadata.modificationTime / 1000000); return JSON.stringify( { - title: uid, + title: fileSaveName, creationTime: { timestamp: creationTime, formatted: formatDate(creationTime * 1000), @@ -101,17 +115,42 @@ export const getUniqueFileSaveName = ( filename: string, usedFileNamesInCollection: Set ) => { - let fileSaveName = filename; + let fileSaveName = sanitizeName(filename); const count = 1; while ( usedFileNamesInCollection && usedFileNamesInCollection.has(fileSaveName) ) { - fileSaveName = filename + `(${count})`; + fileSaveName = sanitizeName(filename) + `(${count})`; } return fileSaveName; }; -export const sanitizeName = (name: string) => { - return name.replaceAll('/', '_').replaceAll(' ', '_'); -}; +export const getFileMetadataSavePath = ( + collectionFolderPath: string, + fileSaveName: string +) => `${collectionFolderPath}/${METADATA_FOLDER_NAME}/${fileSaveName}.json`; + +export const getFileSavePath = ( + collectionFolderPath: string, + fileSaveName: string +) => `${collectionFolderPath}/${fileSaveName}`; + +export const sanitizeName = (name: string) => + name.replaceAll('/', '_').replaceAll(' ', '_'); + +export const getOldCollectionFolderPath = ( + dir: string, + collection: Collection +) => `${dir}/${collection.id}_${sanitizeName(collection.name)}`; + +export const getOldFileSavePath = (collectionFolderPath: string, file: File) => + `${collectionFolderPath}/${file.id}_${sanitizeName(file.metadata.title)}`; + +export const getOldFileMetadataSavePath = ( + collectionFolderPath: string, + file: File +) => + `${collectionFolderPath}/${METADATA_FOLDER_NAME}/${file.id}_${sanitizeName( + file.metadata.title + )}.json`; From da02db2a49c3d4f1c7dd4a4c4709d4296720f3fb Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 16 Nov 2021 16:46:28 +0530 Subject: [PATCH 18/58] update DateTimeOriginal property with editedTime --- src/services/upload/exifService.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/upload/exifService.ts b/src/services/upload/exifService.ts index d14538e7a..0cd83eb84 100644 --- a/src/services/upload/exifService.ts +++ b/src/services/upload/exifService.ts @@ -42,10 +42,10 @@ export async function updateFileModifyDateInEXIF( 'data:image/jpeg;base64' + imageDataURL.slice(imageDataURL.indexOf(',')); const exifObj = piexif.load(imageDataURL); - if (!exifObj['0th']) { - exifObj['0th'] = {}; + if (!exifObj['Exif']) { + exifObj['Exif'] = {}; } - exifObj['0th'][piexif.ImageIFD.DateTime] = + exifObj['Exif'][piexif.ExifIFD.DateTimeOriginal] = convertToExifDateFormat(updatedDate); const exifBytes = piexif.dump(exifObj); From 670dbe2d0f241de4bff40696c3fc3a2ef3e902b2 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 16 Nov 2021 17:21:26 +0530 Subject: [PATCH 19/58] download from photoswipe contain updated exif --- src/utils/file/index.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/utils/file/index.ts b/src/utils/file/index.ts index 95b642920..05d1635ed 100644 --- a/src/utils/file/index.ts +++ b/src/utils/file/index.ts @@ -16,6 +16,7 @@ import { logError } from 'utils/sentry'; import { User } from 'services/userService'; import CryptoWorker from 'utils/crypto'; import { getData, LS_KEYS } from 'utils/storage/localStorage'; +import { updateFileModifyDateInEXIF } from 'services/upload/exifService'; export const TYPE_HEIC = 'heic'; export const TYPE_HEIF = 'heif'; @@ -42,7 +43,13 @@ export function downloadAsFile(filename: string, content: string) { export async function downloadFile(file) { const a = document.createElement('a'); a.style.display = 'none'; - a.href = await DownloadManger.getFile(file); + const fileURL = await DownloadManger.getFile(file); + const fileBlob = await (await fetch(fileURL)).blob(); + const updatedFileBlob = await updateFileModifyDateInEXIF( + fileBlob, + new Date(file.pubMagicMetadata.data.editedTime / 1000) + ); + a.href = URL.createObjectURL(updatedFileBlob); if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { a.download = fileNameWithoutExtension(file.metadata.title) + '.zip'; } else { From de2acbcf32f12fee29338118fc113325b0eb139b Mon Sep 17 00:00:00 2001 From: Abhinav Date: Wed, 17 Nov 2021 11:58:50 +0530 Subject: [PATCH 20/58] fix updateExportTime --- src/components/ExportModal.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/ExportModal.tsx b/src/components/ExportModal.tsx index 029ebb675..f30956646 100644 --- a/src/components/ExportModal.tsx +++ b/src/components/ExportModal.tsx @@ -167,7 +167,7 @@ export default function ExportModal(props: Props) { const updateExportTime = (newTime: number) => { setLastExportTime(newTime); - exportService.updateExportRecord({ time: newTime }); + exportService.updateExportRecord({ lastAttemptTimestamp: newTime }); }; const updateExportProgress = (newProgress: ExportProgress) => { From a4654413d4c1e7c11622d5663155fbfb75b76586 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Wed, 17 Nov 2021 13:58:08 +0530 Subject: [PATCH 21/58] no need to migrate metadata folder --- src/services/exportService.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index 0df5dbfaa..3260c3e27 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -485,10 +485,6 @@ class ExportService { oldCollectionFolderPath, newCollectionFolderPath ); - await this.ElectronAPIs.checkExistsAndRename( - `${oldCollectionFolderPath}/${METADATA_FOLDER_NAME}/`, - `${newCollectionFolderPath}/${METADATA_FOLDER_NAME}` - ); } } From eed7f5e288b9a8be00a7751fe04938a59b7e5d40 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Wed, 17 Nov 2021 14:06:53 +0530 Subject: [PATCH 22/58] change updateFileModifyDateInEXIF to updateFileCreationDateInEXIF --- src/services/exportService.ts | 4 ++-- src/services/upload/exifService.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index 3260c3e27..7f5df6cc9 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -35,7 +35,7 @@ import { TYPE_JPG, } from 'utils/file'; import { User } from './userService'; -import { updateFileModifyDateInEXIF } from './upload/exifService'; +import { updateFileCreationDateInEXIF } from './upload/exifService'; import { MetadataObject } from './upload/uploadService'; export interface ExportProgress { @@ -374,7 +374,7 @@ class ExportService { (fileType === TYPE_JPEG || fileType === TYPE_JPG) ) { const fileBlob = await new Response(fileStream).blob(); - const updatedFileBlob = await updateFileModifyDateInEXIF( + const updatedFileBlob = await updateFileCreationDateInEXIF( fileBlob, new Date(file.pubMagicMetadata.data.editedTime / 1000) ); diff --git a/src/services/upload/exifService.ts b/src/services/upload/exifService.ts index 0cd83eb84..6b670c55a 100644 --- a/src/services/upload/exifService.ts +++ b/src/services/upload/exifService.ts @@ -31,7 +31,7 @@ export async function getExifData( return parsedEXIFData; } -export async function updateFileModifyDateInEXIF( +export async function updateFileCreationDateInEXIF( fileBlob: Blob, updatedDate: Date ) { From 2273dcdc76770c7d0c521a14edb3b1a6c2682876 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Wed, 17 Nov 2021 14:11:04 +0530 Subject: [PATCH 23/58] adds util for getting metadata folderpath --- src/services/exportService.ts | 3 ++- src/utils/export/index.ts | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index 7f5df6cc9..11f7782fb 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -14,6 +14,7 @@ import { getFileSavePath, getOldFileMetadataSavePath, getExportedFiles, + getMetadataFolderPath, } from 'utils/export'; import { retryAsyncFunction } from 'utils/network'; import { logError } from 'utils/sentry'; @@ -221,7 +222,7 @@ class ExportService { collectionFolderPath ); await this.ElectronAPIs.checkExistsAndCreateCollectionDir( - `${collectionFolderPath}/${METADATA_FOLDER_NAME}` + getMetadataFolderPath(collectionFolderPath) ); } for (const [index, file] of files.entries()) { diff --git a/src/utils/export/index.ts b/src/utils/export/index.ts index 11fdb9a2d..5f8d90ad5 100644 --- a/src/utils/export/index.ts +++ b/src/utils/export/index.ts @@ -92,6 +92,9 @@ export const getGoogleLikeMetadataFile = ( ); }; +export const sanitizeName = (name: string) => + name.replaceAll('/', '_').replaceAll(' ', '_'); + export const getUniqueCollectionFolderPath = ( dir: string, collectionName: string, @@ -111,6 +114,9 @@ export const getUniqueCollectionFolderPath = ( return collectionFolderPath; }; +export const getMetadataFolderPath = (collectionFolderPath: string) => + `${collectionFolderPath}/${METADATA_FOLDER_NAME}`; + export const getUniqueFileSaveName = ( filename: string, usedFileNamesInCollection: Set @@ -136,9 +142,6 @@ export const getFileSavePath = ( fileSaveName: string ) => `${collectionFolderPath}/${fileSaveName}`; -export const sanitizeName = (name: string) => - name.replaceAll('/', '_').replaceAll(' ', '_'); - export const getOldCollectionFolderPath = ( dir: string, collection: Collection From 8d4dff4c3804d270e64953165edfebdd8d39c857 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Wed, 17 Nov 2021 14:54:26 +0530 Subject: [PATCH 24/58] fix collection unique name and file name --- src/services/exportService.ts | 21 +++++++++++++-------- src/utils/export/index.ts | 23 ++++++++++++----------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index 11f7782fb..8e2227ae8 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -96,7 +96,7 @@ class ExportService { private recordUpdateInProgress = Promise.resolve(); private stopExport: boolean = false; private pauseExport: boolean = false; - private usedFilenames = new Map>(); + private usedFileNames = new Map>(); constructor() { this.ElectronAPIs = runningInBrowser() && window['ElectronAPIs']; @@ -216,7 +216,6 @@ class ExportService { collection.name, usedCollectionPaths ); - usedCollectionPaths.add(collectionFolderPath); collectionIDPathMap.set(collection.id, collectionFolderPath); await this.ElectronAPIs.checkExistsAndCreateCollectionDir( collectionFolderPath @@ -358,7 +357,10 @@ class ExportService { } async downloadAndSave(file: File, collectionPath: string) { - const usedFileNamesInCollection = this.usedFilenames.get( + if (!this.usedFileNames.has(file.collectionID)) { + this.usedFileNames.set(file.collectionID, new Set()); + } + const usedFileNamesInCollection = this.usedFileNames.get( file.collectionID ); file.metadata = mergeMetadata([file])[0].metadata; @@ -382,7 +384,12 @@ class ExportService { fileStream = updatedFileBlob.stream(); } if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { - this.exportMotionPhoto(fileStream, file, collectionPath); + this.exportMotionPhoto( + fileStream, + file, + collectionPath, + usedFileNamesInCollection + ); } else { this.saveMediaFile(collectionPath, fileSaveName, fileStream); this.saveMetadataFile(collectionPath, fileSaveName, file.metadata); @@ -392,14 +399,12 @@ class ExportService { private async exportMotionPhoto( fileStream: ReadableStream, file: File, - collectionPath: string + collectionPath: string, + usedFileNamesInCollection: Set ) { const fileBlob = await new Response(fileStream).blob(); const originalName = fileNameWithoutExtension(file.metadata.title); const motionPhoto = await decodeMotionPhoto(fileBlob, originalName); - const usedFileNamesInCollection = this.usedFilenames.get( - file.collectionID - ); const imageStream = generateStreamFromArrayBuffer(motionPhoto.image); const imageSaveName = getUniqueFileSaveName( motionPhoto.imageNameTitle, diff --git a/src/utils/export/index.ts b/src/utils/export/index.ts index 5f8d90ad5..593a12db6 100644 --- a/src/utils/export/index.ts +++ b/src/utils/export/index.ts @@ -2,7 +2,7 @@ import { Collection } from 'services/collectionService'; import { ExportRecord, METADATA_FOLDER_NAME } from 'services/exportService'; import { File } from 'services/fileService'; import { MetadataObject } from 'services/upload/uploadService'; -import { formatDate } from 'utils/file'; +import { formatDate, splitFilenameAndExtension } from 'utils/file'; export const getExportRecordFileUID = (file: File) => `${file.id}_${file.collectionID}_${file.updationTime}`; @@ -102,15 +102,13 @@ export const getUniqueCollectionFolderPath = ( ): string => { let collectionFolderPath = `${dir}/${sanitizeName(collectionName)}`; let count = 1; - while ( - usedCollectionPaths && - usedCollectionPaths.has(collectionFolderPath) - ) { + while (usedCollectionPaths.has(collectionFolderPath)) { collectionFolderPath = `${dir}/${sanitizeName( collectionName )}(${count})`; count++; } + usedCollectionPaths.add(collectionFolderPath); return collectionFolderPath; }; @@ -119,16 +117,19 @@ export const getMetadataFolderPath = (collectionFolderPath: string) => export const getUniqueFileSaveName = ( filename: string, - usedFileNamesInCollection: Set + usedFilenamesInCollection: Set ) => { let fileSaveName = sanitizeName(filename); const count = 1; - while ( - usedFileNamesInCollection && - usedFileNamesInCollection.has(fileSaveName) - ) { - fileSaveName = sanitizeName(filename) + `(${count})`; + while (usedFilenamesInCollection.has(fileSaveName)) { + const filenameParts = splitFilenameAndExtension(fileSaveName); + if (filenameParts[1]) { + fileSaveName = `${filenameParts[0]}(${count}).${filenameParts[1]}`; + } else { + fileSaveName = `${filenameParts[0]}(${count})`; + } } + usedFilenamesInCollection.add(fileSaveName); return fileSaveName; }; From 8c1e7249c65fd929595fa1b0d72c890ef253ae1b Mon Sep 17 00:00:00 2001 From: Abhinav Date: Wed, 17 Nov 2021 22:33:15 +0530 Subject: [PATCH 25/58] build fix --- src/utils/file/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/file/index.ts b/src/utils/file/index.ts index 05d1635ed..c20fbea8d 100644 --- a/src/utils/file/index.ts +++ b/src/utils/file/index.ts @@ -16,7 +16,7 @@ import { logError } from 'utils/sentry'; import { User } from 'services/userService'; import CryptoWorker from 'utils/crypto'; import { getData, LS_KEYS } from 'utils/storage/localStorage'; -import { updateFileModifyDateInEXIF } from 'services/upload/exifService'; +import { updateFileCreationDateInEXIF } from 'services/upload/exifService'; export const TYPE_HEIC = 'heic'; export const TYPE_HEIF = 'heif'; @@ -45,7 +45,7 @@ export async function downloadFile(file) { a.style.display = 'none'; const fileURL = await DownloadManger.getFile(file); const fileBlob = await (await fetch(fileURL)).blob(); - const updatedFileBlob = await updateFileModifyDateInEXIF( + const updatedFileBlob = await updateFileCreationDateInEXIF( fileBlob, new Date(file.pubMagicMetadata.data.editedTime / 1000) ); From dcc13dc3175bc71062cc272685593e1b4d7de11e Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 22 Nov 2021 11:40:04 +0530 Subject: [PATCH 26/58] fix UI --- src/components/ExportFinished.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/components/ExportFinished.tsx b/src/components/ExportFinished.tsx index dedae4d34..5fc263d88 100644 --- a/src/components/ExportFinished.tsx +++ b/src/components/ExportFinished.tsx @@ -29,8 +29,8 @@ export default function ExportFinished(props: Props) { padding: '0 5%', }}> - - + + {formatDateTime(props.lastExportTime)} @@ -38,7 +38,7 @@ export default function ExportFinished(props: Props) { - + {props.exportStats.success} / {totalFiles} From 66eb9478aa55e687c1f730bd26491ec292efb318 Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Tue, 23 Nov 2021 21:43:37 +0530 Subject: [PATCH 27/58] add debug logs --- src/services/exportService.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index 8e2227ae8..8626f35a1 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -250,6 +250,10 @@ class ExportService { file, RecordType.FAILED ); + console.log( + `export failed for fileID:${file.id}, reason:`, + e + ); logError( e, 'download and save failed for file during export' From e4e58fd9051ca9ccfca59a11c98715eceff6bc41 Mon Sep 17 00:00:00 2001 From: Abhinav-grd Date: Tue, 23 Nov 2021 22:08:51 +0530 Subject: [PATCH 28/58] fallback to creationTime if modification time missing --- src/utils/export/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/utils/export/index.ts b/src/utils/export/index.ts index 593a12db6..c138ac38d 100644 --- a/src/utils/export/index.ts +++ b/src/utils/export/index.ts @@ -70,7 +70,9 @@ export const getGoogleLikeMetadataFile = ( metadata: MetadataObject ) => { const creationTime = Math.floor(metadata.creationTime / 1000000); - const modificationTime = Math.floor(metadata.modificationTime / 1000000); + const modificationTime = Math.floor( + (metadata.modificationTime ?? metadata.creationTime) / 1000000 + ); return JSON.stringify( { title: fileSaveName, From 33bd80f571cdcdbb04aaf387cc827fb7d3c087d6 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 26 Nov 2021 11:47:32 +0530 Subject: [PATCH 29/58] fix metadata json map setting --- src/services/upload/uploadManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/upload/uploadManager.ts b/src/services/upload/uploadManager.ts index 1c67cbb6d..fe9db1016 100644 --- a/src/services/upload/uploadManager.ts +++ b/src/services/upload/uploadManager.ts @@ -125,7 +125,7 @@ class UploadManager { fileWithCollection.collectionID, title ), - parsedMetaDataJSON + { ...parsedMetaDataJSON } ); UIService.increaseFileUploaded(); } From c443bae39adbe9b6cc4a1f8ad9fbd3c05bc41065 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 29 Nov 2021 14:39:11 +0530 Subject: [PATCH 30/58] add null check before trying to update exif with edited value --- src/utils/file/index.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/utils/file/index.ts b/src/utils/file/index.ts index c0aebca4d..d0fb842bc 100644 --- a/src/utils/file/index.ts +++ b/src/utils/file/index.ts @@ -44,12 +44,14 @@ export async function downloadFile(file) { const a = document.createElement('a'); a.style.display = 'none'; const fileURL = await DownloadManger.getFile(file); - const fileBlob = await (await fetch(fileURL)).blob(); - const updatedFileBlob = await updateFileCreationDateInEXIF( - fileBlob, - new Date(file.pubMagicMetadata.data.editedTime / 1000) - ); - a.href = URL.createObjectURL(updatedFileBlob); + let fileBlob = await (await fetch(fileURL)).blob(); + if (file.pubMagicMetadata?.data.editedTime) { + fileBlob = await updateFileCreationDateInEXIF( + fileBlob, + new Date(file.pubMagicMetadata.data.editedTime / 1000) + ); + } + a.href = URL.createObjectURL(fileBlob); if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { a.download = fileNameWithoutExtension(file.metadata.title) + '.zip'; } else { From 13cb4338ffb043cc0a3ffc63e83d4d31b6b548e6 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 30 Nov 2021 11:57:50 +0530 Subject: [PATCH 31/58] update sanitize name function --- src/utils/export/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/export/index.ts b/src/utils/export/index.ts index c138ac38d..13c28fbb8 100644 --- a/src/utils/export/index.ts +++ b/src/utils/export/index.ts @@ -95,7 +95,7 @@ export const getGoogleLikeMetadataFile = ( }; export const sanitizeName = (name: string) => - name.replaceAll('/', '_').replaceAll(' ', '_'); + name.replace(/[^a-z0-9.]/gi, '_').toLowerCase(); export const getUniqueCollectionFolderPath = ( dir: string, From c84ee17a7c780823782329f2d26c2344ec3ffd6b Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 30 Nov 2021 12:18:02 +0530 Subject: [PATCH 32/58] move version check logic to migrateExport function --- src/services/exportService.ts | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index 8626f35a1..bc8b2d8aa 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -144,18 +144,7 @@ class ExportService { collectionA.id - collectionB.id ); const exportRecord = await this.getExportRecord(exportDir); - if ( - !exportRecord.version || - exportRecord.version < LATEST_EXPORT_VERSION - ) { - const exportedFiles = getExportedFiles(allFiles, exportRecord); - await this.migrateExport( - exportDir, - exportRecord.version ?? 0, - collections, - exportedFiles - ); - } + await this.migrateExport(exportDir, collections, allFiles); if (exportType === ExportType.NEW) { filesToExport = getFilesUploadedAfterLastExport( @@ -453,10 +442,11 @@ class ExportService { private async migrateExport( exportDir: string, - currentVersion: number, collections: Collection[], - files: File[] + allFiles: File[] ) { + const exportRecord = await this.getExportRecord(exportDir); + const currentVersion = exportRecord?.version ?? 0; if (currentVersion === 0) { const collectionIDPathMap = new Map(); @@ -465,7 +455,10 @@ class ExportService { exportDir, collectionIDPathMap ); - await this.migrateFiles(files, collectionIDPathMap); + await this.migrateFiles( + getExportedFiles(allFiles, exportRecord), + collectionIDPathMap + ); await this.updateExportRecord({ version: LATEST_EXPORT_VERSION, @@ -473,6 +466,10 @@ class ExportService { } } + /* + This updates the folder name of already exported folders from the earlier format of + `collectionID_collectionName` to newer `collectionName(numbered)` format + */ private async migrateCollectionFolders( collections: Collection[], dir: string, @@ -498,6 +495,10 @@ class ExportService { } } + /* + This updates the file name of already exported files from the earlier format of + `fileID_fileName` to newer `fileName(numbered)` format + */ private async migrateFiles( files: File[], collectionIDPathMap: Map From 9dfcbacaeaa68ccfed8ad877fc715021c23aeef3 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 30 Nov 2021 12:23:11 +0530 Subject: [PATCH 33/58] adds comments --- src/services/exportService.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index bc8b2d8aa..d2e3e1fd4 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -440,6 +440,13 @@ class ExportService { return this.exportInProgress !== null; }; + /* + this function migrates the exportRecord file to apply any schema changes. + currently we apply only a single migration to update file and collection name to newer format + so there is just a if condition check, + later this will be converted to a loop which applies the migration one by one + till the files reaches the latest version + */ private async migrateExport( exportDir: string, collections: Collection[], From e9e777f5eb196c5bc89b1d9c16f69372f749ae2a Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 30 Nov 2021 22:51:47 +0530 Subject: [PATCH 34/58] better function name --- src/services/exportService.ts | 4 ++-- src/utils/export/index.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index d2e3e1fd4..415ccccd3 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -1,6 +1,6 @@ import { runningInBrowser } from 'utils/common'; import { - getExportPendingFiles, + getExportQueuedFiles, getExportFailedFiles, getFilesUploadedAfterLastExport, dedupe, @@ -154,7 +154,7 @@ class ExportService { } else if (exportType === ExportType.RETRY_FAILED) { filesToExport = getExportFailedFiles(allFiles, exportRecord); } else { - filesToExport = getExportPendingFiles(allFiles, exportRecord); + filesToExport = getExportQueuedFiles(allFiles, exportRecord); } this.exportInProgress = this.fileExporter( filesToExport, diff --git a/src/utils/export/index.ts b/src/utils/export/index.ts index 13c28fbb8..2837afc0b 100644 --- a/src/utils/export/index.ts +++ b/src/utils/export/index.ts @@ -7,7 +7,7 @@ import { formatDate, splitFilenameAndExtension } from 'utils/file'; export const getExportRecordFileUID = (file: File) => `${file.id}_${file.collectionID}_${file.updationTime}`; -export const getExportPendingFiles = ( +export const getExportQueuedFiles = ( allFiles: File[], exportRecord: ExportRecord ) => { From 8be93db2db5ee51fc08e54f240dceb979a69b2cd Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 7 Dec 2021 11:53:48 +0530 Subject: [PATCH 35/58] use exists electron API to remove the need of in memory used fileName and collectionName maps --- src/services/exportService.ts | 52 ++++++++--------------------------- src/utils/export/index.ts | 19 ++++++------- 2 files changed, 20 insertions(+), 51 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index 415ccccd3..e8220d0fb 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -96,7 +96,6 @@ class ExportService { private recordUpdateInProgress = Promise.resolve(); private stopExport: boolean = false; private pauseExport: boolean = false; - private usedFileNames = new Map>(); constructor() { this.ElectronAPIs = runningInBrowser() && window['ElectronAPIs']; @@ -198,12 +197,10 @@ class ExportService { }); this.ElectronAPIs.sendNotification(ExportNotification.START); const collectionIDPathMap = new Map(); - const usedCollectionPaths = new Set(); for (const collection of collections) { const collectionFolderPath = getUniqueCollectionFolderPath( dir, - collection.name, - usedCollectionPaths + collection.name ); collectionIDPathMap.set(collection.id, collectionFolderPath); await this.ElectronAPIs.checkExistsAndCreateCollectionDir( @@ -350,17 +347,8 @@ class ExportService { } async downloadAndSave(file: File, collectionPath: string) { - if (!this.usedFileNames.has(file.collectionID)) { - this.usedFileNames.set(file.collectionID, new Set()); - } - const usedFileNamesInCollection = this.usedFileNames.get( - file.collectionID - ); file.metadata = mergeMetadata([file])[0].metadata; - const fileSaveName = getUniqueFileSaveName( - file.metadata.title, - usedFileNamesInCollection - ); + const fileSaveName = getUniqueFileSaveName(file.metadata.title); let fileStream = await retryAsyncFunction(() => downloadManager.downloadFile(file) ); @@ -377,12 +365,7 @@ class ExportService { fileStream = updatedFileBlob.stream(); } if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { - this.exportMotionPhoto( - fileStream, - file, - collectionPath, - usedFileNamesInCollection - ); + this.exportMotionPhoto(fileStream, file, collectionPath); } else { this.saveMediaFile(collectionPath, fileSaveName, fileStream); this.saveMetadataFile(collectionPath, fileSaveName, file.metadata); @@ -392,25 +375,18 @@ class ExportService { private async exportMotionPhoto( fileStream: ReadableStream, file: File, - collectionPath: string, - usedFileNamesInCollection: Set + collectionPath: string ) { const fileBlob = await new Response(fileStream).blob(); const originalName = fileNameWithoutExtension(file.metadata.title); const motionPhoto = await decodeMotionPhoto(fileBlob, originalName); const imageStream = generateStreamFromArrayBuffer(motionPhoto.image); - const imageSaveName = getUniqueFileSaveName( - motionPhoto.imageNameTitle, - usedFileNamesInCollection - ); + const imageSaveName = getUniqueFileSaveName(motionPhoto.imageNameTitle); this.saveMediaFile(collectionPath, imageSaveName, imageStream); this.saveMetadataFile(collectionPath, imageSaveName, file.metadata); const videoStream = generateStreamFromArrayBuffer(motionPhoto.video); - const videoSaveName = getUniqueFileSaveName( - motionPhoto.videoNameTitle, - usedFileNamesInCollection - ); + const videoSaveName = getUniqueFileSaveName(motionPhoto.videoNameTitle); this.saveMediaFile(collectionPath, videoSaveName, videoStream); this.saveMetadataFile(collectionPath, videoSaveName, file.metadata); } @@ -440,6 +416,10 @@ class ExportService { return this.exportInProgress !== null; }; + exists = (path: string) => { + return this.ElectronAPIs.exists(path); + }; + /* this function migrates the exportRecord file to apply any schema changes. currently we apply only a single migration to update file and collection name to newer format @@ -482,7 +462,6 @@ class ExportService { dir: string, collectionIDPathMap: Map ) { - const tempUsedCollectionPaths = new Set(); for (const collection of collections) { const oldCollectionFolderPath = getOldCollectionFolderPath( dir, @@ -490,10 +469,8 @@ class ExportService { ); const newCollectionFolderPath = getUniqueCollectionFolderPath( dir, - collection.name, - tempUsedCollectionPaths + collection.name ); - tempUsedCollectionPaths.add(newCollectionFolderPath); collectionIDPathMap.set(collection.id, newCollectionFolderPath); await this.ElectronAPIs.checkExistsAndRename( oldCollectionFolderPath, @@ -510,7 +487,6 @@ class ExportService { files: File[], collectionIDPathMap: Map ) { - const tempUsedFileSaveNames = new Set(); for (let file of files) { const oldFileSavePath = getOldFileSavePath( collectionIDPathMap.get(file.collectionID), @@ -521,11 +497,7 @@ class ExportService { file ); file = mergeMetadata([file])[0]; - const newFileSaveName = getUniqueFileSaveName( - file.metadata.title, - tempUsedFileSaveNames - ); - tempUsedFileSaveNames.add(newFileSaveName); + const newFileSaveName = getUniqueFileSaveName(file.metadata.title); const newFileSavePath = getFileSavePath( collectionIDPathMap.get(file.collectionID), diff --git a/src/utils/export/index.ts b/src/utils/export/index.ts index 2837afc0b..9efc84826 100644 --- a/src/utils/export/index.ts +++ b/src/utils/export/index.ts @@ -1,5 +1,8 @@ import { Collection } from 'services/collectionService'; -import { ExportRecord, METADATA_FOLDER_NAME } from 'services/exportService'; +import exportService, { + ExportRecord, + METADATA_FOLDER_NAME, +} from 'services/exportService'; import { File } from 'services/fileService'; import { MetadataObject } from 'services/upload/uploadService'; import { formatDate, splitFilenameAndExtension } from 'utils/file'; @@ -99,31 +102,26 @@ export const sanitizeName = (name: string) => export const getUniqueCollectionFolderPath = ( dir: string, - collectionName: string, - usedCollectionPaths: Set + collectionName: string ): string => { let collectionFolderPath = `${dir}/${sanitizeName(collectionName)}`; let count = 1; - while (usedCollectionPaths.has(collectionFolderPath)) { + while (exportService.exists(collectionFolderPath)) { collectionFolderPath = `${dir}/${sanitizeName( collectionName )}(${count})`; count++; } - usedCollectionPaths.add(collectionFolderPath); return collectionFolderPath; }; export const getMetadataFolderPath = (collectionFolderPath: string) => `${collectionFolderPath}/${METADATA_FOLDER_NAME}`; -export const getUniqueFileSaveName = ( - filename: string, - usedFilenamesInCollection: Set -) => { +export const getUniqueFileSaveName = (filename: string) => { let fileSaveName = sanitizeName(filename); const count = 1; - while (usedFilenamesInCollection.has(fileSaveName)) { + while (exportService.exists(fileSaveName)) { const filenameParts = splitFilenameAndExtension(fileSaveName); if (filenameParts[1]) { fileSaveName = `${filenameParts[0]}(${count}).${filenameParts[1]}`; @@ -131,7 +129,6 @@ export const getUniqueFileSaveName = ( fileSaveName = `${filenameParts[0]}(${count})`; } } - usedFilenamesInCollection.add(fileSaveName); return fileSaveName; }; From 42817eb8d2f575a94ed185af1a7d2a1213c06714 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 7 Dec 2021 12:53:32 +0530 Subject: [PATCH 36/58] better function name --- src/services/exportService.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index e8220d0fb..a799b03cc 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -225,13 +225,13 @@ class ExportService { ); try { await this.downloadAndSave(file, collectionPath); - await this.addFileExportRecord( + await this.addFileExportedRecord( dir, file, RecordType.SUCCESS ); } catch (e) { - await this.addFileExportRecord( + await this.addFileExportedRecord( dir, file, RecordType.FAILED @@ -279,7 +279,7 @@ class ExportService { await this.updateExportRecord(exportRecord, folder); } - async addFileExportRecord(folder: string, file: File, type: RecordType) { + async addFileExportedRecord(folder: string, file: File, type: RecordType) { const fileUID = getExportRecordFileUID(file); const exportRecord = await this.getExportRecord(folder); exportRecord.queuedFiles = exportRecord.queuedFiles.filter( From 2424767e79c1699b19b27683baca72a406657a35 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 7 Dec 2021 12:54:11 +0530 Subject: [PATCH 37/58] null safe variable --- src/components/ExportModal.tsx | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/components/ExportModal.tsx b/src/components/ExportModal.tsx index f30956646..f7c3c6c80 100644 --- a/src/components/ExportModal.tsx +++ b/src/components/ExportModal.tsx @@ -115,8 +115,8 @@ export default function ExportModal(props: Props) { (file) => file.ownerID === user?.id ); const exportRecord = await exportService.getExportRecord(); - const exportedFileCnt = exportRecord.exportedFiles.length; - const failedFilesCnt = exportRecord.failedFiles.length; + const exportedFileCnt = exportRecord.exportedFiles?.length; + const failedFilesCnt = exportRecord.failedFiles?.length; const syncedFilesCnt = userPersonalFiles.length; if (syncedFilesCnt > exportedFileCnt + failedFilesCnt) { updateExportProgress({ @@ -191,8 +191,8 @@ export default function ExportModal(props: Props) { updateExportStage(ExportStage.INPROGRESS); await sleep(100); }; - const postExportRun = async (paused: Boolean) => { - if (!paused) { + const postExportRun = async (exportResult?: { paused?: boolean }) => { + if (!exportResult?.paused) { updateExportStage(ExportStage.FINISHED); await sleep(100); updateExportTime(Date.now()); @@ -202,22 +202,22 @@ export default function ExportModal(props: Props) { const startExport = async () => { await preExportRun(); updateExportProgress({ current: 0, total: 0 }); - const { paused } = await exportService.exportFiles( + const exportResult = await exportService.exportFiles( updateExportProgress, ExportType.NEW ); - await postExportRun(paused); + await postExportRun(exportResult); }; const stopExport = async () => { exportService.stopRunningExport(); - postExportRun(false); + postExportRun(); }; const pauseExport = () => { updateExportStage(ExportStage.PAUSED); exportService.pauseRunningExport(); - postExportRun(true); + postExportRun({ paused: true }); }; const resumeExport = async () => { @@ -232,23 +232,23 @@ export default function ExportModal(props: Props) { current: pausedStageProgress.current + progress.current, total: pausedStageProgress.current + progress.total, }); - const { paused } = await exportService.exportFiles( + const exportResult = await exportService.exportFiles( updateExportStatsWithOffset, ExportType.PENDING ); - await postExportRun(paused); + await postExportRun(exportResult); }; const retryFailedExport = async () => { await preExportRun(); updateExportProgress({ current: 0, total: exportStats.failed }); - const { paused } = await exportService.exportFiles( + const exportResult = await exportService.exportFiles( updateExportProgress, ExportType.RETRY_FAILED ); - await postExportRun(paused); + await postExportRun(exportResult); }; const syncExportStatsWithReport = async () => { From 710ffd202bf3c705525a8191e4ce229e5381c13e Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 7 Dec 2021 12:56:50 +0530 Subject: [PATCH 38/58] add missing await for saveFileToDisk --- src/services/exportService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index a799b03cc..c228931e1 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -401,12 +401,12 @@ class ExportService { fileStream ); } - private saveMetadataFile( + private async saveMetadataFile( collectionFolderPath: string, fileSaveName: string, metadata: MetadataObject ) { - this.ElectronAPIs.saveFileToDisk( + await this.ElectronAPIs.saveFileToDisk( getFileMetadataSavePath(collectionFolderPath, fileSaveName), getGoogleLikeMetadataFile(fileSaveName, metadata) ); From 1476e68e3cf7791f9a3edef31077553945ae8613 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 7 Dec 2021 13:14:21 +0530 Subject: [PATCH 39/58] use queueProcessor to make update sequential --- src/services/exportService.ts | 37 +++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index c228931e1..6a063dabc 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -38,6 +38,7 @@ import { import { User } from './userService'; import { updateFileCreationDateInEXIF } from './upload/exifService'; import { MetadataObject } from './upload/uploadService'; +import QueueProcessor from './upload/queueProcessor'; export interface ExportProgress { current: number; @@ -93,7 +94,7 @@ class ExportService { ElectronAPIs: any; private exportInProgress: Promise<{ paused: boolean }> = null; - private recordUpdateInProgress = Promise.resolve(); + private exportRecordUpdater = new QueueProcessor(1); private stopExport: boolean = false; private pauseExport: boolean = false; @@ -309,27 +310,29 @@ class ExportService { } async updateExportRecord(newData: ExportRecord, folder?: string) { - await this.recordUpdateInProgress; - this.recordUpdateInProgress = (async () => { - try { - if (!folder) { - folder = getData(LS_KEYS.EXPORT)?.folder; - } - const exportRecord = await this.getExportRecord(folder); - const newRecord = { ...exportRecord, ...newData }; - await this.ElectronAPIs.setExportRecord( - `${folder}/${EXPORT_RECORD_FILE_NAME}`, - JSON.stringify(newRecord, null, 2) - ); - } catch (e) { - logError(e, 'error updating Export Record'); + const response = this.exportRecordUpdater.queueUpRequest(() => + this.updateExportRecordHelper(folder, newData) + ); + response.promise; + } + async updateExportRecordHelper(folder: string, newData: ExportRecord) { + try { + if (!folder) { + folder = getData(LS_KEYS.EXPORT)?.folder; } - })(); + const exportRecord = await this.getExportRecord(folder); + const newRecord = { ...exportRecord, ...newData }; + await this.ElectronAPIs.setExportRecord( + `${folder}/${EXPORT_RECORD_FILE_NAME}`, + JSON.stringify(newRecord, null, 2) + ); + } catch (e) { + logError(e, 'error updating Export Record'); + } } async getExportRecord(folder?: string): Promise { try { - await this.recordUpdateInProgress; if (!folder) { folder = getData(LS_KEYS.EXPORT)?.folder; } From 3dd90ba538d62aded22d23ed573044b7321eb913 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 7 Dec 2021 14:16:37 +0530 Subject: [PATCH 40/58] fix getUniqueFileSaveName , by providing collectionPath too for checking if file already exists at the fileSavePath --- src/services/exportService.ts | 20 ++++++++++++++++---- src/utils/export/index.ts | 14 ++++++++++---- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index 6a063dabc..7e33b22a7 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -351,7 +351,10 @@ class ExportService { async downloadAndSave(file: File, collectionPath: string) { file.metadata = mergeMetadata([file])[0].metadata; - const fileSaveName = getUniqueFileSaveName(file.metadata.title); + const fileSaveName = getUniqueFileSaveName( + collectionPath, + file.metadata.title + ); let fileStream = await retryAsyncFunction(() => downloadManager.downloadFile(file) ); @@ -384,12 +387,18 @@ class ExportService { const originalName = fileNameWithoutExtension(file.metadata.title); const motionPhoto = await decodeMotionPhoto(fileBlob, originalName); const imageStream = generateStreamFromArrayBuffer(motionPhoto.image); - const imageSaveName = getUniqueFileSaveName(motionPhoto.imageNameTitle); + const imageSaveName = getUniqueFileSaveName( + collectionPath, + motionPhoto.imageNameTitle + ); this.saveMediaFile(collectionPath, imageSaveName, imageStream); this.saveMetadataFile(collectionPath, imageSaveName, file.metadata); const videoStream = generateStreamFromArrayBuffer(motionPhoto.video); - const videoSaveName = getUniqueFileSaveName(motionPhoto.videoNameTitle); + const videoSaveName = getUniqueFileSaveName( + collectionPath, + motionPhoto.videoNameTitle + ); this.saveMediaFile(collectionPath, videoSaveName, videoStream); this.saveMetadataFile(collectionPath, videoSaveName, file.metadata); } @@ -500,7 +509,10 @@ class ExportService { file ); file = mergeMetadata([file])[0]; - const newFileSaveName = getUniqueFileSaveName(file.metadata.title); + const newFileSaveName = getUniqueFileSaveName( + collectionIDPathMap.get(file.collectionID), + file.metadata.title + ); const newFileSavePath = getFileSavePath( collectionIDPathMap.get(file.collectionID), diff --git a/src/utils/export/index.ts b/src/utils/export/index.ts index 9efc84826..fcb7fdc03 100644 --- a/src/utils/export/index.ts +++ b/src/utils/export/index.ts @@ -118,16 +118,22 @@ export const getUniqueCollectionFolderPath = ( export const getMetadataFolderPath = (collectionFolderPath: string) => `${collectionFolderPath}/${METADATA_FOLDER_NAME}`; -export const getUniqueFileSaveName = (filename: string) => { +export const getUniqueFileSaveName = ( + collectionPath: string, + filename: string +) => { let fileSaveName = sanitizeName(filename); - const count = 1; - while (exportService.exists(fileSaveName)) { - const filenameParts = splitFilenameAndExtension(fileSaveName); + let count = 1; + while ( + exportService.exists(getFileSavePath(collectionPath, fileSaveName)) + ) { + const filenameParts = splitFilenameAndExtension(filename); if (filenameParts[1]) { fileSaveName = `${filenameParts[0]}(${count}).${filenameParts[1]}`; } else { fileSaveName = `${filenameParts[0]}(${count})`; } + count++; } return fileSaveName; }; From cf09858fb79272ceea9becf378ac6ce0c1e38020 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 7 Dec 2021 14:24:28 +0530 Subject: [PATCH 41/58] only export user personal files --- src/services/exportService.ts | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index 7e33b22a7..c03b3d62d 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -128,13 +128,15 @@ class ExportService { return; } let filesToExport: File[]; - const allFiles = (await getLocalFiles()).sort( - (fileA, fileB) => fileA.id - fileB.id - ); + const localFiles = await getLocalFiles(); + const userPersonalFiles = localFiles + .filter((file) => file.ownerID === user?.id) + .sort((fileA, fileB) => fileA.id - fileB.id); + const collections = await getLocalCollections(); const nonEmptyCollections = getNonEmptyCollections( collections, - allFiles + userPersonalFiles ); const user: User = getData(LS_KEYS.USER); const userCollections = nonEmptyCollections @@ -144,17 +146,23 @@ class ExportService { collectionA.id - collectionB.id ); const exportRecord = await this.getExportRecord(exportDir); - await this.migrateExport(exportDir, collections, allFiles); + await this.migrateExport(exportDir, collections, userPersonalFiles); if (exportType === ExportType.NEW) { filesToExport = getFilesUploadedAfterLastExport( - allFiles, + userPersonalFiles, exportRecord ); } else if (exportType === ExportType.RETRY_FAILED) { - filesToExport = getExportFailedFiles(allFiles, exportRecord); + filesToExport = getExportFailedFiles( + userPersonalFiles, + exportRecord + ); } else { - filesToExport = getExportQueuedFiles(allFiles, exportRecord); + filesToExport = getExportQueuedFiles( + userPersonalFiles, + exportRecord + ); } this.exportInProgress = this.fileExporter( filesToExport, From 55b2de1a535f2c4ead58f062dcfe24b31c468910 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 7 Dec 2021 15:41:38 +0530 Subject: [PATCH 42/58] fix collection folder creation --- src/services/exportService.ts | 81 +++++++++++++++++++++++++++-------- src/utils/export/index.ts | 15 +++++++ 2 files changed, 79 insertions(+), 17 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index c03b3d62d..9b6eb7fcd 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -15,6 +15,7 @@ import { getOldFileMetadataSavePath, getExportedFiles, getMetadataFolderPath, + getCollectionsCreatedAfterLastExport, } from 'utils/export'; import { retryAsyncFunction } from 'utils/network'; import { logError } from 'utils/sentry'; @@ -40,10 +41,15 @@ import { updateFileCreationDateInEXIF } from './upload/exifService'; import { MetadataObject } from './upload/uploadService'; import QueueProcessor from './upload/queueProcessor'; +type CollectionIDPathMap = Map; export interface ExportProgress { current: number; total: number; } +export interface ExportedCollection { + collectionID: number; + folderPath: string; +} export interface ExportStats { failed: number; success: number; @@ -59,6 +65,7 @@ export interface ExportRecord { queuedFiles?: string[]; exportedFiles?: string[]; failedFiles?: string[]; + exportedCollections?: ExportedCollection[]; } export enum ExportStage { INIT, @@ -127,6 +134,8 @@ class ExportService { // no-export folder set return; } + const user: User = getData(LS_KEYS.USER); + let filesToExport: File[]; const localFiles = await getLocalFiles(); const userPersonalFiles = localFiles @@ -138,15 +147,14 @@ class ExportService { collections, userPersonalFiles ); - const user: User = getData(LS_KEYS.USER); const userCollections = nonEmptyCollections .filter((collection) => collection.owner.id === user?.id) .sort( (collectionA, collectionB) => collectionA.id - collectionB.id ); - const exportRecord = await this.getExportRecord(exportDir); await this.migrateExport(exportDir, collections, userPersonalFiles); + const exportRecord = await this.getExportRecord(exportDir); if (exportType === ExportType.NEW) { filesToExport = getFilesUploadedAfterLastExport( @@ -164,9 +172,23 @@ class ExportService { exportRecord ); } + const collectionIDPathMap: CollectionIDPathMap = new Map< + number, + string + >( + (exportRecord.exportedCollections ?? []).map((c) => [ + c.collectionID, + c.folderPath, + ]) + ); + const newCollections = getCollectionsCreatedAfterLastExport( + userCollections, + exportRecord + ); this.exportInProgress = this.fileExporter( filesToExport, - userCollections, + newCollections, + collectionIDPathMap, updateProgress, exportDir ); @@ -182,10 +204,24 @@ class ExportService { async fileExporter( files: File[], collections: Collection[], + collectionIDPathMap: CollectionIDPathMap, updateProgress: (progress: ExportProgress) => void, dir: string ): Promise<{ paused: boolean }> { try { + for (const collection of collections) { + const collectionFolderPath = getUniqueCollectionFolderPath( + dir, + collection.name + ); + collectionIDPathMap.set(collection.id, collectionFolderPath); + await this.ElectronAPIs.checkExistsAndCreateCollectionDir( + collectionFolderPath + ); + await this.ElectronAPIs.checkExistsAndCreateCollectionDir( + getMetadataFolderPath(collectionFolderPath) + ); + } if (!files?.length) { this.ElectronAPIs.sendNotification( ExportNotification.UP_TO_DATE @@ -205,20 +241,7 @@ class ExportService { total: files.length, }); this.ElectronAPIs.sendNotification(ExportNotification.START); - const collectionIDPathMap = new Map(); - for (const collection of collections) { - const collectionFolderPath = getUniqueCollectionFolderPath( - dir, - collection.name - ); - collectionIDPathMap.set(collection.id, collectionFolderPath); - await this.ElectronAPIs.checkExistsAndCreateCollectionDir( - collectionFolderPath - ); - await this.ElectronAPIs.checkExistsAndCreateCollectionDir( - getMetadataFolderPath(collectionFolderPath) - ); - } + for (const [index, file] of files.entries()) { if (this.stopExport || this.pauseExport) { if (this.pauseExport) { @@ -317,6 +340,25 @@ class ExportService { await this.updateExportRecord(exportRecord, folder); } + async addCollectionExportedRecord( + folder: string, + collection: Collection, + collectionFolderPath: string + ) { + const exportRecord = await this.getExportRecord(folder); + if (!exportRecord?.exportedCollections) { + exportRecord.exportedCollections = []; + } + exportRecord.exportedCollections.push({ + collectionID: collection.id, + folderPath: collectionFolderPath, + }); + exportRecord.exportedCollections = dedupe( + exportRecord.exportedCollections + ); + await this.updateExportRecord(exportRecord, folder); + } + async updateExportRecord(newData: ExportRecord, folder?: string) { const response = this.exportRecordUpdater.queueUpRequest(() => this.updateExportRecordHelper(folder, newData) @@ -496,6 +538,11 @@ class ExportService { oldCollectionFolderPath, newCollectionFolderPath ); + await this.addCollectionExportedRecord( + dir, + collection, + newCollectionFolderPath + ); } } diff --git a/src/utils/export/index.ts b/src/utils/export/index.ts index fcb7fdc03..8cf4bc600 100644 --- a/src/utils/export/index.ts +++ b/src/utils/export/index.ts @@ -23,6 +23,21 @@ export const getExportQueuedFiles = ( return unExportedFiles; }; +export const getCollectionsCreatedAfterLastExport = ( + collections: Collection[], + exportRecord: ExportRecord +) => { + const exportedCollections = new Set( + (exportRecord?.exportedCollections ?? []).map((c) => c.collectionID) + ); + const unExportedCollections = collections.filter((collection) => { + if (!exportedCollections.has(collection.id)) { + return collection; + } + }); + return unExportedCollections; +}; + export const getFilesUploadedAfterLastExport = ( allFiles: File[], exportRecord: ExportRecord From 6074200b673fe864e10fa3dd66143fecbdd39d78 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 7 Dec 2021 15:41:50 +0530 Subject: [PATCH 43/58] better local errors --- src/utils/sentry/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/sentry/index.ts b/src/utils/sentry/index.ts index dae3f9996..b3d4c87ca 100644 --- a/src/utils/sentry/index.ts +++ b/src/utils/sentry/index.ts @@ -9,7 +9,7 @@ export const logError = ( ) => { const err = errorWithContext(error, msg); if (!process.env.NEXT_PUBLIC_SENTRY_ENV) { - console.log({ error, msg, info }); + console.log(error, { msg, info }); } Sentry.captureException(err, { level: Sentry.Severity.Info, From 5257a918642c703f7cb351288ffd7b381ec338c0 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 7 Dec 2021 16:02:55 +0530 Subject: [PATCH 44/58] fix addCollectionExportedRecord calls --- src/services/exportService.ts | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index 9b6eb7fcd..a9947f7d1 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -221,6 +221,11 @@ class ExportService { await this.ElectronAPIs.checkExistsAndCreateCollectionDir( getMetadataFolderPath(collectionFolderPath) ); + await this.addCollectionExportedRecord( + dir, + collection, + collectionFolderPath + ); } if (!files?.length) { this.ElectronAPIs.sendNotification( @@ -522,7 +527,7 @@ class ExportService { private async migrateCollectionFolders( collections: Collection[], dir: string, - collectionIDPathMap: Map + collectionIDPathMap: CollectionIDPathMap ) { for (const collection of collections) { const oldCollectionFolderPath = getOldCollectionFolderPath( @@ -534,15 +539,17 @@ class ExportService { collection.name ); collectionIDPathMap.set(collection.id, newCollectionFolderPath); - await this.ElectronAPIs.checkExistsAndRename( - oldCollectionFolderPath, - newCollectionFolderPath - ); - await this.addCollectionExportedRecord( - dir, - collection, - newCollectionFolderPath - ); + if (this.ElectronAPIs.exists(oldCollectionFolderPath)) { + await this.ElectronAPIs.checkExistsAndRename( + oldCollectionFolderPath, + newCollectionFolderPath + ); + await this.addCollectionExportedRecord( + dir, + collection, + newCollectionFolderPath + ); + } } } From 27d9d703c739f5a5c098f636f4ce1bf93548a5c9 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 7 Dec 2021 16:23:56 +0530 Subject: [PATCH 45/58] renamed exported Collection if collection name changed --- src/services/exportService.ts | 118 ++++++++++++++++++++++++---------- src/utils/export/index.ts | 28 ++++++++ 2 files changed, 112 insertions(+), 34 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index a9947f7d1..5f54f0300 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -16,6 +16,8 @@ import { getExportedFiles, getMetadataFolderPath, getCollectionsCreatedAfterLastExport, + getCollectionsRenamedAfterLastExport, + getCollectionIDPathMapFromExportRecord, } from 'utils/export'; import { retryAsyncFunction } from 'utils/network'; import { logError } from 'utils/sentry'; @@ -41,7 +43,7 @@ import { updateFileCreationDateInEXIF } from './upload/exifService'; import { MetadataObject } from './upload/uploadService'; import QueueProcessor from './upload/queueProcessor'; -type CollectionIDPathMap = Map; +export type CollectionIDPathMap = Map; export interface ExportProgress { current: number; total: number; @@ -172,22 +174,21 @@ class ExportService { exportRecord ); } - const collectionIDPathMap: CollectionIDPathMap = new Map< - number, - string - >( - (exportRecord.exportedCollections ?? []).map((c) => [ - c.collectionID, - c.folderPath, - ]) - ); + const collectionIDPathMap: CollectionIDPathMap = + getCollectionIDPathMapFromExportRecord(exportRecord); const newCollections = getCollectionsCreatedAfterLastExport( userCollections, exportRecord ); + + const renamedCollections = getCollectionsRenamedAfterLastExport( + userCollections, + exportRecord + ); this.exportInProgress = this.fileExporter( filesToExport, newCollections, + renamedCollections, collectionIDPathMap, updateProgress, exportDir @@ -203,28 +204,25 @@ class ExportService { async fileExporter( files: File[], - collections: Collection[], + newCollections: Collection[], + renamedCollections: Collection[], collectionIDPathMap: CollectionIDPathMap, updateProgress: (progress: ExportProgress) => void, - dir: string + exportDir: string ): Promise<{ paused: boolean }> { try { - for (const collection of collections) { - const collectionFolderPath = getUniqueCollectionFolderPath( - dir, - collection.name + if (!newCollections?.length) { + await this.createNewCollectionFolders( + newCollections, + exportDir, + collectionIDPathMap ); - collectionIDPathMap.set(collection.id, collectionFolderPath); - await this.ElectronAPIs.checkExistsAndCreateCollectionDir( - collectionFolderPath - ); - await this.ElectronAPIs.checkExistsAndCreateCollectionDir( - getMetadataFolderPath(collectionFolderPath) - ); - await this.addCollectionExportedRecord( - dir, - collection, - collectionFolderPath + } + if (!renamedCollections?.length) { + await this.renamedFolderForRenamedCollections( + renamedCollections, + exportDir, + collectionIDPathMap ); } if (!files?.length) { @@ -235,7 +233,7 @@ class ExportService { } this.stopExport = false; this.pauseExport = false; - this.addFilesQueuedRecord(dir, files); + this.addFilesQueuedRecord(exportDir, files); const failedFileCount = 0; this.ElectronAPIs.showOnTray({ @@ -263,13 +261,13 @@ class ExportService { try { await this.downloadAndSave(file, collectionPath); await this.addFileExportedRecord( - dir, + exportDir, file, RecordType.SUCCESS ); } catch (e) { await this.addFileExportedRecord( - dir, + exportDir, file, RecordType.FAILED ); @@ -404,6 +402,58 @@ class ExportService { } } + async createNewCollectionFolders( + newCollections: Collection[], + exportFolder: string, + collectionIDPathMap: CollectionIDPathMap + ) { + for (const collection of newCollections) { + const collectionFolderPath = getUniqueCollectionFolderPath( + exportFolder, + collection.name + ); + await this.ElectronAPIs.checkExistsAndCreateCollectionDir( + collectionFolderPath + ); + await this.ElectronAPIs.checkExistsAndCreateCollectionDir( + getMetadataFolderPath(collectionFolderPath) + ); + await this.addCollectionExportedRecord( + exportFolder, + collection, + collectionFolderPath + ); + collectionIDPathMap.set(collection.id, collectionFolderPath); + } + } + async renamedFolderForRenamedCollections( + renamedCollections: Collection[], + exportFolder: string, + collectionIDPathMap: CollectionIDPathMap + ) { + for (const collection of renamedCollections) { + const oldCollectionFolderPath = collectionIDPathMap.get( + collection.id + ); + + const newCollectionFolderPath = getUniqueCollectionFolderPath( + exportFolder, + collection.name + ); + await this.ElectronAPIs.checkExistsAndRename( + oldCollectionFolderPath, + newCollectionFolderPath + ); + + await this.addCollectionExportedRecord( + exportFolder, + collection, + newCollectionFolderPath + ); + collectionIDPathMap.set(collection.id, newCollectionFolderPath); + } + } + async downloadAndSave(file: File, collectionPath: string) { file.metadata = mergeMetadata([file])[0].metadata; const fileSaveName = getUniqueFileSaveName( @@ -526,16 +576,16 @@ class ExportService { */ private async migrateCollectionFolders( collections: Collection[], - dir: string, + exportDir: string, collectionIDPathMap: CollectionIDPathMap ) { for (const collection of collections) { const oldCollectionFolderPath = getOldCollectionFolderPath( - dir, + exportDir, collection ); const newCollectionFolderPath = getUniqueCollectionFolderPath( - dir, + exportDir, collection.name ); collectionIDPathMap.set(collection.id, newCollectionFolderPath); @@ -545,7 +595,7 @@ class ExportService { newCollectionFolderPath ); await this.addCollectionExportedRecord( - dir, + exportDir, collection, newCollectionFolderPath ); diff --git a/src/utils/export/index.ts b/src/utils/export/index.ts index 8cf4bc600..7ca477fce 100644 --- a/src/utils/export/index.ts +++ b/src/utils/export/index.ts @@ -1,5 +1,6 @@ import { Collection } from 'services/collectionService'; import exportService, { + CollectionIDPathMap, ExportRecord, METADATA_FOLDER_NAME, } from 'services/exportService'; @@ -37,6 +38,33 @@ export const getCollectionsCreatedAfterLastExport = ( }); return unExportedCollections; }; +export const getCollectionIDPathMapFromExportRecord = ( + exportRecord: ExportRecord +): CollectionIDPathMap => { + return new Map( + (exportRecord.exportedCollections ?? []).map((c) => [ + c.collectionID, + c.folderPath, + ]) + ); +}; + +export const getCollectionsRenamedAfterLastExport = ( + collections: Collection[], + exportRecord: ExportRecord +) => { + const collectionIDPathMap = + getCollectionIDPathMapFromExportRecord(exportRecord); + const renamedCollections = collections.filter((collection) => { + if ( + collectionIDPathMap.has(collection.id) && + collectionIDPathMap.get(collection.id) !== collection.name + ) { + return collection; + } + }); + return renamedCollections; +}; export const getFilesUploadedAfterLastExport = ( allFiles: File[], From 85d06fc5288d6132c8c771dcb8f61d1c34fa64c3 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 7 Dec 2021 17:33:11 +0530 Subject: [PATCH 46/58] fix bugs --- src/services/exportService.ts | 33 ++++++++++++------------- src/utils/export/index.ts | 46 +++++++++++++++++++++++------------ 2 files changed, 45 insertions(+), 34 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index 5f54f0300..1ad0e007a 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -48,9 +48,8 @@ export interface ExportProgress { current: number; total: number; } -export interface ExportedCollection { - collectionID: number; - folderPath: string; +export interface ExportedCollectionPaths { + [collectionID: number]: string; } export interface ExportStats { failed: number; @@ -67,7 +66,7 @@ export interface ExportRecord { queuedFiles?: string[]; exportedFiles?: string[]; failedFiles?: string[]; - exportedCollections?: ExportedCollection[]; + exportedCollectionPaths?: ExportedCollectionPaths; } export enum ExportStage { INIT, @@ -211,15 +210,15 @@ class ExportService { exportDir: string ): Promise<{ paused: boolean }> { try { - if (!newCollections?.length) { + if (newCollections?.length) { await this.createNewCollectionFolders( newCollections, exportDir, collectionIDPathMap ); } - if (!renamedCollections?.length) { - await this.renamedFolderForRenamedCollections( + if (renamedCollections?.length) { + await this.renameCollectionFolders( renamedCollections, exportDir, collectionIDPathMap @@ -349,16 +348,14 @@ class ExportService { collectionFolderPath: string ) { const exportRecord = await this.getExportRecord(folder); - if (!exportRecord?.exportedCollections) { - exportRecord.exportedCollections = []; + if (!exportRecord?.exportedCollectionPaths) { + exportRecord.exportedCollectionPaths = {}; } - exportRecord.exportedCollections.push({ - collectionID: collection.id, - folderPath: collectionFolderPath, - }); - exportRecord.exportedCollections = dedupe( - exportRecord.exportedCollections - ); + exportRecord.exportedCollectionPaths = { + ...exportRecord.exportedCollectionPaths, + [collection.id]: collectionFolderPath, + }; + await this.updateExportRecord(exportRecord, folder); } @@ -366,7 +363,7 @@ class ExportService { const response = this.exportRecordUpdater.queueUpRequest(() => this.updateExportRecordHelper(folder, newData) ); - response.promise; + await response.promise; } async updateExportRecordHelper(folder: string, newData: ExportRecord) { try { @@ -426,7 +423,7 @@ class ExportService { collectionIDPathMap.set(collection.id, collectionFolderPath); } } - async renamedFolderForRenamedCollections( + async renameCollectionFolders( renamedCollections: Collection[], exportFolder: string, collectionIDPathMap: CollectionIDPathMap diff --git a/src/utils/export/index.ts b/src/utils/export/index.ts index 7ca477fce..06a949eac 100644 --- a/src/utils/export/index.ts +++ b/src/utils/export/index.ts @@ -18,8 +18,9 @@ export const getExportQueuedFiles = ( const queuedFiles = new Set(exportRecord?.queuedFiles); const unExportedFiles = allFiles.filter((file) => { if (queuedFiles.has(getExportRecordFileUID(file))) { - return file; + return true; } + return false; }); return unExportedFiles; }; @@ -29,12 +30,13 @@ export const getCollectionsCreatedAfterLastExport = ( exportRecord: ExportRecord ) => { const exportedCollections = new Set( - (exportRecord?.exportedCollections ?? []).map((c) => c.collectionID) + Object.keys(exportRecord?.exportedCollectionPaths).map((x) => Number(x)) ); const unExportedCollections = collections.filter((collection) => { if (!exportedCollections.has(collection.id)) { - return collection; + return true; } + return false; }); return unExportedCollections; }; @@ -42,10 +44,11 @@ export const getCollectionIDPathMapFromExportRecord = ( exportRecord: ExportRecord ): CollectionIDPathMap => { return new Map( - (exportRecord.exportedCollections ?? []).map((c) => [ - c.collectionID, - c.folderPath, - ]) + (Object.entries(exportRecord.exportedCollectionPaths) ?? []).map( + (e) => { + return [Number(e[0]), String(e[1])]; + } + ) ); }; @@ -56,12 +59,20 @@ export const getCollectionsRenamedAfterLastExport = ( const collectionIDPathMap = getCollectionIDPathMapFromExportRecord(exportRecord); const renamedCollections = collections.filter((collection) => { - if ( - collectionIDPathMap.has(collection.id) && - collectionIDPathMap.get(collection.id) !== collection.name - ) { - return collection; + if (collectionIDPathMap.has(collection.id)) { + const currentFolderName = collectionIDPathMap.get(collection.id); + const startIndex = currentFolderName.lastIndexOf('/'); + const lastIndex = currentFolderName.lastIndexOf('('); + const nameRoot = currentFolderName.slice( + startIndex + 1, + lastIndex !== -1 ? lastIndex : currentFolderName.length + ); + + if (nameRoot !== sanitizeName(collection.name)) { + return true; + } } + return false; }); return renamedCollections; }; @@ -73,8 +84,9 @@ export const getFilesUploadedAfterLastExport = ( const exportedFiles = new Set(exportRecord?.exportedFiles); const unExportedFiles = allFiles.filter((file) => { if (!exportedFiles.has(getExportRecordFileUID(file))) { - return file; + return true; } + return false; }); return unExportedFiles; }; @@ -86,8 +98,9 @@ export const getExportedFiles = ( const exportedFileIds = new Set(exportRecord?.exportedFiles); const exportedFiles = allFiles.filter((file) => { if (exportedFileIds.has(getExportRecordFileUID(file))) { - return file; + return true; } + return false; }); return exportedFiles; }; @@ -99,8 +112,9 @@ export const getExportFailedFiles = ( const failedFiles = new Set(exportRecord?.failedFiles); const filesToExport = allFiles.filter((file) => { if (failedFiles.has(getExportRecordFileUID(file))) { - return file; + return true; } + return false; }); return filesToExport; }; @@ -170,7 +184,7 @@ export const getUniqueFileSaveName = ( while ( exportService.exists(getFileSavePath(collectionPath, fileSaveName)) ) { - const filenameParts = splitFilenameAndExtension(filename); + const filenameParts = splitFilenameAndExtension(sanitizeName(filename)); if (filenameParts[1]) { fileSaveName = `${filenameParts[0]}(${count}).${filenameParts[1]}`; } else { From f13549b2bff8a4c0b791deb4fb48f8720c6825f5 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 7 Dec 2021 17:40:56 +0530 Subject: [PATCH 47/58] null safety --- src/utils/export/index.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/utils/export/index.ts b/src/utils/export/index.ts index 06a949eac..cce4a5868 100644 --- a/src/utils/export/index.ts +++ b/src/utils/export/index.ts @@ -30,7 +30,9 @@ export const getCollectionsCreatedAfterLastExport = ( exportRecord: ExportRecord ) => { const exportedCollections = new Set( - Object.keys(exportRecord?.exportedCollectionPaths).map((x) => Number(x)) + Object.keys(exportRecord?.exportedCollectionPaths ?? {}).map((x) => + Number(x) + ) ); const unExportedCollections = collections.filter((collection) => { if (!exportedCollections.has(collection.id)) { @@ -44,11 +46,9 @@ export const getCollectionIDPathMapFromExportRecord = ( exportRecord: ExportRecord ): CollectionIDPathMap => { return new Map( - (Object.entries(exportRecord.exportedCollectionPaths) ?? []).map( - (e) => { - return [Number(e[0]), String(e[1])]; - } - ) + Object.entries(exportRecord.exportedCollectionPaths ?? {}).map((e) => { + return [Number(e[0]), String(e[1])]; + }) ); }; From ba1fb670e84dff11690d141496735436e7de73dc Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 7 Dec 2021 17:57:57 +0530 Subject: [PATCH 48/58] add old sanitizer to get oldname for migration --- src/utils/export/index.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/utils/export/index.ts b/src/utils/export/index.ts index cce4a5868..40e96c9ea 100644 --- a/src/utils/export/index.ts +++ b/src/utils/export/index.ts @@ -154,6 +154,9 @@ export const getGoogleLikeMetadataFile = ( ); }; +export const oldSanitizeName = (name: string) => + name.replaceAll('/', '_').replaceAll(' ', '_'); + export const sanitizeName = (name: string) => name.replace(/[^a-z0-9.]/gi, '_').toLowerCase(); @@ -208,15 +211,17 @@ export const getFileSavePath = ( export const getOldCollectionFolderPath = ( dir: string, collection: Collection -) => `${dir}/${collection.id}_${sanitizeName(collection.name)}`; +) => `${dir}/${collection.id}_${oldSanitizeName(collection.name)}`; export const getOldFileSavePath = (collectionFolderPath: string, file: File) => - `${collectionFolderPath}/${file.id}_${sanitizeName(file.metadata.title)}`; + `${collectionFolderPath}/${file.id}_${oldSanitizeName( + file.metadata.title + )}`; export const getOldFileMetadataSavePath = ( collectionFolderPath: string, file: File ) => - `${collectionFolderPath}/${METADATA_FOLDER_NAME}/${file.id}_${sanitizeName( - file.metadata.title - )}.json`; + `${collectionFolderPath}/${METADATA_FOLDER_NAME}/${ + file.id + }_${oldSanitizeName(file.metadata.title)}.json`; From 2e927845699d6c08afaf260ad0d0385c4cdeb052 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 7 Dec 2021 18:17:20 +0530 Subject: [PATCH 49/58] add missing await to save saveMetadataFile --- src/services/exportService.ts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index 1ad0e007a..c2be52209 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -473,10 +473,14 @@ class ExportService { fileStream = updatedFileBlob.stream(); } if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { - this.exportMotionPhoto(fileStream, file, collectionPath); + await this.exportMotionPhoto(fileStream, file, collectionPath); } else { this.saveMediaFile(collectionPath, fileSaveName, fileStream); - this.saveMetadataFile(collectionPath, fileSaveName, file.metadata); + await this.saveMetadataFile( + collectionPath, + fileSaveName, + file.metadata + ); } } @@ -494,7 +498,11 @@ class ExportService { motionPhoto.imageNameTitle ); this.saveMediaFile(collectionPath, imageSaveName, imageStream); - this.saveMetadataFile(collectionPath, imageSaveName, file.metadata); + await this.saveMetadataFile( + collectionPath, + imageSaveName, + file.metadata + ); const videoStream = generateStreamFromArrayBuffer(motionPhoto.video); const videoSaveName = getUniqueFileSaveName( @@ -502,7 +510,11 @@ class ExportService { motionPhoto.videoNameTitle ); this.saveMediaFile(collectionPath, videoSaveName, videoStream); - this.saveMetadataFile(collectionPath, videoSaveName, file.metadata); + await this.saveMetadataFile( + collectionPath, + videoSaveName, + file.metadata + ); } private saveMediaFile( From ca05d41cb43b102632f7d0a2ac0dfae478bfcd83 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Dec 2021 05:50:56 +0000 Subject: [PATCH 50/58] Bump next from 11.1.2 to 11.1.3 Bumps [next](https://github.com/vercel/next.js) from 11.1.2 to 11.1.3. - [Release notes](https://github.com/vercel/next.js/releases) - [Changelog](https://github.com/vercel/next.js/blob/canary/release.js) - [Commits](https://github.com/vercel/next.js/compare/v11.1.2...v11.1.3) --- updated-dependencies: - dependency-name: next dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- package.json | 2 +- yarn.lock | 88 ++++++++++++++++++++++++++-------------------------- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/package.json b/package.json index b2ce5421a..83f50f9d7 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "jszip": "3.7.1", "libsodium-wrappers": "^0.7.8", "localforage": "^1.9.0", - "next": "^11.1.2", + "next": "^11.1.3", "node-forge": "^0.10.0", "photoswipe": "file:./thirdparty/photoswipe", "react": "^17.0.2", diff --git a/yarn.lock b/yarn.lock index aefbf3e4d..67a62af1f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1104,20 +1104,20 @@ dependencies: webpack-bundle-analyzer "3.6.1" -"@next/env@11.1.2": - version "11.1.2" - resolved "https://registry.npmjs.org/@next/env/-/env-11.1.2.tgz" - integrity sha512-+fteyVdQ7C/OoulfcF6vd1Yk0FEli4453gr8kSFbU8sKseNSizYq6df5MKz/AjwLptsxrUeIkgBdAzbziyJ3mA== +"@next/env@11.1.3": + version "11.1.3" + resolved "https://registry.yarnpkg.com/@next/env/-/env-11.1.3.tgz#dc698e00259242012955e43a40788fcf21ba9e37" + integrity sha512-5+vaeooJuWmICSlmVaAC8KG3O8hwKasACVfkHj58xQuCB5SW0TKW3hWxgxkBuefMBn1nM0yEVPKokXCsYjBtng== -"@next/polyfill-module@11.1.2": - version "11.1.2" - resolved "https://registry.npmjs.org/@next/polyfill-module/-/polyfill-module-11.1.2.tgz" - integrity sha512-xZmixqADM3xxtqBV0TpAwSFzWJP0MOQzRfzItHXf1LdQHWb0yofHHC+7eOrPFic8+ZGz5y7BdPkkgR1S25OymA== +"@next/polyfill-module@11.1.3": + version "11.1.3" + resolved "https://registry.yarnpkg.com/@next/polyfill-module/-/polyfill-module-11.1.3.tgz#95163973fe19f1827da32703d1fcb8198fb2c79a" + integrity sha512-7yr9cr4a0SrBoVE8psxXWK1wTFc8UzsY8Wc2cWGL7qA0hgtqACHaXC47M1ByJB410hFZenGrpE+KFaT1unQMyw== -"@next/react-dev-overlay@11.1.2": - version "11.1.2" - resolved "https://registry.npmjs.org/@next/react-dev-overlay/-/react-dev-overlay-11.1.2.tgz" - integrity sha512-rDF/mGY2NC69mMg2vDqzVpCOlWqnwPUXB2zkARhvknUHyS6QJphPYv9ozoPJuoT/QBs49JJd9KWaAzVBvq920A== +"@next/react-dev-overlay@11.1.3": + version "11.1.3" + resolved "https://registry.yarnpkg.com/@next/react-dev-overlay/-/react-dev-overlay-11.1.3.tgz#5d08336931e48ebdb07d82b566223d0ee5941d2a" + integrity sha512-zIwtMliSUR+IKl917ToFNB+0fD7bI5kYMdjHU/UEKpfIXAZPnXRHHISCvPDsczlr+bRsbjlUFW1CsNiuFedeuQ== dependencies: "@babel/code-frame" "7.12.11" anser "1.4.9" @@ -1131,30 +1131,30 @@ stacktrace-parser "0.1.10" strip-ansi "6.0.0" -"@next/react-refresh-utils@11.1.2": - version "11.1.2" - resolved "https://registry.npmjs.org/@next/react-refresh-utils/-/react-refresh-utils-11.1.2.tgz" - integrity sha512-hsoJmPfhVqjZ8w4IFzoo8SyECVnN+8WMnImTbTKrRUHOVJcYMmKLL7xf7T0ft00tWwAl/3f3Q3poWIN2Ueql/Q== +"@next/react-refresh-utils@11.1.3": + version "11.1.3" + resolved "https://registry.yarnpkg.com/@next/react-refresh-utils/-/react-refresh-utils-11.1.3.tgz#fc2c1a4f2403db1a0179d31caa0a4cc811b8ab58" + integrity sha512-144kD8q2nChw67V3AJJlPQ6NUJVFczyn10bhTynn9o2rY5DEnkzuBipcyMuQl2DqfxMkV7sn+yOCOYbrLCk9zg== -"@next/swc-darwin-arm64@11.1.2": - version "11.1.2" - resolved "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-11.1.2.tgz" - integrity sha512-hZuwOlGOwBZADA8EyDYyjx3+4JGIGjSHDHWrmpI7g5rFmQNltjlbaefAbiU5Kk7j3BUSDwt30quJRFv3nyJQ0w== +"@next/swc-darwin-arm64@11.1.3": + version "11.1.3" + resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-11.1.3.tgz#062eb7871048fdb313304e42ace5f91402dbc39f" + integrity sha512-TwP4krjhs+uU9pesDYCShEXZrLSbJr78p12e7XnLBBaNf20SgWLlVmQUT9gX9KbWan5V0sUbJfmcS8MRNHgYuA== -"@next/swc-darwin-x64@11.1.2": - version "11.1.2" - resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-11.1.2.tgz#792003989f560c00677b5daeff360b35b510db83" - integrity sha512-PGOp0E1GisU+EJJlsmJVGE+aPYD0Uh7zqgsrpD3F/Y3766Ptfbe1lEPPWnRDl+OzSSrSrX1lkyM/Jlmh5OwNvA== +"@next/swc-darwin-x64@11.1.3": + version "11.1.3" + resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-11.1.3.tgz#8bd515768d02e4c1e0cd80d33f3f29456ee890ee" + integrity sha512-ZSWmkg/PxccHFNUSeBdrfaH8KwSkoeUtewXKvuYYt7Ph0yRsbqSyNIvhUezDua96lApiXXq6EL2d1THfeWomvw== -"@next/swc-linux-x64-gnu@11.1.2": - version "11.1.2" - resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-11.1.2.tgz#8216b2ae1f21f0112958735c39dd861088108f37" - integrity sha512-YcDHTJjn/8RqvyJVB6pvEKXihDcdrOwga3GfMv/QtVeLphTouY4BIcEUfrG5+26Nf37MP1ywN3RRl1TxpurAsQ== +"@next/swc-linux-x64-gnu@11.1.3": + version "11.1.3" + resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-11.1.3.tgz#40030577e6ee272afb0080b45468bea73208f46d" + integrity sha512-PrTBN0iZudAuj4jSbtXcdBdmfpaDCPIneG4Oms4zcs93KwMgLhivYW082Mvlgx9QVEiRm7+RkFpIVtG/i7JitA== -"@next/swc-win32-x64-msvc@11.1.2": - version "11.1.2" - resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-11.1.2.tgz#e15824405df137129918205e43cb5e9339589745" - integrity sha512-e/pIKVdB+tGQYa1cW3sAeHm8gzEri/HYLZHT4WZojrUxgWXqx8pk7S7Xs47uBcFTqBDRvK3EcQpPLf3XdVsDdg== +"@next/swc-win32-x64-msvc@11.1.3": + version "11.1.3" + resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-11.1.3.tgz#2951cbc127f6ea57032a241fb94439cddb5d2482" + integrity sha512-mRwbscVjRoHk+tDY7XbkT5d9FCwujFIQJpGp0XNb1i5OHCSDO8WW/C9cLEWS4LxKRbIZlTLYg1MTXqLQkvva8w== "@node-rs/helper@1.2.1": version "1.2.1" @@ -4613,17 +4613,17 @@ negotiator@0.6.2: resolved "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz" integrity sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw== -next@^11.1.2: - version "11.1.2" - resolved "https://registry.npmjs.org/next/-/next-11.1.2.tgz" - integrity sha512-azEYL0L+wFjv8lstLru3bgvrzPvK0P7/bz6B/4EJ9sYkXeW8r5Bjh78D/Ol7VOg0EIPz0CXoe72hzAlSAXo9hw== +next@^11.1.3: + version "11.1.3" + resolved "https://registry.yarnpkg.com/next/-/next-11.1.3.tgz#0226b283cb9890e446aea759db8a867de2b279ef" + integrity sha512-ud/gKmnKQ8wtHC+pd1ZiqPRa7DdgulPkAk94MbpsspfNliwZkYs9SIYWhlLSyg+c661LzdUI2nZshvrtggSYWA== dependencies: "@babel/runtime" "7.15.3" "@hapi/accept" "5.0.2" - "@next/env" "11.1.2" - "@next/polyfill-module" "11.1.2" - "@next/react-dev-overlay" "11.1.2" - "@next/react-refresh-utils" "11.1.2" + "@next/env" "11.1.3" + "@next/polyfill-module" "11.1.3" + "@next/react-dev-overlay" "11.1.3" + "@next/react-refresh-utils" "11.1.3" "@node-rs/helper" "1.2.1" assert "2.0.0" ast-types "0.13.2" @@ -4669,10 +4669,10 @@ next@^11.1.2: vm-browserify "1.1.2" watchpack "2.1.1" optionalDependencies: - "@next/swc-darwin-arm64" "11.1.2" - "@next/swc-darwin-x64" "11.1.2" - "@next/swc-linux-x64-gnu" "11.1.2" - "@next/swc-win32-x64-msvc" "11.1.2" + "@next/swc-darwin-arm64" "11.1.3" + "@next/swc-darwin-x64" "11.1.3" + "@next/swc-linux-x64-gnu" "11.1.3" + "@next/swc-win32-x64-msvc" "11.1.3" node-fetch@2.6.1: version "2.6.1" From d814359e10426bfdeabd93da92ff7d6a0f5ddadb Mon Sep 17 00:00:00 2001 From: Abhinav Date: Sun, 19 Dec 2021 12:24:58 +0530 Subject: [PATCH 51/58] add olderClient check and use olderClient logic --- src/services/exportService.ts | 29 +++++++++++++++++++++-------- src/utils/export/index.ts | 18 ++++++++++++++---- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index c2be52209..749c2361d 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -105,9 +105,11 @@ class ExportService { private exportRecordUpdater = new QueueProcessor(1); private stopExport: boolean = false; private pauseExport: boolean = false; + private oldClient: boolean = false; constructor() { this.ElectronAPIs = runningInBrowser() && window['ElectronAPIs']; + this.oldClient = !this.ElectronAPIs.exists; } async selectExportDirectory() { return await this.ElectronAPIs.selectRootDirectory(); @@ -154,7 +156,13 @@ class ExportService { (collectionA, collectionB) => collectionA.id - collectionB.id ); - await this.migrateExport(exportDir, collections, userPersonalFiles); + if (!this.isOldClient()) { + await this.migrateExport( + exportDir, + collections, + userPersonalFiles + ); + } const exportRecord = await this.getExportRecord(exportDir); if (exportType === ExportType.NEW) { @@ -407,7 +415,7 @@ class ExportService { for (const collection of newCollections) { const collectionFolderPath = getUniqueCollectionFolderPath( exportFolder, - collection.name + collection ); await this.ElectronAPIs.checkExistsAndCreateCollectionDir( collectionFolderPath @@ -435,7 +443,7 @@ class ExportService { const newCollectionFolderPath = getUniqueCollectionFolderPath( exportFolder, - collection.name + collection ); await this.ElectronAPIs.checkExistsAndRename( oldCollectionFolderPath, @@ -455,7 +463,8 @@ class ExportService { file.metadata = mergeMetadata([file])[0].metadata; const fileSaveName = getUniqueFileSaveName( collectionPath, - file.metadata.title + file.metadata.title, + file.id ); let fileStream = await retryAsyncFunction(() => downloadManager.downloadFile(file) @@ -495,7 +504,8 @@ class ExportService { const imageStream = generateStreamFromArrayBuffer(motionPhoto.image); const imageSaveName = getUniqueFileSaveName( collectionPath, - motionPhoto.imageNameTitle + motionPhoto.imageNameTitle, + file.id ); this.saveMediaFile(collectionPath, imageSaveName, imageStream); await this.saveMetadataFile( @@ -507,7 +517,8 @@ class ExportService { const videoStream = generateStreamFromArrayBuffer(motionPhoto.video); const videoSaveName = getUniqueFileSaveName( collectionPath, - motionPhoto.videoNameTitle + motionPhoto.videoNameTitle, + file.id ); this.saveMediaFile(collectionPath, videoSaveName, videoStream); await this.saveMetadataFile( @@ -545,6 +556,7 @@ class ExportService { exists = (path: string) => { return this.ElectronAPIs.exists(path); }; + isOldClient = () => this.oldClient; /* this function migrates the exportRecord file to apply any schema changes. @@ -595,7 +607,7 @@ class ExportService { ); const newCollectionFolderPath = getUniqueCollectionFolderPath( exportDir, - collection.name + collection ); collectionIDPathMap.set(collection.id, newCollectionFolderPath); if (this.ElectronAPIs.exists(oldCollectionFolderPath)) { @@ -632,7 +644,8 @@ class ExportService { file = mergeMetadata([file])[0]; const newFileSaveName = getUniqueFileSaveName( collectionIDPathMap.get(file.collectionID), - file.metadata.title + file.metadata.title, + file.id ); const newFileSavePath = getFileSavePath( diff --git a/src/utils/export/index.ts b/src/utils/export/index.ts index 40e96c9ea..e3e2dd0d5 100644 --- a/src/utils/export/index.ts +++ b/src/utils/export/index.ts @@ -162,13 +162,16 @@ export const sanitizeName = (name: string) => export const getUniqueCollectionFolderPath = ( dir: string, - collectionName: string + collection: Collection ): string => { - let collectionFolderPath = `${dir}/${sanitizeName(collectionName)}`; + if (exportService.isOldClient()) { + return getOldCollectionFolderPath(dir, collection); + } + let collectionFolderPath = `${dir}/${sanitizeName(collection.name)}`; let count = 1; while (exportService.exists(collectionFolderPath)) { collectionFolderPath = `${dir}/${sanitizeName( - collectionName + collection.name )}(${count})`; count++; } @@ -180,8 +183,12 @@ export const getMetadataFolderPath = (collectionFolderPath: string) => export const getUniqueFileSaveName = ( collectionPath: string, - filename: string + filename: string, + fileID: number ) => { + if (exportService.isOldClient()) { + return getOldFileSaveName(filename, fileID); + } let fileSaveName = sanitizeName(filename); let count = 1; while ( @@ -198,6 +205,9 @@ export const getUniqueFileSaveName = ( return fileSaveName; }; +export const getOldFileSaveName = (filename: string, fileID: number) => + `${fileID}_${oldSanitizeName(filename)}`; + export const getFileMetadataSavePath = ( collectionFolderPath: string, fileSaveName: string From e2b3c157f538f47915a1b07aacb00771fdb4e4c1 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Sun, 19 Dec 2021 14:38:10 +0530 Subject: [PATCH 52/58] renamed oldClient to better allElectronAPIsExist --- src/services/exportService.ts | 9 +++++---- src/utils/export/index.ts | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index 749c2361d..855fd4a41 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -105,11 +105,11 @@ class ExportService { private exportRecordUpdater = new QueueProcessor(1); private stopExport: boolean = false; private pauseExport: boolean = false; - private oldClient: boolean = false; + private allElectronAPIsExist: boolean = false; constructor() { this.ElectronAPIs = runningInBrowser() && window['ElectronAPIs']; - this.oldClient = !this.ElectronAPIs.exists; + this.allElectronAPIsExist = this.ElectronAPIs.exists; } async selectExportDirectory() { return await this.ElectronAPIs.selectRootDirectory(); @@ -156,7 +156,7 @@ class ExportService { (collectionA, collectionB) => collectionA.id - collectionB.id ); - if (!this.isOldClient()) { + if (!this.checkAllElectronAPIsExists()) { await this.migrateExport( exportDir, collections, @@ -556,7 +556,8 @@ class ExportService { exists = (path: string) => { return this.ElectronAPIs.exists(path); }; - isOldClient = () => this.oldClient; + + checkAllElectronAPIsExists = () => this.allElectronAPIsExist; /* this function migrates the exportRecord file to apply any schema changes. diff --git a/src/utils/export/index.ts b/src/utils/export/index.ts index e3e2dd0d5..b31a1f575 100644 --- a/src/utils/export/index.ts +++ b/src/utils/export/index.ts @@ -164,7 +164,7 @@ export const getUniqueCollectionFolderPath = ( dir: string, collection: Collection ): string => { - if (exportService.isOldClient()) { + if (exportService.checkAllElectronAPIsExists()) { return getOldCollectionFolderPath(dir, collection); } let collectionFolderPath = `${dir}/${sanitizeName(collection.name)}`; @@ -186,7 +186,7 @@ export const getUniqueFileSaveName = ( filename: string, fileID: number ) => { - if (exportService.isOldClient()) { + if (exportService.checkAllElectronAPIsExists()) { return getOldFileSaveName(filename, fileID); } let fileSaveName = sanitizeName(filename); From 9088942e16862e37f6631850bc4a11f225018e54 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Sun, 19 Dec 2021 14:43:30 +0530 Subject: [PATCH 53/58] fix checks --- src/services/exportService.ts | 9 ++++++--- src/utils/export/index.ts | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index 855fd4a41..40a989b90 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -109,7 +109,7 @@ class ExportService { constructor() { this.ElectronAPIs = runningInBrowser() && window['ElectronAPIs']; - this.allElectronAPIsExist = this.ElectronAPIs.exists; + this.allElectronAPIsExist = !!this.ElectronAPIs.exists; } async selectExportDirectory() { return await this.ElectronAPIs.selectRootDirectory(); @@ -156,7 +156,7 @@ class ExportService { (collectionA, collectionB) => collectionA.id - collectionB.id ); - if (!this.checkAllElectronAPIsExists()) { + if (this.checkAllElectronAPIsExists()) { await this.migrateExport( exportDir, collections, @@ -225,7 +225,10 @@ class ExportService { collectionIDPathMap ); } - if (renamedCollections?.length) { + if ( + renamedCollections?.length && + this.checkAllElectronAPIsExists() + ) { await this.renameCollectionFolders( renamedCollections, exportDir, diff --git a/src/utils/export/index.ts b/src/utils/export/index.ts index b31a1f575..ffad98a7f 100644 --- a/src/utils/export/index.ts +++ b/src/utils/export/index.ts @@ -164,7 +164,7 @@ export const getUniqueCollectionFolderPath = ( dir: string, collection: Collection ): string => { - if (exportService.checkAllElectronAPIsExists()) { + if (!exportService.checkAllElectronAPIsExists()) { return getOldCollectionFolderPath(dir, collection); } let collectionFolderPath = `${dir}/${sanitizeName(collection.name)}`; @@ -186,7 +186,7 @@ export const getUniqueFileSaveName = ( filename: string, fileID: number ) => { - if (exportService.checkAllElectronAPIsExists()) { + if (!exportService.checkAllElectronAPIsExists()) { return getOldFileSaveName(filename, fileID); } let fileSaveName = sanitizeName(filename); From f45fd0d0e4794e7cefce81957899150fb832aa68 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Sun, 19 Dec 2021 15:23:36 +0530 Subject: [PATCH 54/58] fix allElectronAPIsExist intialisation --- src/services/exportService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/exportService.ts b/src/services/exportService.ts index 40a989b90..e6e8a1425 100644 --- a/src/services/exportService.ts +++ b/src/services/exportService.ts @@ -109,7 +109,7 @@ class ExportService { constructor() { this.ElectronAPIs = runningInBrowser() && window['ElectronAPIs']; - this.allElectronAPIsExist = !!this.ElectronAPIs.exists; + this.allElectronAPIsExist = !!this.ElectronAPIs?.exists; } async selectExportDirectory() { return await this.ElectronAPIs.selectRootDirectory(); From e066ee5ff1f96c727a02459a83176d083c088107 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 20 Dec 2021 12:59:39 +0530 Subject: [PATCH 55/58] added start and finish loading helper function --- src/pages/gallery/index.tsx | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/pages/gallery/index.tsx b/src/pages/gallery/index.tsx index 3f3976357..df8bf8982 100644 --- a/src/pages/gallery/index.tsx +++ b/src/pages/gallery/index.tsx @@ -362,12 +362,17 @@ export default function Gallery() { setSelected({ count: 0, collectionID: 0 }); }; + const startLoading = () => + !syncInProgress.current && loadingBar.current?.continuousStart(); + const finishLoading = () => + !syncInProgress.current && loadingBar.current.complete(); + if (!files) { return
; } const collectionOpsHelper = (ops: COLLECTION_OPS_TYPE) => async (collection: Collection) => { - loadingBar.current?.continuousStart(); + startLoading(); try { await handleCollectionOps( ops, @@ -388,14 +393,14 @@ export default function Gallery() { }); } finally { await syncWithRemote(false, true); - loadingBar.current.complete(); + finishLoading(); } }; const changeFilesVisibilityHelper = async ( visibility: VISIBILITY_STATE ) => { - loadingBar.current?.continuousStart(); + startLoading(); try { const updatedFiles = await changeFilesVisibility( files, @@ -424,7 +429,7 @@ export default function Gallery() { }); } finally { await syncWithRemote(false, true); - loadingBar.current.complete(); + finishLoading(); } }; @@ -458,7 +463,7 @@ export default function Gallery() { }; const deleteFileHelper = async (permanent?: boolean) => { - loadingBar.current?.continuousStart(); + startLoading(); try { const selectedFiles = getSelectedFiles(selected, files); if (permanent) { @@ -489,7 +494,7 @@ export default function Gallery() { }); } finally { await syncWithRemote(false, true); - loadingBar.current.complete(); + finishLoading(); } }; @@ -519,7 +524,7 @@ export default function Gallery() { close: { text: constants.CANCEL }, }); const emptyTrashHelper = async () => { - loadingBar.current?.continuousStart(); + startLoading(); try { await emptyTrash(); if (selected.collectionID === TRASH_SECTION) { @@ -536,7 +541,7 @@ export default function Gallery() { }); } finally { await syncWithRemote(false, true); - loadingBar.current.complete(); + finishLoading(); } }; @@ -549,9 +554,9 @@ export default function Gallery() { const downloadHelper = async () => { const selectedFiles = getSelectedFiles(selected, files); clearSelection(); - !syncInProgress.current && loadingBar.current?.continuousStart(); + startLoading(); await downloadFiles(selectedFiles); - !syncInProgress.current && loadingBar.current.complete(); + finishLoading(); }; return ( @@ -609,7 +614,8 @@ export default function Gallery() { syncWithRemote={syncWithRemote} setDialogMessage={setDialogMessage} setCollectionNamerAttributes={setCollectionNamerAttributes} - startLoadingBar={loadingBar.current?.continuousStart} + startLoading={startLoading} + finishLoading={finishLoading} collectionFilesCount={collectionFilesCount} /> Date: Mon, 20 Dec 2021 13:00:56 +0530 Subject: [PATCH 56/58] adds download collection option --- .../pages/gallery/CollectionOptions.tsx | 45 ++++++++++++++++--- src/components/pages/gallery/Collections.tsx | 6 ++- src/utils/collection/index.ts | 27 ++++++++++- src/utils/strings/englishConstants.tsx | 8 ++++ 4 files changed, 76 insertions(+), 10 deletions(-) diff --git a/src/components/pages/gallery/CollectionOptions.tsx b/src/components/pages/gallery/CollectionOptions.tsx index 041d58f07..e34be88f4 100644 --- a/src/components/pages/gallery/CollectionOptions.tsx +++ b/src/components/pages/gallery/CollectionOptions.tsx @@ -6,18 +6,20 @@ import { deleteCollection, renameCollection, } from 'services/collectionService'; -import { getSelectedCollection } from 'utils/collection'; +import { downloadCollection, getSelectedCollection } from 'utils/collection'; import constants from 'utils/strings/constants'; import { SetCollectionNamerAttributes } from './CollectionNamer'; import LinkButton, { ButtonVariant, LinkButtonProps } from './LinkButton'; +import { sleep } from 'utils/common'; -interface Props { +interface CollectionOptionsProps { syncWithRemote: () => Promise; setCollectionNamerAttributes: SetCollectionNamerAttributes; collections: Collection[]; selectedCollectionID: number; setDialogMessage: SetDialogMessage; - startLoadingBar: () => void; + startLoading: () => void; + finishLoading: () => void; showCollectionShareModal: () => void; redirectToAll: () => void; } @@ -43,7 +45,7 @@ export const MenuItem = (props: { children: any }) => ( ); -const CollectionOptions = (props: Props) => { +const CollectionOptions = (props: CollectionOptionsProps) => { const collectionRename = async ( selectedCollection: Collection, newName: string @@ -62,7 +64,7 @@ const CollectionOptions = (props: Props) => { props.collections )?.name, callback: (newName) => { - props.startLoadingBar(); + props.startLoading(); collectionRename( getSelectedCollection( props.selectedCollectionID, @@ -81,7 +83,7 @@ const CollectionOptions = (props: Props) => { proceed: { text: constants.DELETE_COLLECTION, action: () => { - props.startLoadingBar(); + props.startLoading(); deleteCollection( props.selectedCollectionID, props.syncWithRemote, @@ -97,6 +99,32 @@ const CollectionOptions = (props: Props) => { }); }; + const confirmDownloadCollection = () => { + props.setDialogMessage({ + title: constants.CONFIRM_DOWNLOAD_COLLECTION, + content: constants.DOWNLOAD_COLLECTION_MESSAGE(), + staticBackdrop: true, + proceed: { + text: constants.DOWNLOAD, + action: downloadCollectionHelper, + variant: 'success', + }, + close: { + text: constants.CANCEL, + }, + }); + }; + + const downloadCollectionHelper = async () => { + props.startLoading(); + await downloadCollection( + props.selectedCollectionID, + props.setDialogMessage + ); + await sleep(1000); + props.finishLoading(); + }; + return ( @@ -111,6 +139,11 @@ const CollectionOptions = (props: Props) => { {constants.SHARE} + + + {constants.DOWNLOAD} + + Promise; setCollectionNamerAttributes: SetCollectionNamerAttributes; - startLoadingBar: () => void; + startLoading: () => void; + finishLoading: () => void; isInSearchMode: boolean; collectionFilesCount: Map; } @@ -169,7 +170,8 @@ export default function Collections(props: CollectionProps) { collections: props.collections, selectedCollectionID, setDialogMessage: props.setDialogMessage, - startLoadingBar: props.startLoadingBar, + startLoading: props.startLoading, + finishLoading: props.finishLoading, showCollectionShareModal: setCollectionShareModalView.bind(null, true), redirectToAll: setActiveCollection.bind(null, ALL_SECTION), }); diff --git a/src/utils/collection/index.ts b/src/utils/collection/index.ts index 506344deb..46aae90e9 100644 --- a/src/utils/collection/index.ts +++ b/src/utils/collection/index.ts @@ -6,12 +6,15 @@ import { removeFromCollection, restoreToCollection, } from 'services/collectionService'; -import { getSelectedFiles } from 'utils/file'; -import { File } from 'services/fileService'; +import { downloadFiles, getSelectedFiles } from 'utils/file'; +import { File, getLocalFiles } from 'services/fileService'; import { CustomError } from 'utils/common/errorUtil'; import { SelectedState } from 'pages/gallery'; import { User } from 'services/userService'; import { getData, LS_KEYS } from 'utils/storage/localStorage'; +import { SetDialogMessage } from 'components/MessageDialog'; +import { logError } from 'utils/sentry'; +import constants from 'utils/strings/constants'; export enum COLLECTION_OPS_TYPE { ADD, @@ -83,3 +86,23 @@ export function isFavoriteCollection( return collection.type === CollectionType.favorites; } } + +export async function downloadCollection( + collectionID: number, + setDialogMessage: SetDialogMessage +) { + try { + const allFiles = await getLocalFiles(); + const collectionFiles = allFiles.filter( + (file) => file.collectionID === collectionID + ); + await downloadFiles(collectionFiles); + } catch (e) { + logError(e, 'download collection failed '); + setDialogMessage({ + title: constants.ERROR, + content: constants.DELETE_COLLECTION_FAILED, + close: { variant: 'danger' }, + }); + } +} diff --git a/src/utils/strings/englishConstants.tsx b/src/utils/strings/englishConstants.tsx index 27e82c1aa..e64872c3b 100644 --- a/src/utils/strings/englishConstants.tsx +++ b/src/utils/strings/englishConstants.tsx @@ -382,6 +382,14 @@ const englishConstants = { `oops, you're already sharing this with ${email}`, SHARING_BAD_REQUEST_ERROR: 'sharing album not allowed', SHARING_DISABLED_FOR_FREE_ACCOUNTS: 'sharing is disabled for free accounts', + CONFIRM_DOWNLOAD_COLLECTION: 'download album', + DOWNLOAD_COLLECTION_MESSAGE: () => ( + <> +

are you sure you want to download the complete album?

+

all files will be queued for download sententially

+ + ), + DOWNLOAD_COLLECTION_FAILED: 'album downloading failed, please try again', CREATE_ALBUM_FAILED: 'failed to create album , please try again', SEARCH_HINT: () => ( try searching for New York, April 14, Christmas... From 40058f08839d5f22fafd56e5b83ed25ec6dcede9 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 20 Dec 2021 13:21:12 +0530 Subject: [PATCH 57/58] fix typo --- src/utils/strings/englishConstants.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/strings/englishConstants.tsx b/src/utils/strings/englishConstants.tsx index e64872c3b..7ada30d45 100644 --- a/src/utils/strings/englishConstants.tsx +++ b/src/utils/strings/englishConstants.tsx @@ -386,7 +386,7 @@ const englishConstants = { DOWNLOAD_COLLECTION_MESSAGE: () => ( <>

are you sure you want to download the complete album?

-

all files will be queued for download sententially

+

all files will be queued for download sequentially

), DOWNLOAD_COLLECTION_FAILED: 'album downloading failed, please try again', From d5c7ebe3468a98fefbf77cc3be05858924fc9209 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 20 Dec 2021 13:52:43 +0530 Subject: [PATCH 58/58] v0.4.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1674865f5..f06fcb6f2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bada-frame", - "version": "0.3.44", + "version": "0.4.0", "private": true, "scripts": { "dev": "next dev",