diff --git a/package.json b/package.json index 83f50f9d7..1674865f5 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "next": "^11.1.3", "node-forge": "^0.10.0", "photoswipe": "file:./thirdparty/photoswipe", + "piexifjs": "^1.0.6", "react": "^17.0.2", "react-bootstrap": "^1.3.0", "react-burger-menu": "^3.0.4", diff --git a/src/components/ExportFinished.tsx b/src/components/ExportFinished.tsx index dedae4d34..5fc263d88 100644 --- a/src/components/ExportFinished.tsx +++ b/src/components/ExportFinished.tsx @@ -29,8 +29,8 @@ export default function ExportFinished(props: Props) { padding: '0 5%', }}> - - + + {formatDateTime(props.lastExportTime)} @@ -38,7 +38,7 @@ export default function ExportFinished(props: Props) { - + {props.exportStats.success} / {totalFiles} diff --git a/src/components/ExportModal.tsx b/src/components/ExportModal.tsx index a03d4b722..f7c3c6c80 100644 --- a/src/components/ExportModal.tsx +++ b/src/components/ExportModal.tsx @@ -8,9 +8,11 @@ import exportService, { ExportType, } from 'services/exportService'; import { getLocalFiles } from 'services/fileService'; +import { User } from 'services/userService'; import styled from 'styled-components'; import { sleep } from 'utils/common'; import { getExportRecordFileUID } from 'utils/export'; +import { logError } from 'utils/sentry'; import { getData, LS_KEYS, setData } from 'utils/storage/localStorage'; import constants from 'utils/strings/constants'; import { Label, Row, Value } from './Container'; @@ -105,30 +107,41 @@ export default function ExportModal(props: Props) { return; } const main = async () => { + const user: User = getData(LS_KEYS.USER); if (exportStage === ExportStage.FINISHED) { - const localFiles = await getLocalFiles(); - const exportRecord = await exportService.getExportRecord(); - const exportedFileCnt = exportRecord.exportedFiles.length; - const failedFilesCnt = exportRecord.failedFiles.length; - const syncedFilesCnt = localFiles.length; - if (syncedFilesCnt > exportedFileCnt + failedFilesCnt) { - updateExportProgress({ - current: exportedFileCnt + failedFilesCnt, - total: syncedFilesCnt, - }); - const exportFileUIDs = new Set([ - ...exportRecord.exportedFiles, - ...exportRecord.failedFiles, - ]); - const unExportedFiles = localFiles.filter( - (file) => - !exportFileUIDs.has(getExportRecordFileUID(file)) + try { + const localFiles = await getLocalFiles(); + const userPersonalFiles = localFiles.filter( + (file) => file.ownerID === user?.id ); - exportService.addFilesQueuedRecord( - exportFolder, - unExportedFiles - ); - updateExportStage(ExportStage.PAUSED); + const exportRecord = await exportService.getExportRecord(); + const exportedFileCnt = exportRecord.exportedFiles?.length; + const failedFilesCnt = exportRecord.failedFiles?.length; + const syncedFilesCnt = userPersonalFiles.length; + if (syncedFilesCnt > exportedFileCnt + failedFilesCnt) { + updateExportProgress({ + current: exportedFileCnt + failedFilesCnt, + total: syncedFilesCnt, + }); + const exportFileUIDs = new Set([ + ...exportRecord.exportedFiles, + ...exportRecord.failedFiles, + ]); + const unExportedFiles = userPersonalFiles.filter( + (file) => + !exportFileUIDs.has( + getExportRecordFileUID(file) + ) + ); + exportService.addFilesQueuedRecord( + exportFolder, + unExportedFiles + ); + updateExportStage(ExportStage.PAUSED); + } + } catch (e) { + setExportStage(ExportStage.INIT); + logError(e, 'error while updating exportModal on reopen'); } } }; @@ -154,7 +167,7 @@ export default function ExportModal(props: Props) { const updateExportTime = (newTime: number) => { setLastExportTime(newTime); - exportService.updateExportRecord({ time: newTime }); + exportService.updateExportRecord({ lastAttemptTimestamp: newTime }); }; const updateExportProgress = (newProgress: ExportProgress) => { @@ -178,8 +191,8 @@ export default function ExportModal(props: Props) { updateExportStage(ExportStage.INPROGRESS); await sleep(100); }; - const postExportRun = async (paused: Boolean) => { - if (!paused) { + const postExportRun = async (exportResult?: { paused?: boolean }) => { + if (!exportResult?.paused) { updateExportStage(ExportStage.FINISHED); await sleep(100); updateExportTime(Date.now()); @@ -189,22 +202,22 @@ export default function ExportModal(props: Props) { const startExport = async () => { await preExportRun(); updateExportProgress({ current: 0, total: 0 }); - const { paused } = await exportService.exportFiles( + const exportResult = await exportService.exportFiles( updateExportProgress, ExportType.NEW ); - await postExportRun(paused); + await postExportRun(exportResult); }; const stopExport = async () => { exportService.stopRunningExport(); - postExportRun(false); + postExportRun(); }; const pauseExport = () => { updateExportStage(ExportStage.PAUSED); exportService.pauseRunningExport(); - postExportRun(true); + postExportRun({ paused: true }); }; const resumeExport = async () => { @@ -219,23 +232,23 @@ export default function ExportModal(props: Props) { current: pausedStageProgress.current + progress.current, total: pausedStageProgress.current + progress.total, }); - const { paused } = await exportService.exportFiles( + const exportResult = await exportService.exportFiles( updateExportStatsWithOffset, ExportType.PENDING ); - await postExportRun(paused); + await postExportRun(exportResult); }; const retryFailedExport = async () => { await preExportRun(); updateExportProgress({ current: 0, total: exportStats.failed }); - const { paused } = await exportService.exportFiles( + const exportResult = await exportService.exportFiles( updateExportProgress, ExportType.RETRY_FAILED ); - await postExportRun(paused); + await postExportRun(exportResult); }; const syncExportStatsWithReport = async () => { @@ -327,11 +340,9 @@ export default function ExportModal(props: Props) { ) : ( <> - {/* */} {exportFolder} - {/* */} {(exportStage === ExportStage.FINISHED || exportStage === ExportStage.INIT) && ( ; export interface ExportProgress { current: number; total: number; } +export interface ExportedCollectionPaths { + [collectionID: number]: string; +} export interface ExportStats { failed: number; success: number; } +const LATEST_EXPORT_VERSION = 1; + export interface ExportRecord { - stage: ExportStage; - lastAttemptTimestamp: number; - progress: ExportProgress; - queuedFiles: string[]; - exportedFiles: string[]; - failedFiles: string[]; + version?: number; + stage?: ExportStage; + lastAttemptTimestamp?: number; + progress?: ExportProgress; + queuedFiles?: string[]; + exportedFiles?: string[]; + failedFiles?: string[]; + exportedCollectionPaths?: ExportedCollectionPaths; } export enum ExportStage { INIT, @@ -74,12 +102,14 @@ class ExportService { ElectronAPIs: any; private exportInProgress: Promise<{ paused: boolean }> = null; - private recordUpdateInProgress = Promise.resolve(); + private exportRecordUpdater = new QueueProcessor(1); private stopExport: boolean = false; private pauseExport: boolean = false; + private allElectronAPIsExist: boolean = false; constructor() { this.ElectronAPIs = runningInBrowser() && window['ElectronAPIs']; + this.allElectronAPIsExist = !!this.ElectronAPIs.exists; } async selectExportDirectory() { return await this.ElectronAPIs.selectRootDirectory(); @@ -94,53 +124,117 @@ class ExportService { updateProgress: (progress: ExportProgress) => void, exportType: ExportType ) { - if (this.exportInProgress) { - this.ElectronAPIs.sendNotification(ExportNotification.IN_PROGRESS); - return this.exportInProgress; - } - this.ElectronAPIs.showOnTray('starting export'); - const exportDir = getData(LS_KEYS.EXPORT)?.folder; - if (!exportDir) { - // no-export folder set - return; - } - let filesToExport: File[]; - const allFiles = await getLocalFiles(); - const collections = await getLocalCollections(); - const nonEmptyCollections = getNonEmptyCollections( - collections, - allFiles - ); - const exportRecord = await this.getExportRecord(exportDir); + try { + if (this.exportInProgress) { + this.ElectronAPIs.sendNotification( + ExportNotification.IN_PROGRESS + ); + return await this.exportInProgress; + } + this.ElectronAPIs.showOnTray('starting export'); + const exportDir = getData(LS_KEYS.EXPORT)?.folder; + if (!exportDir) { + // no-export folder set + return; + } + const user: User = getData(LS_KEYS.USER); - if (exportType === ExportType.NEW) { - filesToExport = await getFilesUploadedAfterLastExport( - allFiles, + let filesToExport: File[]; + const localFiles = await getLocalFiles(); + const userPersonalFiles = localFiles + .filter((file) => file.ownerID === user?.id) + .sort((fileA, fileB) => fileA.id - fileB.id); + + const collections = await getLocalCollections(); + const nonEmptyCollections = getNonEmptyCollections( + collections, + userPersonalFiles + ); + const userCollections = nonEmptyCollections + .filter((collection) => collection.owner.id === user?.id) + .sort( + (collectionA, collectionB) => + collectionA.id - collectionB.id + ); + if (this.checkAllElectronAPIsExists()) { + await this.migrateExport( + exportDir, + collections, + userPersonalFiles + ); + } + const exportRecord = await this.getExportRecord(exportDir); + + if (exportType === ExportType.NEW) { + filesToExport = getFilesUploadedAfterLastExport( + userPersonalFiles, + exportRecord + ); + } else if (exportType === ExportType.RETRY_FAILED) { + filesToExport = getExportFailedFiles( + userPersonalFiles, + exportRecord + ); + } else { + filesToExport = getExportQueuedFiles( + userPersonalFiles, + exportRecord + ); + } + const collectionIDPathMap: CollectionIDPathMap = + getCollectionIDPathMapFromExportRecord(exportRecord); + const newCollections = getCollectionsCreatedAfterLastExport( + userCollections, exportRecord ); - } else if (exportType === ExportType.RETRY_FAILED) { - filesToExport = await getExportFailedFiles(allFiles, exportRecord); - } else { - filesToExport = await getExportPendingFiles(allFiles, exportRecord); + + const renamedCollections = getCollectionsRenamedAfterLastExport( + userCollections, + exportRecord + ); + this.exportInProgress = this.fileExporter( + filesToExport, + newCollections, + renamedCollections, + collectionIDPathMap, + updateProgress, + exportDir + ); + const resp = await this.exportInProgress; + this.exportInProgress = null; + return resp; + } catch (e) { + logError(e, 'exportFiles failed'); + return { paused: false }; } - this.exportInProgress = this.fileExporter( - filesToExport, - nonEmptyCollections, - updateProgress, - exportDir - ); - const resp = await this.exportInProgress; - this.exportInProgress = null; - return resp; } async fileExporter( files: File[], - collections: Collection[], + newCollections: Collection[], + renamedCollections: Collection[], + collectionIDPathMap: CollectionIDPathMap, updateProgress: (progress: ExportProgress) => void, - dir: string + exportDir: string ): Promise<{ paused: boolean }> { try { + if (newCollections?.length) { + await this.createNewCollectionFolders( + newCollections, + exportDir, + collectionIDPathMap + ); + } + if ( + renamedCollections?.length && + this.checkAllElectronAPIsExists() + ) { + await this.renameCollectionFolders( + renamedCollections, + exportDir, + collectionIDPathMap + ); + } if (!files?.length) { this.ElectronAPIs.sendNotification( ExportNotification.UP_TO_DATE @@ -149,7 +243,7 @@ class ExportService { } this.stopExport = false; this.pauseExport = false; - this.addFilesQueuedRecord(dir, files); + this.addFilesQueuedRecord(exportDir, files); const failedFileCount = 0; this.ElectronAPIs.showOnTray({ @@ -161,19 +255,6 @@ class ExportService { }); this.ElectronAPIs.sendNotification(ExportNotification.START); - const collectionIDMap = new Map(); - for (const collection of collections) { - const collectionFolderPath = `${dir}/${ - collection.id - }_${this.sanitizeName(collection.name)}`; - await this.ElectronAPIs.checkExistsAndCreateCollectionDir( - collectionFolderPath - ); - await this.ElectronAPIs.checkExistsAndCreateCollectionDir( - `${collectionFolderPath}/${METADATA_FOLDER_NAME}` - ); - collectionIDMap.set(collection.id, collectionFolderPath); - } for (const [index, file] of files.entries()) { if (this.stopExport || this.pauseExport) { if (this.pauseExport) { @@ -184,20 +265,26 @@ class ExportService { } break; } - const collectionPath = collectionIDMap.get(file.collectionID); + const collectionPath = collectionIDPathMap.get( + file.collectionID + ); try { await this.downloadAndSave(file, collectionPath); - await this.addFileExportRecord( - dir, + await this.addFileExportedRecord( + exportDir, file, RecordType.SUCCESS ); } catch (e) { - await this.addFileExportRecord( - dir, + await this.addFileExportedRecord( + exportDir, file, RecordType.FAILED ); + console.log( + `export failed for fileID:${file.id}, reason:`, + e + ); logError( e, 'download and save failed for file during export' @@ -227,7 +314,8 @@ class ExportService { } return { paused: false }; } catch (e) { - logError(e, 'export failed '); + logError(e, 'fileExporter failed'); + throw e; } } async addFilesQueuedRecord(folder: string, files: File[]) { @@ -236,7 +324,7 @@ class ExportService { await this.updateExportRecord(exportRecord, folder); } - async addFileExportRecord(folder: string, file: File, type: RecordType) { + async addFileExportedRecord(folder: string, file: File, type: RecordType) { const fileUID = getExportRecordFileUID(file); const exportRecord = await this.getExportRecord(folder); exportRecord.queuedFiles = exportRecord.queuedFiles.filter( @@ -265,28 +353,47 @@ class ExportService { await this.updateExportRecord(exportRecord, folder); } - async updateExportRecord(newData: Record, folder?: string) { - await this.recordUpdateInProgress; - this.recordUpdateInProgress = (async () => { - try { - if (!folder) { - folder = getData(LS_KEYS.EXPORT)?.folder; - } - const exportRecord = await this.getExportRecord(folder); - const newRecord = { ...exportRecord, ...newData }; - await this.ElectronAPIs.setExportRecord( - `${folder}/${EXPORT_RECORD_FILE_NAME}`, - JSON.stringify(newRecord, null, 2) - ); - } catch (e) { - logError(e, 'error updating Export Record'); + async addCollectionExportedRecord( + folder: string, + collection: Collection, + collectionFolderPath: string + ) { + const exportRecord = await this.getExportRecord(folder); + if (!exportRecord?.exportedCollectionPaths) { + exportRecord.exportedCollectionPaths = {}; + } + exportRecord.exportedCollectionPaths = { + ...exportRecord.exportedCollectionPaths, + [collection.id]: collectionFolderPath, + }; + + await this.updateExportRecord(exportRecord, folder); + } + + async updateExportRecord(newData: ExportRecord, folder?: string) { + const response = this.exportRecordUpdater.queueUpRequest(() => + this.updateExportRecordHelper(folder, newData) + ); + await response.promise; + } + async updateExportRecordHelper(folder: string, newData: ExportRecord) { + try { + if (!folder) { + folder = getData(LS_KEYS.EXPORT)?.folder; } - })(); + const exportRecord = await this.getExportRecord(folder); + const newRecord = { ...exportRecord, ...newData }; + await this.ElectronAPIs.setExportRecord( + `${folder}/${EXPORT_RECORD_FILE_NAME}`, + JSON.stringify(newRecord, null, 2) + ); + } catch (e) { + logError(e, 'error updating Export Record'); + } } async getExportRecord(folder?: string): Promise { try { - await this.recordUpdateInProgress; if (!folder) { folder = getData(LS_KEYS.EXPORT)?.folder; } @@ -303,16 +410,89 @@ class ExportService { } } + async createNewCollectionFolders( + newCollections: Collection[], + exportFolder: string, + collectionIDPathMap: CollectionIDPathMap + ) { + for (const collection of newCollections) { + const collectionFolderPath = getUniqueCollectionFolderPath( + exportFolder, + collection + ); + await this.ElectronAPIs.checkExistsAndCreateCollectionDir( + collectionFolderPath + ); + await this.ElectronAPIs.checkExistsAndCreateCollectionDir( + getMetadataFolderPath(collectionFolderPath) + ); + await this.addCollectionExportedRecord( + exportFolder, + collection, + collectionFolderPath + ); + collectionIDPathMap.set(collection.id, collectionFolderPath); + } + } + async renameCollectionFolders( + renamedCollections: Collection[], + exportFolder: string, + collectionIDPathMap: CollectionIDPathMap + ) { + for (const collection of renamedCollections) { + const oldCollectionFolderPath = collectionIDPathMap.get( + collection.id + ); + + const newCollectionFolderPath = getUniqueCollectionFolderPath( + exportFolder, + collection + ); + await this.ElectronAPIs.checkExistsAndRename( + oldCollectionFolderPath, + newCollectionFolderPath + ); + + await this.addCollectionExportedRecord( + exportFolder, + collection, + newCollectionFolderPath + ); + collectionIDPathMap.set(collection.id, newCollectionFolderPath); + } + } + async downloadAndSave(file: File, collectionPath: string) { - const uid = `${file.id}_${this.sanitizeName(file.metadata.title)}`; - const fileStream = await retryAsyncFunction(() => + file.metadata = mergeMetadata([file])[0].metadata; + const fileSaveName = getUniqueFileSaveName( + collectionPath, + file.metadata.title, + file.id + ); + let fileStream = await retryAsyncFunction(() => downloadManager.downloadFile(file) ); + const fileType = getFileExtension(file.metadata.title); + if ( + file.pubMagicMetadata?.data.editedTime && + (fileType === TYPE_JPEG || fileType === TYPE_JPG) + ) { + const fileBlob = await new Response(fileStream).blob(); + const updatedFileBlob = await updateFileCreationDateInEXIF( + fileBlob, + new Date(file.pubMagicMetadata.data.editedTime / 1000) + ); + fileStream = updatedFileBlob.stream(); + } if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { - this.exportMotionPhoto(fileStream, file, collectionPath); + await this.exportMotionPhoto(fileStream, file, collectionPath); } else { - this.saveMediaFile(collectionPath, uid, fileStream); - this.saveMetadataFile(collectionPath, uid, file.metadata); + this.saveMediaFile(collectionPath, fileSaveName, fileStream); + await this.saveMetadataFile( + collectionPath, + fileSaveName, + file.metadata + ); } } @@ -324,37 +504,173 @@ class ExportService { const fileBlob = await new Response(fileStream).blob(); const originalName = fileNameWithoutExtension(file.metadata.title); const motionPhoto = await decodeMotionPhoto(fileBlob, originalName); - const imageStream = generateStreamFromArrayBuffer(motionPhoto.image); - const imageUID = `${file.id}_${motionPhoto.imageNameTitle}`; - this.saveMediaFile(collectionPath, imageUID, imageStream); - this.saveMetadataFile(collectionPath, imageUID, file.metadata); + const imageSaveName = getUniqueFileSaveName( + collectionPath, + motionPhoto.imageNameTitle, + file.id + ); + this.saveMediaFile(collectionPath, imageSaveName, imageStream); + await this.saveMetadataFile( + collectionPath, + imageSaveName, + file.metadata + ); const videoStream = generateStreamFromArrayBuffer(motionPhoto.video); - const videoUID = `${file.id}_${motionPhoto.videoNameTitle}`; - this.saveMediaFile(collectionPath, videoUID, videoStream); - this.saveMetadataFile(collectionPath, videoUID, file.metadata); + const videoSaveName = getUniqueFileSaveName( + collectionPath, + motionPhoto.videoNameTitle, + file.id + ); + this.saveMediaFile(collectionPath, videoSaveName, videoStream); + await this.saveMetadataFile( + collectionPath, + videoSaveName, + file.metadata + ); } - private saveMediaFile(collectionPath, uid, fileStream) { + private saveMediaFile( + collectionFolderPath: string, + fileSaveName: string, + fileStream: ReadableStream + ) { this.ElectronAPIs.saveStreamToDisk( - `${collectionPath}/${uid}`, + getFileSavePath(collectionFolderPath, fileSaveName), fileStream ); } - private saveMetadataFile(collectionPath, uid, metadata) { - this.ElectronAPIs.saveFileToDisk( - `${collectionPath}/${METADATA_FOLDER_NAME}/${uid}.json`, - getGoogleLikeMetadataFile(uid, metadata) + private async saveMetadataFile( + collectionFolderPath: string, + fileSaveName: string, + metadata: MetadataObject + ) { + await this.ElectronAPIs.saveFileToDisk( + getFileMetadataSavePath(collectionFolderPath, fileSaveName), + getGoogleLikeMetadataFile(fileSaveName, metadata) ); } - private sanitizeName(name) { - return name.replaceAll('/', '_').replaceAll(' ', '_'); - } - isExportInProgress = () => { return this.exportInProgress !== null; }; + + exists = (path: string) => { + return this.ElectronAPIs.exists(path); + }; + + checkAllElectronAPIsExists = () => this.allElectronAPIsExist; + + /* + this function migrates the exportRecord file to apply any schema changes. + currently we apply only a single migration to update file and collection name to newer format + so there is just a if condition check, + later this will be converted to a loop which applies the migration one by one + till the files reaches the latest version + */ + private async migrateExport( + exportDir: string, + collections: Collection[], + allFiles: File[] + ) { + const exportRecord = await this.getExportRecord(exportDir); + const currentVersion = exportRecord?.version ?? 0; + if (currentVersion === 0) { + const collectionIDPathMap = new Map(); + + await this.migrateCollectionFolders( + collections, + exportDir, + collectionIDPathMap + ); + await this.migrateFiles( + getExportedFiles(allFiles, exportRecord), + collectionIDPathMap + ); + + await this.updateExportRecord({ + version: LATEST_EXPORT_VERSION, + }); + } + } + + /* + This updates the folder name of already exported folders from the earlier format of + `collectionID_collectionName` to newer `collectionName(numbered)` format + */ + private async migrateCollectionFolders( + collections: Collection[], + exportDir: string, + collectionIDPathMap: CollectionIDPathMap + ) { + for (const collection of collections) { + const oldCollectionFolderPath = getOldCollectionFolderPath( + exportDir, + collection + ); + const newCollectionFolderPath = getUniqueCollectionFolderPath( + exportDir, + collection + ); + collectionIDPathMap.set(collection.id, newCollectionFolderPath); + if (this.ElectronAPIs.exists(oldCollectionFolderPath)) { + await this.ElectronAPIs.checkExistsAndRename( + oldCollectionFolderPath, + newCollectionFolderPath + ); + await this.addCollectionExportedRecord( + exportDir, + collection, + newCollectionFolderPath + ); + } + } + } + + /* + This updates the file name of already exported files from the earlier format of + `fileID_fileName` to newer `fileName(numbered)` format + */ + private async migrateFiles( + files: File[], + collectionIDPathMap: Map + ) { + for (let file of files) { + const oldFileSavePath = getOldFileSavePath( + collectionIDPathMap.get(file.collectionID), + file + ); + const oldFileMetadataSavePath = getOldFileMetadataSavePath( + collectionIDPathMap.get(file.collectionID), + file + ); + file = mergeMetadata([file])[0]; + const newFileSaveName = getUniqueFileSaveName( + collectionIDPathMap.get(file.collectionID), + file.metadata.title, + file.id + ); + + const newFileSavePath = getFileSavePath( + collectionIDPathMap.get(file.collectionID), + newFileSaveName + ); + + const newFileMetadataSavePath = getFileMetadataSavePath( + collectionIDPathMap.get(file.collectionID), + newFileSaveName + ); + await this.ElectronAPIs.checkExistsAndRename( + oldFileSavePath, + newFileSavePath + ); + console.log(oldFileMetadataSavePath, newFileMetadataSavePath); + await this.ElectronAPIs.checkExistsAndRename( + oldFileMetadataSavePath, + newFileMetadataSavePath + ); + } + } } export default new ExportService(); diff --git a/src/services/motionPhotoService.ts b/src/services/motionPhotoService.ts index 425a86fcd..27b8440da 100644 --- a/src/services/motionPhotoService.ts +++ b/src/services/motionPhotoService.ts @@ -4,8 +4,8 @@ import { fileExtensionWithDot } from 'utils/file'; class MotionPhoto { image: Uint8Array; video: Uint8Array; - imageNameTitle: String; - videoNameTitle: String; + imageNameTitle: string; + videoNameTitle: string; } export const decodeMotionPhoto = async ( diff --git a/src/services/upload/exifService.ts b/src/services/upload/exifService.ts index d93a49caf..5792267c8 100644 --- a/src/services/upload/exifService.ts +++ b/src/services/upload/exifService.ts @@ -1,6 +1,6 @@ import exifr from 'exifr'; +import piexif from 'piexifjs'; import { logError } from 'utils/sentry'; - import { NULL_LOCATION, Location } from './metadataService'; import { FileTypeInfo } from './readFileService'; @@ -55,6 +55,65 @@ export async function getExifData( } } +export async function updateFileCreationDateInEXIF( + fileBlob: Blob, + updatedDate: Date +) { + try { + const fileURL = URL.createObjectURL(fileBlob); + let imageDataURL = await convertImageToDataURL(fileURL); + imageDataURL = + 'data:image/jpeg;base64' + + imageDataURL.slice(imageDataURL.indexOf(',')); + const exifObj = piexif.load(imageDataURL); + if (!exifObj['Exif']) { + exifObj['Exif'] = {}; + } + exifObj['Exif'][piexif.ExifIFD.DateTimeOriginal] = + convertToExifDateFormat(updatedDate); + + const exifBytes = piexif.dump(exifObj); + const exifInsertedFile = piexif.insert(exifBytes, imageDataURL); + return dataURIToBlob(exifInsertedFile); + } catch (e) { + logError(e, 'updateFileModifyDateInEXIF failed'); + return fileBlob; + } +} + +export async function convertImageToDataURL(url: string) { + const blob = await fetch(url).then((r) => r.blob()); + const dataUrl = await new Promise((resolve) => { + const reader = new FileReader(); + reader.onload = () => resolve(reader.result as string); + reader.readAsDataURL(blob); + }); + return dataUrl; +} + +function dataURIToBlob(dataURI) { + // convert base64 to raw binary data held in a string + // doesn't handle URLEncoded DataURIs - see SO answer #6850276 for code that does this + const byteString = atob(dataURI.split(',')[1]); + + // separate out the mime component + const mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0]; + + // write the bytes of the string to an ArrayBuffer + const ab = new ArrayBuffer(byteString.length); + + // create a view into the buffer + const ia = new Uint8Array(ab); + + // set the bytes of the buffer to the correct values + for (let i = 0; i < byteString.length; i++) { + ia[i] = byteString.charCodeAt(i); + } + + // write the ArrayBuffer to a blob, and you're done + const blob = new Blob([ab], { type: mimeString }); + return blob; +} export async function getRawExif( receivedFile: File, fileTypeInfo: FileTypeInfo @@ -93,3 +152,9 @@ function getEXIFLocation(exifData): Location { } return { latitude: exifData.latitude, longitude: exifData.longitude }; } + +function convertToExifDateFormat(date: Date) { + return `${date.getFullYear()}:${ + date.getMonth() + 1 + }:${date.getDate()} ${date.getHours()}:${date.getMinutes()}:${date.getSeconds()}`; +} diff --git a/src/services/upload/readFileService.ts b/src/services/upload/readFileService.ts index 3342fd572..b7742b67b 100644 --- a/src/services/upload/readFileService.ts +++ b/src/services/upload/readFileService.ts @@ -6,6 +6,7 @@ import { logError } from 'utils/sentry'; import { FILE_READER_CHUNK_SIZE, MULTIPART_PART_SIZE } from './uploadService'; import FileType from 'file-type/browser'; import { CustomError } from 'utils/common/errorUtil'; +import { getFileExtension } from 'utils/file'; const TYPE_VIDEO = 'video'; const TYPE_IMAGE = 'image'; @@ -48,7 +49,7 @@ export async function getFileType( } return { fileType, exactType: typeParts[1] }; } catch (e) { - const fileFormat = receivedFile.name.split('.').pop(); + const fileFormat = getFileExtension(receivedFile.name); const formatMissedByTypeDetection = FORMAT_MISSED_BY_FILE_TYPE_LIB.find( (a) => a.exactType === fileFormat ); diff --git a/src/services/upload/uploadManager.ts b/src/services/upload/uploadManager.ts index 1c67cbb6d..fe9db1016 100644 --- a/src/services/upload/uploadManager.ts +++ b/src/services/upload/uploadManager.ts @@ -125,7 +125,7 @@ class UploadManager { fileWithCollection.collectionID, title ), - parsedMetaDataJSON + { ...parsedMetaDataJSON } ); UIService.increaseFileUploaded(); } diff --git a/src/utils/export/index.ts b/src/utils/export/index.ts index 1aca1446b..ffad98a7f 100644 --- a/src/utils/export/index.ts +++ b/src/utils/export/index.ts @@ -1,46 +1,120 @@ -import { ExportRecord } from 'services/exportService'; +import { Collection } from 'services/collectionService'; +import exportService, { + CollectionIDPathMap, + ExportRecord, + METADATA_FOLDER_NAME, +} from 'services/exportService'; import { File } from 'services/fileService'; import { MetadataObject } from 'services/upload/uploadService'; -import { formatDate } from 'utils/file'; +import { formatDate, splitFilenameAndExtension } from 'utils/file'; export const getExportRecordFileUID = (file: File) => `${file.id}_${file.collectionID}_${file.updationTime}`; -export const getExportPendingFiles = async ( +export const getExportQueuedFiles = ( allFiles: File[], exportRecord: ExportRecord ) => { const queuedFiles = new Set(exportRecord?.queuedFiles); const unExportedFiles = allFiles.filter((file) => { if (queuedFiles.has(getExportRecordFileUID(file))) { - return file; + return true; } + return false; }); return unExportedFiles; }; -export const getFilesUploadedAfterLastExport = async ( +export const getCollectionsCreatedAfterLastExport = ( + collections: Collection[], + exportRecord: ExportRecord +) => { + const exportedCollections = new Set( + Object.keys(exportRecord?.exportedCollectionPaths ?? {}).map((x) => + Number(x) + ) + ); + const unExportedCollections = collections.filter((collection) => { + if (!exportedCollections.has(collection.id)) { + return true; + } + return false; + }); + return unExportedCollections; +}; +export const getCollectionIDPathMapFromExportRecord = ( + exportRecord: ExportRecord +): CollectionIDPathMap => { + return new Map( + Object.entries(exportRecord.exportedCollectionPaths ?? {}).map((e) => { + return [Number(e[0]), String(e[1])]; + }) + ); +}; + +export const getCollectionsRenamedAfterLastExport = ( + collections: Collection[], + exportRecord: ExportRecord +) => { + const collectionIDPathMap = + getCollectionIDPathMapFromExportRecord(exportRecord); + const renamedCollections = collections.filter((collection) => { + if (collectionIDPathMap.has(collection.id)) { + const currentFolderName = collectionIDPathMap.get(collection.id); + const startIndex = currentFolderName.lastIndexOf('/'); + const lastIndex = currentFolderName.lastIndexOf('('); + const nameRoot = currentFolderName.slice( + startIndex + 1, + lastIndex !== -1 ? lastIndex : currentFolderName.length + ); + + if (nameRoot !== sanitizeName(collection.name)) { + return true; + } + } + return false; + }); + return renamedCollections; +}; + +export const getFilesUploadedAfterLastExport = ( allFiles: File[], exportRecord: ExportRecord ) => { const exportedFiles = new Set(exportRecord?.exportedFiles); const unExportedFiles = allFiles.filter((file) => { if (!exportedFiles.has(getExportRecordFileUID(file))) { - return file; + return true; } + return false; }); return unExportedFiles; }; -export const getExportFailedFiles = async ( +export const getExportedFiles = ( + allFiles: File[], + exportRecord: ExportRecord +) => { + const exportedFileIds = new Set(exportRecord?.exportedFiles); + const exportedFiles = allFiles.filter((file) => { + if (exportedFileIds.has(getExportRecordFileUID(file))) { + return true; + } + return false; + }); + return exportedFiles; +}; + +export const getExportFailedFiles = ( allFiles: File[], exportRecord: ExportRecord ) => { const failedFiles = new Set(exportRecord?.failedFiles); const filesToExport = allFiles.filter((file) => { if (failedFiles.has(getExportRecordFileUID(file))) { - return file; + return true; } + return false; }); return filesToExport; }; @@ -52,14 +126,16 @@ export const dedupe = (files: any[]) => { }; export const getGoogleLikeMetadataFile = ( - uid: string, + fileSaveName: string, metadata: MetadataObject ) => { const creationTime = Math.floor(metadata.creationTime / 1000000); - const modificationTime = Math.floor(metadata.modificationTime / 1000000); + const modificationTime = Math.floor( + (metadata.modificationTime ?? metadata.creationTime) / 1000000 + ); return JSON.stringify( { - title: uid, + title: fileSaveName, creationTime: { timestamp: creationTime, formatted: formatDate(creationTime * 1000), @@ -77,3 +153,85 @@ export const getGoogleLikeMetadataFile = ( 2 ); }; + +export const oldSanitizeName = (name: string) => + name.replaceAll('/', '_').replaceAll(' ', '_'); + +export const sanitizeName = (name: string) => + name.replace(/[^a-z0-9.]/gi, '_').toLowerCase(); + +export const getUniqueCollectionFolderPath = ( + dir: string, + collection: Collection +): string => { + if (!exportService.checkAllElectronAPIsExists()) { + return getOldCollectionFolderPath(dir, collection); + } + let collectionFolderPath = `${dir}/${sanitizeName(collection.name)}`; + let count = 1; + while (exportService.exists(collectionFolderPath)) { + collectionFolderPath = `${dir}/${sanitizeName( + collection.name + )}(${count})`; + count++; + } + return collectionFolderPath; +}; + +export const getMetadataFolderPath = (collectionFolderPath: string) => + `${collectionFolderPath}/${METADATA_FOLDER_NAME}`; + +export const getUniqueFileSaveName = ( + collectionPath: string, + filename: string, + fileID: number +) => { + if (!exportService.checkAllElectronAPIsExists()) { + return getOldFileSaveName(filename, fileID); + } + let fileSaveName = sanitizeName(filename); + let count = 1; + while ( + exportService.exists(getFileSavePath(collectionPath, fileSaveName)) + ) { + const filenameParts = splitFilenameAndExtension(sanitizeName(filename)); + if (filenameParts[1]) { + fileSaveName = `${filenameParts[0]}(${count}).${filenameParts[1]}`; + } else { + fileSaveName = `${filenameParts[0]}(${count})`; + } + count++; + } + return fileSaveName; +}; + +export const getOldFileSaveName = (filename: string, fileID: number) => + `${fileID}_${oldSanitizeName(filename)}`; + +export const getFileMetadataSavePath = ( + collectionFolderPath: string, + fileSaveName: string +) => `${collectionFolderPath}/${METADATA_FOLDER_NAME}/${fileSaveName}.json`; + +export const getFileSavePath = ( + collectionFolderPath: string, + fileSaveName: string +) => `${collectionFolderPath}/${fileSaveName}`; + +export const getOldCollectionFolderPath = ( + dir: string, + collection: Collection +) => `${dir}/${collection.id}_${oldSanitizeName(collection.name)}`; + +export const getOldFileSavePath = (collectionFolderPath: string, file: File) => + `${collectionFolderPath}/${file.id}_${oldSanitizeName( + file.metadata.title + )}`; + +export const getOldFileMetadataSavePath = ( + collectionFolderPath: string, + file: File +) => + `${collectionFolderPath}/${METADATA_FOLDER_NAME}/${ + file.id + }_${oldSanitizeName(file.metadata.title)}.json`; diff --git a/src/utils/file/index.ts b/src/utils/file/index.ts index 1e7b08f25..ea911e08a 100644 --- a/src/utils/file/index.ts +++ b/src/utils/file/index.ts @@ -16,9 +16,12 @@ import { logError } from 'utils/sentry'; import { User } from 'services/userService'; import CryptoWorker from 'utils/crypto'; import { getData, LS_KEYS } from 'utils/storage/localStorage'; +import { updateFileCreationDateInEXIF } from 'services/upload/exifService'; export const TYPE_HEIC = 'heic'; export const TYPE_HEIF = 'heif'; +export const TYPE_JPEG = 'jpeg'; +export const TYPE_JPG = 'jpg'; const UNSUPPORTED_FORMATS = ['flv', 'mkv', '3gp', 'avi', 'wmv']; export function downloadAsFile(filename: string, content: string) { @@ -40,13 +43,32 @@ export function downloadAsFile(filename: string, content: string) { export async function downloadFile(file: File) { const a = document.createElement('a'); a.style.display = 'none'; - const cachedFileUrl = await DownloadManager.getCachedOriginalFile(file); - const fileURL = - cachedFileUrl ?? - URL.createObjectURL( + let fileURL = await DownloadManager.getCachedOriginalFile(file); + let tempURL; + if (!fileURL) { + tempURL = URL.createObjectURL( await new Response(await DownloadManager.downloadFile(file)).blob() ); + fileURL = tempURL; + } + const fileType = getFileExtension(file.metadata.title); + let tempEditedFileURL; + if ( + file.pubMagicMetadata?.data.editedTime && + (fileType === TYPE_JPEG || fileType === TYPE_JPG) + ) { + let fileBlob = await (await fetch(fileURL)).blob(); + + fileBlob = await updateFileCreationDateInEXIF( + fileBlob, + new Date(file.pubMagicMetadata.data.editedTime / 1000) + ); + tempEditedFileURL = URL.createObjectURL(fileBlob); + fileURL = tempEditedFileURL; + } + a.href = fileURL; + if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { a.download = fileNameWithoutExtension(file.metadata.title) + '.zip'; } else { @@ -55,6 +77,8 @@ export async function downloadFile(file: File) { document.body.appendChild(a); a.click(); a.remove(); + tempURL && URL.revokeObjectURL(tempURL); + tempEditedFileURL && URL.revokeObjectURL(tempEditedFileURL); } export function isFileHEIC(mimeType: string) { @@ -257,6 +281,10 @@ export function splitFilenameAndExtension(filename): [string, string] { ]; } +export function getFileExtension(filename) { + return splitFilenameAndExtension(filename)[1]; +} + export function generateStreamFromArrayBuffer(data: Uint8Array) { return new ReadableStream({ async start(controller: ReadableStreamDefaultController) { @@ -273,7 +301,7 @@ export async function convertForPreview(file: File, fileBlob: Blob) { fileBlob = new Blob([motionPhoto.image]); } - const typeFromExtension = file.metadata.title.split('.')[-1]; + const typeFromExtension = getFileExtension(file.metadata.title); const worker = await new CryptoWorker(); const mimeType = diff --git a/src/utils/sentry/index.ts b/src/utils/sentry/index.ts index dae3f9996..b3d4c87ca 100644 --- a/src/utils/sentry/index.ts +++ b/src/utils/sentry/index.ts @@ -9,7 +9,7 @@ export const logError = ( ) => { const err = errorWithContext(error, msg); if (!process.env.NEXT_PUBLIC_SENTRY_ENV) { - console.log({ error, msg, info }); + console.log(error, { msg, info }); } Sentry.captureException(err, { level: Sentry.Severity.Info, diff --git a/yarn.lock b/yarn.lock index 67a62af1f..e75a5a4d2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5055,6 +5055,11 @@ picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.2, picomatch@^2.2.3: resolved "https://registry.npmjs.org/picomatch/-/picomatch-2.3.0.tgz" integrity sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw== +piexifjs@^1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/piexifjs/-/piexifjs-1.0.6.tgz#883811d73f447218d0d06e9ed7866d04533e59e0" + integrity sha512-0wVyH0cKohzBQ5Gi2V1BuxYpxWfxF3cSqfFXfPIpl5tl9XLS5z4ogqhUCD20AbHi0h9aJkqXNJnkVev6gwh2ag== + pify@^2.0.0: version "2.3.0" resolved "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz"