Merge branch 'add-remote-ente-file-type' into add-encrypted-collection-type

src/services/collectionService.ts

@@ -523,7 +523,7 @@ export const updateCollectionMagicMetadata = async (collection: Collection) => {
 
     const worker = await new CryptoWorker();
 
-    const { file: encryptedMagicMetadata }: EncryptionResult =
+    const { file: encryptedMagicMetadata }: EncryptionResult<string> =
         await worker.encryptMetadata(
             collection.magicMetadata.data,
             collection.key
@@ -534,7 +534,7 @@ export const updateCollectionMagicMetadata = async (collection: Collection) => {
         magicMetadata: {
             version: collection.magicMetadata.version,
             count: collection.magicMetadata.count,
-            data: encryptedMagicMetadata.encryptedData as unknown as string,
+            data: encryptedMagicMetadata.encryptedData,
             header: encryptedMagicMetadata.decryptionHeader,
         },
     };

src/services/fileService.ts

@@ -253,14 +253,14 @@ export const updateFileMagicMetadata = async (files: EnteFile[]) => {
     const reqBody: BulkUpdateMagicMetadataRequest = { metadataList: [] };
     const worker = await new CryptoWorker();
     for (const file of files) {
-        const { file: encryptedMagicMetadata }: EncryptionResult =
+        const { file: encryptedMagicMetadata }: EncryptionResult<string> =
             await worker.encryptMetadata(file.magicMetadata.data, file.key);
         reqBody.metadataList.push({
             id: file.id,
             magicMetadata: {
                 version: file.magicMetadata.version,
                 count: file.magicMetadata.count,
-                data: encryptedMagicMetadata.encryptedData as unknown as string,
+                data: encryptedMagicMetadata.encryptedData,
                 header: encryptedMagicMetadata.decryptionHeader,
             },
         });
@@ -287,14 +287,14 @@ export const updateFilePublicMagicMetadata = async (files: EnteFile[]) => {
     const reqBody: BulkUpdateMagicMetadataRequest = { metadataList: [] };
     const worker = await new CryptoWorker();
     for (const file of files) {
-        const { file: encryptedPubMagicMetadata }: EncryptionResult =
+        const { file: encryptedPubMagicMetadata }: EncryptionResult<string> =
             await worker.encryptMetadata(file.pubMagicMetadata.data, file.key);
         reqBody.metadataList.push({
             id: file.id,
             magicMetadata: {
                 version: file.pubMagicMetadata.version,
                 count: file.pubMagicMetadata.count,
-                data: encryptedPubMagicMetadata.encryptedData as unknown as string,
+                data: encryptedPubMagicMetadata.encryptedData,
                 header: encryptedPubMagicMetadata.decryptionHeader,
             },
         });

src/services/migrateThumbnailService.ts

@@ -11,7 +11,7 @@ import { SetProgressTracker } from 'components/FixLargeThumbnail';
 import { getFileType } from 'services/typeDetectionService';
 import { getLocalTrash, getTrashedFiles } from './trashService';
 import { EncryptionResult, UploadURL } from 'types/upload';
-import { S3FileAttribute } from 'types/file';
+import { FileAttributes } from 'types/file';
 import { USE_CF_PROXY } from 'constants/upload';
 
 const ENDPOINT = getEndpoint();
@@ -106,20 +106,20 @@ export async function uploadThumbnail(
     fileKey: string,
     updatedThumbnail: Uint8Array,
     uploadURL: UploadURL
-): Promise<S3FileAttribute> {
-    const { file: encryptedThumbnail }: EncryptionResult =
+): Promise<FileAttributes> {
+    const { file: encryptedThumbnail }: EncryptionResult<Uint8Array> =
         await worker.encryptThumbnail(updatedThumbnail, fileKey);
     let thumbnailObjectKey: string = null;
     if (USE_CF_PROXY) {
         thumbnailObjectKey = await uploadHttpClient.putFileV2(
             uploadURL,
-            encryptedThumbnail.encryptedData as Uint8Array,
+            encryptedThumbnail.encryptedData,
             () => {}
         );
     } else {
         thumbnailObjectKey = await uploadHttpClient.putFile(
             uploadURL,
-            encryptedThumbnail.encryptedData as Uint8Array,
+            encryptedThumbnail.encryptedData,
             () => {}
         );
     }
@@ -131,7 +131,7 @@ export async function uploadThumbnail(
 
 export async function updateThumbnail(
     fileID: number,
-    newThumbnail: S3FileAttribute
+    newThumbnail: FileAttributes
 ) {
     try {
         const token = getToken();

src/services/publicCollectionService.ts

@@ -262,8 +262,9 @@ const getPublicFiles = async (
                     resp.data.diff.map(async (file: EncryptedEnteFile) => {
                         if (!file.isDeleted) {
                             return await decryptFile(file, collection.key);
+                        } else {
+                            return file;
                         }
-                        return file;
                     }) as Promise<EnteFile>[]
                 )),
             ];

src/services/upload/encryptionService.ts

@@ -33,7 +33,7 @@ async function encryptFileStream(worker, fileData: DataStream) {
 export async function encryptFiledata(
     worker,
     filedata: Uint8Array | DataStream
-): Promise<EncryptionResult> {
+): Promise<EncryptionResult<Uint8Array | DataStream>> {
     return isDataStream(filedata)
         ? await encryptFileStream(worker, filedata)
         : await worker.encryptFile(filedata);

src/services/upload/fileService.ts

@@ -10,7 +10,6 @@ import {
     ParsedMetadataJSONMap,
     DataStream,
     ElectronFile,
-    MetadataEncryptionResult,
 } from 'types/upload';
 import { splitFilenameAndExtension } from 'utils/file';
 import { logError } from 'utils/sentry';
@@ -23,7 +22,7 @@ import {
     getUint8ArrayView,
 } from '../readerService';
 import { generateThumbnail } from './thumbnailService';
-import { EncryptedMagicMetadataCore } from 'types/magicMetadata';
+import { EncryptedMagicMetadata } from 'types/magicMetadata';
 
 const EDITED_FILE_SUFFIX = '-edited';
 
@@ -107,22 +106,23 @@ export async function encryptFile(
             file.filedata
         );
 
-        const { file: encryptedThumbnail }: EncryptionResult =
+        const { file: encryptedThumbnail }: EncryptionResult<Uint8Array> =
             await worker.encryptThumbnail(file.thumbnail, fileKey);
-        const { file: encryptedMetadata }: MetadataEncryptionResult =
+        const { file: encryptedMetadata }: EncryptionResult<string> =
             await worker.encryptMetadata(file.metadata, fileKey);
 
-        let encryptedPubMagicMetadata: EncryptedMagicMetadataCore;
+        let encryptedPubMagicMetadata: EncryptedMagicMetadata;
         if (file.pubMagicMetadata) {
-            const { file: encryptedPubMagicMetadataData }: EncryptionResult =
-                await worker.encryptMetadata(
-                    file.pubMagicMetadata.data,
-                    fileKey
-                );
+            const {
+                file: encryptedPubMagicMetadataData,
+            }: EncryptionResult<string> = await worker.encryptMetadata(
+                file.pubMagicMetadata.data,
+                fileKey
+            );
             encryptedPubMagicMetadata = {
                 version: file.pubMagicMetadata.version,
                 count: file.pubMagicMetadata.count,
-                data: encryptedPubMagicMetadataData.encryptedData as unknown as string,
+                data: encryptedPubMagicMetadataData.encryptedData,
                 header: encryptedPubMagicMetadataData.decryptionHeader,
             };
         }

src/services/upload/magicMetadataService.ts

@@ -1,8 +1,8 @@
 import {
+    NEW_FILE_MAGIC_METADATA,
     FilePublicMagicMetadataProps,
     FilePublicMagicMetadata,
-} from 'types/file';
-import { NEW_FILE_MAGIC_METADATA } from 'types/magicMetadata';
+} from 'types/magicMetadata';
 import { updateMagicMetadataProps } from 'utils/magicMetadata';
 
 export async function constructPublicMagicMetadata(

src/services/upload/uploadManager.ts

@@ -360,7 +360,7 @@ class UploadManager {
             await this.watchFolderCallback(
                 fileUploadResult,
                 fileWithCollection,
-                decryptedFile
+                uploadedFile as EncryptedEnteFile
             );
             return fileUploadResult;
         } catch (e) {
@@ -372,7 +372,7 @@ class UploadManager {
     private async watchFolderCallback(
         fileUploadResult: UPLOAD_RESULT,
         fileWithCollection: FileWithCollection,
-        uploadedFile: EnteFile
+        uploadedFile: EncryptedEnteFile
     ) {
         if (isElectron()) {
             await watchFolderService.onFileUpload(

src/services/upload/uploadService.ts

@@ -35,7 +35,7 @@ import UIService from './uiService';
 import { USE_CF_PROXY } from 'constants/upload';
 import publicUploadHttpClient from './publicUploadHttpClient';
 import { constructPublicMagicMetadata } from './magicMetadataService';
-import { FilePublicMagicMetadataProps } from 'types/file';
+import { FilePublicMagicMetadataProps } from 'types/magicMetadata';
 
 class UploadService {
     private uploadURLs: UploadURL[] = [];
@@ -175,13 +175,13 @@ class UploadService {
             if (USE_CF_PROXY) {
                 thumbnailObjectKey = await UploadHttpClient.putFileV2(
                     thumbnailUploadURL,
-                    file.thumbnail.encryptedData as Uint8Array,
+                    file.thumbnail.encryptedData,
                     null
                 );
             } else {
                 thumbnailObjectKey = await UploadHttpClient.putFile(
                     thumbnailUploadURL,
-                    file.thumbnail.encryptedData as Uint8Array,
+                    file.thumbnail.encryptedData,
                     null
                 );
             }

src/services/upload/uploader.ts

@@ -1,4 +1,4 @@
-import { EnteFile, FilePublicMagicMetadata } from 'types/file';
+import { EnteFile } from 'types/file';
 import { handleUploadError, CustomError } from 'utils/error';
 import { logError } from 'utils/sentry';
 import { findMatchingExistingFiles } from 'utils/upload';
@@ -19,6 +19,7 @@ import { sleep } from 'utils/common';
 import { addToCollection } from 'services/collectionService';
 import uploadCancelService from './uploadCancelService';
 import uploadService from './uploadService';
+import { FilePublicMagicMetadata } from 'types/magicMetadata';
 
 interface UploadResponse {
     fileUploadResult: UPLOAD_RESULT;

src/services/watchFolder/watchFolderService.ts

@@ -1,5 +1,5 @@
 import { Collection } from 'types/collection';
-import { EnteFile } from 'types/file';
+import { EncryptedEnteFile } from 'types/file';
 import { ElectronFile, FileWithCollection } from 'types/upload';
 import { runningInBrowser } from 'utils/common';
 import { removeFromCollection } from '../collectionService';
@@ -33,7 +33,7 @@ class watchFolderService {
     private trashingDirQueue: string[] = [];
     private isEventRunning: boolean = false;
     private uploadRunning: boolean = false;
-    private filePathToUploadedFileIDMap = new Map<string, EnteFile>();
+    private filePathToUploadedFileIDMap = new Map<string, EncryptedEnteFile>();
     private unUploadableFilePaths = new Set<string>();
     private isPaused = false;
     private setElectronFiles: (files: ElectronFile[]) => void;
@@ -295,7 +295,7 @@ class watchFolderService {
     async onFileUpload(
         fileUploadResult: UPLOAD_RESULT,
         fileWithCollection: FileWithCollection,
-        file: EnteFile
+        file: EncryptedEnteFile
     ) {
         addLocalLog(() => `onFileUpload called`);
         if (!this.isUploadRunning()) {

src/types/file/index.ts

@@ -1,26 +1,26 @@
 import {
     EncryptedMagicMetadata,
-    MagicMetadataCore,
-    VISIBILITY_STATE,
+    FileMagicMetadata,
+    FilePublicMagicMetadata,
 } from 'types/magicMetadata';
-import { DataStream, Metadata } from 'types/upload';
+import { Metadata } from 'types/upload';
 
-export interface FileAttribute {
-    encryptedData: DataStream | Uint8Array;
+interface FileAttributesBase {
     decryptionHeader: string;
 }
 
-export interface B64FileAttribute {
+interface MetadataFileAttributes extends FileAttributesBase {
     encryptedData: string;
-    decryptionHeader: string;
+    objectKey?: string;
 }
-
-export interface S3FileAttribute {
+interface S3FileAttributes extends FileAttributesBase {
     objectKey: string;
-    decryptionHeader: string;
+    encryptedData?: string;
 }
 
-export interface EnteFileInfo {
+export type FileAttributes = MetadataFileAttributes | S3FileAttributes;
+
+export interface FileInfo {
     fileSize: number;
     thumbSize: number;
 }
@@ -29,10 +29,10 @@ export interface EncryptedEnteFile {
     id: number;
     collectionID: number;
     ownerID: number;
-    file: S3FileAttribute;
-    thumbnail: S3FileAttribute;
-    metadata: B64FileAttribute;
-    info: EnteFileInfo;
+    file: FileAttributes;
+    thumbnail: FileAttributes;
+    metadata: FileAttributes;
+    info: FileInfo;
     magicMetadata: EncryptedMagicMetadata;
     pubMagicMetadata: EncryptedMagicMetadata;
     encryptedKey: string;
@@ -68,27 +68,6 @@ export interface EnteFile
     dataIndex?: number;
 }
 
-export interface FileMagicMetadataProps {
-    visibility?: VISIBILITY_STATE;
-    filePaths?: string[];
-}
-
-export interface FileMagicMetadata extends Omit<MagicMetadataCore, 'data'> {
-    data: FileMagicMetadataProps;
-}
-
-export interface FilePublicMagicMetadataProps {
-    editedTime?: number;
-    editedName?: string;
-    caption?: string;
-    uploaderName?: string;
-}
-
-export interface FilePublicMagicMetadata
-    extends Omit<MagicMetadataCore, 'data'> {
-    data: FilePublicMagicMetadataProps;
-}
-
 export interface TrashRequest {
     items: TrashRequestItems[];
 }

src/types/magicMetadata/index.ts

@@ -10,6 +10,27 @@ export interface EncryptedMagicMetadata
     data: string;
 }
 
+export interface FileMagicMetadataProps {
+    visibility?: VISIBILITY_STATE;
+    filePaths?: string[];
+}
+
+export interface FileMagicMetadata extends Omit<MagicMetadataCore, 'data'> {
+    data: FileMagicMetadataProps;
+}
+
+export interface FilePublicMagicMetadataProps {
+    editedTime?: number;
+    editedName?: string;
+    caption?: string;
+    uploaderName?: string;
+}
+
+export interface FilePublicMagicMetadata
+    extends Omit<MagicMetadataCore, 'data'> {
+    data: FilePublicMagicMetadataProps;
+}
+
 export enum VISIBILITY_STATE {
     VISIBLE = 0,
     ARCHIVED = 1,

src/types/upload/index.ts

@@ -1,12 +1,10 @@
 import { FILE_TYPE } from 'constants/file';
 import { Collection } from 'types/collection';
+import { FileAttributes } from 'types/file';
 import {
-    B64FileAttribute,
-    FileAttribute,
-    S3FileAttribute,
+    EncryptedMagicMetadata,
     FilePublicMagicMetadata,
-} from 'types/file';
-import { EncryptedMagicMetadata } from 'types/magicMetadata';
+} from 'types/magicMetadata';
 
 export interface DataStream {
     stream: ReadableStream<Uint8Array>;
@@ -17,13 +15,15 @@ export function isDataStream(object: any): object is DataStream {
     return 'stream' in object;
 }
 
-export interface EncryptionResult {
-    file: FileAttribute;
-    key: string;
+export interface LocalFileAttributes<
+    T extends string | Uint8Array | DataStream
+> {
+    encryptedData: T;
+    decryptionHeader: string;
 }
 
-export interface MetadataEncryptionResult {
-    file: B64FileAttribute;
+export interface EncryptionResult<T extends string | Uint8Array | DataStream> {
+    file: LocalFileAttributes<T>;
     key: string;
 }
 
@@ -132,17 +132,17 @@ export interface EncryptedFile {
     fileKey: B64EncryptionResult;
 }
 export interface ProcessedFile {
-    file: FileAttribute;
-    thumbnail: FileAttribute;
-    metadata: FileAttribute;
+    file: LocalFileAttributes<Uint8Array | DataStream>;
+    thumbnail: LocalFileAttributes<Uint8Array>;
+    metadata: LocalFileAttributes<string>;
     pubMagicMetadata: EncryptedMagicMetadata;
     localID: number;
 }
-
 export interface BackupedFile {
-    file: S3FileAttribute;
-    thumbnail: S3FileAttribute;
-    metadata: B64FileAttribute;
+    file: FileAttributes;
+    thumbnail: FileAttributes;
+    metadata: FileAttributes;
+    pubMagicMetadata: EncryptedMagicMetadata;
 }
 
 export interface UploadFile extends BackupedFile {

src/utils/file/index.ts

@@ -1,12 +1,5 @@
 import { SelectedState } from 'types/gallery';
-import {
-    EnteFile,
-    EncryptedEnteFile,
-    FileMagicMetadata,
-    FileMagicMetadataProps,
-    FilePublicMagicMetadata,
-    FilePublicMagicMetadataProps,
-} from 'types/file';
+import { EnteFile, EncryptedEnteFile } from 'types/file';
 import { decodeMotionPhoto } from 'services/motionPhotoService';
 import { getFileType } from 'services/typeDetectionService';
 import DownloadManager from 'services/downloadManager';
@@ -25,7 +18,14 @@ import {
 import PublicCollectionDownloadManager from 'services/publicCollectionDownloadManager';
 import heicConversionService from 'services/heicConversionService';
 import * as ffmpegService from 'services/ffmpeg/ffmpegService';
-import { NEW_FILE_MAGIC_METADATA, VISIBILITY_STATE } from 'types/magicMetadata';
+import {
+    FileMagicMetadata,
+    FileMagicMetadataProps,
+    FilePublicMagicMetadata,
+    FilePublicMagicMetadataProps,
+    NEW_FILE_MAGIC_METADATA,
+    VISIBILITY_STATE,
+} from 'types/magicMetadata';
 import { IsArchived, updateMagicMetadataProps } from 'utils/magicMetadata';
 
 import { addLogLine } from 'utils/logging';

src/utils/magicMetadata/index.ts

@@ -1,6 +1,10 @@
 import { Collection } from 'types/collection';
-import { EnteFile, FileMagicMetadataProps } from 'types/file';
-import { MagicMetadataCore, VISIBILITY_STATE } from 'types/magicMetadata';
+import { EnteFile } from 'types/file';
+import {
+    FileMagicMetadataProps,
+    MagicMetadataCore,
+    VISIBILITY_STATE,
+} from 'types/magicMetadata';
 import CryptoWorker from 'utils/crypto';
 
 export function IsArchived(item: Collection | EnteFile) {