beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #810 from ente-io/add-remote-ente-file-type

Browse source 
add encryptedEnteFile type
This commit is contained in:
Abhinav Kumar 2022-12-25 10:28:42 +05:30 committed by GitHub
parent a3d3fea1c7 19c36861ae
commit 8b829fd840
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 189 additions and 146 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/services/collectionService.ts
View file

@ -463,13 +463,13 @@ const encryptWithNewCollectionKey = async (
file.key,
newCollection.key
);
file.encryptedKey = newEncryptedKey.encryptedData;
file.keyDecryptionNonce = newEncryptedKey.nonce;
const encryptedKey = newEncryptedKey.encryptedData;
const keyDecryptionNonce = newEncryptedKey.nonce;
fileKeysEncryptedWithNewCollection.push({
id: file.id,
encryptedKey: file.encryptedKey,
keyDecryptionNonce: file.keyDecryptionNonce,
encryptedKey,
keyDecryptionNonce,
});
}
return fileKeysEncryptedWithNewCollection;

9
src/services/fileService.ts
View file

@ -13,7 +13,7 @@ import {
sortFiles,
} from 'utils/file';
import CryptoWorker from 'utils/crypto';
import { EnteFile, TrashRequest } from 'types/file';
import { EnteFile, EncryptedEnteFile, TrashRequest } from 'types/file';
import { SetFiles } from 'types/gallery';
import { MAX_TRASH_BATCH_SIZE } from 'constants/file';
import { BulkUpdateMagicMetadataRequest } from 'types/magicMetadata';
@ -132,11 +132,12 @@ export const getFiles = async (
decryptedFiles = [
...decryptedFiles,
...(await Promise.all(
resp.data.diff.map(async (file: EnteFile) => {
resp.data.diff.map(async (file: EncryptedEnteFile) => {
if (!file.isDeleted) {
file = await decryptFile(file, collection.key);
return await decryptFile(file, collection.key);
} else {
return file;
}
return file;
}) as Promise<EnteFile>[]
)),
];

6
src/services/migrateThumbnailService.ts
View file

@ -11,7 +11,7 @@ import { SetProgressTracker } from 'components/FixLargeThumbnail';
import { getFileType } from 'services/typeDetectionService';
import { getLocalTrash, getTrashedFiles } from './trashService';
import { EncryptionResult, UploadURL } from 'types/upload';
import { fileAttribute } from 'types/file';
import { FileAttributes } from 'types/file';
import { USE_CF_PROXY } from 'constants/upload';
const ENDPOINT = getEndpoint();
@ -106,7 +106,7 @@ export async function uploadThumbnail(
fileKey: string,
updatedThumbnail: Uint8Array,
uploadURL: UploadURL
): Promise<fileAttribute> {
): Promise<FileAttributes> {
const { file: encryptedThumbnail }: EncryptionResult<Uint8Array> =
await worker.encryptThumbnail(updatedThumbnail, fileKey);
let thumbnailObjectKey: string = null;
@ -131,7 +131,7 @@ export async function uploadThumbnail(
export async function updateThumbnail(
fileID: number,
newThumbnail: fileAttribute
newThumbnail: FileAttributes
) {
try {
const token = getToken();

9
src/services/publicCollectionService.ts
View file

@ -4,7 +4,7 @@ import { Collection } from 'types/collection';
import HTTPService from './HTTPService';
import { logError } from 'utils/sentry';
import { decryptFile, mergeMetadata, sortFiles } from 'utils/file';
import { EnteFile } from 'types/file';
import { EncryptedEnteFile, EnteFile } from 'types/file';
import {
AbuseReportDetails,
AbuseReportRequest,
@ -259,11 +259,12 @@ const getPublicFiles = async (
decryptedFiles = [
...decryptedFiles,
...(await Promise.all(
resp.data.diff.map(async (file: EnteFile) => {
resp.data.diff.map(async (file: EncryptedEnteFile) => {
if (!file.isDeleted) {
file = await decryptFile(file, collection.key);
return await decryptFile(file, collection.key);
} else {
return file;
}
return file;
}) as Promise<EnteFile>[]
)),
];

37
src/services/trashService.ts
View file

@ -14,7 +14,7 @@ import { getCollection } from './collectionService';
import { EnteFile } from 'types/file';
import HTTPService from './HTTPService';
import { Trash, TrashItem } from 'types/trash';
import { EncryptedTrashItem, Trash } from 'types/trash';
const TRASH = 'file-trash';
const TRASH_TIME = 'trash-time';
@ -99,7 +99,8 @@ export const updateTrash = async (
'X-Auth-Token': token,
}
);
for (const trashItem of resp.data.diff as TrashItem[]) {
// #Perf: This can be optimized by running the decryption in parallel
for (const trashItem of resp.data.diff as EncryptedTrashItem[]) {
const collectionID = trashItem.file.collectionID;
let collection = collections.get(collectionID);
if (!collection) {
@ -110,19 +111,21 @@ export const updateTrash = async (
]);
}
if (!trashItem.isDeleted && !trashItem.isRestored) {
trashItem.file = await decryptFile(
const decryptedFile = await decryptFile(
trashItem.file,
collection.key
);
updatedTrash.push({ ...trashItem, file: decryptedFile });
} else {
updatedTrash = updatedTrash.filter(
(item) => item.file.id !== trashItem.file.id
);
}
updatedTrash.push(trashItem);
}
if (resp.data.diff.length) {
time = resp.data.diff.slice(-1)[0].updatedAt;
}
updatedTrash = removeDuplicates(updatedTrash);
updatedTrash = removeRestoredOrDeletedTrashItems(updatedTrash);
setFiles(
preservePhotoswipeProps(
@ -145,28 +148,6 @@ export const updateTrash = async (
return currentTrash;
};
function removeDuplicates(trash: Trash) {
const latestVersionTrashItems = new Map<number, TrashItem>();
trash.forEach(({ file, updatedAt, ...rest }) => {
if (
!latestVersionTrashItems.has(file.id) ||
latestVersionTrashItems.get(file.id).updatedAt < updatedAt
) {
latestVersionTrashItems.set(file.id, { file, updatedAt, ...rest });
}
});
trash = [];
// eslint-disable-next-line @typescript-eslint/no-unused-vars
for (const [_, trashedFile] of latestVersionTrashItems) {
trash.push(trashedFile);
}
return trash;
}
function removeRestoredOrDeletedTrashItems(trash: Trash) {
return trash.filter((item) => !item.isDeleted && !item.isRestored);
}
export function getTrashedFiles(trash: Trash) {
return mergeMetadata(
trash.map((trashedFile) => ({

4
src/services/upload/fileService.ts
View file

@ -22,7 +22,7 @@ import {
getUint8ArrayView,
} from '../readerService';
import { generateThumbnail } from './thumbnailService';
import { EncryptedMagicMetadataCore } from 'types/magicMetadata';
import { EncryptedMagicMetadata } from 'types/magicMetadata';
const EDITED_FILE_SUFFIX = '-edited';
@ -111,7 +111,7 @@ export async function encryptFile(
const { file: encryptedMetadata }: EncryptionResult<string> =
await worker.encryptMetadata(file.metadata, fileKey);
let encryptedPubMagicMetadata: EncryptedMagicMetadataCore;
let encryptedPubMagicMetadata: EncryptedMagicMetadata;
if (file.pubMagicMetadata) {
const {
file: encryptedPubMagicMetadataData,

4
src/services/upload/magicMetadataService.ts
View file

@ -1,8 +1,8 @@
import {
NEW_FILE_MAGIC_METADATA,
FilePublicMagicMetadataProps,
FilePublicMagicMetadata,
} from 'types/file';
import { NEW_FILE_MAGIC_METADATA } from 'types/magicMetadata';
} from 'types/magicMetadata';
import { updateMagicMetadataProps } from 'utils/magicMetadata';
export async function constructPublicMagicMetadata(

14
src/services/upload/uploadManager.ts
View file

@ -18,7 +18,7 @@ import UIService from './uiService';
import UploadService from './uploadService';
import { CustomError } from 'utils/error';
import { Collection } from 'types/collection';
import { EnteFile } from 'types/file';
import { EncryptedEnteFile, EnteFile } from 'types/file';
import {
FileWithCollection,
ParsedMetadataJSON,
@ -311,7 +311,7 @@ class UploadManager {
async postUploadTask(
fileUploadResult: UPLOAD_RESULT,
uploadedFile: EnteFile | null,
uploadedFile: EncryptedEnteFile | EnteFile | null,
fileWithCollection: FileWithCollection
) {
try {
@ -326,16 +326,16 @@ class UploadManager {
this.failedFiles.push(fileWithCollection);
break;
case UPLOAD_RESULT.ALREADY_UPLOADED:
decryptedFile = uploadedFile;
decryptedFile = uploadedFile as EnteFile;
break;
case UPLOAD_RESULT.ADDED_SYMLINK:
decryptedFile = uploadedFile;
decryptedFile = uploadedFile as EnteFile;
fileUploadResult = UPLOAD_RESULT.UPLOADED;
break;
case UPLOAD_RESULT.UPLOADED:
case UPLOAD_RESULT.UPLOADED_WITH_STATIC_THUMBNAIL:
decryptedFile = await decryptFile(
uploadedFile,
uploadedFile as EncryptedEnteFile,
fileWithCollection.collection.key
);
break;
@ -360,7 +360,7 @@ class UploadManager {
await this.watchFolderCallback(
fileUploadResult,
fileWithCollection,
uploadedFile
uploadedFile as EncryptedEnteFile
);
return fileUploadResult;
} catch (e) {
@ -372,7 +372,7 @@ class UploadManager {
private async watchFolderCallback(
fileUploadResult: UPLOAD_RESULT,
fileWithCollection: FileWithCollection,
uploadedFile: EnteFile
uploadedFile: EncryptedEnteFile
) {
if (isElectron()) {
await watchFolderService.onFileUpload(

2
src/services/upload/uploadService.ts
View file

@ -35,7 +35,7 @@ import UIService from './uiService';
import { USE_CF_PROXY } from 'constants/upload';
import publicUploadHttpClient from './publicUploadHttpClient';
import { constructPublicMagicMetadata } from './magicMetadataService';
import { FilePublicMagicMetadataProps } from 'types/file';
import { FilePublicMagicMetadataProps } from 'types/magicMetadata';
class UploadService {
private uploadURLs: UploadURL[] = [];

3
src/services/upload/uploader.ts
View file

@ -1,4 +1,4 @@
import { EnteFile, FilePublicMagicMetadata } from 'types/file';
import { EnteFile } from 'types/file';
import { handleUploadError, CustomError } from 'utils/error';
import { logError } from 'utils/sentry';
import { findMatchingExistingFiles } from 'utils/upload';
@ -19,6 +19,7 @@ import { sleep } from 'utils/common';
import { addToCollection } from 'services/collectionService';
import uploadCancelService from './uploadCancelService';
import uploadService from './uploadService';
import { FilePublicMagicMetadata } from 'types/magicMetadata';
interface UploadResponse {
fileUploadResult: UPLOAD_RESULT;

6
src/services/watchFolder/watchFolderService.ts
View file

@ -1,5 +1,5 @@
import { Collection } from 'types/collection';
import { EnteFile } from 'types/file';
import { EncryptedEnteFile } from 'types/file';
import { ElectronFile, FileWithCollection } from 'types/upload';
import { runningInBrowser } from 'utils/common';
import { removeFromCollection } from '../collectionService';
@ -33,7 +33,7 @@ class watchFolderService {
private trashingDirQueue: string[] = [];
private isEventRunning: boolean = false;
private uploadRunning: boolean = false;
private filePathToUploadedFileIDMap = new Map<string, EnteFile>();
private filePathToUploadedFileIDMap = new Map<string, EncryptedEnteFile>();
private unUploadableFilePaths = new Set<string>();
private isPaused = false;
private setElectronFiles: (files: ElectronFile[]) => void;
@ -295,7 +295,7 @@ class watchFolderService {
async onFileUpload(
fileUploadResult: UPLOAD_RESULT,
fileWithCollection: FileWithCollection,
file: EnteFile
file: EncryptedEnteFile
) {
addLocalLog(() => `onFileUpload called`);
if (!this.isUploadRunning()) {

84
src/types/file/index.ts
View file

@ -1,65 +1,71 @@
import { MagicMetadataCore, VISIBILITY_STATE } from 'types/magicMetadata';
import {
EncryptedMagicMetadata,
FileMagicMetadata,
FilePublicMagicMetadata,
} from 'types/magicMetadata';
import { Metadata } from 'types/upload';
export interface fileAttribute {
encryptedData?: string;
objectKey?: string;
interface FileAttributesBase {
decryptionHeader: string;
}
export interface FileMagicMetadataProps {
visibility?: VISIBILITY_STATE;
filePaths?: string[];
interface MetadataFileAttributes extends FileAttributesBase {
encryptedData: string;
objectKey?: string;
}
interface S3FileAttributes extends FileAttributesBase {
objectKey: string;
encryptedData?: string;
}
export interface FileMagicMetadata extends Omit<MagicMetadataCore, 'data'> {
data: FileMagicMetadataProps;
}
export type FileAttributes = MetadataFileAttributes | S3FileAttributes;
export interface FilePublicMagicMetadataProps {
editedTime?: number;
editedName?: string;
caption?: string;
uploaderName?: string;
}
export interface FilePublicMagicMetadata
extends Omit<MagicMetadataCore, 'data'> {
data: FilePublicMagicMetadataProps;
}
export interface EnteFileInfo {
export interface FileInfo {
fileSize: number;
thumbSize: number;
}
export interface EnteFile {
export interface EncryptedEnteFile {
id: number;
collectionID: number;
ownerID: number;
file: fileAttribute;
thumbnail: fileAttribute;
metadata: Metadata;
info: EnteFileInfo;
magicMetadata: FileMagicMetadata;
pubMagicMetadata: FilePublicMagicMetadata;
file: FileAttributes;
thumbnail: FileAttributes;
metadata: FileAttributes;
info: FileInfo;
magicMetadata: EncryptedMagicMetadata;
pubMagicMetadata: EncryptedMagicMetadata;
encryptedKey: string;
keyDecryptionNonce: string;
key: string;
src: string;
msrc: string;
html: string;
w: number;
h: number;
title: string;
isDeleted: boolean;
updationTime: number;
}
export interface EnteFile
extends Omit<
EncryptedEnteFile,
| 'metadata'
| 'pubMagicMetadata'
| 'magicMetadata'
| 'encryptedKey'
| 'keyDecryptionNonce'
> {
metadata: Metadata;
magicMetadata: FileMagicMetadata;
pubMagicMetadata: FilePublicMagicMetadata;
key: string;
src?: string;
msrc?: string;
html?: string;
w?: number;
h?: number;
title?: string;
isTrashed?: boolean;
deleteBy?: number;
isSourceLoaded?: boolean;
originalVideoURL?: string;
originalImageURL?: string;
dataIndex: number;
updationTime: number;
dataIndex?: number;
}
export interface TrashRequest {

25
src/types/magicMetadata/index.ts
View file

@ -5,11 +5,32 @@ export interface MagicMetadataCore {
data: Record<string, any>;
}
export interface EncryptedMagicMetadataCore
export interface EncryptedMagicMetadata
extends Omit<MagicMetadataCore, 'data'> {
data: string;
}
export interface FileMagicMetadataProps {
visibility?: VISIBILITY_STATE;
filePaths?: string[];
}
export interface FileMagicMetadata extends Omit<MagicMetadataCore, 'data'> {
data: FileMagicMetadataProps;
}
export interface FilePublicMagicMetadataProps {
editedTime?: number;
editedName?: string;
caption?: string;
uploaderName?: string;
}
export interface FilePublicMagicMetadata
extends Omit<MagicMetadataCore, 'data'> {
data: FilePublicMagicMetadataProps;
}
export enum VISIBILITY_STATE {
VISIBLE = 0,
ARCHIVED = 1,
@ -40,5 +61,5 @@ export interface BulkUpdateMagicMetadataRequest {
export interface UpdateMagicMetadataRequest {
id: number;
magicMetadata: EncryptedMagicMetadataCore;
magicMetadata: EncryptedMagicMetadata;
}

9
src/types/trash/index.ts
View file

@ -1,11 +1,16 @@
import { EnteFile } from 'types/file';
import { EncryptedEnteFile, EnteFile } from 'types/file';
export interface TrashItem {
export interface TrashItem extends Omit<EncryptedTrashItem, 'file'> {
file: EnteFile;
}
export interface EncryptedTrashItem {
file: EncryptedEnteFile;
isDeleted: boolean;
isRestored: boolean;
deleteBy: number;
createdAt: number;
updatedAt: number;
}
export type Trash = TrashItem[];

17
src/types/upload/index.ts
View file

@ -1,7 +1,10 @@
import { FILE_TYPE } from 'constants/file';
import { Collection } from 'types/collection';
import { fileAttribute, FilePublicMagicMetadata } from 'types/file';
import { EncryptedMagicMetadataCore } from 'types/magicMetadata';
import { FileAttributes } from 'types/file';
import {
EncryptedMagicMetadata,
FilePublicMagicMetadata,
} from 'types/magicMetadata';
export interface DataStream {
stream: ReadableStream<Uint8Array>;
@ -132,14 +135,14 @@ export interface ProcessedFile {
file: LocalFileAttributes<Uint8Array | DataStream>;
thumbnail: LocalFileAttributes<Uint8Array>;
metadata: LocalFileAttributes<string>;
pubMagicMetadata: EncryptedMagicMetadataCore;
pubMagicMetadata: EncryptedMagicMetadata;
localID: number;
}
export interface BackupedFile {
file: fileAttribute;
thumbnail: fileAttribute;
metadata: fileAttribute;
pubMagicMetadata: EncryptedMagicMetadataCore;
file: FileAttributes;
thumbnail: FileAttributes;
metadata: FileAttributes;
pubMagicMetadata: EncryptedMagicMetadata;
}
export interface UploadFile extends BackupedFile {

90
src/utils/file/index.ts
View file

@ -1,10 +1,5 @@
import { SelectedState } from 'types/gallery';
import {
EnteFile,
fileAttribute,
FileMagicMetadataProps,
FilePublicMagicMetadataProps,
} from 'types/file';
import { EnteFile, EncryptedEnteFile } from 'types/file';
import { decodeMotionPhoto } from 'services/motionPhotoService';
import { getFileType } from 'services/typeDetectionService';
import DownloadManager from 'services/downloadManager';
@ -23,7 +18,14 @@ import {
import PublicCollectionDownloadManager from 'services/publicCollectionDownloadManager';
import heicConversionService from 'services/heicConversionService';
import * as ffmpegService from 'services/ffmpeg/ffmpegService';
import { NEW_FILE_MAGIC_METADATA, VISIBILITY_STATE } from 'types/magicMetadata';
import {
FileMagicMetadata,
FileMagicMetadataProps,
FilePublicMagicMetadata,
FilePublicMagicMetadataProps,
NEW_FILE_MAGIC_METADATA,
VISIBILITY_STATE,
} from 'types/magicMetadata';
import { IsArchived, updateMagicMetadataProps } from 'utils/magicMetadata';
import { addLogLine } from 'utils/logging';
@ -191,41 +193,59 @@ export function sortFiles(files: EnteFile[]) {
return files;
}
export async function decryptFile(file: EnteFile, collectionKey: string) {
export async function decryptFile(
file: EncryptedEnteFile,
collectionKey: string
): Promise<EnteFile> {
try {
const worker = await new CryptoWorker();
file.key = await worker.decryptB64(
file.encryptedKey,
file.keyDecryptionNonce,
const {
encryptedKey,
keyDecryptionNonce,
metadata,
magicMetadata,
pubMagicMetadata,
...restFileProps
} = file;
const fileKey = await worker.decryptB64(
encryptedKey,
keyDecryptionNonce,
collectionKey
);
const encryptedMetadata = file.metadata as unknown as fileAttribute;
file.metadata = await worker.decryptMetadata(
encryptedMetadata.encryptedData,
encryptedMetadata.decryptionHeader,
file.key
const fileMetadata = await worker.decryptMetadata(
metadata.encryptedData,
metadata.decryptionHeader,
fileKey
);
if (
file.magicMetadata?.data &&
typeof file.magicMetadata.data === 'string'
) {
file.magicMetadata.data = await worker.decryptMetadata(
file.magicMetadata.data,
file.magicMetadata.header,
file.key
);
let fileMagicMetadata: FileMagicMetadata;
let filePubMagicMetadata: FilePublicMagicMetadata;
if (magicMetadata?.data) {
fileMagicMetadata = {
...file.magicMetadata,
data: await worker.decryptMetadata(
magicMetadata.data,
magicMetadata.header,
fileKey
),
};
}
if (
file.pubMagicMetadata?.data &&
typeof file.pubMagicMetadata.data === 'string'
) {
file.pubMagicMetadata.data = await worker.decryptMetadata(
file.pubMagicMetadata.data,
file.pubMagicMetadata.header,
file.key
);
if (pubMagicMetadata?.data) {
filePubMagicMetadata = {
...pubMagicMetadata,
data: await worker.decryptMetadata(
pubMagicMetadata.data,
pubMagicMetadata.header,
fileKey
),
};
}
return file;
return {
...restFileProps,
key: fileKey,
metadata: fileMetadata,
magicMetadata: fileMagicMetadata,
pubMagicMetadata: filePubMagicMetadata,
};
} catch (e) {
logError(e, 'file decryption failed');
throw e;

8
src/utils/magicMetadata/index.ts
View file

@ -1,6 +1,10 @@
import { Collection } from 'types/collection';
import { EnteFile, FileMagicMetadataProps } from 'types/file';
import { MagicMetadataCore, VISIBILITY_STATE } from 'types/magicMetadata';
import { EnteFile } from 'types/file';
import {
FileMagicMetadataProps,
MagicMetadataCore,
VISIBILITY_STATE,
} from 'types/magicMetadata';
import CryptoWorker from 'utils/crypto';
export function IsArchived(item: Collection | EnteFile) {