beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #614 from ente-io/refactor-watch

Browse source 
Refactor watch
This commit is contained in:
Abhinav Kumar 2022-06-24 12:45:54 +05:30 committed by GitHub
parent 40412f83fa 0a1076c584
commit 49ad023412
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 71 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/components/WatchFolder/index.tsx
View file

@ -9,7 +9,7 @@ import constants from 'utils/strings/constants';
import DialogBoxBase from 'components/DialogBox/base';
import DialogTitleWithCloseButton from 'components/DialogBox/titleWithCloseButton';
import UploadStrategyChoiceModal from 'components/pages/gallery/UploadStrategyChoiceModal';
import { UPLOAD_STRATEGY } from 'components/pages/gallery/Upload';
import { UPLOAD_STRATEGY } from 'constants/upload';
interface Iprops {
open: boolean;

7
src/components/pages/gallery/Upload.tsx
View file

@ -33,7 +33,7 @@ import {
SegregatedFinishedUploads,
InProgressUpload,
} from 'types/upload/ui';
import { UPLOAD_STAGES } from 'constants/upload';
import { UPLOAD_STAGES, UPLOAD_STRATEGY } from 'constants/upload';
const FIRST_ALBUM_NAME = 'My First Album';
@ -59,11 +59,6 @@ interface Props {
showSessionExpiredMessage: () => void;
}
export enum UPLOAD_STRATEGY {
SINGLE_COLLECTION,
COLLECTION_PER_FOLDER,
}
export enum DESKTOP_UPLOAD_TYPE {
FILES = 'files',
FOLDERS = 'folders',

5
src/constants/upload/index.ts
View file

@ -31,6 +31,11 @@ export enum UPLOAD_STAGES {
FINISH,
}
export enum UPLOAD_STRATEGY {
SINGLE_COLLECTION,
COLLECTION_PER_FOLDER,
}
export enum UPLOAD_RESULT {
FAILED,
ALREADY_UPLOADED,

14
src/services/upload/livePhotoService.ts
View file

@ -1,7 +1,6 @@
import { FILE_TYPE } from 'constants/file';
import { LIVE_PHOTO_ASSET_SIZE_LIMIT } from 'constants/upload';
import { encodeMotionPhoto } from 'services/motionPhotoService';
import { FileMagicMetadata } from 'types/file';
import {
ElectronFile,
FileTypeInfo,
@ -62,12 +61,8 @@ export function getLivePhotoMetadata(
};
}
export function getLivePhotoMagicMetadata(
imageFile: FileWithCollection
): FileMagicMetadata['data'] {
return {
filePaths: [getLivePhotoName((imageFile.file as any).path as string)],
};
export function getLivePhotoFilePath(imageAsset: Asset): string {
return getLivePhotoName((imageAsset.file as any).path);
}
export function getLivePhotoSize(livePhotoAssets: LivePhotoAssets) {
@ -198,12 +193,11 @@ export function clusterLivePhotoFiles(mediaFiles: FileWithCollection[]) {
imageAsset.metadata,
videoAsset.metadata
);
const livePhotoMagicMetadata: FileMagicMetadata['data'] =
getLivePhotoMagicMetadata(firstMediaFile);
const livePhotoPath = getLivePhotoFilePath(imageAsset);
uploadService.setFileMetadataAndFileTypeInfo(livePhotoLocalID, {
fileTypeInfo: { ...livePhotoFileTypeInfo },
metadata: { ...livePhotoMetadata },
magicMetadata: { ...livePhotoMagicMetadata },
filePath: livePhotoPath,
});
index += 2;
} else {

63
src/services/upload/uploadManager.ts
View file

@ -1,4 +1,8 @@
import { getLocalFiles, setLocalFiles } from '../fileService';
import {
getLocalFiles,
setLocalFiles,
updateFileMagicMetadata,
} from '../fileService';
import { SetFiles } from 'types/gallery';
import { getDedicatedCryptoWorker } from 'utils/crypto';
import {
@ -6,7 +10,7 @@ import {
sortFiles,
preservePhotoswipeProps,
decryptFile,
changeFilePaths,
appendNewFilePath,
} from 'utils/file';
import { logError } from 'utils/sentry';
import { getMetadataJSONMapKey, parseMetadataJSON } from './metadataService';
@ -20,7 +24,7 @@ import UIService from './uiService';
import UploadService from './uploadService';
import { CustomError } from 'utils/error';
import { Collection } from 'types/collection';
import { EnteFile, FileMagicMetadata } from 'types/file';
import { EnteFile } from 'types/file';
import {
FileWithCollection,
MetadataAndFileTypeInfo,
@ -230,7 +234,7 @@ class UploadManager {
UIService.reset(mediaFiles.length);
for (const { file, localID, collectionID } of mediaFiles) {
try {
const { fileTypeInfo, metadata, magicMetadata } =
const { fileTypeInfo, metadata, filePath } =
await (async () => {
if (file.size >= MAX_FILE_SIZE_SUPPORTED) {
logUploadInfo(
@ -260,10 +264,8 @@ class UploadManager {
collectionID,
fileTypeInfo
)) || null;
const magicMetadata = {
filePaths: [(file as any).path as string],
} as FileMagicMetadata['data'];
return { fileTypeInfo, metadata, magicMetadata };
const filePath = (file as any).path as string;
return { fileTypeInfo, metadata, filePath };
})();
logUploadInfo(
@ -274,7 +276,7 @@ class UploadManager {
this.metadataAndFileTypeInfoMap.set(localID, {
fileTypeInfo: fileTypeInfo && { ...fileTypeInfo },
metadata: metadata && { ...metadata },
magicMetadata: magicMetadata && { ...magicMetadata },
filePath: filePath,
});
UIService.increaseFileUploaded();
} catch (e) {
@ -340,25 +342,18 @@ class UploadManager {
this.existingFiles,
fileWithCollection
);
const filePaths = UploadService.getFileMetadataAndFileTypeInfo(
fileWithCollection.localID
).magicMetadata.filePaths;
await changeFilePaths(
fileUploadResult,
uploadedFile,
fileWithCollection.collection.key,
filePaths
);
UIService.moveFileToResultList(
fileWithCollection.localID,
fileUploadResult
);
UploadService.reducePendingUploadCount();
await this.postUploadTask(
const finalUploadResult = await this.postUploadTask(
fileUploadResult,
uploadedFile,
fileWithCollection
);
UIService.moveFileToResultList(
fileWithCollection.localID,
finalUploadResult
);
UploadService.reducePendingUploadCount();
}
}
@ -383,6 +378,10 @@ class UploadManager {
uploadedFile
);
}
await this.updateFilePaths(
uploadedFile,
fileWithCollection
);
break;
case UPLOAD_RESULT.ADDED_SYMLINK:
decryptedFile = uploadedFile;
@ -404,14 +403,16 @@ class UploadManager {
fileWithCollection,
uploadedFile
);
await this.updateFilePaths(decryptedFile, fileWithCollection);
}
return fileUploadResult;
} catch (e) {
logError(e, 'failed to do post file upload action');
logUploadInfo(
`failed to do post file upload action -> ${e.message}
${(e as Error).stack}`
);
throw e;
return UPLOAD_RESULT.FAILED;
}
}
@ -457,6 +458,18 @@ class UploadManager {
}
}
private async updateFilePaths(
decryptedFile: EnteFile,
fileWithCollection: FileWithCollection
) {
const filePath = UploadService.getFileMetadataAndFileTypeInfo(
fileWithCollection.localID
).filePath;
const updatedFile = await appendNewFilePath(decryptedFile, filePath);
await updateFileMagicMetadata([updatedFile]);
}
async retryFailedFiles() {
await this.queueFilesForUpload(this.failedFiles, [
...this.collections.values(),

2
src/services/watchFolder/watchFolderService.ts
View file

@ -16,8 +16,8 @@ import {
diskFileRemovedCallback,
diskFolderRemovedCallback,
} from './watchFolderEventHandlers';
import { UPLOAD_STRATEGY } from 'components/pages/gallery/Upload';
import { getParentFolderName } from './utils';
import { UPLOAD_STRATEGY } from 'constants/upload';
class watchFolderService {
private ElectronAPIs: ElectronAPIsInterface;

4
src/types/upload/index.ts
View file

@ -1,6 +1,6 @@
import { FILE_TYPE } from 'constants/file';
import { Collection } from 'types/collection';
import { fileAttribute, FileMagicMetadata } from 'types/file';
import { fileAttribute } from 'types/file';
export interface DataStream {
stream: ReadableStream<Uint8Array>;
@ -92,7 +92,7 @@ export interface FileWithCollection extends UploadAsset {
export interface MetadataAndFileTypeInfo {
metadata: Metadata;
fileTypeInfo: FileTypeInfo;
magicMetadata: FileMagicMetadata['data'];
filePath: string;
}
export type MetadataAndFileTypeInfoMap = Map<number, MetadataAndFileTypeInfo>;

2
src/types/watchFolder/index.ts
View file

@ -1,4 +1,4 @@
import { UPLOAD_STRATEGY } from 'components/pages/gallery/Upload';
import { UPLOAD_STRATEGY } from 'constants/upload';
import { ElectronFile } from 'types/upload';
export interface WatchMapping {

48
src/utils/file/index.ts
View file

@ -26,8 +26,7 @@ import ffmpegService from 'services/ffmpeg/ffmpegService';
import { NEW_FILE_MAGIC_METADATA, VISIBILITY_STATE } from 'types/magicMetadata';
import { IsArchived, updateMagicMetadataProps } from 'utils/magicMetadata';
import { ARCHIVE_SECTION, TRASH_SECTION } from 'constants/collection';
import { UPLOAD_RESULT } from 'constants/upload';
import { updateFileMagicMetadata } from 'services/fileService';
export function downloadAsFile(filename: string, content: string) {
const file = new Blob([content], {
type: 'text/plain',
@ -454,35 +453,24 @@ export async function changeFileName(file: EnteFile, editedName: string) {
return file;
}
export const changeFilePaths = async (
fileUploadResult: UPLOAD_RESULT,
file: EnteFile,
collectionKey: string,
filePaths: string[]
) => {
if (
fileUploadResult === UPLOAD_RESULT.UPLOADED ||
fileUploadResult === UPLOAD_RESULT.ALREADY_UPLOADED ||
fileUploadResult === UPLOAD_RESULT.UPLOADED_WITH_STATIC_THUMBNAIL
) {
let mergedMetadataFilePaths = [...filePaths];
if (file.magicMetadata?.data.filePaths?.length > 0) {
mergedMetadataFilePaths = [
...new Set([
...file.magicMetadata.data.filePaths,
...filePaths,
]),
];
}
file.key = await getFileKey(file, collectionKey);
const updatedMagicMetadata = await updateMagicMetadataProps(
file.magicMetadata ?? NEW_FILE_MAGIC_METADATA,
file.key,
{ filePaths: mergedMetadataFilePaths }
);
file.magicMetadata = updatedMagicMetadata;
await updateFileMagicMetadata([file]);
export const appendNewFilePath = async (file: EnteFile, filePath: string) => {
let mergedMetadataFilePaths = [filePath];
if (file.magicMetadata?.data.filePaths?.length > 0) {
mergedMetadataFilePaths = [
...new Set([
...file.magicMetadata.data.filePaths,
...mergedMetadataFilePaths,
]),
];
}
const updatedMagicMetadata = await updateMagicMetadataProps(
file.magicMetadata ?? NEW_FILE_MAGIC_METADATA,
file.key,
{ filePaths: mergedMetadataFilePaths }
);
file.magicMetadata = updatedMagicMetadata;
return file;
};
export function isSharedFile(file: EnteFile) {