fix minor bugs

This commit is contained in:
Abhinav 2022-02-13 13:43:14 +05:30
parent 87d635be58
commit 0fff2ee674
9 changed files with 107 additions and 75 deletions

View file

@ -77,6 +77,7 @@ export default function Upload(props: Props) {
setFileProgress, setFileProgress,
setUploadResult, setUploadResult,
setUploadStage, setUploadStage,
setFilenames,
}, },
props.setFiles props.setFiles
); );
@ -121,11 +122,6 @@ export default function Upload(props: Props) {
if (props.acceptedFiles.length === 0) { if (props.acceptedFiles.length === 0) {
return null; return null;
} }
setFilenames(
new Map<number, string>(
props.acceptedFiles.map((file, index) => [index, file.name])
)
);
const paths: string[] = props.acceptedFiles.map((file) => file['path']); const paths: string[] = props.acceptedFiles.map((file) => file['path']);
const getCharCount = (str: string) => (str.match(/\//g) ?? []).length; const getCharCount = (str: string) => (str.match(/\//g) ?? []).length;
paths.sort((path1, path2) => getCharCount(path1) - getCharCount(path2)); paths.sort((path1, path2) => getCharCount(path1) - getCharCount(path2));

View file

@ -6,6 +6,7 @@ import {
EncryptedFile, EncryptedFile,
EncryptionResult, EncryptionResult,
FileWithMetadata, FileWithMetadata,
ParsedMetadataJSONMap,
} from 'types/upload'; } from 'types/upload';
import { logError } from 'utils/sentry'; import { logError } from 'utils/sentry';
import { encryptFiledata } from './encryptionService'; import { encryptFiledata } from './encryptionService';
@ -40,13 +41,14 @@ export async function readFile(
} }
export async function getFileMetadata( export async function getFileMetadata(
parsedMetadataJSONMap: ParsedMetadataJSONMap,
rawFile: File, rawFile: File,
collectionID: number, collectionID: number,
fileTypeInfo: FileTypeInfo fileTypeInfo: FileTypeInfo
) { ) {
const originalName = getFileOriginalName(rawFile); const originalName = getFileOriginalName(rawFile);
const googleMetadata = const googleMetadata =
this.metadataMap.get( parsedMetadataJSONMap.get(
getMetadataJSONMapKey(collectionID, originalName) getMetadataJSONMapKey(collectionID, originalName)
) ?? {}; ) ?? {};
const extractedMetadata: Metadata = await extractMetadata( const extractedMetadata: Metadata = await extractMetadata(

View file

@ -9,23 +9,33 @@ import {
} from 'types/upload'; } from 'types/upload';
import { splitFilenameAndExtension } from 'utils/file'; import { splitFilenameAndExtension } from 'utils/file';
import { readFile } from './fileService'; import { readFile } from './fileService';
import { getFileType } from './readFileService';
import uploadService from './uploadService'; import uploadService from './uploadService';
import UploadService from './uploadService'; import UploadService from './uploadService';
export async function getLivePhotoFileType(
worker, export function getLivePhotoFileType(
livePhotoAssets: LivePhotoAssets file1TypeInfo: FileTypeInfo,
file2TypeInfo: FileTypeInfo
) { ) {
const file1TypeInfo = await getFileType(worker, livePhotoAssets[0]);
const file2TypeInfo = await getFileType(worker, livePhotoAssets[1]);
return { return {
fileType: FILE_TYPE.LIVE_PHOTO, fileType: FILE_TYPE.LIVE_PHOTO,
exactType: `${file1TypeInfo.exactType}+${file2TypeInfo.exactType}`, exactType: `${file1TypeInfo.exactType}+${file2TypeInfo.exactType}`,
}; };
} }
export function getLivePhotoMetadata(
file1Metadata: Metadata,
file2Metadata: Metadata
) {
return {
...file1Metadata,
...file2Metadata,
title: splitFilenameAndExtension(file1Metadata.title)[0],
fileType: FILE_TYPE.LIVE_PHOTO,
};
}
export function getLivePhotoSize(livePhotoAssets: LivePhotoAssets) { export function getLivePhotoSize(livePhotoAssets: LivePhotoAssets) {
return livePhotoAssets[0].size + livePhotoAssets[1].size; return livePhotoAssets.image.size + livePhotoAssets.video.size;
} }
export async function readLivePhoto( export async function readLivePhoto(
@ -37,13 +47,13 @@ export async function readLivePhoto(
const image = await readFile( const image = await readFile(
worker, worker,
reader, reader,
fileTypeInfo, { exactType: fileTypeInfo.exactType, fileType: FILE_TYPE.IMAGE },
livePhotoAssets.image livePhotoAssets.image
); );
const video = await readFile( const video = await readFile(
worker, worker,
reader, reader,
fileTypeInfo, { exactType: fileTypeInfo.exactType, fileType: FILE_TYPE.VIDEO },
livePhotoAssets.video livePhotoAssets.video
); );
@ -66,9 +76,14 @@ export async function readLivePhoto(
export function clusterLivePhotoFiles(mediaFiles: FileWithCollection[]) { export function clusterLivePhotoFiles(mediaFiles: FileWithCollection[]) {
const analysedMediaFiles: FileWithCollection[] = []; const analysedMediaFiles: FileWithCollection[] = [];
for (let i = 0; i < mediaFiles.length - 1; i++) { mediaFiles.sort((media1Files, media2Files) =>
splitFilenameAndExtension(media1Files.file.name)[0].localeCompare(
splitFilenameAndExtension(media2Files.file.name)[0]
)
);
for (let i = 0; i < mediaFiles.length - 1; i += 2) {
const mediaFile1 = mediaFiles[i]; const mediaFile1 = mediaFiles[i];
const mediaFile2 = mediaFiles[i]; const mediaFile2 = mediaFiles[i + 1];
const { fileTypeInfo: file1TypeInfo, metadata: file1Metadata } = const { fileTypeInfo: file1TypeInfo, metadata: file1Metadata } =
UploadService.getFileMetadataAndFileTypeInfo(mediaFile1.localID); UploadService.getFileMetadataAndFileTypeInfo(mediaFile1.localID);
const { fileTypeInfo: file2TypeInfo, metadata: file2Metadata } = const { fileTypeInfo: file2TypeInfo, metadata: file2Metadata } =
@ -93,15 +108,14 @@ export function clusterLivePhotoFiles(mediaFiles: FileWithCollection[]) {
isLivePhoto: true, isLivePhoto: true,
livePhotoAssets: { image: imageFile, video: videoFile }, livePhotoAssets: { image: imageFile, video: videoFile },
}); });
const livePhotoFileTypeInfo: FileTypeInfo = { const livePhotoFileTypeInfo: FileTypeInfo = getLivePhotoFileType(
fileType: FILE_TYPE.LIVE_PHOTO, file1TypeInfo,
exactType: `${file1TypeInfo.exactType} - ${file2TypeInfo.exactType}`, file2TypeInfo
}; );
const livePhotoMetadata: Metadata = { const livePhotoMetadata: Metadata = getLivePhotoMetadata(
...file1Metadata, file1Metadata,
...file2Metadata, file2Metadata
title: splitFilenameAndExtension(file1Metadata.title)[0], );
};
uploadService.setFileMetadataAndFileTypeInfo(livePhotoLocalID, { uploadService.setFileMetadataAndFileTypeInfo(livePhotoLocalID, {
fileTypeInfo: { ...livePhotoFileTypeInfo }, fileTypeInfo: { ...livePhotoFileTypeInfo },
metadata: { ...livePhotoMetadata }, metadata: { ...livePhotoMetadata },
@ -121,10 +135,22 @@ function areFilesLivePhotoAssets(
mediaFile1: FileWithCollection, mediaFile1: FileWithCollection,
mediaFile2: FileWithCollection mediaFile2: FileWithCollection
) { ) {
const { collectionID: file1collectionID, file: file1 } = mediaFile1; const {
const { collectionID: file2collectionID, file: file2 } = mediaFile2; collectionID: file1collectionID,
const file1Type = FILE_TYPE.OTHERS; file: file1,
const file2Type = FILE_TYPE.OTHERS; localID: localID1,
} = mediaFile1;
const {
collectionID: file2collectionID,
file: file2,
localID: localID2,
} = mediaFile2;
const {
fileTypeInfo: { fileType: file1Type },
} = UploadService.getFileMetadataAndFileTypeInfo(localID1);
const {
fileTypeInfo: { fileType: file2Type },
} = UploadService.getFileMetadataAndFileTypeInfo(localID2);
return ( return (
file1collectionID === file2collectionID && file1collectionID === file2collectionID &&
file1Type !== file2Type && file1Type !== file2Type &&

View file

@ -1,6 +1,10 @@
import { FILE_TYPE } from 'constants/file'; import { FILE_TYPE } from 'constants/file';
import { logError } from 'utils/sentry'; import { logError } from 'utils/sentry';
import { FILE_READER_CHUNK_SIZE, MULTIPART_PART_SIZE } from 'constants/upload'; import {
FILE_READER_CHUNK_SIZE,
FORMAT_MISSED_BY_FILE_TYPE_LIB,
MULTIPART_PART_SIZE,
} from 'constants/upload';
import FileType from 'file-type/browser'; import FileType from 'file-type/browser';
import { CustomError } from 'utils/error'; import { CustomError } from 'utils/error';
import { getFileExtension, splitFilenameAndExtension } from 'utils/file'; import { getFileExtension, splitFilenameAndExtension } from 'utils/file';
@ -43,6 +47,15 @@ export async function getFileType(
return { fileType, exactType: typeParts[1] }; return { fileType, exactType: typeParts[1] };
} catch (e) { } catch (e) {
const fileFormat = getFileExtension(receivedFile.name); const fileFormat = getFileExtension(receivedFile.name);
const formatMissedByTypeDetection = FORMAT_MISSED_BY_FILE_TYPE_LIB.find(
(a) => a.exactType === fileFormat
);
if (formatMissedByTypeDetection) {
return formatMissedByTypeDetection;
}
logError(e, CustomError.TYPE_DETECTION_FAILED, {
fileType: fileFormat,
});
return { fileType: FILE_TYPE.OTHERS, exactType: fileFormat }; return { fileType: FILE_TYPE.OTHERS, exactType: fileFormat };
} }
} }

View file

@ -43,6 +43,10 @@ class UIService {
this.progressUpdater.setPercentComplete(percent); this.progressUpdater.setPercentComplete(percent);
} }
setFilenames(filenames: Map<number, string>) {
this.progressUpdater.setFilenames(filenames);
}
increaseFileUploaded() { increaseFileUploaded() {
this.filesUploaded++; this.filesUploaded++;
this.updateProgressBarUI(); this.updateProgressBarUI();
@ -97,18 +101,16 @@ class UIService {
return { return {
cancel, cancel,
onUploadProgress: (event) => { onUploadProgress: (event) => {
fileLocalID && this.fileProgress.set(
this.fileProgress.set( fileLocalID,
fileLocalID, Math.min(
Math.min( Math.round(
Math.round( percentPerPart * index +
percentPerPart * index + (percentPerPart * event.loaded) / event.total
(percentPerPart * event.loaded) / ),
event.total 98
), )
98 );
)
);
this.updateProgressBarUI(); this.updateProgressBarUI();
if (event.loaded === event.total) { if (event.loaded === event.total) {
clearTimeout(timeout); clearTimeout(timeout);

View file

@ -31,6 +31,7 @@ import {
} from 'constants/upload'; } from 'constants/upload';
import { ComlinkWorker } from 'utils/comlink'; import { ComlinkWorker } from 'utils/comlink';
import { FILE_TYPE } from 'constants/file'; import { FILE_TYPE } from 'constants/file';
import uiService from './uiService';
const MAX_CONCURRENT_UPLOADS = 4; const MAX_CONCURRENT_UPLOADS = 4;
const FILE_UPLOAD_COMPLETED = 100; const FILE_UPLOAD_COMPLETED = 100;
@ -96,7 +97,19 @@ class UploadManager {
); );
UIService.setUploadStage(UPLOAD_STAGES.START); UIService.setUploadStage(UPLOAD_STAGES.START);
const analysedMediaFiles = const analysedMediaFiles =
UploadService.clusterLivePhotoFiles(mediaFiles); mediaFiles.length > 1
? UploadService.clusterLivePhotoFiles(mediaFiles)
: mediaFiles;
uiService.setFilenames(
new Map<number, string>(
mediaFiles.map(({ localID }) => [
localID,
UploadService.getFileMetadataAndFileTypeInfo(
localID
).metadata.title,
])
)
);
await this.uploadMediaFiles(analysedMediaFiles); await this.uploadMediaFiles(analysedMediaFiles);
} }
UIService.setUploadStage(UPLOAD_STAGES.FINISH); UIService.setUploadStage(UPLOAD_STAGES.FINISH);
@ -174,6 +187,7 @@ class UploadManager {
} }
private async uploadMediaFiles(mediaFiles: FileWithCollection[]) { private async uploadMediaFiles(mediaFiles: FileWithCollection[]) {
this.filesToBeUploaded.push(...mediaFiles);
UIService.reset(mediaFiles.length); UIService.reset(mediaFiles.length);
await UploadService.setFileCount(mediaFiles.length); await UploadService.setFileCount(mediaFiles.length);

View file

@ -3,7 +3,7 @@ import { logError } from 'utils/sentry';
import UploadHttpClient from './uploadHttpClient'; import UploadHttpClient from './uploadHttpClient';
import { getFileMetadata } from './fileService'; import { getFileMetadata } from './fileService';
import { getFileType } from './readFileService'; import { getFileType } from './readFileService';
import { CustomError, handleUploadError } from 'utils/error'; import { handleUploadError } from 'utils/error';
import { import {
B64EncryptionResult, B64EncryptionResult,
BackupedFile, BackupedFile,
@ -22,8 +22,6 @@ import {
UploadFile, UploadFile,
UploadURL, UploadURL,
} from 'types/upload'; } from 'types/upload';
import { FILE_TYPE } from 'constants/file';
import { FORMAT_MISSED_BY_FILE_TYPE_LIB } from 'constants/upload';
import { import {
clusterLivePhotoFiles, clusterLivePhotoFiles,
getLivePhotoSize, getLivePhotoSize,
@ -71,25 +69,9 @@ class UploadService {
} }
async getFileType(reader: FileReader, file: File) { async getFileType(reader: FileReader, file: File) {
const fileTypeInfo = await getFileType(reader, file); return getFileType(reader, file);
if (fileTypeInfo.fileType !== FILE_TYPE.OTHERS) {
return fileTypeInfo;
}
try {
const formatMissedByTypeDetection =
FORMAT_MISSED_BY_FILE_TYPE_LIB.find(
(a) => a.exactType === fileTypeInfo.exactType
);
if (formatMissedByTypeDetection) {
return formatMissedByTypeDetection;
}
throw Error(CustomError.UNSUPPORTED_FILE_FORMAT);
} catch (e) {
logError(e, CustomError.TYPE_DETECTION_FAILED, {
fileType: fileTypeInfo.exactType,
});
}
} }
async readAsset( async readAsset(
worker: any, worker: any,
reader: FileReader, reader: FileReader,
@ -106,7 +88,12 @@ class UploadService {
collectionID: number, collectionID: number,
fileTypeInfo: FileTypeInfo fileTypeInfo: FileTypeInfo
): Promise<Metadata> { ): Promise<Metadata> {
return getFileMetadata(file, collectionID, fileTypeInfo); return getFileMetadata(
this.parsedMetadataJSONMap,
file,
collectionID,
fileTypeInfo
);
} }
getFileMetadataAndFileTypeInfo(localID: number) { getFileMetadataAndFileTypeInfo(localID: number) {

View file

@ -37,14 +37,6 @@ export default async function uploader(
throw Error(CustomError.NO_METADATA); throw Error(CustomError.NO_METADATA);
} }
// fileTypeInfo = await UploadService.getAssetType(worker, uploadAsset);
// metadata = await UploadService.getAssetMetadata(
// uploadAsset,
// collection,
// fileTypeInfo
// );
if (fileAlreadyInCollection(existingFilesInCollection, metadata)) { if (fileAlreadyInCollection(existingFilesInCollection, metadata)) {
return { fileUploadResult: FileUploadResults.ALREADY_UPLOADED }; return { fileUploadResult: FileUploadResults.ALREADY_UPLOADED };
} }

View file

@ -61,6 +61,7 @@ export interface ProgressUpdater {
setUploadStage: React.Dispatch<React.SetStateAction<UPLOAD_STAGES>>; setUploadStage: React.Dispatch<React.SetStateAction<UPLOAD_STAGES>>;
setFileProgress: React.Dispatch<React.SetStateAction<Map<number, number>>>; setFileProgress: React.Dispatch<React.SetStateAction<Map<number, number>>>;
setUploadResult: React.Dispatch<React.SetStateAction<Map<number, number>>>; setUploadResult: React.Dispatch<React.SetStateAction<Map<number, number>>>;
setFilenames: React.Dispatch<React.SetStateAction<Map<number, string>>>;
} }
export interface UploadAsset { export interface UploadAsset {
@ -78,7 +79,6 @@ export interface FileWithCollection extends UploadAsset {
collection?: Collection; collection?: Collection;
collectionID?: number; collectionID?: number;
} }
export type MetadataMap = Map<string, ParsedMetadataJSON>;
export interface MetadataAndFileTypeInfo { export interface MetadataAndFileTypeInfo {
metadata: Metadata; metadata: Metadata;
fileTypeInfo: FileTypeInfo; fileTypeInfo: FileTypeInfo;