beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

add upload logs

Browse source
This commit is contained in:
Abhinav 2022-03-04 08:40:14 +05:30
parent 91ad161e05
commit 72e0f7318c
7 changed files with 90 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/services/upload/metadataService.ts
View file

@ -11,6 +11,8 @@ import {
import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload';
import { splitFilenameAndExtension } from 'utils/file';
import { getVideoMetadata } from './videoMetadataService';
import { getFileNameSize } from 'utils/upload';
import { saveLogLine } from 'utils/storage';
interface ParsedMetadataJSONWithTitle {
title: string;
@ -31,7 +33,15 @@ export async function extractMetadata(
if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) {
extractedMetadata = await getExifData(receivedFile, fileTypeInfo);
} else if (fileTypeInfo.fileType === FILE_TYPE.VIDEO) {
saveLogLine(
`getVideoMetadata called for ${getFileNameSize(receivedFile)}`
);
extractedMetadata = await getVideoMetadata(receivedFile);
saveLogLine(
`videoMetadata successfully extracted ${getFileNameSize(
receivedFile
)}`
);
}
const metadata: Metadata = {

16
src/services/upload/thumbnailService.ts
View file

@ -8,6 +8,8 @@ import { isFileHEIC } from 'utils/file';
import { FileTypeInfo } from 'types/upload';
import { getUint8ArrayView } from './readFileService';
import HEICConverter from 'services/HEICConverter';
import { saveLogLine } from 'utils/storage';
import { getFileNameSize } from 'utils/upload';
const MAX_THUMBNAIL_DIMENSION = 720;
const MIN_COMPRESSION_PERCENTAGE_SIZE_DIFF = 10;
@ -28,6 +30,7 @@ export async function generateThumbnail(
fileTypeInfo: FileTypeInfo
): Promise<{ thumbnail: Uint8Array; hasStaticThumbnail: boolean }> {
try {
saveLogLine(`generating thumbnail for ${getFileNameSize(file)}`);
let hasStaticThumbnail = false;
let canvas = document.createElement('canvas');
let thumbnail: Uint8Array;
@ -37,7 +40,18 @@ export async function generateThumbnail(
canvas = await generateImageThumbnail(file, isHEIC);
} else {
try {
saveLogLine(
`ffmpeg generateThumbnail called for ${getFileNameSize(
file
)}`
);
const thumb = await FFmpegService.generateThumbnail(file);
saveLogLine(
`ffmpeg thumbnail successfully generated ${getFileNameSize(
file
)}`
);
const dummyImageFile = new File([thumb], file.name);
canvas = await generateImageThumbnail(
dummyImageFile,
@ -79,7 +93,9 @@ export async function generateImageThumbnail(file: File, isHEIC: boolean) {
let timeout = null;
if (isHEIC) {
saveLogLine(`HEICConverter called for ${getFileNameSize(file)}`);
file = new File([await HEICConverter.convert(file)], null, null);
saveLogLine(`${getFileNameSize(file)} successfully converted`);
}
let image = new Image();
imageURL = URL.createObjectURL(file);

36
src/services/upload/uploadManager.ts
View file

@ -9,7 +9,7 @@ import {
} from 'utils/file';
import { logError } from 'utils/sentry';
import { getMetadataJSONMapKey, parseMetadataJSON } from './metadataService';
import { segregateMetadataAndMediaFiles } from 'utils/upload';
import { getFileNameSize, segregateMetadataAndMediaFiles } from 'utils/upload';
import uploader from './uploader';
import UIService from './uiService';
import UploadService from './uploadService';
@ -35,6 +35,7 @@ import uiService from './uiService';
import { getData, LS_KEYS, setData } from 'utils/storage/localStorage';
import { dedupe } from 'utils/export';
import { convertToHumanReadable } from 'utils/billing';
import { saveLogLine } from 'utils/storage';
const MAX_CONCURRENT_UPLOADS = 4;
const FILE_UPLOAD_COMPLETED = 100;
@ -81,8 +82,13 @@ class UploadManager {
) {
try {
await this.init(newCreatedCollections);
saveLogLine(
`received ${fileWithCollectionToBeUploaded.length} files to upload`
);
const { metadataJSONFiles, mediaFiles } =
segregateMetadataAndMediaFiles(fileWithCollectionToBeUploaded);
saveLogLine(`has ${metadataJSONFiles.length} metadata json files`);
saveLogLine(`has ${metadataJSONFiles.length} media files`);
if (metadataJSONFiles.length) {
UIService.setUploadStage(
UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES
@ -99,6 +105,7 @@ class UploadManager {
this.metadataAndFileTypeInfoMap
);
UIService.setUploadStage(UPLOAD_STAGES.START);
saveLogLine(`clusterLivePhotoFiles called`);
const analysedMediaFiles =
UploadService.clusterLivePhotoFiles(mediaFiles);
uiService.setFilenames(
@ -109,9 +116,16 @@ class UploadManager {
])
)
);
UIService.setHasLivePhoto(
mediaFiles.length !== analysedMediaFiles.length
);
saveLogLine(
`got live photos: ${
mediaFiles.length !== analysedMediaFiles.length
}`
);
await this.uploadMediaFiles(analysedMediaFiles);
}
UIService.setUploadStage(UPLOAD_STAGES.FINISH);
@ -128,10 +142,14 @@ class UploadManager {
private async parseMetadataJSONFiles(metadataFiles: FileWithCollection[]) {
try {
saveLogLine(`parseMetadataJSONFiles function executed `);
UIService.reset(metadataFiles.length);
const reader = new FileReader();
for (const { file, collectionID } of metadataFiles) {
try {
saveLogLine(`parsing file ${getFileNameSize(file)}`);
const parsedMetadataJSONWithTitle = await parseMetadataJSON(
reader,
file
@ -157,12 +175,19 @@ class UploadManager {
private async extractMetadataFromFiles(mediaFiles: FileWithCollection[]) {
try {
saveLogLine(`extractMetadataFromFiles executed`);
UIService.reset(mediaFiles.length);
const reader = new FileReader();
for (const { file, localID, collectionID } of mediaFiles) {
try {
const { fileTypeInfo, metadata } = await (async () => {
if (file.size >= MAX_FILE_SIZE_SUPPORTED) {
saveLogLine(
`${getFileNameSize(
file
)} rejected because of large size`
);
return { fileTypeInfo: null, metadata: null };
}
const fileTypeInfo = await UploadService.getFileType(
@ -170,8 +195,16 @@ class UploadManager {
file
);
if (fileTypeInfo.fileType === FILE_TYPE.OTHERS) {
saveLogLine(
`${getFileNameSize(
file
)} rejected because of unknown file format`
);
return { fileTypeInfo, metadata: null };
}
saveLogLine(
` extracting ${getFileNameSize(file)} metadata`
);
const metadata =
(await UploadService.extractFileMetadata(
file,
@ -197,6 +230,7 @@ class UploadManager {
}
private async uploadMediaFiles(mediaFiles: FileWithCollection[]) {
saveLogLine(`uploadMediaFiles called`);
this.filesToBeUploaded.push(...mediaFiles);
UIService.reset(mediaFiles.length);

11
src/services/upload/uploader.ts
View file

@ -9,6 +9,7 @@ import UploadService from './uploadService';
import { FILE_TYPE } from 'constants/file';
import { FileUploadResults, MAX_FILE_SIZE_SUPPORTED } from 'constants/upload';
import { FileWithCollection, BackupedFile, UploadFile } from 'types/upload';
import { saveLogLine } from 'utils/storage';
interface UploadResponse {
fileUploadResult: FileUploadResults;
@ -21,7 +22,11 @@ export default async function uploader(
fileWithCollection: FileWithCollection
): Promise<UploadResponse> {
const { collection, localID, ...uploadAsset } = fileWithCollection;
const fileNameSize = `${UploadService.getAssetName(
fileWithCollection
)}_${UploadService.getAssetSize(uploadAsset)}`;
saveLogLine(`uploader called for ${fileNameSize}`);
UIService.setFileProgress(localID, 0);
const { fileTypeInfo, metadata } =
UploadService.getFileMetadataAndFileTypeInfo(localID);
@ -40,6 +45,7 @@ export default async function uploader(
if (fileAlreadyInCollection(existingFilesInCollection, metadata)) {
return { fileUploadResult: FileUploadResults.ALREADY_UPLOADED };
}
saveLogLine(`reading asset ${fileNameSize}`);
const file = await UploadService.readAsset(
reader,
@ -57,12 +63,16 @@ export default async function uploader(
metadata,
};
saveLogLine(`encryptAsset ${fileNameSize}`);
const encryptedFile = await UploadService.encryptAsset(
worker,
fileWithMetadata,
collection.key
);
saveLogLine(`uploadToBucket ${fileNameSize}`);
const backupedFile: BackupedFile = await UploadService.uploadToBucket(
encryptedFile.file
);
@ -72,6 +82,7 @@ export default async function uploader(
backupedFile,
encryptedFile.fileKey
);
saveLogLine(`uploadFile ${fileNameSize}`);
const uploadedFile = await UploadHttpClient.uploadFile(uploadFile);
const decryptedFile = await decryptFile(uploadedFile, collection.key);

12
src/utils/storage/index.ts
View file

@ -1,4 +1,5 @@
import { convertToHumanReadable } from 'utils/billing';
import { formatDate } from 'utils/file';
import { getData, LS_KEYS, setData } from './localStorage';
export const isFirstLogin = () =>
@ -32,3 +33,14 @@ export function setLastAttemptedFile(file: File) {
export function getLastAttemptedFile() {
return getData(LS_KEYS.LAST_ATTEMPTED_FILE)?.file;
}
export function saveLogLine(line: string) {
const log = { time: formatDate(Date.now()), line };
setData(LS_KEYS.UPLOAD_LOGS, {
logs: [...getData(LS_KEYS.UPLOAD_LOGS), log],
});
}
export function getLogs() {
return getData(LS_KEYS.UPLOAD_LOGS)?.logs;
}

1
src/utils/storage/localStorage.ts
View file

@ -16,6 +16,7 @@ export enum LS_KEYS {
LIVE_PHOTO_INFO_SHOWN_COUNT = 'livePhotoInfoShownCount',
FAILED_UPLOADS = 'failedUploads',
LAST_ATTEMPTED_FILE = 'lastAttemptedFile',
UPLOAD_LOGS = 'uploadLogs',
}
export const setData = (key: LS_KEYS, value: object) => {

5
src/utils/upload/index.ts
View file

@ -1,5 +1,6 @@
import { FileWithCollection, Metadata } from 'types/upload';
import { EnteFile } from 'types/file';
import { convertToHumanReadable } from 'utils/billing';
const TYPE_JSON = 'json';
@ -49,3 +50,7 @@ export function segregateMetadataAndMediaFiles(
});
return { mediaFiles, metadataJSONFiles };
}
export function getFileNameSize(file: File) {
return `${file.name}_${convertToHumanReadable(file.size)}`;
}