beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #418 from ente-io/upload-logs

Browse source 
Upload logs
This commit is contained in:
Abhinav Kumar 2022-03-04 11:15:01 +05:30 committed by GitHub
parent f24f2bd00f 76776fcf5d
commit 2ca5b05949
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 135 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
src/components/FailedUploads.tsx
View file

@ -1,42 +0,0 @@
import React, { useEffect, useState } from 'react';
import { getData, LS_KEYS } from 'utils/storage/localStorage';
import constants from 'utils/strings/constants';
import MessageDialog from './MessageDialog';
import { FileList } from 'components/pages/gallery/UploadProgress';
import LinkButton from './pages/gallery/LinkButton';
export default function FailedUploads() {
const [listView, setListView] = useState(false);
const [failedFiles, setFailedFiles] = useState([]);
const hideList = () => setListView(false);
const showList = () => setListView(true);
useEffect(() => {
const failedFiles = getData(LS_KEYS.FAILED_UPLOADS)?.files ?? [];
setFailedFiles(failedFiles);
}, [listView]);
return (
failedFiles.length > 0 && (
<>
<LinkButton style={{ marginTop: '30px' }} onClick={showList}>
{constants.FAILED_UPLOADS}
</LinkButton>
<MessageDialog
show={listView}
onHide={hideList}
attributes={{
title: constants.FAILED_UPLOADS,
staticBackdrop: true,
close: { text: constants.CLOSE },
}}>
<FileList>
{failedFiles.map((file) => (
<li key={file}> {file}</li>
))}
</FileList>
</MessageDialog>
</>
)
);
}

16
src/components/Sidebar.tsx
View file

@ -35,7 +35,8 @@ import { PAGES } from 'constants/pages';
import { ARCHIVE_SECTION, TRASH_SECTION } from 'constants/collection';
import FixLargeThumbnails from './FixLargeThumbnail';
import { SetLoading } from 'types/gallery';
import FailedUploads from './FailedUploads';
import { downloadAsFile } from 'utils/file';
import { getUploadLogs } from 'utils/upload';
interface Props {
collections: Collection[];
setDialogMessage: SetDialogMessage;
@ -55,6 +56,7 @@ export default function Sidebar(props: Props) {
const [exportModalView, setExportModalView] = useState(false);
const [fixLargeThumbsView, setFixLargeThumbsView] = useState(false);
const galleryContext = useContext(GalleryContext);
useEffect(() => {
const main = async () => {
if (!isOpen) {
@ -109,6 +111,12 @@ export default function Sidebar(props: Props) {
}
}
const downloadUploadLogs = () => {
const logs = getUploadLogs();
const logString = logs.join('\n');
downloadAsFile(`upload_logs_${Date.now()}.txt`, logString);
};
const router = useRouter();
function onManageClick() {
setIsOpen(false);
@ -293,7 +301,11 @@ export default function Sidebar(props: Props) {
{constants.FIX_LARGE_THUMBNAILS}
</LinkButton>
</>
<FailedUploads />
<LinkButton
style={{ marginTop: '30px' }}
onClick={downloadUploadLogs}>
{constants.DOWNLOAD_UPLOAD_LOGS}
</LinkButton>
<LinkButton
style={{ marginTop: '30px' }}
onClick={openFeedbackURL}>

10
src/services/upload/metadataService.ts
View file

@ -11,6 +11,8 @@ import {
import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload';
import { splitFilenameAndExtension } from 'utils/file';
import { getVideoMetadata } from './videoMetadataService';
import { getFileNameSize } from 'utils/upload';
import { logUploadInfo } from 'utils/upload';
interface ParsedMetadataJSONWithTitle {
title: string;
@ -31,7 +33,15 @@ export async function extractMetadata(
if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) {
extractedMetadata = await getExifData(receivedFile, fileTypeInfo);
} else if (fileTypeInfo.fileType === FILE_TYPE.VIDEO) {
logUploadInfo(
`getVideoMetadata called for ${getFileNameSize(receivedFile)}`
);
extractedMetadata = await getVideoMetadata(receivedFile);
logUploadInfo(
`videoMetadata successfully extracted ${getFileNameSize(
receivedFile
)}`
);
}
const metadata: Metadata = {

17
src/services/upload/thumbnailService.ts
View file

@ -8,6 +8,7 @@ import { isFileHEIC } from 'utils/file';
import { FileTypeInfo } from 'types/upload';
import { getUint8ArrayView } from './readFileService';
import HEICConverter from 'services/HEICConverter';
import { getFileNameSize, logUploadInfo } from 'utils/upload';
const MAX_THUMBNAIL_DIMENSION = 720;
const MIN_COMPRESSION_PERCENTAGE_SIZE_DIFF = 10;
@ -28,6 +29,7 @@ export async function generateThumbnail(
fileTypeInfo: FileTypeInfo
): Promise<{ thumbnail: Uint8Array; hasStaticThumbnail: boolean }> {
try {
logUploadInfo(`generating thumbnail for ${getFileNameSize(file)}`);
let hasStaticThumbnail = false;
let canvas = document.createElement('canvas');
let thumbnail: Uint8Array;
@ -37,7 +39,18 @@ export async function generateThumbnail(
canvas = await generateImageThumbnail(file, isHEIC);
} else {
try {
logUploadInfo(
`ffmpeg generateThumbnail called for ${getFileNameSize(
file
)}`
);
const thumb = await FFmpegService.generateThumbnail(file);
logUploadInfo(
`ffmpeg thumbnail successfully generated ${getFileNameSize(
file
)}`
);
const dummyImageFile = new File([thumb], file.name);
canvas = await generateImageThumbnail(
dummyImageFile,
@ -79,7 +92,9 @@ export async function generateImageThumbnail(file: File, isHEIC: boolean) {
let timeout = null;
if (isHEIC) {
file = new File([await HEICConverter.convert(file)], null, null);
logUploadInfo(`HEICConverter called for ${getFileNameSize(file)}`);
file = new File([await HEICConverter.convert(file)], file.name);
logUploadInfo(`${getFileNameSize(file)} successfully converted`);
}
let image = new Image();
imageURL = URL.createObjectURL(file);

40
src/services/upload/uploadManager.ts
View file

@ -9,7 +9,7 @@ import {
} from 'utils/file';
import { logError } from 'utils/sentry';
import { getMetadataJSONMapKey, parseMetadataJSON } from './metadataService';
import { segregateMetadataAndMediaFiles } from 'utils/upload';
import { getFileNameSize, segregateMetadataAndMediaFiles } from 'utils/upload';
import uploader from './uploader';
import UIService from './uiService';
import UploadService from './uploadService';
@ -35,6 +35,7 @@ import uiService from './uiService';
import { getData, LS_KEYS, setData } from 'utils/storage/localStorage';
import { dedupe } from 'utils/export';
import { convertToHumanReadable } from 'utils/billing';
import { logUploadInfo } from 'utils/upload';
const MAX_CONCURRENT_UPLOADS = 4;
const FILE_UPLOAD_COMPLETED = 100;
@ -81,8 +82,15 @@ class UploadManager {
) {
try {
await this.init(newCreatedCollections);
logUploadInfo(
`received ${fileWithCollectionToBeUploaded.length} files to upload`
);
const { metadataJSONFiles, mediaFiles } =
segregateMetadataAndMediaFiles(fileWithCollectionToBeUploaded);
logUploadInfo(
`has ${metadataJSONFiles.length} metadata json files`
);
logUploadInfo(`has ${mediaFiles.length} media files`);
if (metadataJSONFiles.length) {
UIService.setUploadStage(
UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES
@ -99,6 +107,7 @@ class UploadManager {
this.metadataAndFileTypeInfoMap
);
UIService.setUploadStage(UPLOAD_STAGES.START);
logUploadInfo(`clusterLivePhotoFiles called`);
const analysedMediaFiles =
UploadService.clusterLivePhotoFiles(mediaFiles);
uiService.setFilenames(
@ -109,9 +118,16 @@ class UploadManager {
])
)
);
UIService.setHasLivePhoto(
mediaFiles.length !== analysedMediaFiles.length
);
logUploadInfo(
`got live photos: ${
mediaFiles.length !== analysedMediaFiles.length
}`
);
await this.uploadMediaFiles(analysedMediaFiles);
}
UIService.setUploadStage(UPLOAD_STAGES.FINISH);
@ -128,10 +144,14 @@ class UploadManager {
private async parseMetadataJSONFiles(metadataFiles: FileWithCollection[]) {
try {
logUploadInfo(`parseMetadataJSONFiles function executed `);
UIService.reset(metadataFiles.length);
const reader = new FileReader();
for (const { file, collectionID } of metadataFiles) {
try {
logUploadInfo(`parsing file ${getFileNameSize(file)}`);
const parsedMetadataJSONWithTitle = await parseMetadataJSON(
reader,
file
@ -157,12 +177,19 @@ class UploadManager {
private async extractMetadataFromFiles(mediaFiles: FileWithCollection[]) {
try {
logUploadInfo(`extractMetadataFromFiles executed`);
UIService.reset(mediaFiles.length);
const reader = new FileReader();
for (const { file, localID, collectionID } of mediaFiles) {
try {
const { fileTypeInfo, metadata } = await (async () => {
if (file.size >= MAX_FILE_SIZE_SUPPORTED) {
logUploadInfo(
`${getFileNameSize(
file
)} rejected because of large size`
);
return { fileTypeInfo: null, metadata: null };
}
const fileTypeInfo = await UploadService.getFileType(
@ -170,8 +197,16 @@ class UploadManager {
file
);
if (fileTypeInfo.fileType === FILE_TYPE.OTHERS) {
logUploadInfo(
`${getFileNameSize(
file
)} rejected because of unknown file format`
);
return { fileTypeInfo, metadata: null };
}
logUploadInfo(
` extracting ${getFileNameSize(file)} metadata`
);
const metadata =
(await UploadService.extractFileMetadata(
file,
@ -197,6 +232,7 @@ class UploadManager {
}
private async uploadMediaFiles(mediaFiles: FileWithCollection[]) {
logUploadInfo(`uploadMediaFiles called`);
this.filesToBeUploaded.push(...mediaFiles);
UIService.reset(mediaFiles.length);
@ -232,13 +268,13 @@ class UploadManager {
const existingFilesInCollection =
this.existingFilesCollectionWise.get(collectionID) ?? [];
const collection = this.collections.get(collectionID);
const { fileUploadResult, file } = await uploader(
worker,
reader,
existingFilesInCollection,
{ ...fileWithCollection, collection }
);
if (fileUploadResult === FileUploadResults.UPLOADED) {
this.existingFiles.push(file);
this.existingFiles = sortFiles(this.existingFiles);

19
src/services/upload/uploader.ts
View file

@ -9,6 +9,8 @@ import UploadService from './uploadService';
import { FILE_TYPE } from 'constants/file';
import { FileUploadResults, MAX_FILE_SIZE_SUPPORTED } from 'constants/upload';
import { FileWithCollection, BackupedFile, UploadFile } from 'types/upload';
import { logUploadInfo } from 'utils/upload';
import { convertToHumanReadable } from 'utils/billing';
interface UploadResponse {
fileUploadResult: FileUploadResults;
@ -21,7 +23,11 @@ export default async function uploader(
fileWithCollection: FileWithCollection
): Promise<UploadResponse> {
const { collection, localID, ...uploadAsset } = fileWithCollection;
const fileNameSize = `${UploadService.getAssetName(
fileWithCollection
)}_${convertToHumanReadable(UploadService.getAssetSize(uploadAsset))}`;
logUploadInfo(`uploader called for ${fileNameSize}`);
UIService.setFileProgress(localID, 0);
const { fileTypeInfo, metadata } =
UploadService.getFileMetadataAndFileTypeInfo(localID);
@ -38,8 +44,10 @@ export default async function uploader(
}
if (fileAlreadyInCollection(existingFilesInCollection, metadata)) {
logUploadInfo(`skipped upload for ${fileNameSize}`);
return { fileUploadResult: FileUploadResults.ALREADY_UPLOADED };
}
logUploadInfo(`reading asset ${fileNameSize}`);
const file = await UploadService.readAsset(
reader,
@ -57,12 +65,16 @@ export default async function uploader(
metadata,
};
logUploadInfo(`encryptAsset ${fileNameSize}`);
const encryptedFile = await UploadService.encryptAsset(
worker,
fileWithMetadata,
collection.key
);
logUploadInfo(`uploadToBucket ${fileNameSize}`);
const backupedFile: BackupedFile = await UploadService.uploadToBucket(
encryptedFile.file
);
@ -72,16 +84,23 @@ export default async function uploader(
backupedFile,
encryptedFile.fileKey
);
logUploadInfo(`uploadFile ${fileNameSize}`);
const uploadedFile = await UploadHttpClient.uploadFile(uploadFile);
const decryptedFile = await decryptFile(uploadedFile, collection.key);
UIService.increaseFileUploaded();
logUploadInfo(`${fileNameSize} successfully uploaded`);
return {
fileUploadResult: FileUploadResults.UPLOADED,
file: decryptedFile,
};
} catch (e) {
logUploadInfo(
`upload failed for ${fileNameSize} ,error: ${e.message}`
);
logError(e, 'file upload failed', {
fileFormat: fileTypeInfo.exactType,
});

15
src/utils/storage/index.ts
View file

@ -1,5 +1,10 @@
import { getData, LS_KEYS, setData } from './localStorage';
export interface Log {
type: string;
timestamp: number;
logLine: string;
}
export const isFirstLogin = () =>
getData(LS_KEYS.IS_FIRST_LOGIN)?.status ?? false;
@ -21,3 +26,13 @@ export function getLivePhotoInfoShownCount() {
export function setLivePhotoInfoShownCount(count) {
setData(LS_KEYS.LIVE_PHOTO_INFO_SHOWN_COUNT, { count });
}
export function saveLogLine(log: Log) {
setData(LS_KEYS.LOGS, {
logs: [...getLogs(), log],
});
}
export function getLogs(): Log[] {
return getData(LS_KEYS.LOGS)?.logs ?? [];
}

1
src/utils/storage/localStorage.ts
View file

@ -15,6 +15,7 @@ export enum LS_KEYS {
THUMBNAIL_FIX_STATE = 'thumbnailFixState',
LIVE_PHOTO_INFO_SHOWN_COUNT = 'livePhotoInfoShownCount',
FAILED_UPLOADS = 'failedUploads',
LOGS = 'logs',
}
export const setData = (key: LS_KEYS, value: object) => {

1
src/utils/strings/englishConstants.tsx
View file

@ -674,6 +674,7 @@ const englishConstants = {
PLAYBACK_SUPPORT_COMING: 'playback support coming soon...',
LIVE_PHOTO: 'this is a live photo',
LIVE: 'LIVE',
DOWNLOAD_UPLOAD_LOGS: 'debug logs',
};
export default englishConstants;

21
src/utils/upload/index.ts
View file

@ -1,5 +1,8 @@
import { FileWithCollection, Metadata } from 'types/upload';
import { EnteFile } from 'types/file';
import { convertToHumanReadable } from 'utils/billing';
import { formatDateTime } from 'utils/file';
import { getLogs, saveLogLine } from 'utils/storage';
const TYPE_JSON = 'json';
@ -49,3 +52,21 @@ export function segregateMetadataAndMediaFiles(
});
return { mediaFiles, metadataJSONFiles };
}
export function logUploadInfo(log: string) {
saveLogLine({
type: 'upload',
timestamp: Date.now(),
logLine: log,
});
}
export function getUploadLogs() {
return getLogs()
.filter((log) => log.type === 'upload')
.map((log) => `[${formatDateTime(log.timestamp)}] ${log.logLine}`);
}
export function getFileNameSize(file: File) {
return `${file.name}_${convertToHumanReadable(file.size)}`;
}