beenull/ente
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add export logs (#1304)

Browse source
This commit is contained in:
Abhinav Kumar 2023-08-08 16:57:55 +05:30 committed by GitHub
parent 00790aa559 bfe878dab0
commit 93e8e1fec2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 233 additions and 121 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

273
apps/photos/src/services/downloadManager.ts
View file

@ -17,6 +17,8 @@ import { CACHES } from 'constants/cache';
import { Remote } from 'comlink'; import { Remote } from 'comlink';
import { DedicatedCryptoWorker } from 'worker/crypto.worker'; import { DedicatedCryptoWorker } from 'worker/crypto.worker';
import { LimitedCache } from 'types/cache'; import { LimitedCache } from 'types/cache';
import { retryAsyncFunction } from 'utils/network';
import { addLogLine } from 'utils/logging';
class DownloadManager { class DownloadManager {
private fileObjectURLPromise = new Map< private fileObjectURLPromise = new Map<
@ -170,96 +172,195 @@ class DownloadManager {
usingWorker?: Remote<DedicatedCryptoWorker>, usingWorker?: Remote<DedicatedCryptoWorker>,
timeout?: number timeout?: number
) { ) {
const cryptoWorker = try {
usingWorker || (await ComlinkCryptoWorker.getInstance()); const cryptoWorker =
const token = tokenOverride || getToken(); usingWorker || (await ComlinkCryptoWorker.getInstance());
if (!token) { const token = tokenOverride || getToken();
return null; if (!token) {
} return null;
if (
file.metadata.fileType === FILE_TYPE.IMAGE ||
file.metadata.fileType === FILE_TYPE.LIVE_PHOTO
) {
const resp = await HTTPService.get(
getFileURL(file.id),
null,
{ 'X-Auth-Token': token },
{ responseType: 'arraybuffer', timeout }
);
if (typeof resp.data === 'undefined') {
throw Error(CustomError.REQUEST_FAILED);
} }
const decrypted = await cryptoWorker.decryptFile( if (
new Uint8Array(resp.data), file.metadata.fileType === FILE_TYPE.IMAGE ||
await cryptoWorker.fromB64(file.file.decryptionHeader), file.metadata.fileType === FILE_TYPE.LIVE_PHOTO
file.key ) {
); const resp = await retryAsyncFunction(() =>
return generateStreamFromArrayBuffer(decrypted); HTTPService.get(
} getFileURL(file.id),
const resp = await fetch(getFileURL(file.id), { null,
headers: { { 'X-Auth-Token': token },
'X-Auth-Token': token, { responseType: 'arraybuffer', timeout }
}, )
});
const reader = resp.body.getReader();
const stream = new ReadableStream({
async start(controller) {
const decryptionHeader = await cryptoWorker.fromB64(
file.file.decryptionHeader
); );
const fileKey = await cryptoWorker.fromB64(file.key); if (typeof resp.data === 'undefined') {
const { pullState, decryptionChunkSize } = throw Error(CustomError.REQUEST_FAILED);
await cryptoWorker.initChunkDecryption(
decryptionHeader,
fileKey
);
let data = new Uint8Array();
// The following function handles each data chunk
function push() {
// "done" is a Boolean and value a "Uint8Array"
reader.read().then(async ({ done, value }) => {
// Is there more data to read?
if (!done) {
const buffer = new Uint8Array(
data.byteLength + value.byteLength
);
buffer.set(new Uint8Array(data), 0);
buffer.set(new Uint8Array(value), data.byteLength);
if (buffer.length > decryptionChunkSize) {
const fileData = buffer.slice(
0,
decryptionChunkSize
);
const { decryptedData } =
await cryptoWorker.decryptFileChunk(
fileData,
pullState
);
controller.enqueue(decryptedData);
data = buffer.slice(decryptionChunkSize);
} else {
data = buffer;
}
push();
} else {
if (data) {
const { decryptedData } =
await cryptoWorker.decryptFileChunk(
data,
pullState
);
controller.enqueue(decryptedData);
data = null;
}
controller.close();
}
});
} }
try {
const decrypted = await cryptoWorker.decryptFile(
new Uint8Array(resp.data),
await cryptoWorker.fromB64(file.file.decryptionHeader),
file.key
);
return generateStreamFromArrayBuffer(decrypted);
} catch (e) {
if (e.message === CustomError.PROCESSING_FAILED) {
logError(e, 'Failed to process file', {
fileID: file.id,
fromMobile:
!!file.metadata.localID ||
!!file.metadata.deviceFolder ||
!!file.metadata.version,
});
addLogLine(
`Failed to process file with fileID:${file.id}, localID: ${file.metadata.localID}, version: ${file.metadata.version}, deviceFolder:${file.metadata.deviceFolder} with error: ${e.message}`
);
}
throw e;
}
}
const resp = await retryAsyncFunction(() =>
fetch(getFileURL(file.id), {
headers: {
'X-Auth-Token': token,
},
})
);
const reader = resp.body.getReader();
const stream = new ReadableStream({
async start(controller) {
try {
const decryptionHeader = await cryptoWorker.fromB64(
file.file.decryptionHeader
);
const fileKey = await cryptoWorker.fromB64(file.key);
const { pullState, decryptionChunkSize } =
await cryptoWorker.initChunkDecryption(
decryptionHeader,
fileKey
);
let data = new Uint8Array();
// The following function handles each data chunk
const push = () => {
// "done" is a Boolean and value a "Uint8Array"
reader.read().then(async ({ done, value }) => {
try {
// Is there more data to read?
if (!done) {
const buffer = new Uint8Array(
data.byteLength + value.byteLength
);
buffer.set(new Uint8Array(data), 0);
buffer.set(
new Uint8Array(value),
data.byteLength
);
if (
buffer.length > decryptionChunkSize
) {
const fileData = buffer.slice(
0,
decryptionChunkSize
);
try {
const { decryptedData } =
await cryptoWorker.decryptFileChunk(
fileData,
pullState
);
controller.enqueue(
decryptedData
);
data =
buffer.slice(
decryptionChunkSize
);
} catch (e) {
if (
e.message ===
CustomError.PROCESSING_FAILED
) {
logError(
e,
'Failed to process file',
{
fileID: file.id,
fromMobile:
!!file.metadata
.localID ||
!!file.metadata
.deviceFolder ||
!!file.metadata
.version,
}
);
addLogLine(
`Failed to process file ${file.id} from localID: ${file.metadata.localID} version: ${file.metadata.version} deviceFolder:${file.metadata.deviceFolder} with error: ${e.message}`
);
}
throw e;
}
} else {
data = buffer;
}
push();
} else {
if (data) {
try {
const { decryptedData } =
await cryptoWorker.decryptFileChunk(
data,
pullState
);
controller.enqueue(
decryptedData
);
data = null;
} catch (e) {
if (
e.message ===
CustomError.PROCESSING_FAILED
) {
logError(
e,
'Failed to process file',
{
fileID: file.id,
fromMobile:
!!file.metadata
.localID ||
!!file.metadata
.deviceFolder ||
!!file.metadata
.version,
}
);
addLogLine(
`Failed to process file ${file.id} from localID: ${file.metadata.localID} version: ${file.metadata.version} deviceFolder:${file.metadata.deviceFolder} with error: ${e.message}`
);
}
throw e;
}
}
controller.close();
}
} catch (e) {
logError(e, 'Failed to process file chunk');
controller.error(e);
}
});
};
push(); push();
}, } catch (e) {
}); logError(e, 'Failed to process file stream');
return stream; controller.error(e);
}
},
});
return stream;
} catch (e) {
logError(e, 'Failed to download file');
throw e;
}
} }
} }

65
apps/photos/src/services/export/index.ts
View file

@ -22,7 +22,6 @@ import {
parseLivePhotoExportName, parseLivePhotoExportName,
getCollectionIDFromFileUID, getCollectionIDFromFileUID,
} from 'utils/export'; } from 'utils/export';
import { retryAsyncFunction } from 'utils/network';
import { logError } from 'utils/sentry'; import { logError } from 'utils/sentry';
import { getData, LS_KEYS, setData } from 'utils/storage/localStorage'; import { getData, LS_KEYS, setData } from 'utils/storage/localStorage';
import { getLocalCollections } from '../collectionService'; import { getLocalCollections } from '../collectionService';
@ -435,9 +434,8 @@ class ExportService {
exportRecord exportRecord
); );
addLocalLog( addLogLine(
() => `personal files:${personalFiles.length} unexported files: ${filesToExport.length}, deleted exported files: ${removedFileUIDs.length}, renamed collections: ${renamedCollections.length}, deleted collections: ${deletedExportedCollections.length}`
`personal files:${personalFiles.length} unexported files: ${filesToExport.length}, deleted exported files: ${removedFileUIDs.length}, renamed collections: ${renamedCollections.length}, deleted collections: ${deletedExportedCollections.length}`
); );
let success = 0; let success = 0;
let failed = 0; let failed = 0;
@ -557,10 +555,8 @@ class ExportService {
exportFolder, exportFolder,
collection.name collection.name
); );
addLocalLog( addLogLine(
() => `renaming collection with id ${collection.id} from ${oldCollectionExportName} to ${newCollectionExportName}`
`renaming collection with id ${collection.id} from ${oldCollectionExportName} to ${newCollectionExportName}
`
); );
const newCollectionExportPath = getCollectionExportPath( const newCollectionExportPath = getCollectionExportPath(
exportFolder, exportFolder,
@ -581,6 +577,9 @@ class ExportService {
collection.id, collection.id,
newCollectionExportName newCollectionExportName
); );
addLogLine(
`renaming collection with id ${collection.id} from ${oldCollectionExportName} to ${newCollectionExportName} successful`
);
incrementSuccess(); incrementSuccess();
} catch (e) { } catch (e) {
incrementFailed(); incrementFailed();
@ -626,9 +625,8 @@ class ExportService {
throw Error(CustomError.EXPORT_STOPPED); throw Error(CustomError.EXPORT_STOPPED);
} }
this.verifyExportFolderExists(exportFolder); this.verifyExportFolderExists(exportFolder);
addLocalLog( addLogLine(
() => `removing collection with id ${collectionID} from export folder`
`removing collection with id ${collectionID} from export folder`
); );
const collectionExportName = const collectionExportName =
collectionIDPathMap.get(collectionID); collectionIDPathMap.get(collectionID);
@ -656,6 +654,9 @@ class ExportService {
exportFolder, exportFolder,
collectionID collectionID
); );
addLogLine(
`removing collection with id ${collectionID} from export folder successful`
);
incrementSuccess(); incrementSuccess();
} catch (e) { } catch (e) {
incrementFailed(); incrementFailed();
@ -693,13 +694,12 @@ class ExportService {
): Promise<void> { ): Promise<void> {
try { try {
for (const file of files) { for (const file of files) {
addLocalLog( addLogLine(
() => `exporting file ${file.metadata.title} with id ${
`exporting file ${file.metadata.title} with id ${ file.id
file.id } from collection ${collectionIDNameMap.get(
} from collection ${collectionIDNameMap.get( file.collectionID
file.collectionID )}`
)}`
); );
if (isCanceled.status) { if (isCanceled.status) {
throw Error(CustomError.EXPORT_STOPPED); throw Error(CustomError.EXPORT_STOPPED);
@ -743,6 +743,13 @@ class ExportService {
fileExportName fileExportName
); );
incrementSuccess(); incrementSuccess();
addLogLine(
`exporting file ${file.metadata.title} with id ${
file.id
} from collection ${collectionIDNameMap.get(
file.collectionID
)} successful`
);
} catch (e) { } catch (e) {
incrementFailed(); incrementFailed();
logError(e, 'export failed for a file'); logError(e, 'export failed for a file');
@ -783,7 +790,7 @@ class ExportService {
); );
for (const fileUID of removedFileUIDs) { for (const fileUID of removedFileUIDs) {
this.verifyExportFolderExists(exportDir); this.verifyExportFolderExists(exportDir);
addLocalLog(() => `trashing file with id ${fileUID}`); addLogLine(`trashing file with id ${fileUID}`);
if (isCanceled.status) { if (isCanceled.status) {
throw Error(CustomError.EXPORT_STOPPED); throw Error(CustomError.EXPORT_STOPPED);
} }
@ -804,9 +811,8 @@ class ExportService {
collectionExportPath, collectionExportPath,
imageExportName imageExportName
); );
addLocalLog( addLogLine(
() => `moving image file ${imageExportPath} to trash folder`
`moving image file ${imageExportPath} to trash folder`
); );
await this.electronAPIs.moveFile( await this.electronAPIs.moveFile(
imageExportPath, imageExportPath,
@ -828,9 +834,8 @@ class ExportService {
collectionExportPath, collectionExportPath,
videoExportName videoExportName
); );
addLocalLog( addLogLine(
() => `moving video file ${videoExportPath} to trash folder`
`moving video file ${videoExportPath} to trash folder`
); );
await this.electronAPIs.moveFile( await this.electronAPIs.moveFile(
videoExportPath, videoExportPath,
@ -854,9 +859,8 @@ class ExportService {
exportDir, exportDir,
fileExportPath fileExportPath
); );
addLocalLog( addLogLine(
() => `moving file ${fileExportPath} to ${trashedFilePath} trash folder`
`moving file ${fileExportPath} to ${trashedFilePath} trash folder`
); );
await this.electronAPIs.moveFile( await this.electronAPIs.moveFile(
fileExportPath, fileExportPath,
@ -873,6 +877,7 @@ class ExportService {
); );
} }
await this.removeFileExportedRecord(exportDir, fileUID); await this.removeFileExportedRecord(exportDir, fileUID);
addLogLine(`trashing file with id ${fileUID} successful`);
incrementSuccess(); incrementSuccess();
} catch (e) { } catch (e) {
incrementFailed(); incrementFailed();
@ -1062,9 +1067,7 @@ class ExportService {
collectionExportPath, collectionExportPath,
file.metadata.title file.metadata.title
); );
let fileStream = await retryAsyncFunction(() => let fileStream = await downloadManager.downloadFile(file);
downloadManager.downloadFile(file)
);
const fileType = getFileExtension(file.metadata.title); const fileType = getFileExtension(file.metadata.title);
if ( if (
file.pubMagicMetadata?.data.editedTime && file.pubMagicMetadata?.data.editedTime &&

5
apps/photos/src/services/export/migration.ts
View file

@ -38,7 +38,6 @@ import {
import { FILE_TYPE } from 'constants/file'; import { FILE_TYPE } from 'constants/file';
import { decodeLivePhoto } from 'services/livePhotoService'; import { decodeLivePhoto } from 'services/livePhotoService';
import downloadManager from 'services/downloadManager'; import downloadManager from 'services/downloadManager';
import { retryAsyncFunction } from 'utils/network';
import { sleep } from 'utils/common'; import { sleep } from 'utils/common';
export async function migrateExport( export async function migrateExport(
@ -326,9 +325,7 @@ async function getFileExportNamesFromExportedFiles(
For Live Photos we need to download the file to get the image and video name For Live Photos we need to download the file to get the image and video name
*/ */
if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) { if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
const fileStream = await retryAsyncFunction(() => const fileStream = await downloadManager.downloadFile(file);
downloadManager.downloadFile(file)
);
const fileBlob = await new Response(fileStream).blob(); const fileBlob = await new Response(fileStream).blob();
const livePhoto = await decodeLivePhoto(file, fileBlob); const livePhoto = await decodeLivePhoto(file, fileBlob);
const imageExportName = getUniqueFileExportNameForMigration( const imageExportName = getUniqueFileExportNameForMigration(

3
apps/photos/src/types/upload/index.ts
View file

@ -29,6 +29,9 @@ export interface Metadata {
hash?: string; hash?: string;
imageHash?: string; imageHash?: string;
videoHash?: string; videoHash?: string;
localID?: number;
version?: number;
deviceFolder?: string;
} }
export interface Location { export interface Location {

7
apps/photos/src/utils/crypto/libsodium.ts
View file

@ -1,6 +1,7 @@
import sodium, { StateAddress } from 'libsodium-wrappers'; import sodium, { StateAddress } from 'libsodium-wrappers';
import { ENCRYPTION_CHUNK_SIZE } from 'constants/crypto'; import { ENCRYPTION_CHUNK_SIZE } from 'constants/crypto';
import { B64EncryptionResult } from 'types/crypto'; import { B64EncryptionResult } from 'types/crypto';
import { CustomError } from 'utils/error';
export async function decryptChaChaOneShot( export async function decryptChaChaOneShot(
data: Uint8Array, data: Uint8Array,
@ -46,6 +47,9 @@ export async function decryptChaCha(
pullState, pullState,
buffer buffer
); );
if (!pullResult.message) {
throw new Error(CustomError.PROCESSING_FAILED);
}
for (let index = 0; index < pullResult.message.length; index++) { for (let index = 0; index < pullResult.message.length; index++) {
decryptedData.push(pullResult.message[index]); decryptedData.push(pullResult.message[index]);
} }
@ -77,6 +81,9 @@ export async function decryptFileChunk(
pullState, pullState,
data data
); );
if (!pullResult.message) {
throw new Error(CustomError.PROCESSING_FAILED);
}
const newTag = pullResult.tag; const newTag = pullResult.tag;
return { decryptedData: pullResult.message, newTag }; return { decryptedData: pullResult.message, newTag };
} }

1
apps/photos/src/utils/error/index.ts
View file

@ -69,6 +69,7 @@ export const CustomError = {
NOT_AVAILABLE_ON_WEB: 'not available on web', NOT_AVAILABLE_ON_WEB: 'not available on web',
UNSUPPORTED_RAW_FORMAT: 'unsupported raw format', UNSUPPORTED_RAW_FORMAT: 'unsupported raw format',
NON_PREVIEWABLE_FILE: 'non previewable file', NON_PREVIEWABLE_FILE: 'non previewable file',
PROCESSING_FAILED: 'processing failed',
}; };
export function parseUploadErrorCodes(error) { export function parseUploadErrorCodes(error) {