Add export logs (#1304)
This commit is contained in:
Abhinav Kumar
GitHub
commit
93e8e1fec2
|
|
@ -17,6 +17,8 @@ import { CACHES } from 'constants/cache';
|
|||
import { Remote } from 'comlink';
|
||||
import { DedicatedCryptoWorker } from 'worker/crypto.worker';
|
||||
import { LimitedCache } from 'types/cache';
|
||||
import { retryAsyncFunction } from 'utils/network';
|
||||
import { addLogLine } from 'utils/logging';
|
||||
|
||||
class DownloadManager {
|
||||
private fileObjectURLPromise = new Map<
|
||||
|
|
@ -170,96 +172,195 @@ class DownloadManager {
|
|||
usingWorker?: Remote<DedicatedCryptoWorker>,
|
||||
timeout?: number
|
||||
) {
|
||||
const cryptoWorker =
|
||||
usingWorker || (await ComlinkCryptoWorker.getInstance());
|
||||
const token = tokenOverride || getToken();
|
||||
if (!token) {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
file.metadata.fileType === FILE_TYPE.IMAGE ||
|
||||
file.metadata.fileType === FILE_TYPE.LIVE_PHOTO
|
||||
) {
|
||||
const resp = await HTTPService.get(
|
||||
getFileURL(file.id),
|
||||
null,
|
||||
{ 'X-Auth-Token': token },
|
||||
{ responseType: 'arraybuffer', timeout }
|
||||
);
|
||||
if (typeof resp.data === 'undefined') {
|
||||
throw Error(CustomError.REQUEST_FAILED);
|
||||
try {
|
||||
const cryptoWorker =
|
||||
usingWorker || (await ComlinkCryptoWorker.getInstance());
|
||||
const token = tokenOverride || getToken();
|
||||
if (!token) {
|
||||
return null;
|
||||
}
|
||||
const decrypted = await cryptoWorker.decryptFile(
|
||||
new Uint8Array(resp.data),
|
||||
await cryptoWorker.fromB64(file.file.decryptionHeader),
|
||||
file.key
|
||||
);
|
||||
return generateStreamFromArrayBuffer(decrypted);
|
||||
}
|
||||
const resp = await fetch(getFileURL(file.id), {
|
||||
headers: {
|
||||
'X-Auth-Token': token,
|
||||
},
|
||||
});
|
||||
const reader = resp.body.getReader();
|
||||
const stream = new ReadableStream({
|
||||
async start(controller) {
|
||||
const decryptionHeader = await cryptoWorker.fromB64(
|
||||
file.file.decryptionHeader
|
||||
if (
|
||||
file.metadata.fileType === FILE_TYPE.IMAGE ||
|
||||
file.metadata.fileType === FILE_TYPE.LIVE_PHOTO
|
||||
) {
|
||||
const resp = await retryAsyncFunction(() =>
|
||||
HTTPService.get(
|
||||
getFileURL(file.id),
|
||||
null,
|
||||
{ 'X-Auth-Token': token },
|
||||
{ responseType: 'arraybuffer', timeout }
|
||||
)
|
||||
);
|
||||
const fileKey = await cryptoWorker.fromB64(file.key);
|
||||
const { pullState, decryptionChunkSize } =
|
||||
await cryptoWorker.initChunkDecryption(
|
||||
decryptionHeader,
|
||||
fileKey
|
||||
);
|
||||
let data = new Uint8Array();
|
||||
// The following function handles each data chunk
|
||||
function push() {
|
||||
// "done" is a Boolean and value a "Uint8Array"
|
||||
reader.read().then(async ({ done, value }) => {
|
||||
// Is there more data to read?
|
||||
if (!done) {
|
||||
const buffer = new Uint8Array(
|
||||
data.byteLength + value.byteLength
|
||||
);
|
||||
buffer.set(new Uint8Array(data), 0);
|
||||
buffer.set(new Uint8Array(value), data.byteLength);
|
||||
if (buffer.length > decryptionChunkSize) {
|
||||
const fileData = buffer.slice(
|
||||
0,
|
||||
decryptionChunkSize
|
||||
);
|
||||
const { decryptedData } =
|
||||
await cryptoWorker.decryptFileChunk(
|
||||
fileData,
|
||||
pullState
|
||||
);
|
||||
controller.enqueue(decryptedData);
|
||||
data = buffer.slice(decryptionChunkSize);
|
||||
} else {
|
||||
data = buffer;
|
||||
}
|
||||
push();
|
||||
} else {
|
||||
if (data) {
|
||||
const { decryptedData } =
|
||||
await cryptoWorker.decryptFileChunk(
|
||||
data,
|
||||
pullState
|
||||
);
|
||||
controller.enqueue(decryptedData);
|
||||
data = null;
|
||||
}
|
||||
controller.close();
|
||||
}
|
||||
});
|
||||
if (typeof resp.data === 'undefined') {
|
||||
throw Error(CustomError.REQUEST_FAILED);
|
||||
}
|
||||
try {
|
||||
const decrypted = await cryptoWorker.decryptFile(
|
||||
new Uint8Array(resp.data),
|
||||
await cryptoWorker.fromB64(file.file.decryptionHeader),
|
||||
file.key
|
||||
);
|
||||
return generateStreamFromArrayBuffer(decrypted);
|
||||
} catch (e) {
|
||||
if (e.message === CustomError.PROCESSING_FAILED) {
|
||||
logError(e, 'Failed to process file', {
|
||||
fileID: file.id,
|
||||
fromMobile:
|
||||
!!file.metadata.localID ||
|
||||
!!file.metadata.deviceFolder ||
|
||||
!!file.metadata.version,
|
||||
});
|
||||
addLogLine(
|
||||
`Failed to process file with fileID:${file.id}, localID: ${file.metadata.localID}, version: ${file.metadata.version}, deviceFolder:${file.metadata.deviceFolder} with error: ${e.message}`
|
||||
);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
const resp = await retryAsyncFunction(() =>
|
||||
fetch(getFileURL(file.id), {
|
||||
headers: {
|
||||
'X-Auth-Token': token,
|
||||
},
|
||||
})
|
||||
);
|
||||
const reader = resp.body.getReader();
|
||||
const stream = new ReadableStream({
|
||||
async start(controller) {
|
||||
try {
|
||||
const decryptionHeader = await cryptoWorker.fromB64(
|
||||
file.file.decryptionHeader
|
||||
);
|
||||
const fileKey = await cryptoWorker.fromB64(file.key);
|
||||
const { pullState, decryptionChunkSize } =
|
||||
await cryptoWorker.initChunkDecryption(
|
||||
decryptionHeader,
|
||||
fileKey
|
||||
);
|
||||
let data = new Uint8Array();
|
||||
// The following function handles each data chunk
|
||||
const push = () => {
|
||||
// "done" is a Boolean and value a "Uint8Array"
|
||||
reader.read().then(async ({ done, value }) => {
|
||||
try {
|
||||
// Is there more data to read?
|
||||
if (!done) {
|
||||
const buffer = new Uint8Array(
|
||||
data.byteLength + value.byteLength
|
||||
);
|
||||
buffer.set(new Uint8Array(data), 0);
|
||||
buffer.set(
|
||||
new Uint8Array(value),
|
||||
data.byteLength
|
||||
);
|
||||
if (
|
||||
buffer.length > decryptionChunkSize
|
||||
) {
|
||||
const fileData = buffer.slice(
|
||||
0,
|
||||
decryptionChunkSize
|
||||
);
|
||||
try {
|
||||
const { decryptedData } =
|
||||
await cryptoWorker.decryptFileChunk(
|
||||
fileData,
|
||||
pullState
|
||||
);
|
||||
controller.enqueue(
|
||||
decryptedData
|
||||
);
|
||||
data =
|
||||
buffer.slice(
|
||||
decryptionChunkSize
|
||||
);
|
||||
} catch (e) {
|
||||
if (
|
||||
e.message ===
|
||||
CustomError.PROCESSING_FAILED
|
||||
) {
|
||||
logError(
|
||||
e,
|
||||
'Failed to process file',
|
||||
{
|
||||
fileID: file.id,
|
||||
fromMobile:
|
||||
!!file.metadata
|
||||
.localID ||
|
||||
!!file.metadata
|
||||
.deviceFolder ||
|
||||
!!file.metadata
|
||||
.version,
|
||||
}
|
||||
);
|
||||
addLogLine(
|
||||
`Failed to process file ${file.id} from localID: ${file.metadata.localID} version: ${file.metadata.version} deviceFolder:${file.metadata.deviceFolder} with error: ${e.message}`
|
||||
);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
} else {
|
||||
data = buffer;
|
||||
}
|
||||
push();
|
||||
} else {
|
||||
if (data) {
|
||||
try {
|
||||
const { decryptedData } =
|
||||
await cryptoWorker.decryptFileChunk(
|
||||
data,
|
||||
pullState
|
||||
);
|
||||
controller.enqueue(
|
||||
decryptedData
|
||||
);
|
||||
data = null;
|
||||
} catch (e) {
|
||||
if (
|
||||
e.message ===
|
||||
CustomError.PROCESSING_FAILED
|
||||
) {
|
||||
logError(
|
||||
e,
|
||||
'Failed to process file',
|
||||
{
|
||||
fileID: file.id,
|
||||
fromMobile:
|
||||
!!file.metadata
|
||||
.localID ||
|
||||
!!file.metadata
|
||||
.deviceFolder ||
|
||||
!!file.metadata
|
||||
.version,
|
||||
}
|
||||
);
|
||||
addLogLine(
|
||||
`Failed to process file ${file.id} from localID: ${file.metadata.localID} version: ${file.metadata.version} deviceFolder:${file.metadata.deviceFolder} with error: ${e.message}`
|
||||
);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
controller.close();
|
||||
}
|
||||
} catch (e) {
|
||||
logError(e, 'Failed to process file chunk');
|
||||
controller.error(e);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
push();
|
||||
},
|
||||
});
|
||||
return stream;
|
||||
push();
|
||||
} catch (e) {
|
||||
logError(e, 'Failed to process file stream');
|
||||
controller.error(e);
|
||||
}
|
||||
},
|
||||
});
|
||||
return stream;
|
||||
} catch (e) {
|
||||
logError(e, 'Failed to download file');
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ import {
|
|||
parseLivePhotoExportName,
|
||||
getCollectionIDFromFileUID,
|
||||
} from 'utils/export';
|
||||
import { retryAsyncFunction } from 'utils/network';
|
||||
import { logError } from 'utils/sentry';
|
||||
import { getData, LS_KEYS, setData } from 'utils/storage/localStorage';
|
||||
import { getLocalCollections } from '../collectionService';
|
||||
|
|
@ -435,9 +434,8 @@ class ExportService {
|
|||
exportRecord
|
||||
);
|
||||
|
||||
addLocalLog(
|
||||
() =>
|
||||
`personal files:${personalFiles.length} unexported files: ${filesToExport.length}, deleted exported files: ${removedFileUIDs.length}, renamed collections: ${renamedCollections.length}, deleted collections: ${deletedExportedCollections.length}`
|
||||
addLogLine(
|
||||
`personal files:${personalFiles.length} unexported files: ${filesToExport.length}, deleted exported files: ${removedFileUIDs.length}, renamed collections: ${renamedCollections.length}, deleted collections: ${deletedExportedCollections.length}`
|
||||
);
|
||||
let success = 0;
|
||||
let failed = 0;
|
||||
|
|
@ -557,10 +555,8 @@ class ExportService {
|
|||
exportFolder,
|
||||
collection.name
|
||||
);
|
||||
addLocalLog(
|
||||
() =>
|
||||
`renaming collection with id ${collection.id} from ${oldCollectionExportName} to ${newCollectionExportName}
|
||||
`
|
||||
addLogLine(
|
||||
`renaming collection with id ${collection.id} from ${oldCollectionExportName} to ${newCollectionExportName}`
|
||||
);
|
||||
const newCollectionExportPath = getCollectionExportPath(
|
||||
exportFolder,
|
||||
|
|
@ -581,6 +577,9 @@ class ExportService {
|
|||
collection.id,
|
||||
newCollectionExportName
|
||||
);
|
||||
addLogLine(
|
||||
`renaming collection with id ${collection.id} from ${oldCollectionExportName} to ${newCollectionExportName} successful`
|
||||
);
|
||||
incrementSuccess();
|
||||
} catch (e) {
|
||||
incrementFailed();
|
||||
|
|
@ -626,9 +625,8 @@ class ExportService {
|
|||
throw Error(CustomError.EXPORT_STOPPED);
|
||||
}
|
||||
this.verifyExportFolderExists(exportFolder);
|
||||
addLocalLog(
|
||||
() =>
|
||||
`removing collection with id ${collectionID} from export folder`
|
||||
addLogLine(
|
||||
`removing collection with id ${collectionID} from export folder`
|
||||
);
|
||||
const collectionExportName =
|
||||
collectionIDPathMap.get(collectionID);
|
||||
|
|
@ -656,6 +654,9 @@ class ExportService {
|
|||
exportFolder,
|
||||
collectionID
|
||||
);
|
||||
addLogLine(
|
||||
`removing collection with id ${collectionID} from export folder successful`
|
||||
);
|
||||
incrementSuccess();
|
||||
} catch (e) {
|
||||
incrementFailed();
|
||||
|
|
@ -693,13 +694,12 @@ class ExportService {
|
|||
): Promise<void> {
|
||||
try {
|
||||
for (const file of files) {
|
||||
addLocalLog(
|
||||
() =>
|
||||
`exporting file ${file.metadata.title} with id ${
|
||||
file.id
|
||||
} from collection ${collectionIDNameMap.get(
|
||||
file.collectionID
|
||||
)}`
|
||||
addLogLine(
|
||||
`exporting file ${file.metadata.title} with id ${
|
||||
file.id
|
||||
} from collection ${collectionIDNameMap.get(
|
||||
file.collectionID
|
||||
)}`
|
||||
);
|
||||
if (isCanceled.status) {
|
||||
throw Error(CustomError.EXPORT_STOPPED);
|
||||
|
|
@ -743,6 +743,13 @@ class ExportService {
|
|||
fileExportName
|
||||
);
|
||||
incrementSuccess();
|
||||
addLogLine(
|
||||
`exporting file ${file.metadata.title} with id ${
|
||||
file.id
|
||||
} from collection ${collectionIDNameMap.get(
|
||||
file.collectionID
|
||||
)} successful`
|
||||
);
|
||||
} catch (e) {
|
||||
incrementFailed();
|
||||
logError(e, 'export failed for a file');
|
||||
|
|
@ -783,7 +790,7 @@ class ExportService {
|
|||
);
|
||||
for (const fileUID of removedFileUIDs) {
|
||||
this.verifyExportFolderExists(exportDir);
|
||||
addLocalLog(() => `trashing file with id ${fileUID}`);
|
||||
addLogLine(`trashing file with id ${fileUID}`);
|
||||
if (isCanceled.status) {
|
||||
throw Error(CustomError.EXPORT_STOPPED);
|
||||
}
|
||||
|
|
@ -804,9 +811,8 @@ class ExportService {
|
|||
collectionExportPath,
|
||||
imageExportName
|
||||
);
|
||||
addLocalLog(
|
||||
() =>
|
||||
`moving image file ${imageExportPath} to trash folder`
|
||||
addLogLine(
|
||||
`moving image file ${imageExportPath} to trash folder`
|
||||
);
|
||||
await this.electronAPIs.moveFile(
|
||||
imageExportPath,
|
||||
|
|
@ -828,9 +834,8 @@ class ExportService {
|
|||
collectionExportPath,
|
||||
videoExportName
|
||||
);
|
||||
addLocalLog(
|
||||
() =>
|
||||
`moving video file ${videoExportPath} to trash folder`
|
||||
addLogLine(
|
||||
`moving video file ${videoExportPath} to trash folder`
|
||||
);
|
||||
await this.electronAPIs.moveFile(
|
||||
videoExportPath,
|
||||
|
|
@ -854,9 +859,8 @@ class ExportService {
|
|||
exportDir,
|
||||
fileExportPath
|
||||
);
|
||||
addLocalLog(
|
||||
() =>
|
||||
`moving file ${fileExportPath} to ${trashedFilePath} trash folder`
|
||||
addLogLine(
|
||||
`moving file ${fileExportPath} to ${trashedFilePath} trash folder`
|
||||
);
|
||||
await this.electronAPIs.moveFile(
|
||||
fileExportPath,
|
||||
|
|
@ -873,6 +877,7 @@ class ExportService {
|
|||
);
|
||||
}
|
||||
await this.removeFileExportedRecord(exportDir, fileUID);
|
||||
addLogLine(`trashing file with id ${fileUID} successful`);
|
||||
incrementSuccess();
|
||||
} catch (e) {
|
||||
incrementFailed();
|
||||
|
|
@ -1062,9 +1067,7 @@ class ExportService {
|
|||
collectionExportPath,
|
||||
file.metadata.title
|
||||
);
|
||||
let fileStream = await retryAsyncFunction(() =>
|
||||
downloadManager.downloadFile(file)
|
||||
);
|
||||
let fileStream = await downloadManager.downloadFile(file);
|
||||
const fileType = getFileExtension(file.metadata.title);
|
||||
if (
|
||||
file.pubMagicMetadata?.data.editedTime &&
|
||||
|
|
|
|||
|
|
@ -38,7 +38,6 @@ import {
|
|||
import { FILE_TYPE } from 'constants/file';
|
||||
import { decodeLivePhoto } from 'services/livePhotoService';
|
||||
import downloadManager from 'services/downloadManager';
|
||||
import { retryAsyncFunction } from 'utils/network';
|
||||
import { sleep } from 'utils/common';
|
||||
|
||||
export async function migrateExport(
|
||||
|
|
@ -326,9 +325,7 @@ async function getFileExportNamesFromExportedFiles(
|
|||
For Live Photos we need to download the file to get the image and video name
|
||||
*/
|
||||
if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
|
||||
const fileStream = await retryAsyncFunction(() =>
|
||||
downloadManager.downloadFile(file)
|
||||
);
|
||||
const fileStream = await downloadManager.downloadFile(file);
|
||||
const fileBlob = await new Response(fileStream).blob();
|
||||
const livePhoto = await decodeLivePhoto(file, fileBlob);
|
||||
const imageExportName = getUniqueFileExportNameForMigration(
|
||||
|
|
|
|||
|
|
@ -29,6 +29,9 @@ export interface Metadata {
|
|||
hash?: string;
|
||||
imageHash?: string;
|
||||
videoHash?: string;
|
||||
localID?: number;
|
||||
version?: number;
|
||||
deviceFolder?: string;
|
||||
}
|
||||
|
||||
export interface Location {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import sodium, { StateAddress } from 'libsodium-wrappers';
|
||||
import { ENCRYPTION_CHUNK_SIZE } from 'constants/crypto';
|
||||
import { B64EncryptionResult } from 'types/crypto';
|
||||
import { CustomError } from 'utils/error';
|
||||
|
||||
export async function decryptChaChaOneShot(
|
||||
data: Uint8Array,
|
||||
|
|
@ -46,6 +47,9 @@ export async function decryptChaCha(
|
|||
pullState,
|
||||
buffer
|
||||
);
|
||||
if (!pullResult.message) {
|
||||
throw new Error(CustomError.PROCESSING_FAILED);
|
||||
}
|
||||
for (let index = 0; index < pullResult.message.length; index++) {
|
||||
decryptedData.push(pullResult.message[index]);
|
||||
}
|
||||
|
|
@ -77,6 +81,9 @@ export async function decryptFileChunk(
|
|||
pullState,
|
||||
data
|
||||
);
|
||||
if (!pullResult.message) {
|
||||
throw new Error(CustomError.PROCESSING_FAILED);
|
||||
}
|
||||
const newTag = pullResult.tag;
|
||||
return { decryptedData: pullResult.message, newTag };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@ export const CustomError = {
|
|||
NOT_AVAILABLE_ON_WEB: 'not available on web',
|
||||
UNSUPPORTED_RAW_FORMAT: 'unsupported raw format',
|
||||
NON_PREVIEWABLE_FILE: 'non previewable file',
|
||||
PROCESSING_FAILED: 'processing failed',
|
||||
};
|
||||
|
||||
export function parseUploadErrorCodes(error) {
|
||||
|
|
|
|||
Loading…
Reference in a new issue