From f3f7288deb5097e47663b36df414ebc65986ef24 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 2 Jan 2023 21:08:08 +0530 Subject: [PATCH 01/51] add image generation api --- src/api/heicConvert.ts | 9 --- src/api/imageProcessor.ts | 47 ++++++++++++++++ src/preload.ts | 3 +- .../{heicConverter.ts => imageProcessor.ts} | 56 +++++++++++++++++++ src/utils/ipcComms.ts | 9 ++- 5 files changed, 113 insertions(+), 11 deletions(-) delete mode 100644 src/api/heicConvert.ts create mode 100644 src/api/imageProcessor.ts rename src/services/{heicConverter.ts => imageProcessor.ts} (56%) diff --git a/src/api/heicConvert.ts b/src/api/heicConvert.ts deleted file mode 100644 index 68909b678..000000000 --- a/src/api/heicConvert.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { ipcRenderer } from 'electron/renderer'; - -export async function convertHEIC(fileData: Uint8Array): Promise { - const convertedFileData = await ipcRenderer.invoke( - 'convert-heic', - fileData - ); - return convertedFileData; -} diff --git a/src/api/imageProcessor.ts b/src/api/imageProcessor.ts new file mode 100644 index 000000000..ffeb0e60e --- /dev/null +++ b/src/api/imageProcessor.ts @@ -0,0 +1,47 @@ +import { ipcRenderer } from 'electron/renderer'; +import { existsSync } from 'fs'; +import { logError } from '../services/logging'; +import { ElectronFile } from '../types'; + +export async function convertHEIC(fileData: Uint8Array): Promise { + const convertedFileData = await ipcRenderer.invoke( + 'convert-heic', + fileData + ); + return convertedFileData; +} + +export async function generateImageThumbnail( + inputFile: File | ElectronFile, + maxDimension: number +): Promise { + let inputFilePath = null; + let createdTempInputFile = null; + try { + if (!existsSync(inputFile.path)) { + const inputFileData = new Uint8Array(await inputFile.arrayBuffer()); + inputFilePath = await ipcRenderer.invoke( + 'write-temp-file', + inputFileData, + inputFile.name + ); + createdTempInputFile = true; + } else { + inputFilePath = inputFile.path; + } + const thumbnail = await ipcRenderer.invoke( + 'generate-image-thumbnail', + inputFilePath, + maxDimension + ); + return thumbnail; + } finally { + if (createdTempInputFile) { + try { + await ipcRenderer.invoke('remove-temp-file', inputFilePath); + } catch (e) { + logError(e, 'failed to deleteTempFile'); + } + } + } +} diff --git a/src/preload.ts b/src/preload.ts index 504a3016e..03120429b 100644 --- a/src/preload.ts +++ b/src/preload.ts @@ -48,7 +48,7 @@ import { } from './api/common'; import { fixHotReloadNext12 } from './utils/preload'; import { isFolder, getDirFiles } from './api/fs'; -import { convertHEIC } from './api/heicConvert'; +import { convertHEIC, generateImageThumbnail } from './api/imageProcessor'; import { setupLogging } from './utils/logging'; import { setupRendererProcessStatsLogger } from './utils/processStats'; import { runFFmpegCmd } from './api/ffmpeg'; @@ -104,4 +104,5 @@ windowObject['ElectronAPIs'] = { getSentryUserID, getAppVersion, runFFmpegCmd, + generateImageThumbnail, }; diff --git a/src/services/heicConverter.ts b/src/services/imageProcessor.ts similarity index 56% rename from src/services/heicConverter.ts rename to src/services/imageProcessor.ts index e1af0a27f..8dd0f5872 100644 --- a/src/services/heicConverter.ts +++ b/src/services/imageProcessor.ts @@ -70,3 +70,59 @@ async function runConvertCommand( Error(`${process.platform} native heic convert not supported yet`); } } + +export async function generateImageThumbnail( + inputFilePath: string, + width: number +): Promise { + let tempOutputFilePath: string; + try { + tempOutputFilePath = await generateTempFilePath('.jpeg'); + + await runThumbnailGenerationCommand( + inputFilePath, + tempOutputFilePath, + width + ); + + if (!existsSync(tempOutputFilePath)) { + throw new Error('heic convert output file not found'); + } + const convertedFileData = new Uint8Array( + await readFile(tempOutputFilePath) + ); + return convertedFileData; + } catch (e) { + logErrorSentry(e, 'ffmpeg run command error'); + throw e; + } finally { + try { + rmSync(tempOutputFilePath, { force: true }); + } catch (e) { + logErrorSentry(e, 'failed to remove tempOutputFile'); + } + } +} + +async function runThumbnailGenerationCommand( + inputFilePath: string, + tempOutputFilePath: string, + maxDimension: number +) { + if (isPlatform('mac')) { + await asyncExec( + `sips -s format jpeg -Z ${maxDimension} ${inputFilePath} --out ${tempOutputFilePath} ` + ); + } else if (isPlatform('linux')) { + await asyncExec( + `${getImageMagickStaticPath()} -define jpeg:size=${ + 2 * maxDimension + }x${2 * maxDimension} ${inputFilePath} -auto-orient + -thumbnail ${maxDimension}x${maxDimension}> -gravity center -unsharp 0x.5 ${tempOutputFilePath}` + ); + } else { + Error( + `${process.platform} native thumbnail generation not supported yet` + ); + } +} diff --git a/src/utils/ipcComms.ts b/src/utils/ipcComms.ts index debec6273..56a9e565e 100644 --- a/src/utils/ipcComms.ts +++ b/src/utils/ipcComms.ts @@ -14,7 +14,10 @@ import { getSentryUserID, logErrorSentry } from '../services/sentry'; import chokidar from 'chokidar'; import path from 'path'; import { getDirFilePaths } from '../services/fs'; -import { convertHEIC } from '../services/heicConverter'; +import { + convertHEIC, + generateImageThumbnail, +} from '../services/imageProcessor'; import { getAppVersion, skipAppVersion, @@ -146,4 +149,8 @@ export default function setupIpcComs( ipcMain.handle('remove-temp-file', (_, tempFilePath: string) => { return deleteTempFile(tempFilePath); }); + + ipcMain.handle('generate-image-thumbnail', (_, fileData, maxDimension) => { + return generateImageThumbnail(fileData, maxDimension); + }); } From c47e061fdc871d4fc6d706fa9a12e51715f6c54e Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 2 Jan 2023 21:20:04 +0530 Subject: [PATCH 02/51] update UI --- ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui b/ui index a8c520a4b..fe87be3d0 160000 --- a/ui +++ b/ui @@ -1 +1 @@ -Subproject commit a8c520a4b0ff90279cffedf41b9e8f8564d9a753 +Subproject commit fe87be3d02be2a9064f4d8208fe12bd11debbdfc From 43a1af3600a58687e45a524aab413bd273507ca6 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 3 Jan 2023 11:06:50 +0530 Subject: [PATCH 03/51] wrap file paths in quotes --- src/services/imageProcessor.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/services/imageProcessor.ts b/src/services/imageProcessor.ts index 8dd0f5872..67c6f4b12 100644 --- a/src/services/imageProcessor.ts +++ b/src/services/imageProcessor.ts @@ -60,11 +60,11 @@ async function runConvertCommand( ) { if (isPlatform('mac')) { await asyncExec( - `sips -s format jpeg ${tempInputFilePath} --out ${tempOutputFilePath}` + `sips -s format jpeg "${tempInputFilePath}" --out "${tempOutputFilePath}"` ); } else if (isPlatform('linux')) { await asyncExec( - `${getImageMagickStaticPath()} ${tempInputFilePath} -quality 100% ${tempOutputFilePath}` + `${getImageMagickStaticPath()} "${tempInputFilePath}" -quality 100% "${tempOutputFilePath}"` ); } else { Error(`${process.platform} native heic convert not supported yet`); @@ -111,14 +111,14 @@ async function runThumbnailGenerationCommand( ) { if (isPlatform('mac')) { await asyncExec( - `sips -s format jpeg -Z ${maxDimension} ${inputFilePath} --out ${tempOutputFilePath} ` + `sips -s format jpeg -Z ${maxDimension} "${inputFilePath}" --out "${tempOutputFilePath}" ` ); } else if (isPlatform('linux')) { await asyncExec( `${getImageMagickStaticPath()} -define jpeg:size=${ 2 * maxDimension - }x${2 * maxDimension} ${inputFilePath} -auto-orient - -thumbnail ${maxDimension}x${maxDimension}> -gravity center -unsharp 0x.5 ${tempOutputFilePath}` + }x${2 * maxDimension} "${inputFilePath}" -auto-orient + -thumbnail ${maxDimension}x${maxDimension}> -gravity center -unsharp 0x.5 "${tempOutputFilePath}"` ); } else { Error( From 45e400e57b35341759db5e4d8f05ec2f4c659d22 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 3 Jan 2023 11:07:06 +0530 Subject: [PATCH 04/51] update UI --- ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui b/ui index fe87be3d0..e0c245513 160000 --- a/ui +++ b/ui @@ -1 +1 @@ -Subproject commit fe87be3d02be2a9064f4d8208fe12bd11debbdfc +Subproject commit e0c2455132e23ecf4a6f4d889703d5639baf6dda From 5b0fcddab2f6dab3fe6352cfae6a456d65033384 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 3 Jan 2023 11:20:21 +0530 Subject: [PATCH 05/51] update UI --- ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui b/ui index e0c245513..df7e1d0bb 160000 --- a/ui +++ b/ui @@ -1 +1 @@ -Subproject commit e0c2455132e23ecf4a6f4d889703d5639baf6dda +Subproject commit df7e1d0bb64be421cadeba2893d8231fe259e46b From cc8f248e270cd809f078998a52cdf4874483f29a Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 3 Jan 2023 11:28:40 +0530 Subject: [PATCH 06/51] update --- ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui b/ui index df7e1d0bb..7586c391f 160000 --- a/ui +++ b/ui @@ -1 +1 @@ -Subproject commit df7e1d0bb64be421cadeba2893d8231fe259e46b +Subproject commit 7586c391f1b1b1b3c2b222e030fe7aa52d346d89 From e579caca60d0126ca25276b83480296207a5a753 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 3 Jan 2023 11:38:44 +0530 Subject: [PATCH 07/51] better names --- src/services/imageProcessor.ts | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/services/imageProcessor.ts b/src/services/imageProcessor.ts index 67c6f4b12..0c15d7727 100644 --- a/src/services/imageProcessor.ts +++ b/src/services/imageProcessor.ts @@ -23,8 +23,8 @@ export async function convertHEIC( let tempInputFilePath: string; let tempOutputFilePath: string; try { - tempInputFilePath = await generateTempFilePath('.heic'); - tempOutputFilePath = await generateTempFilePath('.jpeg'); + tempInputFilePath = await generateTempFilePath('input.heic'); + tempOutputFilePath = await generateTempFilePath('output.jpeg'); await writeFile(tempInputFilePath, heicFileData); @@ -77,7 +77,7 @@ export async function generateImageThumbnail( ): Promise { let tempOutputFilePath: string; try { - tempOutputFilePath = await generateTempFilePath('.jpeg'); + tempOutputFilePath = await generateTempFilePath('thumb.jpeg'); await runThumbnailGenerationCommand( inputFilePath, @@ -86,14 +86,12 @@ export async function generateImageThumbnail( ); if (!existsSync(tempOutputFilePath)) { - throw new Error('heic convert output file not found'); + throw new Error('output thumbnail file not found'); } - const convertedFileData = new Uint8Array( - await readFile(tempOutputFilePath) - ); - return convertedFileData; + const thumbnail = new Uint8Array(await readFile(tempOutputFilePath)); + return thumbnail; } catch (e) { - logErrorSentry(e, 'ffmpeg run command error'); + logErrorSentry(e, 'generate image thumbnail failed'); throw e; } finally { try { From 03df3faf2d986f5e3630c89cd29064a2d1bb3977 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 3 Jan 2023 11:41:28 +0530 Subject: [PATCH 08/51] v1.6.17-alpha.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 929fc09e3..650e4e404 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "ente", "productName": "ente", - "version": "1.6.16", + "version": "1.6.17-alpha.1", "private": true, "description": "Desktop client for ente.io", "main": "app/main.js", From 0ec70ed77774987d686389532225b8d69986010e Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 3 Jan 2023 13:28:14 +0530 Subject: [PATCH 09/51] update log --- src/services/ffmpeg.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/ffmpeg.ts b/src/services/ffmpeg.ts index dfd24737a..e430346e5 100644 --- a/src/services/ffmpeg.ts +++ b/src/services/ffmpeg.ts @@ -38,7 +38,7 @@ export async function runFFmpegCmd( } }); cmd = shellescape(cmd); - log.info('cmd', cmd); + log.info('running ffmpeg command', cmd); await execAsync(cmd); if (!existsSync(tempOutputFilePath)) { throw new Error('ffmpeg output file not found'); From 28af6873f0b29fecfcf822c327a8dc01e40eb1a3 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 3 Jan 2023 13:29:42 +0530 Subject: [PATCH 10/51] no need to export constants --- src/services/ffmpeg.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/ffmpeg.ts b/src/services/ffmpeg.ts index e430346e5..25df583de 100644 --- a/src/services/ffmpeg.ts +++ b/src/services/ffmpeg.ts @@ -9,9 +9,9 @@ import { existsSync } from 'fs'; const execAsync = util.promisify(require('child_process').exec); -export const INPUT_PATH_PLACEHOLDER = 'INPUT'; -export const FFMPEG_PLACEHOLDER = 'FFMPEG'; -export const OUTPUT_PATH_PLACEHOLDER = 'OUTPUT'; +const INPUT_PATH_PLACEHOLDER = 'INPUT'; +const FFMPEG_PLACEHOLDER = 'FFMPEG'; +const OUTPUT_PATH_PLACEHOLDER = 'OUTPUT'; function getFFmpegStaticPath() { return pathToFfmpeg.replace('app.asar', 'app.asar.unpacked'); From 794c3bf0073337bb0248afddeb1d330ca51b14da Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 3 Jan 2023 13:34:12 +0530 Subject: [PATCH 11/51] refactor code to use command template and escape shell to prevent unescaped characters --- src/services/imageProcessor.ts | 147 ++++++++++++++++++++++++++++++--- 1 file changed, 134 insertions(+), 13 deletions(-) diff --git a/src/services/imageProcessor.ts b/src/services/imageProcessor.ts index 0c15d7727..f4e669d09 100644 --- a/src/services/imageProcessor.ts +++ b/src/services/imageProcessor.ts @@ -8,9 +8,57 @@ import { logErrorSentry } from './sentry'; import { isPlatform } from '../utils/main'; import { isDev } from '../utils/common'; import path from 'path'; +import log from 'electron-log'; +const shellescape = require('any-shell-escape'); const asyncExec = util.promisify(exec); +const IMAGE_MAGICK_PLACEHOLDER = 'IMAGE_MAGICK'; +const MAX_DIMENSION_PLACEHOLDER = 'MAX_DIMENSION'; +const SAMPLE_SIZE_PLACEHOLDER = 'SAMPLE_SIZE'; +const INPUT_PATH_PLACEHOLDER = 'INPUT'; +const OUTPUT_PATH_PLACEHOLDER = 'OUTPUT'; + +const SIPS_HEIC_CONVERT_COMMAND_TEMPLATE = [ + 'sips', + '-s', + 'format', + 'jpeg', + INPUT_PATH_PLACEHOLDER, + '--out', + OUTPUT_PATH_PLACEHOLDER, +]; + +const SIPS_THUMBNAIL_GENERATE_COMMAND_TEMPLATE = [ + 'sips', + '-s', + 'format', + 'jpeg', + INPUT_PATH_PLACEHOLDER, + '-Z', + MAX_DIMENSION_PLACEHOLDER, + '--out', + OUTPUT_PATH_PLACEHOLDER, +]; + +const IMAGEMAGICK_HEIC_CONVERT_COMMAND_TEMPLATE = [ + IMAGE_MAGICK_PLACEHOLDER, + INPUT_PATH_PLACEHOLDER, + '-quality', + '100%', + OUTPUT_PATH_PLACEHOLDER, +]; + +const IMAGE_MAGICK_THUMBNAIL_GENERATE_COMMAND_TEMPLATE = [ + IMAGE_MAGICK_PLACEHOLDER, + '-define', + `jpeg:size=${SAMPLE_SIZE_PLACEHOLDER}x${SAMPLE_SIZE_PLACEHOLDER}`, + INPUT_PATH_PLACEHOLDER, + '-thumbnail', + `${MAX_DIMENSION_PLACEHOLDER}x${MAX_DIMENSION_PLACEHOLDER}>`, + OUTPUT_PATH_PLACEHOLDER, +]; + function getImageMagickStaticPath() { return isDev ? 'build/image-magick' @@ -58,17 +106,49 @@ async function runConvertCommand( tempInputFilePath: string, tempOutputFilePath: string ) { + const convertCmd = constructConvertCommand( + tempInputFilePath, + tempOutputFilePath + ); + const escapedCmd = shellescape(convertCmd); + log.info('running convert command: ' + escapedCmd); + await asyncExec(escapedCmd); +} + +function constructConvertCommand( + tempInputFilePath: string, + tempOutputFilePath: string +) { + let convertCmd: string[]; if (isPlatform('mac')) { - await asyncExec( - `sips -s format jpeg "${tempInputFilePath}" --out "${tempOutputFilePath}"` - ); + convertCmd = SIPS_HEIC_CONVERT_COMMAND_TEMPLATE.map((cmdPart) => { + if (cmdPart === INPUT_PATH_PLACEHOLDER) { + return tempInputFilePath; + } + if (cmdPart === OUTPUT_PATH_PLACEHOLDER) { + return tempOutputFilePath; + } + return cmdPart; + }); } else if (isPlatform('linux')) { - await asyncExec( - `${getImageMagickStaticPath()} "${tempInputFilePath}" -quality 100% "${tempOutputFilePath}"` + convertCmd = IMAGEMAGICK_HEIC_CONVERT_COMMAND_TEMPLATE.map( + (cmdPart) => { + if (cmdPart === IMAGE_MAGICK_PLACEHOLDER) { + return getImageMagickStaticPath(); + } + if (cmdPart === INPUT_PATH_PLACEHOLDER) { + return tempInputFilePath; + } + if (cmdPart === OUTPUT_PATH_PLACEHOLDER) { + return tempOutputFilePath; + } + return cmdPart; + } ); } else { Error(`${process.platform} native heic convert not supported yet`); } + return convertCmd; } export async function generateImageThumbnail( @@ -107,20 +187,61 @@ async function runThumbnailGenerationCommand( tempOutputFilePath: string, maxDimension: number ) { + const thumbnailGenerationCmd: string[] = + constructThumbnailGenerationCommand( + inputFilePath, + tempOutputFilePath, + maxDimension + ); + const escapedCmd = shellescape(thumbnailGenerationCmd); + log.info('running thumbnail generation command: ' + escapedCmd); + await asyncExec(escapedCmd); +} +function constructThumbnailGenerationCommand( + inputFilePath: string, + tempOutputFilePath: string, + maxDimension: number +) { + let thumbnailGenerationCmd: string[]; if (isPlatform('mac')) { - await asyncExec( - `sips -s format jpeg -Z ${maxDimension} "${inputFilePath}" --out "${tempOutputFilePath}" ` + thumbnailGenerationCmd = SIPS_THUMBNAIL_GENERATE_COMMAND_TEMPLATE.map( + (cmdPart) => { + if (cmdPart === INPUT_PATH_PLACEHOLDER) { + return inputFilePath; + } + if (cmdPart === OUTPUT_PATH_PLACEHOLDER) { + return tempOutputFilePath; + } + if (cmdPart === MAX_DIMENSION_PLACEHOLDER) { + return maxDimension.toString(); + } + return cmdPart; + } ); } else if (isPlatform('linux')) { - await asyncExec( - `${getImageMagickStaticPath()} -define jpeg:size=${ - 2 * maxDimension - }x${2 * maxDimension} "${inputFilePath}" -auto-orient - -thumbnail ${maxDimension}x${maxDimension}> -gravity center -unsharp 0x.5 "${tempOutputFilePath}"` - ); + thumbnailGenerationCmd = + IMAGE_MAGICK_THUMBNAIL_GENERATE_COMMAND_TEMPLATE.map((cmdPart) => { + if (cmdPart === IMAGE_MAGICK_PLACEHOLDER) { + return getImageMagickStaticPath(); + } + if (cmdPart === INPUT_PATH_PLACEHOLDER) { + return inputFilePath; + } + if (cmdPart === OUTPUT_PATH_PLACEHOLDER) { + return tempOutputFilePath; + } + if (cmdPart === SAMPLE_SIZE_PLACEHOLDER) { + return (2 * maxDimension).toString(); + } + if (cmdPart === MAX_DIMENSION_PLACEHOLDER) { + return maxDimension.toString(); + } + return cmdPart; + }); } else { Error( `${process.platform} native thumbnail generation not supported yet` ); } + return thumbnailGenerationCmd; } From bfdf1c04d70a5f91dcab71b9f0c9c0a077efca0d Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 3 Jan 2023 14:09:46 +0530 Subject: [PATCH 12/51] update tsconfig to es2021 --- tsconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsconfig.json b/tsconfig.json index 6f29749e1..283ef57b4 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "es2017", + "target": "es2021", "module": "commonjs", "esModuleInterop": true, "noImplicitAny": true, From 6fa796bc0b4ddd7085abc66df181d616f1c6bf4d Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 3 Jan 2023 14:10:51 +0530 Subject: [PATCH 13/51] use find and replaceAll for SAMPLE_SIZE_PLACEHOLDER MAX_DIMENSION_PLACEHOLDER --- src/services/imageProcessor.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/services/imageProcessor.ts b/src/services/imageProcessor.ts index f4e669d09..95f2b753f 100644 --- a/src/services/imageProcessor.ts +++ b/src/services/imageProcessor.ts @@ -230,11 +230,17 @@ function constructThumbnailGenerationCommand( if (cmdPart === OUTPUT_PATH_PLACEHOLDER) { return tempOutputFilePath; } - if (cmdPart === SAMPLE_SIZE_PLACEHOLDER) { - return (2 * maxDimension).toString(); + if (cmdPart.includes(SAMPLE_SIZE_PLACEHOLDER)) { + return cmdPart.replaceAll( + SAMPLE_SIZE_PLACEHOLDER, + (2 * maxDimension).toString() + ); } - if (cmdPart === MAX_DIMENSION_PLACEHOLDER) { - return maxDimension.toString(); + if (cmdPart.includes(MAX_DIMENSION_PLACEHOLDER)) { + return cmdPart.replaceAll( + MAX_DIMENSION_PLACEHOLDER, + maxDimension.toString() + ); } return cmdPart; }); From 9fea5e41497f47ee27960830622d4d007a061068 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 3 Jan 2023 14:11:11 +0530 Subject: [PATCH 14/51] v1.6.17-alpha.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 650e4e404..ed49d043d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "ente", "productName": "ente", - "version": "1.6.17-alpha.1", + "version": "1.6.17-alpha.2", "private": true, "description": "Desktop client for ente.io", "main": "app/main.js", From d8c69c5acc22b4dcb704d1cdfb43893e902d016f Mon Sep 17 00:00:00 2001 From: Abhinav Date: Thu, 5 Jan 2023 10:53:57 +0530 Subject: [PATCH 15/51] add size parameter --- src/api/imageProcessor.ts | 6 ++-- src/services/imageProcessor.ts | 55 +++++++++++++++++++++++++--------- src/utils/ipcComms.ts | 9 ++++-- 3 files changed, 51 insertions(+), 19 deletions(-) diff --git a/src/api/imageProcessor.ts b/src/api/imageProcessor.ts index ffeb0e60e..07389bd71 100644 --- a/src/api/imageProcessor.ts +++ b/src/api/imageProcessor.ts @@ -13,7 +13,8 @@ export async function convertHEIC(fileData: Uint8Array): Promise { export async function generateImageThumbnail( inputFile: File | ElectronFile, - maxDimension: number + maxDimension: number, + maxSize: number ): Promise { let inputFilePath = null; let createdTempInputFile = null; @@ -32,7 +33,8 @@ export async function generateImageThumbnail( const thumbnail = await ipcRenderer.invoke( 'generate-image-thumbnail', inputFilePath, - maxDimension + maxDimension, + maxSize ); return thumbnail; } finally { diff --git a/src/services/imageProcessor.ts b/src/services/imageProcessor.ts index 95f2b753f..198b2916b 100644 --- a/src/services/imageProcessor.ts +++ b/src/services/imageProcessor.ts @@ -18,6 +18,10 @@ const MAX_DIMENSION_PLACEHOLDER = 'MAX_DIMENSION'; const SAMPLE_SIZE_PLACEHOLDER = 'SAMPLE_SIZE'; const INPUT_PATH_PLACEHOLDER = 'INPUT'; const OUTPUT_PATH_PLACEHOLDER = 'OUTPUT'; +const QUALITY_PLACEHOLDER = 'QUALITY'; + +const MAX_QUALITY = 80; +const MIN_QUALITY = 50; const SIPS_HEIC_CONVERT_COMMAND_TEMPLATE = [ 'sips', @@ -56,6 +60,10 @@ const IMAGE_MAGICK_THUMBNAIL_GENERATE_COMMAND_TEMPLATE = [ INPUT_PATH_PLACEHOLDER, '-thumbnail', `${MAX_DIMENSION_PLACEHOLDER}x${MAX_DIMENSION_PLACEHOLDER}>`, + '-unsharp', + '0x.5', + '-quality', + QUALITY_PLACEHOLDER, OUTPUT_PATH_PLACEHOLDER, ]; @@ -153,22 +161,35 @@ function constructConvertCommand( export async function generateImageThumbnail( inputFilePath: string, - width: number + width: number, + maxSize: number ): Promise { let tempOutputFilePath: string; + let quality = MAX_QUALITY; try { tempOutputFilePath = await generateTempFilePath('thumb.jpeg'); + let thumbnail: Uint8Array; + do { + console.log( + 'generating thumbnail', + 'quality', + quality, + 'thumbnail', + thumbnail?.length + ); + await runThumbnailGenerationCommand( + inputFilePath, + tempOutputFilePath, + width, + quality + ); - await runThumbnailGenerationCommand( - inputFilePath, - tempOutputFilePath, - width - ); - - if (!existsSync(tempOutputFilePath)) { - throw new Error('output thumbnail file not found'); - } - const thumbnail = new Uint8Array(await readFile(tempOutputFilePath)); + if (!existsSync(tempOutputFilePath)) { + throw new Error('output thumbnail file not found'); + } + thumbnail = new Uint8Array(await readFile(tempOutputFilePath)); + quality -= 10; + } while (thumbnail.length > maxSize && quality > MIN_QUALITY); return thumbnail; } catch (e) { logErrorSentry(e, 'generate image thumbnail failed'); @@ -185,13 +206,15 @@ export async function generateImageThumbnail( async function runThumbnailGenerationCommand( inputFilePath: string, tempOutputFilePath: string, - maxDimension: number + maxDimension: number, + quality: number ) { const thumbnailGenerationCmd: string[] = constructThumbnailGenerationCommand( inputFilePath, tempOutputFilePath, - maxDimension + maxDimension, + quality ); const escapedCmd = shellescape(thumbnailGenerationCmd); log.info('running thumbnail generation command: ' + escapedCmd); @@ -200,7 +223,8 @@ async function runThumbnailGenerationCommand( function constructThumbnailGenerationCommand( inputFilePath: string, tempOutputFilePath: string, - maxDimension: number + maxDimension: number, + quality: number ) { let thumbnailGenerationCmd: string[]; if (isPlatform('mac')) { @@ -215,6 +239,9 @@ function constructThumbnailGenerationCommand( if (cmdPart === MAX_DIMENSION_PLACEHOLDER) { return maxDimension.toString(); } + if (cmdPart === QUALITY_PLACEHOLDER) { + return quality.toString(); + } return cmdPart; } ); diff --git a/src/utils/ipcComms.ts b/src/utils/ipcComms.ts index 56a9e565e..2bbe39452 100644 --- a/src/utils/ipcComms.ts +++ b/src/utils/ipcComms.ts @@ -150,7 +150,10 @@ export default function setupIpcComs( return deleteTempFile(tempFilePath); }); - ipcMain.handle('generate-image-thumbnail', (_, fileData, maxDimension) => { - return generateImageThumbnail(fileData, maxDimension); - }); + ipcMain.handle( + 'generate-image-thumbnail', + (_, fileData, maxDimension, maxSize) => { + return generateImageThumbnail(fileData, maxDimension, maxSize); + } + ); } From 462b1209fee6d1c51465926f7b826a1edb55336d Mon Sep 17 00:00:00 2001 From: Abhinav Date: Thu, 5 Jan 2023 10:54:32 +0530 Subject: [PATCH 16/51] update UI --- ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui b/ui index 7586c391f..6df2e22e9 160000 --- a/ui +++ b/ui @@ -1 +1 @@ -Subproject commit 7586c391f1b1b1b3c2b222e030fe7aa52d346d89 +Subproject commit 6df2e22e9d03e12fbfb6b7f25a898fcc73872f1d From dc658ca28787e8c442828c9e0b2451837e1409c4 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Thu, 5 Jan 2023 10:55:12 +0530 Subject: [PATCH 17/51] update UI --- ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui b/ui index 6df2e22e9..f54d1a01f 160000 --- a/ui +++ b/ui @@ -1 +1 @@ -Subproject commit 6df2e22e9d03e12fbfb6b7f25a898fcc73872f1d +Subproject commit f54d1a01faf747903cc2213ea08e944d78bf35fb From 7a41aee5ddac7a5be7bd9674a3b4caa42724eaaa Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 6 Jan 2023 10:41:53 +0530 Subject: [PATCH 18/51] fix thumbnail generation cmds --- src/services/imageProcessor.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/services/imageProcessor.ts b/src/services/imageProcessor.ts index 198b2916b..93006c649 100644 --- a/src/services/imageProcessor.ts +++ b/src/services/imageProcessor.ts @@ -38,9 +38,12 @@ const SIPS_THUMBNAIL_GENERATE_COMMAND_TEMPLATE = [ '-s', 'format', 'jpeg', - INPUT_PATH_PLACEHOLDER, + '-s', + 'formatOptions', + QUALITY_PLACEHOLDER, '-Z', MAX_DIMENSION_PLACEHOLDER, + INPUT_PATH_PLACEHOLDER, '--out', OUTPUT_PATH_PLACEHOLDER, ]; @@ -269,6 +272,9 @@ function constructThumbnailGenerationCommand( maxDimension.toString() ); } + if (cmdPart === QUALITY_PLACEHOLDER) { + return quality.toString(); + } return cmdPart; }); } else { From b83959e53a7a6af71d5a5546f4b78bde563121ec Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 6 Jan 2023 10:58:54 +0530 Subject: [PATCH 19/51] update UI --- ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui b/ui index f54d1a01f..6529142ec 160000 --- a/ui +++ b/ui @@ -1 +1 @@ -Subproject commit f54d1a01faf747903cc2213ea08e944d78bf35fb +Subproject commit 6529142ecee5c16071b1597835f3d2565cf0f56a From e736648e9acdb6d908a1077023560915eba19759 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 6 Jan 2023 11:24:24 +0530 Subject: [PATCH 20/51] update UI --- ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui b/ui index 6529142ec..4d868ffe5 160000 --- a/ui +++ b/ui @@ -1 +1 @@ -Subproject commit 6529142ecee5c16071b1597835f3d2565cf0f56a +Subproject commit 4d868ffe5cffb8e71da368fd257d8712f2d56409 From 0a4c1052a3417df1bcc12052ad1657cfd4962256 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 6 Jan 2023 11:24:41 +0530 Subject: [PATCH 21/51] v1.6.17-alpha.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ed49d043d..e1ed6ff2f 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "ente", "productName": "ente", - "version": "1.6.17-alpha.2", + "version": "1.6.17-alpha.3", "private": true, "description": "Desktop client for ente.io", "main": "app/main.js", From 452c63692479eac869f715a825efe87338282a52 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 6 Jan 2023 12:14:02 +0530 Subject: [PATCH 22/51] update UI --- ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui b/ui index 4d868ffe5..afa9ba052 160000 --- a/ui +++ b/ui @@ -1 +1 @@ -Subproject commit 4d868ffe5cffb8e71da368fd257d8712f2d56409 +Subproject commit afa9ba05218aaaa3f9f1ddcecd4cfd00607f5b74 From efbdf66bfde81244c9d82559a02e2467f23ac037 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 6 Jan 2023 12:20:48 +0530 Subject: [PATCH 23/51] reduce max quality --- src/services/imageProcessor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/imageProcessor.ts b/src/services/imageProcessor.ts index 93006c649..7d5fed0a5 100644 --- a/src/services/imageProcessor.ts +++ b/src/services/imageProcessor.ts @@ -20,7 +20,7 @@ const INPUT_PATH_PLACEHOLDER = 'INPUT'; const OUTPUT_PATH_PLACEHOLDER = 'OUTPUT'; const QUALITY_PLACEHOLDER = 'QUALITY'; -const MAX_QUALITY = 80; +const MAX_QUALITY = 70; const MIN_QUALITY = 50; const SIPS_HEIC_CONVERT_COMMAND_TEMPLATE = [ From d8b81028b3231c9dd0fbc9011b3605f8ea5f32b8 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 6 Jan 2023 12:24:35 +0530 Subject: [PATCH 24/51] remove console log --- src/services/imageProcessor.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/services/imageProcessor.ts b/src/services/imageProcessor.ts index 7d5fed0a5..e211e2b64 100644 --- a/src/services/imageProcessor.ts +++ b/src/services/imageProcessor.ts @@ -173,13 +173,6 @@ export async function generateImageThumbnail( tempOutputFilePath = await generateTempFilePath('thumb.jpeg'); let thumbnail: Uint8Array; do { - console.log( - 'generating thumbnail', - 'quality', - quality, - 'thumbnail', - thumbnail?.length - ); await runThumbnailGenerationCommand( inputFilePath, tempOutputFilePath, From 9117926b67f2098a1984855d05f5c390c2482f1b Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 6 Jan 2023 12:38:33 +0530 Subject: [PATCH 25/51] use new variable for escaped command --- src/services/ffmpeg.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/ffmpeg.ts b/src/services/ffmpeg.ts index 25df583de..d8c1853fc 100644 --- a/src/services/ffmpeg.ts +++ b/src/services/ffmpeg.ts @@ -37,9 +37,9 @@ export async function runFFmpegCmd( return cmdPart; } }); - cmd = shellescape(cmd); - log.info('running ffmpeg command', cmd); - await execAsync(cmd); + const escapedCmd = shellescape(cmd); + log.info('running ffmpeg command', escapedCmd); + await execAsync(escapedCmd); if (!existsSync(tempOutputFilePath)) { throw new Error('ffmpeg output file not found'); } From 5980e06da8de91bd1a8b595ff10af47f381d9533 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 6 Jan 2023 12:38:51 +0530 Subject: [PATCH 26/51] update UI --- ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui b/ui index afa9ba052..a955ae39f 160000 --- a/ui +++ b/ui @@ -1 +1 @@ -Subproject commit afa9ba05218aaaa3f9f1ddcecd4cfd00607f5b74 +Subproject commit a955ae39fa97232131682ad72d866bf3155a0906 From fc9823edc770e9a209e8947519a588a28ad3dc31 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 6 Jan 2023 13:09:36 +0530 Subject: [PATCH 27/51] update UI --- ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui b/ui index a955ae39f..86a7f1cda 160000 --- a/ui +++ b/ui @@ -1 +1 @@ -Subproject commit a955ae39fa97232131682ad72d866bf3155a0906 +Subproject commit 86a7f1cdab5c3a9a2cfbd0daecb9f7e1dd06660e From d71d65073f6b5d3aa0d5e3794e131502a18fcd79 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 6 Jan 2023 13:09:57 +0530 Subject: [PATCH 28/51] rename constant --- src/utils/processStats.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/utils/processStats.ts b/src/utils/processStats.ts index 2b8d8f054..6f3baaa30 100644 --- a/src/utils/processStats.ts +++ b/src/utils/processStats.ts @@ -1,7 +1,7 @@ import ElectronLog from 'electron-log'; import { webFrame } from 'electron/renderer'; -const FIVE_MINUTES_IN_MICROSECONDS = 30 * 1000; +const LOGGING_INTERVAL_IN_MICROSECONDS = 1 * 1000; async function logMainProcessStats() { const systemMemoryInfo = process.getSystemMemoryInfo(); @@ -31,9 +31,9 @@ async function logRendererProcessStats() { } export function setupMainProcessStatsLogger() { - setInterval(logMainProcessStats, FIVE_MINUTES_IN_MICROSECONDS); + setInterval(logMainProcessStats, LOGGING_INTERVAL_IN_MICROSECONDS); } export function setupRendererProcessStatsLogger() { - setInterval(logRendererProcessStats, FIVE_MINUTES_IN_MICROSECONDS); + setInterval(logRendererProcessStats, LOGGING_INTERVAL_IN_MICROSECONDS); } From a66d720dc2b461fe753093f15d548f62218b4f14 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 6 Jan 2023 13:18:47 +0530 Subject: [PATCH 29/51] convert process stats to human readable format value --- src/utils/processStats.ts | 58 +++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/src/utils/processStats.ts b/src/utils/processStats.ts index 6f3baaa30..1a4dd7806 100644 --- a/src/utils/processStats.ts +++ b/src/utils/processStats.ts @@ -4,28 +4,31 @@ import { webFrame } from 'electron/renderer'; const LOGGING_INTERVAL_IN_MICROSECONDS = 1 * 1000; async function logMainProcessStats() { - const systemMemoryInfo = process.getSystemMemoryInfo(); - const cpuUsage = process.getCPUUsage(); const processMemoryInfo = await process.getProcessMemoryInfo(); + const normalizedProcessMemoryInfo = await getNormalizedProcessMemoryInfo( + processMemoryInfo + ); + const systemMemoryInfo = process.getSystemMemoryInfo(); + const normalizedSystemMemoryInfo = + getNormalizedSystemMemoryInfo(systemMemoryInfo); + const cpuUsage = process.getCPUUsage(); const heapStatistics = process.getHeapStatistics(); ElectronLog.log('main process stats', { - systemMemoryInfo, - cpuUsage, - processMemoryInfo, + processMemoryInfo: normalizedProcessMemoryInfo, + systemMemoryInfo: normalizedSystemMemoryInfo, heapStatistics, + cpuUsage, }); } async function logRendererProcessStats() { - const blinkMemoryInfo = process.getBlinkMemoryInfo(); + const blinkMemoryInfo = getNormalizedBlinkMemoryInfo(); const heapStatistics = process.getHeapStatistics(); - const processMemoryInfo = process.getProcessMemoryInfo(); const webFrameResourceUsage = webFrame.getResourceUsage(); ElectronLog.log('renderer process stats', { blinkMemoryInfo, heapStatistics, - processMemoryInfo, webFrameResourceUsage, }); } @@ -37,3 +40,42 @@ export function setupMainProcessStatsLogger() { export function setupRendererProcessStatsLogger() { setInterval(logRendererProcessStats, LOGGING_INTERVAL_IN_MICROSECONDS); } + +const getNormalizedProcessMemoryInfo = async ( + processMemoryInfo: Electron.ProcessMemoryInfo +) => { + return { + residentSet: convertBytesToHumanReadable(processMemoryInfo.residentSet), + private: convertBytesToHumanReadable(processMemoryInfo.private), + shared: convertBytesToHumanReadable(processMemoryInfo.shared), + }; +}; + +const getNormalizedSystemMemoryInfo = ( + systemMemoryInfo: Electron.SystemMemoryInfo +) => { + return { + total: convertBytesToHumanReadable(systemMemoryInfo.total), + free: convertBytesToHumanReadable(systemMemoryInfo.free), + swapTotal: convertBytesToHumanReadable(systemMemoryInfo.swapTotal), + swapFree: convertBytesToHumanReadable(systemMemoryInfo.swapFree), + }; +}; + +const getNormalizedBlinkMemoryInfo = () => { + const blinkMemoryInfo = process.getBlinkMemoryInfo(); + return { + allocated: convertBytesToHumanReadable(blinkMemoryInfo.allocated), + total: convertBytesToHumanReadable(blinkMemoryInfo.total), + }; +}; + +function convertBytesToHumanReadable(bytes: number, precision = 2): string { + if (bytes === 0) { + return '0 MB'; + } + + const i = Math.floor(Math.log(bytes) / Math.log(1024)); + const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']; + return (bytes / Math.pow(1024, i)).toFixed(precision) + ' ' + sizes[i]; +} From 54fc0705cdc4db66eda24f6529801134ddfedce0 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 6 Jan 2023 14:58:23 +0530 Subject: [PATCH 30/51] fix units and add spike logger --- src/utils/processStats.ts | 58 +++++++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/src/utils/processStats.ts b/src/utils/processStats.ts index 1a4dd7806..43ded4976 100644 --- a/src/utils/processStats.ts +++ b/src/utils/processStats.ts @@ -1,7 +1,14 @@ import ElectronLog from 'electron-log'; import { webFrame } from 'electron/renderer'; +import { isPlatform } from './main'; -const LOGGING_INTERVAL_IN_MICROSECONDS = 1 * 1000; +const LOGGING_INTERVAL_IN_MICROSECONDS = 30 * 1000; // 30 seconds + +const SPIKE_DETECTION_INTERVAL_IN_MICROSECONDS = 1 * 1000; // 1 seconds + +const HIGH_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES = 1 * 1024 * 1024; // 1 GB + +const LOW_MEMORY_FREE_THRESHOLD_IN_KILOBYTES = 1 * 1024 * 1024; // 1 GB async function logMainProcessStats() { const processMemoryInfo = await process.getProcessMemoryInfo(); @@ -22,6 +29,30 @@ async function logMainProcessStats() { }); } +async function logSpikeMemoryUsage() { + const processMemoryInfo = await process.getProcessMemoryInfo(); + const systemMemoryInfo = process.getSystemMemoryInfo(); + if ( + processMemoryInfo.residentSet > + HIGH_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES || + systemMemoryInfo.free < LOW_MEMORY_FREE_THRESHOLD_IN_KILOBYTES + ) { + const normalizedProcessMemoryInfo = + await getNormalizedProcessMemoryInfo(processMemoryInfo); + const normalizedSystemMemoryInfo = + getNormalizedSystemMemoryInfo(systemMemoryInfo); + const cpuUsage = process.getCPUUsage(); + const heapStatistics = process.getHeapStatistics(); + + ElectronLog.log('main process stats', { + processMemoryInfo: normalizedProcessMemoryInfo, + systemMemoryInfo: normalizedSystemMemoryInfo, + heapStatistics, + cpuUsage, + }); + } +} + async function logRendererProcessStats() { const blinkMemoryInfo = getNormalizedBlinkMemoryInfo(); const heapStatistics = process.getHeapStatistics(); @@ -34,6 +65,7 @@ async function logRendererProcessStats() { } export function setupMainProcessStatsLogger() { + setInterval(logSpikeMemoryUsage, SPIKE_DETECTION_INTERVAL_IN_MICROSECONDS); setInterval(logMainProcessStats, LOGGING_INTERVAL_IN_MICROSECONDS); } @@ -45,9 +77,11 @@ const getNormalizedProcessMemoryInfo = async ( processMemoryInfo: Electron.ProcessMemoryInfo ) => { return { - residentSet: convertBytesToHumanReadable(processMemoryInfo.residentSet), - private: convertBytesToHumanReadable(processMemoryInfo.private), - shared: convertBytesToHumanReadable(processMemoryInfo.shared), + residentSet: !isPlatform('mac') + ? convertBytesToHumanReadable(processMemoryInfo.residentSet * 1024) + : 0, + private: convertBytesToHumanReadable(processMemoryInfo.private * 10124), + shared: convertBytesToHumanReadable(processMemoryInfo.shared * 1024), }; }; @@ -55,17 +89,23 @@ const getNormalizedSystemMemoryInfo = ( systemMemoryInfo: Electron.SystemMemoryInfo ) => { return { - total: convertBytesToHumanReadable(systemMemoryInfo.total), - free: convertBytesToHumanReadable(systemMemoryInfo.free), - swapTotal: convertBytesToHumanReadable(systemMemoryInfo.swapTotal), - swapFree: convertBytesToHumanReadable(systemMemoryInfo.swapFree), + total: convertBytesToHumanReadable(systemMemoryInfo.total * 1024), + free: convertBytesToHumanReadable(systemMemoryInfo.free * 1024), + swapTotal: !isPlatform('mac') + ? convertBytesToHumanReadable(systemMemoryInfo.swapTotal * 1024) + : 0, + swapFree: !isPlatform('mac') + ? convertBytesToHumanReadable(systemMemoryInfo.swapFree * 1024) + : 0, }; }; const getNormalizedBlinkMemoryInfo = () => { const blinkMemoryInfo = process.getBlinkMemoryInfo(); return { - allocated: convertBytesToHumanReadable(blinkMemoryInfo.allocated), + allocated: convertBytesToHumanReadable( + blinkMemoryInfo.allocated * 1024 + ), total: convertBytesToHumanReadable(blinkMemoryInfo.total), }; }; From d18462535eb5da6b99608651e6451cd1ba09ee0b Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 6 Jan 2023 17:01:30 +0530 Subject: [PATCH 31/51] no need of platform check --- src/utils/processStats.ts | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/utils/processStats.ts b/src/utils/processStats.ts index 43ded4976..f9bb273a9 100644 --- a/src/utils/processStats.ts +++ b/src/utils/processStats.ts @@ -1,6 +1,5 @@ import ElectronLog from 'electron-log'; import { webFrame } from 'electron/renderer'; -import { isPlatform } from './main'; const LOGGING_INTERVAL_IN_MICROSECONDS = 30 * 1000; // 30 seconds @@ -77,9 +76,9 @@ const getNormalizedProcessMemoryInfo = async ( processMemoryInfo: Electron.ProcessMemoryInfo ) => { return { - residentSet: !isPlatform('mac') - ? convertBytesToHumanReadable(processMemoryInfo.residentSet * 1024) - : 0, + residentSet: convertBytesToHumanReadable( + processMemoryInfo.residentSet * 1024 + ), private: convertBytesToHumanReadable(processMemoryInfo.private * 10124), shared: convertBytesToHumanReadable(processMemoryInfo.shared * 1024), }; @@ -91,12 +90,10 @@ const getNormalizedSystemMemoryInfo = ( return { total: convertBytesToHumanReadable(systemMemoryInfo.total * 1024), free: convertBytesToHumanReadable(systemMemoryInfo.free * 1024), - swapTotal: !isPlatform('mac') - ? convertBytesToHumanReadable(systemMemoryInfo.swapTotal * 1024) - : 0, - swapFree: !isPlatform('mac') - ? convertBytesToHumanReadable(systemMemoryInfo.swapFree * 1024) - : 0, + swapTotal: convertBytesToHumanReadable( + systemMemoryInfo.swapTotal * 1024 + ), + swapFree: convertBytesToHumanReadable(systemMemoryInfo.swapFree * 1024), }; }; From 4c36a3b80833938a0d98e2397a4f1600205eb0eb Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 6 Jan 2023 21:56:27 +0530 Subject: [PATCH 32/51] prevent nan --- src/utils/processStats.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/processStats.ts b/src/utils/processStats.ts index f9bb273a9..2361e7036 100644 --- a/src/utils/processStats.ts +++ b/src/utils/processStats.ts @@ -108,7 +108,7 @@ const getNormalizedBlinkMemoryInfo = () => { }; function convertBytesToHumanReadable(bytes: number, precision = 2): string { - if (bytes === 0) { + if (bytes === 0 || isNaN(bytes)) { return '0 MB'; } From 7fabf2c5c91e75b1ea38d08db0e9ec4964061abd Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 6 Jan 2023 22:40:20 +0530 Subject: [PATCH 33/51] add getNormalizedHeapStatistics util --- src/utils/processStats.ts | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/src/utils/processStats.ts b/src/utils/processStats.ts index 2361e7036..2f0dc2ce0 100644 --- a/src/utils/processStats.ts +++ b/src/utils/processStats.ts @@ -18,7 +18,7 @@ async function logMainProcessStats() { const normalizedSystemMemoryInfo = getNormalizedSystemMemoryInfo(systemMemoryInfo); const cpuUsage = process.getCPUUsage(); - const heapStatistics = process.getHeapStatistics(); + const heapStatistics = getNormalizedHeapStatistics(); ElectronLog.log('main process stats', { processMemoryInfo: normalizedProcessMemoryInfo, @@ -41,7 +41,7 @@ async function logSpikeMemoryUsage() { const normalizedSystemMemoryInfo = getNormalizedSystemMemoryInfo(systemMemoryInfo); const cpuUsage = process.getCPUUsage(); - const heapStatistics = process.getHeapStatistics(); + const heapStatistics = getNormalizedHeapStatistics(); ElectronLog.log('main process stats', { processMemoryInfo: normalizedProcessMemoryInfo, @@ -107,6 +107,38 @@ const getNormalizedBlinkMemoryInfo = () => { }; }; +const getNormalizedHeapStatistics = () => { + const heapStatistics = process.getHeapStatistics(); + return { + totalHeapSize: convertBytesToHumanReadable( + heapStatistics.totalHeapSize * 1024 + ), + totalHeapSizeExecutable: convertBytesToHumanReadable( + heapStatistics.totalHeapSizeExecutable * 1024 + ), + totalPhysicalSize: convertBytesToHumanReadable( + heapStatistics.totalPhysicalSize * 1024 + ), + totalAvailableSize: convertBytesToHumanReadable( + heapStatistics.totalAvailableSize * 1024 + ), + usedHeapSize: convertBytesToHumanReadable( + heapStatistics.usedHeapSize * 1024 + ), + + heapSizeLimit: convertBytesToHumanReadable( + heapStatistics.heapSizeLimit * 1024 + ), + mallocedMemory: convertBytesToHumanReadable( + heapStatistics.mallocedMemory * 1024 + ), + peakMallocedMemory: convertBytesToHumanReadable( + heapStatistics.peakMallocedMemory * 1024 + ), + doesZapGarbage: heapStatistics.doesZapGarbage, + }; +}; + function convertBytesToHumanReadable(bytes: number, precision = 2): string { if (bytes === 0 || isNaN(bytes)) { return '0 MB'; From ae8326a773401e583fb4f9e62eb5225f0ad9fb5f Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 6 Jan 2023 22:41:33 +0530 Subject: [PATCH 34/51] remove getSystemMemoryInfo as free memory reported is incorrect --- src/utils/processStats.ts | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/src/utils/processStats.ts b/src/utils/processStats.ts index 2f0dc2ce0..c9b5eaa68 100644 --- a/src/utils/processStats.ts +++ b/src/utils/processStats.ts @@ -7,22 +7,16 @@ const SPIKE_DETECTION_INTERVAL_IN_MICROSECONDS = 1 * 1000; // 1 seconds const HIGH_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES = 1 * 1024 * 1024; // 1 GB -const LOW_MEMORY_FREE_THRESHOLD_IN_KILOBYTES = 1 * 1024 * 1024; // 1 GB - async function logMainProcessStats() { const processMemoryInfo = await process.getProcessMemoryInfo(); const normalizedProcessMemoryInfo = await getNormalizedProcessMemoryInfo( processMemoryInfo ); - const systemMemoryInfo = process.getSystemMemoryInfo(); - const normalizedSystemMemoryInfo = - getNormalizedSystemMemoryInfo(systemMemoryInfo); const cpuUsage = process.getCPUUsage(); const heapStatistics = getNormalizedHeapStatistics(); ElectronLog.log('main process stats', { processMemoryInfo: normalizedProcessMemoryInfo, - systemMemoryInfo: normalizedSystemMemoryInfo, heapStatistics, cpuUsage, }); @@ -30,22 +24,16 @@ async function logMainProcessStats() { async function logSpikeMemoryUsage() { const processMemoryInfo = await process.getProcessMemoryInfo(); - const systemMemoryInfo = process.getSystemMemoryInfo(); if ( - processMemoryInfo.residentSet > - HIGH_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES || - systemMemoryInfo.free < LOW_MEMORY_FREE_THRESHOLD_IN_KILOBYTES + processMemoryInfo.residentSet > HIGH_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES ) { const normalizedProcessMemoryInfo = await getNormalizedProcessMemoryInfo(processMemoryInfo); - const normalizedSystemMemoryInfo = - getNormalizedSystemMemoryInfo(systemMemoryInfo); const cpuUsage = process.getCPUUsage(); const heapStatistics = getNormalizedHeapStatistics(); ElectronLog.log('main process stats', { processMemoryInfo: normalizedProcessMemoryInfo, - systemMemoryInfo: normalizedSystemMemoryInfo, heapStatistics, cpuUsage, }); @@ -84,19 +72,6 @@ const getNormalizedProcessMemoryInfo = async ( }; }; -const getNormalizedSystemMemoryInfo = ( - systemMemoryInfo: Electron.SystemMemoryInfo -) => { - return { - total: convertBytesToHumanReadable(systemMemoryInfo.total * 1024), - free: convertBytesToHumanReadable(systemMemoryInfo.free * 1024), - swapTotal: convertBytesToHumanReadable( - systemMemoryInfo.swapTotal * 1024 - ), - swapFree: convertBytesToHumanReadable(systemMemoryInfo.swapFree * 1024), - }; -}; - const getNormalizedBlinkMemoryInfo = () => { const blinkMemoryInfo = process.getBlinkMemoryInfo(); return { From dd7081478fd0283a727379438defba8ee814aa34 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 6 Jan 2023 22:44:01 +0530 Subject: [PATCH 35/51] increase log size to 50MB --- src/config/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/index.ts b/src/config/index.ts index 6b2e8a1f4..1f7b9bc71 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -1,7 +1,7 @@ const PROD_HOST_URL: string = 'ente://app'; const RENDERER_OUTPUT_DIR: string = './ui/out'; const LOG_FILENAME = 'ente.log'; -const MAX_LOG_SIZE = 5 * 1024 * 1024; // 5MB +const MAX_LOG_SIZE = 50 * 1024 * 1024; // 50MB const FILE_STREAM_CHUNK_SIZE: number = 4 * 1024 * 1024; From ed0568a9a413e8385a9ce5e95bfadce7b5ed40ee Mon Sep 17 00:00:00 2001 From: Abhinav Date: Fri, 6 Jan 2023 23:01:37 +0530 Subject: [PATCH 36/51] add getNormalizedWebFrameResourceUsage util --- src/utils/processStats.ts | 60 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/src/utils/processStats.ts b/src/utils/processStats.ts index c9b5eaa68..b138d1122 100644 --- a/src/utils/processStats.ts +++ b/src/utils/processStats.ts @@ -42,8 +42,8 @@ async function logSpikeMemoryUsage() { async function logRendererProcessStats() { const blinkMemoryInfo = getNormalizedBlinkMemoryInfo(); - const heapStatistics = process.getHeapStatistics(); - const webFrameResourceUsage = webFrame.getResourceUsage(); + const heapStatistics = getNormalizedHeapStatistics(); + const webFrameResourceUsage = getNormalizedWebFrameResourceUsage(); ElectronLog.log('renderer process stats', { blinkMemoryInfo, heapStatistics, @@ -114,6 +114,62 @@ const getNormalizedHeapStatistics = () => { }; }; +const getNormalizedWebFrameResourceUsage = () => { + const webFrameResourceUsage = webFrame.getResourceUsage(); + return { + images: { + count: webFrameResourceUsage.images.count, + size: convertBytesToHumanReadable( + webFrameResourceUsage.images.size + ), + liveSize: convertBytesToHumanReadable( + webFrameResourceUsage.images.liveSize + ), + }, + scripts: { + count: webFrameResourceUsage.scripts.count, + size: convertBytesToHumanReadable( + webFrameResourceUsage.scripts.size + ), + liveSize: convertBytesToHumanReadable( + webFrameResourceUsage.scripts.liveSize + ), + }, + cssStyleSheets: { + count: webFrameResourceUsage.cssStyleSheets.count, + size: convertBytesToHumanReadable( + webFrameResourceUsage.cssStyleSheets.size + ), + liveSize: convertBytesToHumanReadable( + webFrameResourceUsage.cssStyleSheets.liveSize + ), + }, + xslStyleSheets: { + count: webFrameResourceUsage.xslStyleSheets.count, + size: convertBytesToHumanReadable( + webFrameResourceUsage.xslStyleSheets.size + ), + liveSize: convertBytesToHumanReadable( + webFrameResourceUsage.xslStyleSheets.liveSize + ), + }, + fonts: { + count: webFrameResourceUsage.fonts.count, + size: convertBytesToHumanReadable(webFrameResourceUsage.fonts.size), + liveSize: convertBytesToHumanReadable( + webFrameResourceUsage.fonts.liveSize + ), + }, + other: { + count: webFrameResourceUsage.other.count, + size: convertBytesToHumanReadable(webFrameResourceUsage.other.size), + liveSize: convertBytesToHumanReadable( + webFrameResourceUsage.other.liveSize + ), + }, + }; +}; + function convertBytesToHumanReadable(bytes: number, precision = 2): string { if (bytes === 0 || isNaN(bytes)) { return '0 MB'; From a72af289ad4e6212ef47463aacac27c60963f579 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 10 Jan 2023 18:15:54 +0530 Subject: [PATCH 37/51] wait for stream to finish before returning --- src/services/fs.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/services/fs.ts b/src/services/fs.ts index ba22e39ff..0f56368ad 100644 --- a/src/services/fs.ts +++ b/src/services/fs.ts @@ -192,7 +192,9 @@ export async function isFolder(dirPath: string) { .catch(() => false); } -export const convertBrowserStreamToNode = (fileStream: any) => { +export const convertBrowserStreamToNode = ( + fileStream: ReadableStream +) => { const reader = fileStream.getReader(); const rs = new Readable(); @@ -210,10 +212,17 @@ export const convertBrowserStreamToNode = (fileStream: any) => { return rs; }; -export function writeStream(filePath: string, fileStream: any) { +export function writeStream( + filePath: string, + fileStream: ReadableStream +) { const writeable = fs.createWriteStream(filePath); const readable = convertBrowserStreamToNode(fileStream); readable.pipe(writeable); + return new Promise((resolve, reject) => { + writeable.on('finish', resolve); + writeable.on('error', reject); + }); } export async function readTextFile(filePath: string) { From 69392d9ee940bc6f6d1dcebbcb776bb9f494a860 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 10 Jan 2023 18:18:23 +0530 Subject: [PATCH 38/51] added loop to retry if tempPath already exists --- src/utils/temp.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/utils/temp.ts b/src/utils/temp.ts index 838d3692f..35a261e63 100644 --- a/src/utils/temp.ts +++ b/src/utils/temp.ts @@ -28,11 +28,11 @@ function generateTempName(length: number) { } export async function generateTempFilePath(formatSuffix: string) { - const tempDirPath = await getTempDirPath(); - const namePrefix = generateTempName(10); - const tempFilePath = path.join( - tempDirPath, - namePrefix + '-' + formatSuffix - ); + let tempFilePath: string; + do { + const tempDirPath = await getTempDirPath(); + const namePrefix = generateTempName(10); + tempFilePath = path.join(tempDirPath, namePrefix + '-' + formatSuffix); + } while (existsSync(tempFilePath)); return tempFilePath; } From 0e8ff75645b6d7d8e2226617d89a9587e696b170 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 10 Jan 2023 18:23:39 +0530 Subject: [PATCH 39/51] use stream to write file instead of loading whole file --- src/api/ffmpeg.ts | 9 +++------ src/api/imageProcessor.ts | 9 +++------ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/api/ffmpeg.ts b/src/api/ffmpeg.ts index 1b0f57068..dba1a7577 100644 --- a/src/api/ffmpeg.ts +++ b/src/api/ffmpeg.ts @@ -1,5 +1,6 @@ import { ipcRenderer } from 'electron'; import { existsSync } from 'fs'; +import { writeStream } from '../services/fs'; import { logError } from '../services/logging'; import { ElectronFile } from '../types'; @@ -12,12 +13,8 @@ export async function runFFmpegCmd( let createdTempInputFile = null; try { if (!existsSync(inputFile.path)) { - const inputFileData = new Uint8Array(await inputFile.arrayBuffer()); - inputFilePath = await ipcRenderer.invoke( - 'write-temp-file', - inputFileData, - inputFile.name - ); + const tempFilePath = await ipcRenderer.invoke('get-temp-file-path'); + inputFilePath = writeStream(tempFilePath, await inputFile.stream()); createdTempInputFile = true; } else { inputFilePath = inputFile.path; diff --git a/src/api/imageProcessor.ts b/src/api/imageProcessor.ts index 07389bd71..d85895c00 100644 --- a/src/api/imageProcessor.ts +++ b/src/api/imageProcessor.ts @@ -1,5 +1,6 @@ import { ipcRenderer } from 'electron/renderer'; import { existsSync } from 'fs'; +import { writeStream } from '../services/fs'; import { logError } from '../services/logging'; import { ElectronFile } from '../types'; @@ -20,12 +21,8 @@ export async function generateImageThumbnail( let createdTempInputFile = null; try { if (!existsSync(inputFile.path)) { - const inputFileData = new Uint8Array(await inputFile.arrayBuffer()); - inputFilePath = await ipcRenderer.invoke( - 'write-temp-file', - inputFileData, - inputFile.name - ); + const tempFilePath = await ipcRenderer.invoke('get-temp-file-path'); + inputFilePath = writeStream(tempFilePath, await inputFile.stream()); createdTempInputFile = true; } else { inputFilePath = inputFile.path; From 6773c07f2dd1a08a603d3c8bf3f227f32929b92a Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 10 Jan 2023 18:24:03 +0530 Subject: [PATCH 40/51] change write-temp-file to get-temp-file-path --- src/utils/ipcComms.ts | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/utils/ipcComms.ts b/src/utils/ipcComms.ts index 2bbe39452..3acc87141 100644 --- a/src/utils/ipcComms.ts +++ b/src/utils/ipcComms.ts @@ -23,11 +23,8 @@ import { skipAppVersion, updateAndRestart, } from '../services/appUpdater'; -import { - deleteTempFile, - runFFmpegCmd, - writeTempFile, -} from '../services/ffmpeg'; +import { deleteTempFile, runFFmpegCmd } from '../services/ffmpeg'; +import { generateTempFilePath } from './temp'; export default function setupIpcComs( tray: Tray, @@ -140,12 +137,9 @@ export default function setupIpcComs( return runFFmpegCmd(cmd, inputFilePath, outputFileName); } ); - ipcMain.handle( - 'write-temp-file', - (_, fileStream: Uint8Array, fileName: string) => { - return writeTempFile(fileStream, fileName); - } - ); + ipcMain.handle('get-temp-file-path', (_, formatSuffix) => { + return generateTempFilePath(formatSuffix); + }); ipcMain.handle('remove-temp-file', (_, tempFilePath: string) => { return deleteTempFile(tempFilePath); }); From e61b14c3736fd2d58ae3d2bb282f4498d4295204 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 10 Jan 2023 18:50:30 +0530 Subject: [PATCH 41/51] update writeStream to async function --- src/api/export.ts | 6 +++--- src/services/fs.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/api/export.ts b/src/api/export.ts index 93e6b4f4d..84239bea3 100644 --- a/src/api/export.ts +++ b/src/api/export.ts @@ -22,11 +22,11 @@ export const checkExistsAndRename = async ( } }; -export const saveStreamToDisk = ( +export const saveStreamToDisk = async ( filePath: string, - fileStream: ReadableStream + fileStream: ReadableStream ) => { - writeStream(filePath, fileStream); + await writeStream(filePath, fileStream); }; export const saveFileToDisk = async (path: string, fileData: any) => { diff --git a/src/services/fs.ts b/src/services/fs.ts index 0f56368ad..9565cb086 100644 --- a/src/services/fs.ts +++ b/src/services/fs.ts @@ -212,14 +212,14 @@ export const convertBrowserStreamToNode = ( return rs; }; -export function writeStream( +export async function writeStream( filePath: string, fileStream: ReadableStream ) { const writeable = fs.createWriteStream(filePath); const readable = convertBrowserStreamToNode(fileStream); readable.pipe(writeable); - return new Promise((resolve, reject) => { + await new Promise((resolve, reject) => { writeable.on('finish', resolve); writeable.on('error', reject); }); From ab09f5e759e33fd8b63ed670a535e11aaf310cec Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 10 Jan 2023 18:50:49 +0530 Subject: [PATCH 42/51] fix api --- src/api/ffmpeg.ts | 8 ++++++-- src/api/imageProcessor.ts | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/api/ffmpeg.ts b/src/api/ffmpeg.ts index dba1a7577..462a63385 100644 --- a/src/api/ffmpeg.ts +++ b/src/api/ffmpeg.ts @@ -13,8 +13,12 @@ export async function runFFmpegCmd( let createdTempInputFile = null; try { if (!existsSync(inputFile.path)) { - const tempFilePath = await ipcRenderer.invoke('get-temp-file-path'); - inputFilePath = writeStream(tempFilePath, await inputFile.stream()); + const tempFilePath = await ipcRenderer.invoke( + 'get-temp-file-path', + inputFile.name + ); + await writeStream(tempFilePath, await inputFile.stream()); + inputFilePath = tempFilePath; createdTempInputFile = true; } else { inputFilePath = inputFile.path; diff --git a/src/api/imageProcessor.ts b/src/api/imageProcessor.ts index d85895c00..61b0ae529 100644 --- a/src/api/imageProcessor.ts +++ b/src/api/imageProcessor.ts @@ -21,8 +21,12 @@ export async function generateImageThumbnail( let createdTempInputFile = null; try { if (!existsSync(inputFile.path)) { - const tempFilePath = await ipcRenderer.invoke('get-temp-file-path'); - inputFilePath = writeStream(tempFilePath, await inputFile.stream()); + const tempFilePath = await ipcRenderer.invoke( + 'get-temp-file-path', + inputFile.name + ); + await writeStream(tempFilePath, await inputFile.stream()); + inputFilePath = tempFilePath; createdTempInputFile = true; } else { inputFilePath = inputFile.path; From d197e821013f8b5203737df4df6d12eeecc98528 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Tue, 10 Jan 2023 21:58:54 +0530 Subject: [PATCH 43/51] update spike detection logic --- src/utils/processStats.ts | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/utils/processStats.ts b/src/utils/processStats.ts index b138d1122..a552d7aa0 100644 --- a/src/utils/processStats.ts +++ b/src/utils/processStats.ts @@ -5,7 +5,9 @@ const LOGGING_INTERVAL_IN_MICROSECONDS = 30 * 1000; // 30 seconds const SPIKE_DETECTION_INTERVAL_IN_MICROSECONDS = 1 * 1000; // 1 seconds -const HIGH_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES = 1 * 1024 * 1024; // 1 GB +const MEMORY_DIFF_IN_KILOBYTES_CONSIDERED_AS_SPIKE = 500 * 1024; // 500 MB + +let previousMemoryUsage = 0; async function logMainProcessStats() { const processMemoryInfo = await process.getProcessMemoryInfo(); @@ -24,9 +26,15 @@ async function logMainProcessStats() { async function logSpikeMemoryUsage() { const processMemoryInfo = await process.getProcessMemoryInfo(); - if ( - processMemoryInfo.residentSet > HIGH_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES - ) { + const currentMemoryUsage = Math.max( + processMemoryInfo.residentSet, + processMemoryInfo.private + ); + const isSpiking = + currentMemoryUsage - previousMemoryUsage >= + MEMORY_DIFF_IN_KILOBYTES_CONSIDERED_AS_SPIKE; + + if (isSpiking) { const normalizedProcessMemoryInfo = await getNormalizedProcessMemoryInfo(processMemoryInfo); const cpuUsage = process.getCPUUsage(); @@ -38,6 +46,7 @@ async function logSpikeMemoryUsage() { cpuUsage, }); } + previousMemoryUsage = currentMemoryUsage; } async function logRendererProcessStats() { @@ -67,7 +76,7 @@ const getNormalizedProcessMemoryInfo = async ( residentSet: convertBytesToHumanReadable( processMemoryInfo.residentSet * 1024 ), - private: convertBytesToHumanReadable(processMemoryInfo.private * 10124), + private: convertBytesToHumanReadable(processMemoryInfo.private * 1024), shared: convertBytesToHumanReadable(processMemoryInfo.shared * 1024), }; }; @@ -78,7 +87,7 @@ const getNormalizedBlinkMemoryInfo = () => { allocated: convertBytesToHumanReadable( blinkMemoryInfo.allocated * 1024 ), - total: convertBytesToHumanReadable(blinkMemoryInfo.total), + total: convertBytesToHumanReadable(blinkMemoryInfo.total * 1024), }; }; From 6a47f6d7ea428905defc9f1aefe15ffbf255c86d Mon Sep 17 00:00:00 2001 From: Abhinav Date: Wed, 11 Jan 2023 11:19:56 +0530 Subject: [PATCH 44/51] log ffmpeg execution time --- src/services/ffmpeg.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/services/ffmpeg.ts b/src/services/ffmpeg.ts index d8c1853fc..7a9b4e6dc 100644 --- a/src/services/ffmpeg.ts +++ b/src/services/ffmpeg.ts @@ -39,10 +39,18 @@ export async function runFFmpegCmd( }); const escapedCmd = shellescape(cmd); log.info('running ffmpeg command', escapedCmd); + const startTime = Date.now(); await execAsync(escapedCmd); if (!existsSync(tempOutputFilePath)) { throw new Error('ffmpeg output file not found'); } + log.info( + 'ffmpeg command execution time ', + escapedCmd, + Date.now() - startTime, + 'ms' + ); + const outputFile = await readFile(tempOutputFilePath); return new Uint8Array(outputFile); } catch (e) { From 71e96df4548f98bf36a2090972950254e38a2ed9 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Wed, 11 Jan 2023 13:58:46 +0530 Subject: [PATCH 45/51] log sudden increase in memory and app using more than high limit --- src/utils/processStats.ts | 41 +++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/src/utils/processStats.ts b/src/utils/processStats.ts index a552d7aa0..a6d5241fb 100644 --- a/src/utils/processStats.ts +++ b/src/utils/processStats.ts @@ -5,9 +5,9 @@ const LOGGING_INTERVAL_IN_MICROSECONDS = 30 * 1000; // 30 seconds const SPIKE_DETECTION_INTERVAL_IN_MICROSECONDS = 1 * 1000; // 1 seconds -const MEMORY_DIFF_IN_KILOBYTES_CONSIDERED_AS_SPIKE = 500 * 1024; // 500 MB +const MEMORY_DIFF_IN_KILOBYTES_CONSIDERED_AS_SPIKE = 200 * 1024; // 200 MB -let previousMemoryUsage = 0; +const HIGH_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES = 700 * 1024; // 700 MB async function logMainProcessStats() { const processMemoryInfo = await process.getProcessMemoryInfo(); @@ -24,29 +24,54 @@ async function logMainProcessStats() { }); } +let previousProcessMemoryInfo: Electron.ProcessMemoryInfo = { + private: 0, + shared: 0, + residentSet: 0, +}; + +let usingHighMemory = false; + async function logSpikeMemoryUsage() { const processMemoryInfo = await process.getProcessMemoryInfo(); const currentMemoryUsage = Math.max( processMemoryInfo.residentSet, processMemoryInfo.private ); + const previewMemoryUsage = Math.max( + previousProcessMemoryInfo.private, + previousProcessMemoryInfo.residentSet + ); const isSpiking = - currentMemoryUsage - previousMemoryUsage >= + currentMemoryUsage - previewMemoryUsage >= MEMORY_DIFF_IN_KILOBYTES_CONSIDERED_AS_SPIKE; - if (isSpiking) { - const normalizedProcessMemoryInfo = + const isHighMemoryUsage = + currentMemoryUsage >= HIGH_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES; + + const shouldReport = + (isHighMemoryUsage && !usingHighMemory) || + (!isHighMemoryUsage && usingHighMemory); + + if (isSpiking || shouldReport) { + const normalizedCurrentProcessMemoryInfo = await getNormalizedProcessMemoryInfo(processMemoryInfo); + const normalizedPreviousProcessMemoryInfo = + await getNormalizedProcessMemoryInfo(previousProcessMemoryInfo); const cpuUsage = process.getCPUUsage(); const heapStatistics = getNormalizedHeapStatistics(); - ElectronLog.log('main process stats', { - processMemoryInfo: normalizedProcessMemoryInfo, + ElectronLog.log('reporting memory usage spike', { + currentProcessMemoryInfo: normalizedCurrentProcessMemoryInfo, + previousProcessMemoryInfo: normalizedPreviousProcessMemoryInfo, heapStatistics, cpuUsage, }); } - previousMemoryUsage = currentMemoryUsage; + previousProcessMemoryInfo = processMemoryInfo; + if (shouldReport) { + usingHighMemory = !usingHighMemory; + } } async function logRendererProcessStats() { From 03be62a0827964434bf51e101d2a8ca28b1523f3 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Wed, 11 Jan 2023 14:35:40 +0530 Subject: [PATCH 46/51] add renderer spike logging --- src/utils/processStats.ts | 93 +++++++++++++++++++++++++++++++++------ 1 file changed, 80 insertions(+), 13 deletions(-) diff --git a/src/utils/processStats.ts b/src/utils/processStats.ts index a6d5241fb..18c16590b 100644 --- a/src/utils/processStats.ts +++ b/src/utils/processStats.ts @@ -5,20 +5,25 @@ const LOGGING_INTERVAL_IN_MICROSECONDS = 30 * 1000; // 30 seconds const SPIKE_DETECTION_INTERVAL_IN_MICROSECONDS = 1 * 1000; // 1 seconds -const MEMORY_DIFF_IN_KILOBYTES_CONSIDERED_AS_SPIKE = 200 * 1024; // 200 MB +const MEMORY_DIFF_IN_KILOBYTES_CONSIDERED_AS_SPIKE = 10 * 1024; // 10 MB -const HIGH_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES = 700 * 1024; // 700 MB +const HIGH_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES = 200 * 1024; // 200 MB + +const HEAP_SIZE_DIFF_IN_KILOBYTES_CONSIDERED_AS_SPIKE = 200 * 1024; // 200 MB + +const HIGH_HEAP_USAGE_THRESHOLD_IN_KILOBYTES = 700 * 1024; // 700 MB async function logMainProcessStats() { - const processMemoryInfo = await process.getProcessMemoryInfo(); - const normalizedProcessMemoryInfo = await getNormalizedProcessMemoryInfo( - processMemoryInfo + const processMemoryInfo = await getNormalizedProcessMemoryInfo( + await process.getProcessMemoryInfo() ); const cpuUsage = process.getCPUUsage(); - const heapStatistics = getNormalizedHeapStatistics(); + const heapStatistics = getNormalizedHeapStatistics( + process.getHeapStatistics() + ); ElectronLog.log('main process stats', { - processMemoryInfo: normalizedProcessMemoryInfo, + processMemoryInfo, heapStatistics, cpuUsage, }); @@ -32,7 +37,7 @@ let previousProcessMemoryInfo: Electron.ProcessMemoryInfo = { let usingHighMemory = false; -async function logSpikeMemoryUsage() { +async function logSpikeMainMemoryUsage() { const processMemoryInfo = await process.getProcessMemoryInfo(); const currentMemoryUsage = Math.max( processMemoryInfo.residentSet, @@ -59,7 +64,9 @@ async function logSpikeMemoryUsage() { const normalizedPreviousProcessMemoryInfo = await getNormalizedProcessMemoryInfo(previousProcessMemoryInfo); const cpuUsage = process.getCPUUsage(); - const heapStatistics = getNormalizedHeapStatistics(); + const heapStatistics = getNormalizedHeapStatistics( + process.getHeapStatistics() + ); ElectronLog.log('reporting memory usage spike', { currentProcessMemoryInfo: normalizedCurrentProcessMemoryInfo, @@ -74,9 +81,61 @@ async function logSpikeMemoryUsage() { } } +let previousHeapStatistics: Electron.HeapStatistics = { + totalHeapSize: 0, + totalHeapSizeExecutable: 0, + totalPhysicalSize: 0, + totalAvailableSize: 0, + usedHeapSize: 0, + heapSizeLimit: 0, + mallocedMemory: 0, + peakMallocedMemory: 0, + doesZapGarbage: false, +}; + +let usingHighHeapMemory = false; +async function logSpikeRendererMemoryUsage() { + const currentHeapStatistics = process.getHeapStatistics(); + + const isSpiking = + currentHeapStatistics.totalHeapSize - + previousHeapStatistics.totalHeapSize >= + HEAP_SIZE_DIFF_IN_KILOBYTES_CONSIDERED_AS_SPIKE; + + const isHighHeapSize = + currentHeapStatistics.totalHeapSize >= + HIGH_HEAP_USAGE_THRESHOLD_IN_KILOBYTES; + + const shouldReport = + (isHighHeapSize && !usingHighHeapMemory) || + (!isHighHeapSize && usingHighHeapMemory); + + if (isSpiking || shouldReport) { + const normalizedCurrentHeapStatistics = getNormalizedHeapStatistics( + currentHeapStatistics + ); + const normalizedPreviousProcessMemoryInfo = getNormalizedHeapStatistics( + previousHeapStatistics + ); + const cpuUsage = process.getCPUUsage(); + + ElectronLog.log('reporting memory usage spike', { + currentProcessMemoryInfo: normalizedCurrentHeapStatistics, + previousProcessMemoryInfo: normalizedPreviousProcessMemoryInfo, + cpuUsage, + }); + } + previousHeapStatistics = currentHeapStatistics; + if (shouldReport) { + usingHighHeapMemory = !usingHighHeapMemory; + } +} + async function logRendererProcessStats() { const blinkMemoryInfo = getNormalizedBlinkMemoryInfo(); - const heapStatistics = getNormalizedHeapStatistics(); + const heapStatistics = getNormalizedHeapStatistics( + process.getHeapStatistics() + ); const webFrameResourceUsage = getNormalizedWebFrameResourceUsage(); ElectronLog.log('renderer process stats', { blinkMemoryInfo, @@ -86,11 +145,18 @@ async function logRendererProcessStats() { } export function setupMainProcessStatsLogger() { - setInterval(logSpikeMemoryUsage, SPIKE_DETECTION_INTERVAL_IN_MICROSECONDS); + setInterval( + logSpikeMainMemoryUsage, + SPIKE_DETECTION_INTERVAL_IN_MICROSECONDS + ); setInterval(logMainProcessStats, LOGGING_INTERVAL_IN_MICROSECONDS); } export function setupRendererProcessStatsLogger() { + setInterval( + logSpikeRendererMemoryUsage, + SPIKE_DETECTION_INTERVAL_IN_MICROSECONDS + ); setInterval(logRendererProcessStats, LOGGING_INTERVAL_IN_MICROSECONDS); } @@ -116,8 +182,9 @@ const getNormalizedBlinkMemoryInfo = () => { }; }; -const getNormalizedHeapStatistics = () => { - const heapStatistics = process.getHeapStatistics(); +const getNormalizedHeapStatistics = ( + heapStatistics: Electron.HeapStatistics +) => { return { totalHeapSize: convertBytesToHumanReadable( heapStatistics.totalHeapSize * 1024 From faab06506e42f72fe8f37b3b41fbd658b053a03b Mon Sep 17 00:00:00 2001 From: Abhinav Date: Wed, 11 Jan 2023 19:59:16 +0530 Subject: [PATCH 47/51] update main HIGH_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES --- src/utils/processStats.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/processStats.ts b/src/utils/processStats.ts index 18c16590b..f984eddd4 100644 --- a/src/utils/processStats.ts +++ b/src/utils/processStats.ts @@ -7,7 +7,7 @@ const SPIKE_DETECTION_INTERVAL_IN_MICROSECONDS = 1 * 1000; // 1 seconds const MEMORY_DIFF_IN_KILOBYTES_CONSIDERED_AS_SPIKE = 10 * 1024; // 10 MB -const HIGH_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES = 200 * 1024; // 200 MB +const HIGH_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES = 100 * 1024; // 100 MB const HEAP_SIZE_DIFF_IN_KILOBYTES_CONSIDERED_AS_SPIKE = 200 * 1024; // 200 MB From c71d571c8624254a05247fc17ceba5417e122dbe Mon Sep 17 00:00:00 2001 From: Abhinav Date: Wed, 11 Jan 2023 20:57:25 +0530 Subject: [PATCH 48/51] use processMemoryInfo for renderer too --- src/utils/processStats.ts | 116 +++++++++++++++++++++----------------- 1 file changed, 64 insertions(+), 52 deletions(-) diff --git a/src/utils/processStats.ts b/src/utils/processStats.ts index f984eddd4..d5da4d739 100644 --- a/src/utils/processStats.ts +++ b/src/utils/processStats.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ import ElectronLog from 'electron-log'; import { webFrame } from 'electron/renderer'; @@ -5,13 +6,13 @@ const LOGGING_INTERVAL_IN_MICROSECONDS = 30 * 1000; // 30 seconds const SPIKE_DETECTION_INTERVAL_IN_MICROSECONDS = 1 * 1000; // 1 seconds -const MEMORY_DIFF_IN_KILOBYTES_CONSIDERED_AS_SPIKE = 10 * 1024; // 10 MB +const MAIN_MEMORY_USAGE_DIFF_IN_KILOBYTES_CONSIDERED_AS_SPIKE = 10 * 1024; // 10 MB -const HIGH_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES = 100 * 1024; // 100 MB +const HIGH_MAIN_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES = 100 * 1024; // 100 MB -const HEAP_SIZE_DIFF_IN_KILOBYTES_CONSIDERED_AS_SPIKE = 200 * 1024; // 200 MB +const RENDERER_MEMORY_USAGE_DIFF_IN_KILOBYTES_CONSIDERED_AS_SPIKE = 200 * 1024; // 200 MB -const HIGH_HEAP_USAGE_THRESHOLD_IN_KILOBYTES = 700 * 1024; // 700 MB +const HIGH_RENDERER_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES = 700 * 1024; // 700 MB async function logMainProcessStats() { const processMemoryInfo = await getNormalizedProcessMemoryInfo( @@ -29,13 +30,13 @@ async function logMainProcessStats() { }); } -let previousProcessMemoryInfo: Electron.ProcessMemoryInfo = { +let previousMainProcessMemoryInfo: Electron.ProcessMemoryInfo = { private: 0, shared: 0, residentSet: 0, }; -let usingHighMemory = false; +let mainProcessUsingHighMemory = false; async function logSpikeMainMemoryUsage() { const processMemoryInfo = await process.getProcessMemoryInfo(); @@ -44,25 +45,25 @@ async function logSpikeMainMemoryUsage() { processMemoryInfo.private ); const previewMemoryUsage = Math.max( - previousProcessMemoryInfo.private, - previousProcessMemoryInfo.residentSet + previousMainProcessMemoryInfo.private, + previousMainProcessMemoryInfo.residentSet ); const isSpiking = currentMemoryUsage - previewMemoryUsage >= - MEMORY_DIFF_IN_KILOBYTES_CONSIDERED_AS_SPIKE; + MAIN_MEMORY_USAGE_DIFF_IN_KILOBYTES_CONSIDERED_AS_SPIKE; const isHighMemoryUsage = - currentMemoryUsage >= HIGH_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES; + currentMemoryUsage >= HIGH_MAIN_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES; const shouldReport = - (isHighMemoryUsage && !usingHighMemory) || - (!isHighMemoryUsage && usingHighMemory); + (isHighMemoryUsage && !mainProcessUsingHighMemory) || + (!isHighMemoryUsage && mainProcessUsingHighMemory); if (isSpiking || shouldReport) { const normalizedCurrentProcessMemoryInfo = await getNormalizedProcessMemoryInfo(processMemoryInfo); const normalizedPreviousProcessMemoryInfo = - await getNormalizedProcessMemoryInfo(previousProcessMemoryInfo); + await getNormalizedProcessMemoryInfo(previousMainProcessMemoryInfo); const cpuUsage = process.getCPUUsage(); const heapStatistics = getNormalizedHeapStatistics( process.getHeapStatistics() @@ -75,59 +76,63 @@ async function logSpikeMainMemoryUsage() { cpuUsage, }); } - previousProcessMemoryInfo = processMemoryInfo; + previousMainProcessMemoryInfo = processMemoryInfo; if (shouldReport) { - usingHighMemory = !usingHighMemory; + mainProcessUsingHighMemory = !mainProcessUsingHighMemory; } } -let previousHeapStatistics: Electron.HeapStatistics = { - totalHeapSize: 0, - totalHeapSizeExecutable: 0, - totalPhysicalSize: 0, - totalAvailableSize: 0, - usedHeapSize: 0, - heapSizeLimit: 0, - mallocedMemory: 0, - peakMallocedMemory: 0, - doesZapGarbage: false, +let previousRendererProcessMemoryInfo: Electron.ProcessMemoryInfo = { + private: 0, + shared: 0, + residentSet: 0, }; -let usingHighHeapMemory = false; +let rendererUsingHighMemory = false; + async function logSpikeRendererMemoryUsage() { - const currentHeapStatistics = process.getHeapStatistics(); - + const processMemoryInfo = await process.getProcessMemoryInfo(); + const currentMemoryUsage = Math.max( + processMemoryInfo.residentSet, + processMemoryInfo.private + ); + const previewMemoryUsage = Math.max( + previousRendererProcessMemoryInfo.private, + previousRendererProcessMemoryInfo.residentSet + ); const isSpiking = - currentHeapStatistics.totalHeapSize - - previousHeapStatistics.totalHeapSize >= - HEAP_SIZE_DIFF_IN_KILOBYTES_CONSIDERED_AS_SPIKE; + currentMemoryUsage - previewMemoryUsage >= + RENDERER_MEMORY_USAGE_DIFF_IN_KILOBYTES_CONSIDERED_AS_SPIKE; - const isHighHeapSize = - currentHeapStatistics.totalHeapSize >= - HIGH_HEAP_USAGE_THRESHOLD_IN_KILOBYTES; + const isHighMemoryUsage = + currentMemoryUsage >= HIGH_RENDERER_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES; const shouldReport = - (isHighHeapSize && !usingHighHeapMemory) || - (!isHighHeapSize && usingHighHeapMemory); + (isHighMemoryUsage && !rendererUsingHighMemory) || + (!isHighMemoryUsage && rendererUsingHighMemory); if (isSpiking || shouldReport) { - const normalizedCurrentHeapStatistics = getNormalizedHeapStatistics( - currentHeapStatistics - ); - const normalizedPreviousProcessMemoryInfo = getNormalizedHeapStatistics( - previousHeapStatistics - ); + const normalizedCurrentProcessMemoryInfo = + await getNormalizedProcessMemoryInfo(processMemoryInfo); + const normalizedPreviousProcessMemoryInfo = + await getNormalizedProcessMemoryInfo( + previousRendererProcessMemoryInfo + ); const cpuUsage = process.getCPUUsage(); + const heapStatistics = getNormalizedHeapStatistics( + process.getHeapStatistics() + ); ElectronLog.log('reporting memory usage spike', { - currentProcessMemoryInfo: normalizedCurrentHeapStatistics, + currentProcessMemoryInfo: normalizedCurrentProcessMemoryInfo, previousProcessMemoryInfo: normalizedPreviousProcessMemoryInfo, + heapStatistics, cpuUsage, }); } - previousHeapStatistics = currentHeapStatistics; + previousRendererProcessMemoryInfo = processMemoryInfo; if (shouldReport) { - usingHighHeapMemory = !usingHighHeapMemory; + rendererUsingHighMemory = !rendererUsingHighMemory; } } @@ -137,27 +142,34 @@ async function logRendererProcessStats() { process.getHeapStatistics() ); const webFrameResourceUsage = getNormalizedWebFrameResourceUsage(); + const processMemoryInfo = await getNormalizedProcessMemoryInfo( + await process.getProcessMemoryInfo() + ); ElectronLog.log('renderer process stats', { blinkMemoryInfo, heapStatistics, + processMemoryInfo, webFrameResourceUsage, }); } export function setupMainProcessStatsLogger() { - setInterval( - logSpikeMainMemoryUsage, - SPIKE_DETECTION_INTERVAL_IN_MICROSECONDS - ); - setInterval(logMainProcessStats, LOGGING_INTERVAL_IN_MICROSECONDS); + // setInterval( + // logSpikeMainMemoryUsage, + // SPIKE_DETECTION_INTERVAL_IN_MICROSECONDS + // ); + setInterval(logMainProcessStats, SPIKE_DETECTION_INTERVAL_IN_MICROSECONDS); } export function setupRendererProcessStatsLogger() { + // setInterval( + // logSpikeRendererMemoryUsage, + // SPIKE_DETECTION_INTERVAL_IN_MICROSECONDS + // ); setInterval( - logSpikeRendererMemoryUsage, + logRendererProcessStats, SPIKE_DETECTION_INTERVAL_IN_MICROSECONDS ); - setInterval(logRendererProcessStats, LOGGING_INTERVAL_IN_MICROSECONDS); } const getNormalizedProcessMemoryInfo = async ( From 4484977c0775d14485917502a24a8e8db73415e5 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Wed, 11 Jan 2023 22:56:06 +0530 Subject: [PATCH 49/51] increase renderer high limit --- src/utils/processStats.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/processStats.ts b/src/utils/processStats.ts index d5da4d739..e06ef768a 100644 --- a/src/utils/processStats.ts +++ b/src/utils/processStats.ts @@ -12,7 +12,7 @@ const HIGH_MAIN_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES = 100 * 1024; // 100 MB const RENDERER_MEMORY_USAGE_DIFF_IN_KILOBYTES_CONSIDERED_AS_SPIKE = 200 * 1024; // 200 MB -const HIGH_RENDERER_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES = 700 * 1024; // 700 MB +const HIGH_RENDERER_MEMORY_USAGE_THRESHOLD_IN_KILOBYTES = 1024 * 1024; // 1 GB async function logMainProcessStats() { const processMemoryInfo = await getNormalizedProcessMemoryInfo( From 386808bf541613783f964861b9a3d7cfe1556b69 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Wed, 11 Jan 2023 22:57:05 +0530 Subject: [PATCH 50/51] reset testing changes --- src/utils/processStats.ts | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/utils/processStats.ts b/src/utils/processStats.ts index e06ef768a..7bf262cd9 100644 --- a/src/utils/processStats.ts +++ b/src/utils/processStats.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ import ElectronLog from 'electron-log'; import { webFrame } from 'electron/renderer'; @@ -154,22 +153,19 @@ async function logRendererProcessStats() { } export function setupMainProcessStatsLogger() { - // setInterval( - // logSpikeMainMemoryUsage, - // SPIKE_DETECTION_INTERVAL_IN_MICROSECONDS - // ); - setInterval(logMainProcessStats, SPIKE_DETECTION_INTERVAL_IN_MICROSECONDS); + setInterval( + logSpikeMainMemoryUsage, + SPIKE_DETECTION_INTERVAL_IN_MICROSECONDS + ); + setInterval(logMainProcessStats, LOGGING_INTERVAL_IN_MICROSECONDS); } export function setupRendererProcessStatsLogger() { - // setInterval( - // logSpikeRendererMemoryUsage, - // SPIKE_DETECTION_INTERVAL_IN_MICROSECONDS - // ); setInterval( - logRendererProcessStats, + logSpikeRendererMemoryUsage, SPIKE_DETECTION_INTERVAL_IN_MICROSECONDS ); + setInterval(logRendererProcessStats, LOGGING_INTERVAL_IN_MICROSECONDS); } const getNormalizedProcessMemoryInfo = async ( From 29124babb0accdc5da78d221a426252640c93780 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Thu, 12 Jan 2023 09:12:47 +0530 Subject: [PATCH 51/51] add process name in spike log --- src/utils/processStats.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/processStats.ts b/src/utils/processStats.ts index 7bf262cd9..64f900d58 100644 --- a/src/utils/processStats.ts +++ b/src/utils/processStats.ts @@ -68,7 +68,7 @@ async function logSpikeMainMemoryUsage() { process.getHeapStatistics() ); - ElectronLog.log('reporting memory usage spike', { + ElectronLog.log('reporting main memory usage spike', { currentProcessMemoryInfo: normalizedCurrentProcessMemoryInfo, previousProcessMemoryInfo: normalizedPreviousProcessMemoryInfo, heapStatistics, @@ -122,7 +122,7 @@ async function logSpikeRendererMemoryUsage() { process.getHeapStatistics() ); - ElectronLog.log('reporting memory usage spike', { + ElectronLog.log('reporting renderer memory usage spike', { currentProcessMemoryInfo: normalizedCurrentProcessMemoryInfo, previousProcessMemoryInfo: normalizedPreviousProcessMemoryInfo, heapStatistics,