diff --git a/backend/package.json b/backend/package.json index 4b3b0e6..8191964 100644 --- a/backend/package.json +++ b/backend/package.json @@ -35,6 +35,7 @@ "fastify-multipart": "^5.3.1", "fastify-static": "^4.6.1", "file-type": "^17.1.1", + "ms": "^2.1.3", "passport": "^0.5.2", "passport-jwt": "^4.0.0", "passport-local": "^1.0.0", @@ -55,6 +56,7 @@ "@nestjs/testing": "^8.4.4", "@types/bcrypt": "^5.0.0", "@types/cors": "^2.8.12", + "@types/ms": "^0.7.31", "@types/multer": "^1.4.7", "@types/node": "^17.0.24", "@types/passport-jwt": "^3.0.6", diff --git a/backend/src/collections/image-db/image-file-db.service.ts b/backend/src/collections/image-db/image-file-db.service.ts index b07090c..8ec6dcc 100644 --- a/backend/src/collections/image-db/image-file-db.service.ts +++ b/backend/src/collections/image-db/image-file-db.service.ts @@ -1,11 +1,13 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { AsyncFailable, Fail } from 'picsur-shared/dist/types'; -import { Repository } from 'typeorm'; +import { LessThan, Repository } from 'typeorm'; import { ImageFileType } from '../../models/constants/image-file-types.const'; import { EImageDerivativeBackend } from '../../models/entities/image-derivative.entity'; import { EImageFileBackend } from '../../models/entities/image-file.entity'; +const A_DAY_IN_SECONDS = 24 * 60 * 60; + @Injectable() export class ImageFileDBService { constructor( @@ -83,6 +85,7 @@ export class ImageFileDBService { imageDerivative.key = key; imageDerivative.mime = mime; imageDerivative.data = file; + imageDerivative.last_read_unix_sec = Math.floor(Date.now() / 1000); try { return await this.imageDerivativeRepo.save(imageDerivative); @@ -96,9 +99,18 @@ export class ImageFileDBService { key: string, ): AsyncFailable { try { - return await this.imageDerivativeRepo.findOne({ + const derivative = await this.imageDerivativeRepo.findOne({ where: { image_id: imageId, key }, }); + if (!derivative) return null; + + const unix_seconds = Math.floor(Date.now() / 1000); + if (derivative.last_read_unix_sec > unix_seconds - A_DAY_IN_SECONDS) { + derivative.last_read_unix_sec = unix_seconds; + return await this.imageDerivativeRepo.save(derivative); + } + + return derivative; } catch (e) { return Fail(e); } @@ -120,4 +132,19 @@ export class ImageFileDBService { return Fail(e); } } + + public async cleanupDerivatives( + olderThanSeconds: number, + ): AsyncFailable { + try { + const unix_seconds = Math.floor(Date.now() / 1000); + const result = await this.imageDerivativeRepo.delete({ + last_read_unix_sec: LessThan(unix_seconds - olderThanSeconds), + }); + + return result.affected ?? 0; + } catch (e) { + return Fail(e); + } + } } diff --git a/backend/src/collections/preference-db/preference-defaults.service.ts b/backend/src/collections/preference-db/preference-defaults.service.ts index 72261e8..798c608 100644 --- a/backend/src/collections/preference-db/preference-defaults.service.ts +++ b/backend/src/collections/preference-db/preference-defaults.service.ts @@ -37,5 +37,6 @@ export class PreferenceDefaultsService { [SysPreference.JwtExpiresIn]: () => this.jwtConfigService.getJwtExpiresIn() ?? '7d', [SysPreference.BCryptStrength]: () => 12, + [SysPreference.RemoveDerivativesAfter]: () => '7d', }; } diff --git a/backend/src/managers/image/image.module.ts b/backend/src/managers/image/image.module.ts index cd7f398..cbe4426 100644 --- a/backend/src/managers/image/image.module.ts +++ b/backend/src/managers/image/image.module.ts @@ -1,6 +1,11 @@ -import { Module } from '@nestjs/common'; +import { Logger, Module, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; +import ms from 'ms'; +import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.dto'; +import { HasFailed } from 'picsur-shared/dist/types'; import { ImageDBModule } from '../../collections/image-db/image-db.module'; +import { ImageFileDBService } from '../../collections/image-db/image-file-db.service'; import { PreferenceModule } from '../../collections/preference-db/preference-db.module'; +import { SysPreferenceService } from '../../collections/preference-db/sys-preference-db.service'; import { ImageConverterService } from './image-converter.service'; import { ImageProcessorService } from './image-processor.service'; import { ImageManagerService } from './image.service'; @@ -14,4 +19,48 @@ import { ImageManagerService } from './image.service'; ], exports: [ImageManagerService], }) -export class ImageManagerModule {} +export class ImageManagerModule implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger('ImageManagerModule'); + private interval: NodeJS.Timeout; + + constructor( + private prefManager: SysPreferenceService, + private imageFileDB: ImageFileDBService, + ) {} + + async onModuleInit() { + this.interval = setInterval( + // Run demoManagerService.execute() every interval + this.imageManagerCron.bind(this), + 1000 * 60 * 60, + ); + await this.imageManagerCron(); + } + + private async imageManagerCron() { + const remove_derivatives_after = await this.prefManager.getStringPreference( + SysPreference.RemoveDerivativesAfter, + ); + if (HasFailed(remove_derivatives_after)) { + this.logger.warn('Failed to get remove_derivatives_after preference'); + return; + } + + const after_ms = ms(remove_derivatives_after); + if (after_ms === 0) { + this.logger.log('remove_derivatives_after is 0, skipping cron'); + return; + } + + const result = await this.imageFileDB.cleanupDerivatives(after_ms / 1000); + if (HasFailed(result)) { + this.logger.warn(`Failed to cleanup derivatives`); + } + + this.logger.log(`Cleaned up ${result} derivatives`); + } + + onModuleDestroy() { + if (this.interval) clearInterval(this.interval); + } +} diff --git a/backend/src/models/constants/syspreferences.const.ts b/backend/src/models/constants/syspreferences.const.ts index b534c4a..60b10c1 100644 --- a/backend/src/models/constants/syspreferences.const.ts +++ b/backend/src/models/constants/syspreferences.const.ts @@ -11,4 +11,5 @@ export const SysPreferenceValueTypes: { [SysPreference.JwtSecret]: 'string', [SysPreference.JwtExpiresIn]: 'string', [SysPreference.BCryptStrength]: 'number', + [SysPreference.RemoveDerivativesAfter]: 'string', }; diff --git a/backend/src/models/entities/image-derivative.entity.ts b/backend/src/models/entities/image-derivative.entity.ts index 6129ea4..c4a34a7 100644 --- a/backend/src/models/entities/image-derivative.entity.ts +++ b/backend/src/models/entities/image-derivative.entity.ts @@ -17,6 +17,9 @@ export class EImageDerivativeBackend { @Column({ nullable: false }) mime: string; + @Column({ name: 'last_read', nullable: false }) + last_read_unix_sec: number; + // Binary data @Column({ type: 'bytea', nullable: false }) data: Buffer; diff --git a/frontend/src/app/i18n/sys-pref.i18n.ts b/frontend/src/app/i18n/sys-pref.i18n.ts index f03137c..4e2417a 100644 --- a/frontend/src/app/i18n/sys-pref.i18n.ts +++ b/frontend/src/app/i18n/sys-pref.i18n.ts @@ -6,4 +6,5 @@ export const SysPreferenceFriendlyNames: { [SysPreference.JwtSecret]: 'JWT Secret', [SysPreference.JwtExpiresIn]: 'JWT Expiry Time', [SysPreference.BCryptStrength]: 'BCrypt Strength', + [SysPreference.RemoveDerivativesAfter]: 'Cached Images Expiry Time', }; diff --git a/shared/src/dto/sys-preferences.dto.ts b/shared/src/dto/sys-preferences.dto.ts index 2f42b2c..9336347 100644 --- a/shared/src/dto/sys-preferences.dto.ts +++ b/shared/src/dto/sys-preferences.dto.ts @@ -4,4 +4,5 @@ export enum SysPreference { JwtSecret = 'jwt_secret', JwtExpiresIn = 'jwt_expires_in', BCryptStrength = 'bcrypt_strength', + RemoveDerivativesAfter = 'remove_derivatives_after', } diff --git a/yarn.lock b/yarn.lock index b83ade0..78d67f3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1757,6 +1757,11 @@ resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a" integrity sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw== +"@types/ms@^0.7.31": + version "0.7.31" + resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.31.tgz#31b7ca6407128a3d2bbc27fe2d21b345397f6197" + integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA== + "@types/multer@^1.4.7": version "1.4.7" resolved "https://registry.yarnpkg.com/@types/multer/-/multer-1.4.7.tgz#89cf03547c28c7bbcc726f029e2a76a7232cc79e" @@ -3309,9 +3314,9 @@ ee-first@1.1.1: integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= electron-to-chromium@^1.4.118: - version "1.4.118" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.118.tgz#2d917c71712dac9652cc01af46c7d0bd51552974" - integrity sha512-maZIKjnYDvF7Fs35nvVcyr44UcKNwybr93Oba2n3HkKDFAtk0svERkLN/HyczJDS3Fo4wU9th9fUQd09ZLtj1w== + version "1.4.119" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.119.tgz#a1dcef9f9a0283119256030a605da6ae63b0a402" + integrity sha512-HPEmKy+d0xK8oCfEHc5t6wDsSAi1WmE3Ld08QrBjAPxaAzfuKP66VJ77lcTqxTt7GJmSE279s75mhW64Xh+4kw== emoji-regex@^8.0.0: version "8.0.0" @@ -5360,7 +5365,7 @@ ms@2.1.2: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@2.1.3, ms@^2.0.0, ms@^2.1.1: +ms@2.1.3, ms@^2.0.0, ms@^2.1.1, ms@^2.1.3: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==