cleanup unused cached images

backend/package.json

@@ -35,6 +35,7 @@
     "fastify-multipart": "^5.3.1",
     "fastify-static": "^4.6.1",
     "file-type": "^17.1.1",
+    "ms": "^2.1.3",
     "passport": "^0.5.2",
     "passport-jwt": "^4.0.0",
     "passport-local": "^1.0.0",
@@ -55,6 +56,7 @@
     "@nestjs/testing": "^8.4.4",
     "@types/bcrypt": "^5.0.0",
     "@types/cors": "^2.8.12",
+    "@types/ms": "^0.7.31",
     "@types/multer": "^1.4.7",
     "@types/node": "^17.0.24",
     "@types/passport-jwt": "^3.0.6",

backend/src/collections/image-db/image-file-db.service.ts

@@ -1,11 +1,13 @@
 import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { AsyncFailable, Fail } from 'picsur-shared/dist/types';
-import { Repository } from 'typeorm';
+import { LessThan, Repository } from 'typeorm';
 import { ImageFileType } from '../../models/constants/image-file-types.const';
 import { EImageDerivativeBackend } from '../../models/entities/image-derivative.entity';
 import { EImageFileBackend } from '../../models/entities/image-file.entity';
 
+const A_DAY_IN_SECONDS = 24 * 60 * 60;
+
 @Injectable()
 export class ImageFileDBService {
   constructor(
@@ -83,6 +85,7 @@ export class ImageFileDBService {
     imageDerivative.key = key;
     imageDerivative.mime = mime;
     imageDerivative.data = file;
+    imageDerivative.last_read_unix_sec = Math.floor(Date.now() / 1000);
 
     try {
       return await this.imageDerivativeRepo.save(imageDerivative);
@@ -96,9 +99,18 @@ export class ImageFileDBService {
     key: string,
   ): AsyncFailable<EImageDerivativeBackend | null> {
     try {
-      return await this.imageDerivativeRepo.findOne({
+      const derivative = await this.imageDerivativeRepo.findOne({
         where: { image_id: imageId, key },
       });
+      if (!derivative) return null;
+
+      const unix_seconds = Math.floor(Date.now() / 1000);
+      if (derivative.last_read_unix_sec > unix_seconds - A_DAY_IN_SECONDS) {
+        derivative.last_read_unix_sec = unix_seconds;
+        return await this.imageDerivativeRepo.save(derivative);
+      }
+
+      return derivative;
     } catch (e) {
       return Fail(e);
     }
@@ -120,4 +132,19 @@ export class ImageFileDBService {
       return Fail(e);
     }
   }
+
+  public async cleanupDerivatives(
+    olderThanSeconds: number,
+  ): AsyncFailable<number> {
+    try {
+      const unix_seconds = Math.floor(Date.now() / 1000);
+      const result = await this.imageDerivativeRepo.delete({
+        last_read_unix_sec: LessThan(unix_seconds - olderThanSeconds),
+      });
+
+      return result.affected ?? 0;
+    } catch (e) {
+      return Fail(e);
+    }
+  }
 }

backend/src/collections/preference-db/preference-defaults.service.ts

@@ -37,5 +37,6 @@ export class PreferenceDefaultsService {
     [SysPreference.JwtExpiresIn]: () =>
       this.jwtConfigService.getJwtExpiresIn() ?? '7d',
     [SysPreference.BCryptStrength]: () => 12,
+    [SysPreference.RemoveDerivativesAfter]: () => '7d',
   };
 }

backend/src/managers/image/image.module.ts

@@ -1,6 +1,11 @@
-import { Module } from '@nestjs/common';
+import { Logger, Module, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
+import ms from 'ms';
+import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.dto';
+import { HasFailed } from 'picsur-shared/dist/types';
 import { ImageDBModule } from '../../collections/image-db/image-db.module';
+import { ImageFileDBService } from '../../collections/image-db/image-file-db.service';
 import { PreferenceModule } from '../../collections/preference-db/preference-db.module';
+import { SysPreferenceService } from '../../collections/preference-db/sys-preference-db.service';
 import { ImageConverterService } from './image-converter.service';
 import { ImageProcessorService } from './image-processor.service';
 import { ImageManagerService } from './image.service';
@@ -14,4 +19,48 @@ import { ImageManagerService } from './image.service';
   ],
   exports: [ImageManagerService],
 })
-export class ImageManagerModule {}
+export class ImageManagerModule implements OnModuleInit, OnModuleDestroy {
+  private readonly logger = new Logger('ImageManagerModule');
+  private interval: NodeJS.Timeout;
+
+  constructor(
+    private prefManager: SysPreferenceService,
+    private imageFileDB: ImageFileDBService,
+  ) {}
+
+  async onModuleInit() {
+    this.interval = setInterval(
+      // Run demoManagerService.execute() every interval
+      this.imageManagerCron.bind(this),
+      1000 * 60 * 60,
+    );
+    await this.imageManagerCron();
+  }
+
+  private async imageManagerCron() {
+    const remove_derivatives_after = await this.prefManager.getStringPreference(
+      SysPreference.RemoveDerivativesAfter,
+    );
+    if (HasFailed(remove_derivatives_after)) {
+      this.logger.warn('Failed to get remove_derivatives_after preference');
+      return;
+    }
+
+    const after_ms = ms(remove_derivatives_after);
+    if (after_ms === 0) {
+      this.logger.log('remove_derivatives_after is 0, skipping cron');
+      return;
+    }
+
+    const result = await this.imageFileDB.cleanupDerivatives(after_ms / 1000);
+    if (HasFailed(result)) {
+      this.logger.warn(`Failed to cleanup derivatives`);
+    }
+
+    this.logger.log(`Cleaned up ${result} derivatives`);
+  }
+
+  onModuleDestroy() {
+    if (this.interval) clearInterval(this.interval);
+  }
+}

backend/src/models/constants/syspreferences.const.ts

@@ -11,4 +11,5 @@ export const SysPreferenceValueTypes: {
   [SysPreference.JwtSecret]: 'string',
   [SysPreference.JwtExpiresIn]: 'string',
   [SysPreference.BCryptStrength]: 'number',
+  [SysPreference.RemoveDerivativesAfter]: 'string',
 };

backend/src/models/entities/image-derivative.entity.ts

@@ -17,6 +17,9 @@ export class EImageDerivativeBackend {
   @Column({ nullable: false })
   mime: string;
 
+  @Column({ name: 'last_read', nullable: false })
+  last_read_unix_sec: number;
+
   // Binary data
   @Column({ type: 'bytea', nullable: false })
   data: Buffer;

frontend/src/app/i18n/sys-pref.i18n.ts

@@ -6,4 +6,5 @@ export const SysPreferenceFriendlyNames: {
   [SysPreference.JwtSecret]: 'JWT Secret',
   [SysPreference.JwtExpiresIn]: 'JWT Expiry Time',
   [SysPreference.BCryptStrength]: 'BCrypt Strength',
+  [SysPreference.RemoveDerivativesAfter]: 'Cached Images Expiry Time',
 };

shared/src/dto/sys-preferences.dto.ts

@@ -4,4 +4,5 @@ export enum SysPreference {
   JwtSecret = 'jwt_secret',
   JwtExpiresIn = 'jwt_expires_in',
   BCryptStrength = 'bcrypt_strength',
+  RemoveDerivativesAfter = 'remove_derivatives_after',
 }

yarn.lock

@@ -1757,6 +1757,11 @@
   resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a"
   integrity sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==
 
+"@types/ms@^0.7.31":
+  version "0.7.31"
+  resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.31.tgz#31b7ca6407128a3d2bbc27fe2d21b345397f6197"
+  integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==
+
 "@types/multer@^1.4.7":
   version "1.4.7"
   resolved "https://registry.yarnpkg.com/@types/multer/-/multer-1.4.7.tgz#89cf03547c28c7bbcc726f029e2a76a7232cc79e"
@@ -3309,9 +3314,9 @@ ee-first@1.1.1:
   integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=
 
 electron-to-chromium@^1.4.118:
-  version "1.4.118"
+  version "1.4.119"
-  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.118.tgz#2d917c71712dac9652cc01af46c7d0bd51552974"
+  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.119.tgz#a1dcef9f9a0283119256030a605da6ae63b0a402"
-  integrity sha512-maZIKjnYDvF7Fs35nvVcyr44UcKNwybr93Oba2n3HkKDFAtk0svERkLN/HyczJDS3Fo4wU9th9fUQd09ZLtj1w==
+  integrity sha512-HPEmKy+d0xK8oCfEHc5t6wDsSAi1WmE3Ld08QrBjAPxaAzfuKP66VJ77lcTqxTt7GJmSE279s75mhW64Xh+4kw==
 
 emoji-regex@^8.0.0:
   version "8.0.0"
@@ -5360,7 +5365,7 @@ ms@2.1.2:
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
   integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==
 
-ms@2.1.3, ms@^2.0.0, ms@^2.1.1:
+ms@2.1.3, ms@^2.0.0, ms@^2.1.1, ms@^2.1.3:
   version "2.1.3"
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
   integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==