move image storage to s3/local s3

backend/package.json

@@ -22,6 +22,7 @@
     "purge": "rm -rf dist && rm -rf node_modules"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.181.0",
     "@fastify/helmet": "^10.0.1",
     "@fastify/multipart": "^7.2.0",
     "@fastify/reply-from": "^8.3.0",
@@ -44,6 +45,7 @@
     "bull": "^4.10.0",
     "cors": "^2.8.5",
     "file-type": "^18.0.0",
+    "get-stream": "^6.0.1",
     "is-docker": "^3.0.0",
     "ms": "^2.1.3",
     "node-fetch": "^3.2.10",

backend/src/app.module.ts

@@ -4,7 +4,7 @@ import {
   MiddlewareConsumer,
   Module,
   NestModule,
-  OnModuleInit
+  OnModuleInit,
 } from '@nestjs/common';
 import { ScheduleModule } from '@nestjs/schedule';
 import { ServeStaticModule } from '@nestjs/serve-static';

backend/src/collections/file-s3/file-s3.module.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+import { EarlyConfigModule } from '../../config/early/early-config.module';
+import { FileS3Service } from './file-s3.service';
+
+@Module({
+  imports: [EarlyConfigModule],
+  providers: [FileS3Service],
+  exports: [FileS3Service],
+})
+export class FileS3Module {}

backend/src/collections/file-s3/file-s3.service.ts

@@ -0,0 +1,122 @@
+import {
+  CreateBucketCommand,
+  DeleteObjectCommand,
+  DeleteObjectsCommand,
+  GetObjectCommand,
+  ListBucketsCommand,
+  PutObjectCommand,
+  S3Client,
+} from '@aws-sdk/client-s3';
+import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
+import { buffer as streamToBuffer } from 'get-stream';
+import { AsyncFailable, Fail, FT } from 'picsur-shared/dist/types';
+import { Readable } from 'stream';
+import { S3ConfigService } from '../../config/early/s3.config.service';
+
+@Injectable()
+export class FileS3Service implements OnModuleInit {
+  private readonly logger = new Logger(FileS3Service.name);
+
+  private S3: Promise<S3Client> = this.loadS3();
+
+  constructor(private readonly s3config: S3ConfigService) {}
+
+  onModuleInit() {
+    this.loadS3();
+  }
+
+  public async putFile(key: string, data: Buffer): AsyncFailable<string> {
+    const S3 = await this.S3;
+
+    const request = new PutObjectCommand({
+      Bucket: this.s3config.getS3Bucket(),
+      Key: key,
+      Body: data,
+    });
+
+    try {
+      await S3.send(request);
+      return key;
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
+
+  public async getFile(key: string): AsyncFailable<Buffer> {
+    const S3 = await this.S3;
+
+    const request = new GetObjectCommand({
+      Bucket: this.s3config.getS3Bucket(),
+      Key: key,
+    });
+
+    try {
+      const result = await S3.send(request);
+      if (!result.Body) return Fail(FT.NotFound, 'File not found');
+
+      if (result.Body instanceof Blob) {
+        return Buffer.from(await result.Body.arrayBuffer());
+      }
+      return streamToBuffer(result.Body as Readable);
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
+
+  public async deleteFile(key: string): AsyncFailable<true> {
+    const S3 = await this.S3;
+
+    const request = new DeleteObjectCommand({
+      Bucket: this.s3config.getS3Bucket(),
+      Key: key,
+    });
+
+    try {
+      await S3.send(request);
+      return true;
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
+
+  public async deleteFiles(keys: string[]): AsyncFailable<true> {
+    const S3 = await this.S3;
+
+    const request = new DeleteObjectsCommand({
+      Bucket: this.s3config.getS3Bucket(),
+      Delete: {
+        Objects: keys.map((key) => ({ Key: key })),
+      },
+    });
+
+    try {
+      await S3.send(request);
+      return true;
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
+
+  private async loadS3(): Promise<S3Client> {
+    const S3 = new S3Client(this.s3config.getS3Config());
+
+    try {
+      // Create bucket if it doesn't exist
+      const bucket = this.s3config.getS3Bucket();
+
+      // List buckets
+      const listBuckets = await S3.send(new ListBucketsCommand({}));
+
+      const bucketExists = listBuckets.Buckets?.some((b) => b.Name === bucket);
+      if (!bucketExists) {
+        this.logger.verbose(`Creating S3 Bucket ${bucket}`);
+        await S3.send(new CreateBucketCommand({ Bucket: bucket }));
+      } else {
+        this.logger.verbose(`Using existing S3 Bucket ${bucket}`);
+      }
+    } catch (e) {
+      this.logger.error(e);
+    }
+    return S3;
+  }
+}

backend/src/collections/image-db/image-db.module.ts

@@ -3,6 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
 import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity';
 import { EImageFileBackend } from '../../database/entities/images/image-file.entity';
 import { EImageBackend } from '../../database/entities/images/image.entity';
+import { FileS3Module } from '../file-s3/file-s3.module';
 import { ImageDBService } from './image-db.service';
 import { ImageFileDBService } from './image-file-db.service';
 
@@ -13,6 +14,7 @@ import { ImageFileDBService } from './image-file-db.service';
       EImageFileBackend,
       EImageDerivativeBackend,
     ]),
+    FileS3Module,
   ],
   providers: [ImageDBService, ImageFileDBService],
   exports: [ImageDBService, ImageFileDBService],

backend/src/collections/image-db/image-file-db.service.ts

@@ -2,9 +2,11 @@ import { Injectable, Logger } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum';
 import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
-import { LessThan, Repository } from 'typeorm';
+import { In, IsNull, LessThan, Repository } from 'typeorm';
+import { v4 as uuidv4 } from 'uuid';
 import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity';
 import { EImageFileBackend } from '../../database/entities/images/image-file.entity';
+import { FileS3Service } from '../file-s3/file-s3.service';
 
 const A_DAY_IN_SECONDS = 24 * 60 * 60;
 
@@ -18,24 +20,40 @@ export class ImageFileDBService {
 
     @InjectRepository(EImageDerivativeBackend)
     private readonly imageDerivativeRepo: Repository<EImageDerivativeBackend>,
+
+    private readonly s3Service: FileS3Service,
   ) {}
 
+  public async getData(
+    file: EImageFileBackend | EImageDerivativeBackend,
+  ): AsyncFailable<Buffer> {
+    const result = await this.s3Service.getFile(file.s3key);
+    if (HasFailed(result)) return result;
+
+    return result;
+  }
+
   public async setFile(
     imageId: string,
     variant: ImageEntryVariant,
     file: Buffer,
     filetype: string,
   ): AsyncFailable<true> {
+    const s3key = uuidv4();
+
     const imageFile = new EImageFileBackend();
     imageFile.image_id = imageId;
     imageFile.variant = variant;
     imageFile.filetype = filetype;
-    imageFile.data = file;
+    imageFile.s3key = s3key;
 
     try {
       await this.imageFileRepo.upsert(imageFile, {
         conflictPaths: ['image_id', 'variant'],
       });
+
+      const s3result = await this.s3Service.putFile(s3key, file);
+      if (HasFailed(s3result)) return s3result;
     } catch (e) {
       return Fail(FT.Database, e);
     }
@@ -53,11 +71,6 @@ export class ImageFileDBService {
       });
 
       if (!found) return Fail(FT.NotFound, 'Image not found');
-
-      if (!(found.data instanceof Buffer)) {
-        found.data = Buffer.from(found.data);
-      }
-
       return found;
     } catch (e) {
       return Fail(FT.Database, e);
@@ -80,7 +93,7 @@ export class ImageFileDBService {
     }
   }
 
-  public async deleteFile(
+  public async orphanFile(
     imageId: string,
     variant: ImageEntryVariant,
   ): AsyncFailable<EImageFileBackend> {
@@ -91,8 +104,9 @@ export class ImageFileDBService {
 
       if (!found) return Fail(FT.NotFound, 'Image not found');
 
-      await this.imageFileRepo.delete({ image_id: imageId, variant: variant });
-      return found;
+      found.image_id = null;
+
+      return await this.imageFileRepo.save(found);
     } catch (e) {
       return Fail(FT.Database, e);
     }
@@ -127,15 +141,22 @@ export class ImageFileDBService {
     filetype: string,
     file: Buffer,
   ): AsyncFailable<EImageDerivativeBackend> {
+    const s3key = uuidv4();
+
     const imageDerivative = new EImageDerivativeBackend();
     imageDerivative.image_id = imageId;
     imageDerivative.key = key;
     imageDerivative.filetype = filetype;
-    imageDerivative.data = file;
+    imageDerivative.s3key = s3key;
     imageDerivative.last_read = new Date();
 
     try {
-      return await this.imageDerivativeRepo.save(imageDerivative);
+      const result = await this.imageDerivativeRepo.save(imageDerivative);
+
+      const s3result = await this.s3Service.putFile(s3key, file);
+      if (HasFailed(s3result)) return s3result;
+
+      return result;
     } catch (e) {
       return Fail(FT.Database, e);
     }
@@ -156,13 +177,9 @@ export class ImageFileDBService {
       const aMinuteAgo = new Date(Date.now() - 60 * 1000);
       if (derivative.last_read > aMinuteAgo) {
         derivative.last_read = new Date();
-        this.imageDerivativeRepo.save(derivative).then(r => {
+        this.imageDerivativeRepo.save(derivative).then((r) => {
           if (HasFailed(r)) r.print(this.logger);
-        })
-      }
-
-      if (!(derivative.data instanceof Buffer)) {
-        derivative.data = Buffer.from(derivative.data);
+        });
       }
 
       return derivative;
@@ -184,4 +201,47 @@ export class ImageFileDBService {
       return Fail(FT.Database, e);
     }
   }
+
+  public async cleanupOrphanedDerivatives(): AsyncFailable<number> {
+    return this.cleanupRepoWithS3(this.imageDerivativeRepo);
+  }
+
+  public async cleanupOrphanedFiles(): AsyncFailable<number> {
+    return this.cleanupRepoWithS3(this.imageFileRepo);
+  }
+
+  private async cleanupRepoWithS3(
+    repo: Repository<{ image_id: string | null; s3key: string }>,
+  ): AsyncFailable<number> {
+    try {
+      let remaining = Infinity;
+      let processed = 0;
+      while (remaining > 0) {
+        const orphaned = await repo.findAndCount({
+          where: {
+            image_id: IsNull(),
+          },
+          select: ['s3key'],
+          take: 100,
+        });
+        if (orphaned[1] === 0) break;
+        remaining = orphaned[1] - orphaned[0].length;
+
+        const keys = orphaned[0].map((d) => d.s3key);
+
+        const s3result = await this.s3Service.deleteFiles(keys);
+        if (HasFailed(s3result)) return s3result;
+
+        const result = await repo.delete({
+          s3key: In(keys),
+        });
+
+        processed += result.affected ?? 0;
+      }
+
+      return processed;
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
 }

backend/src/collections/preference-db/sys-preference-db.service.ts

@@ -3,17 +3,20 @@ import { InjectRepository } from '@nestjs/typeorm';
 import {
   DecodedSysPref,
   PrefValueType,
-  PrefValueTypeStrings
+  PrefValueTypeStrings,
 } from 'picsur-shared/dist/dto/preferences.dto';
 import {
   SysPreference,
   SysPreferenceList,
   SysPreferenceValidators,
-  SysPreferenceValueTypes
+  SysPreferenceValueTypes,
 } from 'picsur-shared/dist/dto/sys-preferences.enum';
 import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
 import { Repository } from 'typeorm';
-import { ESysPreferenceBackend, ESysPreferenceSchema } from '../../database/entities/system/sys-preference.entity';
+import {
+  ESysPreferenceBackend,
+  ESysPreferenceSchema,
+} from '../../database/entities/system/sys-preference.entity';
 import { MutexFallBack } from '../../util/mutex-fallback';
 import { PreferenceCommonService } from './preference-common.service';
 import { PreferenceDefaultsService } from './preference-defaults.service';

backend/src/collections/preference-db/usr-preference-db.service.ts

@@ -3,19 +3,19 @@ import { InjectRepository } from '@nestjs/typeorm';
 import {
   DecodedUsrPref,
   PrefValueType,
-  PrefValueTypeStrings
+  PrefValueTypeStrings,
 } from 'picsur-shared/dist/dto/preferences.dto';
 import {
   UsrPreference,
   UsrPreferenceList,
   UsrPreferenceValidators,
-  UsrPreferenceValueTypes
+  UsrPreferenceValueTypes,
 } from 'picsur-shared/dist/dto/usr-preferences.enum';
 import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
 import { Repository } from 'typeorm';
 import {
   EUsrPreferenceBackend,
-  EUsrPreferenceSchema
+  EUsrPreferenceSchema,
 } from '../../database/entities/system/usr-preference.entity';
 import { MutexFallBack } from '../../util/mutex-fallback';
 import { PreferenceCommonService } from './preference-common.service';

backend/src/collections/role-db/role-db.module.ts

@@ -7,7 +7,7 @@ import { ERoleBackend } from '../../database/entities/users/role.entity';
 import {
   ImmutableRolesList,
   SystemRoleDefaults,
-  SystemRolesList
+  SystemRolesList,
 } from '../../models/constants/roles.const';
 import { RoleDbService } from './role-db.service';
 

backend/src/collections/role-db/role-db.service.ts

@@ -6,7 +6,7 @@ import {
   Fail,
   FT,
   HasFailed,
-  HasSuccess
+  HasSuccess,
 } from 'picsur-shared/dist/types';
 import { makeUnique } from 'picsur-shared/dist/util/unique';
 import { In, Repository } from 'typeorm';
@@ -14,7 +14,7 @@ import { ERoleBackend } from '../../database/entities/users/role.entity';
 import { Permissions } from '../../models/constants/permissions.const';
 import {
   ImmutableRolesList,
-  UndeletableRolesList
+  UndeletableRolesList,
 } from '../../models/constants/roles.const';
 
 @Injectable()

backend/src/collections/user-db/user-db.service.ts

@@ -7,7 +7,7 @@ import {
   Fail,
   FT,
   HasFailed,
-  HasSuccess
+  HasSuccess,
 } from 'picsur-shared/dist/types';
 import { FindResult } from 'picsur-shared/dist/types/find-result';
 import { makeUnique } from 'picsur-shared/dist/util/unique';
@@ -16,12 +16,12 @@ import { EUserBackend } from '../../database/entities/users/user.entity';
 import { Permissions } from '../../models/constants/permissions.const';
 import {
   DefaultRolesList,
-  SoulBoundRolesList
+  SoulBoundRolesList,
 } from '../../models/constants/roles.const';
 import {
   ImmutableUsersList,
   LockedLoginUsersList,
-  UndeletableUsersList
+  UndeletableUsersList,
 } from '../../models/constants/special-users.const';
 import { GetCols } from '../../util/collection';
 import { SysPreferenceDbService } from '../preference-db/sys-preference-db.service';

backend/src/config/early/bull.config.service.ts

@@ -1,6 +1,6 @@
 import {
   BullRootModuleOptions,
-  SharedBullConfigurationFactory
+  SharedBullConfigurationFactory,
 } from '@nestjs/bull';
 import { Injectable } from '@nestjs/common';
 import { RedisConfigService } from './redis.config.service';

backend/src/config/early/early-config.module.ts

@@ -6,6 +6,7 @@ import { EarlyJwtConfigService } from './early-jwt.config.service';
 import { HostConfigService } from './host.config.service';
 import { MultipartConfigService } from './multipart.config.service';
 import { RedisConfigService } from './redis.config.service';
+import { S3ConfigService } from './s3.config.service';
 import { ServeStaticConfigService } from './serve-static.config.service';
 import { TypeOrmConfigService } from './type-orm.config.service';
 
@@ -25,6 +26,7 @@ import { TypeOrmConfigService } from './type-orm.config.service';
     MultipartConfigService,
     RedisConfigService,
     BullConfigService,
+    S3ConfigService,
   ],
   exports: [
     ConfigModule,
@@ -36,6 +38,7 @@ import { TypeOrmConfigService } from './type-orm.config.service';
     MultipartConfigService,
     RedisConfigService,
     BullConfigService,
+    S3ConfigService,
   ],
 })
 export class EarlyConfigModule {}

backend/src/config/early/s3.config.service.ts

@@ -0,0 +1,73 @@
+import { S3ClientConfig } from '@aws-sdk/client-s3';
+import { Injectable, Logger } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { ParseString } from 'picsur-shared/dist/util/parse-simple';
+import { EnvPrefix } from '../config.static';
+
+@Injectable()
+export class S3ConfigService {
+  private readonly logger = new Logger(S3ConfigService.name);
+
+  constructor(private readonly configService: ConfigService) {
+    if (this.getS3Endpoint())
+      this.logger.log('Custom S3 Endpoint: ' + this.getS3Endpoint());
+
+    this.logger.log('S3 Region: ' + this.getS3Region());
+    this.logger.log('S3 Bucket: ' + this.getS3Bucket());
+
+    this.logger.verbose('S3 Access Key: ' + this.getS3AccessKey());
+    this.logger.verbose('S3 Secret Key: ' + this.getS3SecretKey());
+  }
+
+  public getS3Config(): S3ClientConfig {
+    return {
+      credentials: {
+        accessKeyId: this.getS3AccessKey(),
+        secretAccessKey: this.getS3SecretKey(),
+      },
+      endpoint: this.getS3Endpoint() ?? undefined,
+      region: this.getS3Region(),
+      tls: this.getS3TLS(),
+    };
+  }
+
+  public getS3Endpoint(): string | null {
+    return ParseString(this.configService.get(`${EnvPrefix}S3_ENDPOINT`), null);
+  }
+
+  public getS3TLS(): boolean | undefined {
+    const endpoint = this.getS3Endpoint();
+    if (endpoint) {
+      return endpoint.startsWith('https');
+    }
+    return undefined;
+  }
+
+  public getS3Bucket(): string {
+    return ParseString(
+      this.configService.get(`${EnvPrefix}S3_BUCKET`),
+      'picsur',
+    );
+  }
+
+  public getS3Region(): string {
+    return ParseString(
+      this.configService.get(`${EnvPrefix}S3_REGION`),
+      'us-east-1',
+    );
+  }
+
+  public getS3AccessKey(): string {
+    return ParseString(
+      this.configService.get(`${EnvPrefix}S3_ACCESS_KEY`),
+      'picsur',
+    );
+  }
+
+  public getS3SecretKey(): string {
+    return ParseString(
+      this.configService.get(`${EnvPrefix}S3_SECRET_KEY`),
+      'picsur',
+    );
+  }
+}

backend/src/config/early/type-orm.config.service.ts

@@ -71,7 +71,7 @@ export class TypeOrmConfigService implements TypeOrmOptionsFactory {
       cache: {
         duration: 60000,
         type: 'ioredis',
-        //alwaysEnabled: true,
+        alwaysEnabled: false,
         options: this.redisConfig.getRedisUrl(),
       },
 

backend/src/database/entities/apikey.entity.ts

@@ -4,7 +4,7 @@ import {
   Entity,
   Index,
   ManyToOne,
-  PrimaryGeneratedColumn
+  PrimaryGeneratedColumn,
 } from 'typeorm';
 import { z } from 'zod';
 import { EUserBackend } from './users/user.entity';

backend/src/database/entities/images/image-derivative.entity.ts

@@ -4,30 +4,32 @@ import {
   Index,
   JoinColumn,
   ManyToOne,
-  PrimaryGeneratedColumn,
-  Unique
+  PrimaryColumn,
+  Unique,
 } from 'typeorm';
 import { EImageBackend } from './image.entity';
 
 @Entity()
 @Unique(['image_id', 'key'])
 export class EImageDerivativeBackend {
-  @PrimaryGeneratedColumn('uuid')
-  private _id?: string;
+  @PrimaryColumn({ type: 'uuid', nullable: false })
+  @Index()
+  s3key: string;
 
   // We do a little trickery
   @Index()
   @ManyToOne(() => EImageBackend, (image) => image.derivatives, {
-    nullable: false,
-    onDelete: 'CASCADE',
+    nullable: true,
+    onDelete: 'SET NULL',
   })
   @JoinColumn({ name: 'image_id' })
   private _image?: any;
 
   @Column({
     name: 'image_id',
+    nullable: true,
   })
-  image_id: string;
+  image_id: string | null;
 
   @Index()
   @Column({ nullable: false })
@@ -42,8 +44,4 @@ export class EImageDerivativeBackend {
     nullable: false,
   })
   last_read: Date;
-
-  // Binary data
-  @Column({ type: 'bytea', nullable: false })
-  data: Buffer;
 }

backend/src/database/entities/images/image-file.entity.ts

@@ -5,7 +5,7 @@ import {
   Index,
   JoinColumn,
   ManyToOne,
-  PrimaryGeneratedColumn,
+  PrimaryColumn,
   Unique,
 } from 'typeorm';
 import { EImageBackend } from './image.entity';
@@ -13,22 +13,24 @@ import { EImageBackend } from './image.entity';
 @Entity()
 @Unique(['image_id', 'variant'])
 export class EImageFileBackend {
-  @PrimaryGeneratedColumn('uuid')
-  private _id?: string;
+  @PrimaryColumn({ type: 'uuid', nullable: false })
+  @Index()
+  s3key: string;
 
   // We do a little trickery
   @Index()
   @ManyToOne(() => EImageBackend, (image) => image.files, {
-    nullable: false,
-    onDelete: 'CASCADE',
+    nullable: true,
+    onDelete: 'SET NULL',
   })
   @JoinColumn({ name: 'image_id' })
   private _image?: any;
 
   @Column({
     name: 'image_id',
+    nullable: true,
   })
-  image_id: string;
+  image_id: string | null;
 
   @Index()
   @Column({ nullable: false, enum: ImageEntryVariant })
@@ -36,8 +38,4 @@ export class EImageFileBackend {
 
   @Column({ nullable: false })
   filetype: string;
-
-  // Binary data
-  @Column({ type: 'bytea', nullable: false })
-  data: Buffer;
 }

backend/src/database/entities/system/usr-preference.entity.ts

@@ -6,7 +6,7 @@ import {
   JoinColumn,
   ManyToOne,
   PrimaryGeneratedColumn,
-  Unique
+  Unique,
 } from 'typeorm';
 import z from 'zod';
 import { EUserBackend } from '../users/user.entity';

backend/src/database/entities/users/user.entity.ts

@@ -4,7 +4,7 @@ import {
   Entity,
   Index,
   OneToMany,
-  PrimaryGeneratedColumn
+  PrimaryGeneratedColumn,
 } from 'typeorm';
 import { z } from 'zod';
 import { EApiKeyBackend } from '../apikey.entity';

backend/src/decorators/multipart/multipart.decorator.ts

@@ -4,4 +4,5 @@ import { MultiPartPipe } from './postfiles.pipe';
 
 export const PostFile = () => InjectRequest(PostFilePipe);
 
-export const PostFiles = (maxFiles?: number) => InjectRequest(maxFiles, MultiPartPipe);
+export const PostFiles = (maxFiles?: number) =>
+  InjectRequest(maxFiles, MultiPartPipe);

backend/src/decorators/multipart/postfile.pipe.ts

@@ -12,7 +12,7 @@ export class PostFilePipe implements PipeTransform {
     private readonly multipartConfigService: MultipartConfigService,
   ) {}
 
-  async transform({ request, data }: { data: any; request: FastifyRequest },) {
+  async transform({ request, data }: { data: any; request: FastifyRequest }) {
     if (!request.isMultipart()) throw Fail(FT.UsrValidation, 'Invalid file');
 
     // Only one file is allowed

backend/src/decorators/multipart/postfiles.pipe.ts

@@ -4,7 +4,7 @@ import {
   Injectable,
   Logger,
   PipeTransform,
-  Scope
+  Scope,
 } from '@nestjs/common';
 import { FastifyRequest } from 'fastify';
 import { Fail, FT } from 'picsur-shared/dist/types';

backend/src/layers/PicsurLayers.module.ts

@@ -6,10 +6,12 @@ import { PicsurThrottlerGuard } from './throttler/PicsurThrottler.guard';
 import { ZodValidationPipe } from './validate/zod-validator.pipe';
 
 @Module({
-  imports: [ThrottlerModule.forRoot({
-    ttl: 60,
-    limit: 60,
-  })],
+  imports: [
+    ThrottlerModule.forRoot({
+      ttl: 60,
+      limit: 60,
+    }),
+  ],
   providers: [
     PicsurThrottlerGuard,
     MainExceptionFilter,

backend/src/layers/exception/exception.filter.ts

@@ -6,7 +6,7 @@ import {
   Logger,
   MethodNotAllowedException,
   NotFoundException,
-  UnauthorizedException
+  UnauthorizedException,
 } from '@nestjs/common';
 import { FastifyReply, FastifyRequest } from 'fastify';
 import { ApiErrorResponse } from 'picsur-shared/dist/dto/api/api.dto';
@@ -14,7 +14,7 @@ import {
   Fail,
   Failure,
   FT,
-  IsFailure
+  IsFailure,
 } from 'picsur-shared/dist/types/failable';
 
 // This will catch any exception that is made in any request

backend/src/layers/success/success.interceptor.ts

@@ -4,7 +4,7 @@ import {
   Injectable,
   Logger,
   NestInterceptor,
-  Optional
+  Optional,
 } from '@nestjs/common';
 import { Reflector } from '@nestjs/core';
 import { FastifyReply } from 'fastify';

backend/src/main.ts

@@ -4,7 +4,7 @@ import fastifyReplyFrom from '@fastify/reply-from';
 import { NestFactory } from '@nestjs/core';
 import {
   FastifyAdapter,
-  NestFastifyApplication
+  NestFastifyApplication,
 } from '@nestjs/platform-fastify';
 import { AppModule } from './app.module';
 import { HostConfigService } from './config/early/host.config.service';

backend/src/managers/image/convert.consumer.ts

@@ -48,13 +48,16 @@ export class ConvertConsumer {
     const masterImage = ThrowIfFailed(
       await this.imageService.getMaster(imageId),
     );
+    const masterImageData = ThrowIfFailed(
+      await this.imageService.getData(masterImage),
+    );
     const sourceFileType = ThrowIfFailed(ParseFileType(masterImage.filetype));
 
     // Conver timage
     const startTime = Date.now();
     const convertResult = ThrowIfFailed(
       await this.imageConverter.convert(
-        masterImage.data,
+        masterImageData,
         sourceFileType,
         targetFileType,
         allow_editing ? options : {},

backend/src/managers/image/image-converter.service.ts

@@ -3,7 +3,7 @@ import ms from 'ms';
 import { ImageRequestParams } from 'picsur-shared/dist/dto/api/image.dto';
 import {
   FileType,
-  SupportedFileTypeCategory
+  SupportedFileTypeCategory,
 } from 'picsur-shared/dist/dto/mimes.dto';
 import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
 import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';

backend/src/managers/image/image-manager.module.ts

@@ -13,10 +13,7 @@ import { ConvertConsumer } from './convert.consumer';
 import { ConvertService } from './convert.service';
 import { ImageConverterService } from './image-converter.service';
 import { ImageManagerService } from './image-manager.service';
-import {
-  ImageConvertQueueID,
-  ImageIngestQueueID,
-} from './image.queue';
+import { ImageConvertQueueID, ImageIngestQueueID } from './image.queue';
 import { IngestConsumer } from './ingest.consumer';
 import { IngestService } from './ingest.service';
 
@@ -67,6 +64,7 @@ export class ImageManagerModule implements OnModuleInit {
   private async imageManagerCron() {
     await this.cleanupDerivatives();
     await this.cleanupExpired();
+    await this.cleanupOrphanedFiles();
   }
 
   private async cleanupDerivatives() {
@@ -104,4 +102,25 @@ export class ImageManagerModule implements OnModuleInit {
     if (cleanedUp > 0)
       this.logger.log(`Cleaned up ${cleanedUp} expired images`);
   }
+
+  private async cleanupOrphanedFiles() {
+    const cleanedUpDerivatives =
+      await this.imageFileDB.cleanupOrphanedDerivatives();
+
+    if (HasFailed(cleanedUpDerivatives)) {
+      cleanedUpDerivatives.print(this.logger);
+      return;
+    }
+
+    const cleanedUpFiles = await this.imageFileDB.cleanupOrphanedFiles();
+    if (HasFailed(cleanedUpFiles)) {
+      cleanedUpFiles.print(this.logger);
+      return;
+    }
+
+    if (cleanedUpDerivatives > 0 || cleanedUpFiles > 0)
+      this.logger.log(
+        `Cleaned up ${cleanedUpDerivatives} orphaned derivatives and ${cleanedUpFiles} orphaned files`,
+      );
+  }
 }

backend/src/managers/image/image-manager.service.ts

@@ -6,6 +6,7 @@ import { FindResult } from 'picsur-shared/dist/types/find-result';
 import { ParseFileType } from 'picsur-shared/dist/util/parse-mime';
 import { ImageDBService } from '../../collections/image-db/image-db.service';
 import { ImageFileDBService } from '../../collections/image-db/image-file-db.service';
+import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity';
 import { EImageFileBackend } from '../../database/entities/images/image-file.entity';
 import { EImageBackend } from '../../database/entities/images/image.entity';
 
@@ -87,6 +88,10 @@ export class ImageManagerService {
     return ParseFileType(filetypes['original']);
   }
 
+  public async getData(image: EImageFileBackend | EImageDerivativeBackend) {
+    return await this.imageFilesService.getData(image);
+  }
+
   public async getFileMimes(imageId: string): AsyncFailable<{
     [ImageEntryVariant.MASTER]: string;
     [ImageEntryVariant.ORIGINAL]: string | undefined;

backend/src/managers/image/ingest.consumer.ts

@@ -5,7 +5,7 @@ import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.en
 import {
   FileType,
   ImageFileType,
-  SupportedFileTypeCategory
+  SupportedFileTypeCategory,
 } from 'picsur-shared/dist/dto/mimes.dto';
 import {
   AsyncFailable,
@@ -13,7 +13,7 @@ import {
   FT,
   HasFailed,
   IsFailure,
-  ThrowIfFailed
+  ThrowIfFailed,
 } from 'picsur-shared/dist/types';
 import { ParseFileType } from 'picsur-shared/dist/util/parse-mime';
 import { ImageDBService } from '../../collections/image-db/image-db.service';
@@ -55,11 +55,13 @@ export class IngestConsumer {
     const ingestFile = ThrowIfFailed(
       await this.imageFilesService.getFile(imageID, ImageEntryVariant.INGEST),
     );
-
+    const ingestFileData = ThrowIfFailed(
+      await this.imageFilesService.getData(ingestFile),
+    );
     const ingestFiletype = ThrowIfFailed(ParseFileType(ingestFile.filetype));
 
     const processed = ThrowIfFailed(
-      await this.process(ingestFile.data, ingestFiletype),
+      await this.process(ingestFileData, ingestFiletype),
     );
 
     const masterPromise = this.imageFilesService.setFile(
@@ -75,7 +77,7 @@ export class IngestConsumer {
           ImageEntryVariant.INGEST,
           ImageEntryVariant.ORIGINAL,
         )
-      : this.imageFilesService.deleteFile(imageID, ImageEntryVariant.INGEST);
+      : this.imageFilesService.orphanFile(imageID, ImageEntryVariant.INGEST);
 
     const results = await Promise.all([masterPromise, originalPromise]);
     results.map((r) => ThrowIfFailed(r));

backend/src/routes/api/apikeys/apikeys.controller.ts

@@ -9,14 +9,14 @@ import {
   ApiKeyListRequest,
   ApiKeyListResponse,
   ApiKeyUpdateRequest,
-  ApiKeyUpdateResponse
+  ApiKeyUpdateResponse,
 } from 'picsur-shared/dist/dto/api/apikeys.dto';
 import { Permission } from 'picsur-shared/dist/dto/permissions.enum';
 import { ThrowIfFailed } from 'picsur-shared/dist/types';
 import { ApiKeyDbService } from '../../../collections/apikey-db/apikey-db.service';
 import {
   HasPermission,
-  RequiredPermissions
+  RequiredPermissions,
 } from '../../../decorators/permissions.decorator';
 import { ReqUserID } from '../../../decorators/request-user.decorator';
 import { Returns } from '../../../decorators/returns.decorator';

backend/src/routes/api/experiment/experiment.module.ts

@@ -8,6 +8,6 @@ import { ExperimentController } from './experiment.controller';
 
 @Module({
   imports: [ImageManagerModule, PicsurLoggerModule],
-  controllers: [ExperimentController]
+  controllers: [ExperimentController],
 })
 export class ExperimentModule {}

backend/src/routes/api/pref/sys-pref.controller.ts

@@ -4,7 +4,7 @@ import {
   GetPreferenceResponse,
   MultiplePreferencesResponse,
   UpdatePreferenceRequest,
-  UpdatePreferenceResponse
+  UpdatePreferenceResponse,
 } from 'picsur-shared/dist/dto/api/pref.dto';
 import { ThrowIfFailed } from 'picsur-shared/dist/types';
 import { SysPreferenceDbService } from '../../../collections/preference-db/sys-preference-db.service';

backend/src/routes/api/pref/usr-pref.controller.ts

@@ -4,7 +4,7 @@ import {
   GetPreferenceResponse,
   MultiplePreferencesResponse,
   UpdatePreferenceRequest,
-  UpdatePreferenceResponse
+  UpdatePreferenceResponse,
 } from 'picsur-shared/dist/dto/api/pref.dto';
 import { ThrowIfFailed } from 'picsur-shared/dist/types';
 import { UsrPreferenceDbService } from '../../../collections/preference-db/usr-preference-db.service';

backend/src/routes/api/roles/roles.controller.ts

@@ -10,7 +10,7 @@ import {
   RoleListResponse,
   RoleUpdateRequest,
   RoleUpdateResponse,
-  SpecialRolesResponse
+  SpecialRolesResponse,
 } from 'picsur-shared/dist/dto/api/roles.dto';
 import { Fail, FT, ThrowIfFailed } from 'picsur-shared/dist/types';
 import { RoleDbService } from '../../../collections/role-db/role-db.service';
@@ -22,7 +22,7 @@ import {
   DefaultRolesList,
   ImmutableRolesList,
   SoulBoundRolesList,
-  UndeletableRolesList
+  UndeletableRolesList,
 } from '../../../models/constants/roles.const';
 import { isPermissionsArray } from '../../../models/validators/permissions.validator';
 

backend/src/routes/api/user/user-manage.controller.ts

@@ -11,7 +11,7 @@ import {
   UserListRequest,
   UserListResponse,
   UserUpdateRequest,
-  UserUpdateResponse
+  UserUpdateResponse,
 } from 'picsur-shared/dist/dto/api/user-manage.dto';
 import { ThrowIfFailed } from 'picsur-shared/dist/types';
 import { UserDbService } from '../../../collections/user-db/user-db.service';
@@ -21,7 +21,7 @@ import { Permission } from '../../../models/constants/permissions.const';
 import {
   ImmutableUsersList,
   LockedLoginUsersList,
-  UndeletableUsersList
+  UndeletableUsersList,
 } from '../../../models/constants/special-users.const';
 import { EUserBackend2EUser } from '../../../models/transformers/user.transformer';
 

backend/src/routes/api/user/user.controller.ts

@@ -7,7 +7,7 @@ import {
   UserMePermissionsResponse,
   UserMeResponse,
   UserRegisterRequest,
-  UserRegisterResponse
+  UserRegisterResponse,
 } from 'picsur-shared/dist/dto/api/user.dto';
 import type { EUser } from 'picsur-shared/dist/entities/user.entity';
 import { ThrowIfFailed } from 'picsur-shared/dist/types';
@@ -15,7 +15,7 @@ import { UserDbService } from '../../../collections/user-db/user-db.service';
 import {
   NoPermissions,
   RequiredPermissions,
-  UseLocalAuth
+  UseLocalAuth,
 } from '../../../decorators/permissions.decorator';
 import { ReqUser, ReqUserID } from '../../../decorators/request-user.decorator';
 import { Returns } from '../../../decorators/returns.decorator';

backend/src/routes/image/image.controller.ts

@@ -63,9 +63,10 @@ export class ImageController {
         const image = ThrowIfFailed(
           await this.imagesService.getOriginal(fullid.id),
         );
+        const data = ThrowIfFailed(await this.imagesService.getData(image));
 
         res.type(ThrowIfFailed(FileType2Mime(image.filetype)));
-        return image.data;
+        return data;
       }
 
       const image = ThrowIfFailed(
@@ -75,9 +76,10 @@ export class ImageController {
           params,
         ),
       );
+      const data = ThrowIfFailed(await this.imagesService.getData(image));
 
       res.type(ThrowIfFailed(FileType2Mime(image.filetype)));
-      return image.data;
+      return data;
     } catch (e) {
       if (!IsFailure(e) || e.getType() !== FT.NotFound) throw e;
 

backend/src/routes/image/image.module.ts

@@ -6,11 +6,7 @@ import { ImageManageController } from './image-manage.controller';
 import { ImageController } from './image.controller';
 
 @Module({
-  imports: [
-    ImageManagerModule,
-    UserDbModule,
-    DecoratorsModule,
-  ],
+  imports: [ImageManagerModule, UserDbModule, DecoratorsModule],
   controllers: [ImageController, ImageManageController],
 })
 export class ImageModule {}

backend/src/workers/sharp/universal-sharp.ts

@@ -2,7 +2,7 @@ import { BMPdecode, BMPencode } from 'bmp-img';
 import {
   AnimFileType,
   FileType,
-  ImageFileType
+  ImageFileType,
 } from 'picsur-shared/dist/dto/mimes.dto';
 import { QOIdecode, QOIencode } from 'qoi-img';
 import sharp, { Sharp, SharpOptions } from 'sharp';

frontend/src/app/components/picsur-img/picsur-img.component.ts

@@ -6,7 +6,7 @@ import {
   Input,
   OnChanges,
   SimpleChanges,
-  ViewChild
+  ViewChild,
 } from '@angular/core';
 import { FileType, ImageFileType } from 'picsur-shared/dist/dto/mimes.dto';
 import { AsyncFailable, HasFailed } from 'picsur-shared/dist/types';

frontend/src/app/routes/images/images.component.ts

@@ -11,7 +11,7 @@ import {
   merge,
   Observable,
   switchMap,
-  timer
+  timer,
 } from 'rxjs';
 import { ImageService } from 'src/app/services/api/image.service';
 import { UserService } from 'src/app/services/api/user.service';

frontend/src/app/routes/view/view-speeddial/view-speeddial.component.ts

@@ -17,11 +17,11 @@ import { ErrorService } from 'src/app/util/error-manager/error.service';
 import { UtilService } from 'src/app/util/util.service';
 import {
   CustomizeDialogComponent,
-  CustomizeDialogData
+  CustomizeDialogData,
 } from '../customize-dialog/customize-dialog.component';
 import {
   EditDialogComponent,
-  EditDialogData
+  EditDialogData,
 } from '../edit-dialog/edit-dialog.component';
 
 @Component({

frontend/src/app/routes/view/view.component.ts

@@ -3,7 +3,7 @@ import {
   ChangeDetectorRef,
   Component,
   OnDestroy,
-  OnInit
+  OnInit,
 } from '@angular/core';
 import { ActivatedRoute, Router } from '@angular/router';
 import { ImageMetaResponse } from 'picsur-shared/dist/dto/api/image.dto';
@@ -11,7 +11,7 @@ import { ImageLinks } from 'picsur-shared/dist/dto/image-links.class';
 import {
   AnimFileType,
   ImageFileType,
-  SupportedFileTypeCategory
+  SupportedFileTypeCategory,
 } from 'picsur-shared/dist/dto/mimes.dto';
 import { EImage } from 'picsur-shared/dist/entities/image.entity';
 import { EUser } from 'picsur-shared/dist/entities/user.entity';

frontend/src/app/services/api/api.service.ts

@@ -3,7 +3,7 @@ import { WINDOW } from '@ng-web-apis/common';
 import axios, {
   AxiosRequestConfig,
   AxiosResponse,
-  AxiosResponseHeaders
+  AxiosResponseHeaders,
 } from 'axios';
 import { ApiResponseSchema } from 'picsur-shared/dist/dto/api/api.dto';
 import { FileType2Ext } from 'picsur-shared/dist/dto/mimes.dto';
@@ -13,7 +13,7 @@ import {
   Failure,
   FT,
   HasFailed,
-  HasSuccess
+  HasSuccess,
 } from 'picsur-shared/dist/types';
 import { ZodDtoStatic } from 'picsur-shared/dist/util/create-zod-dto';
 import { ParseMime2FileType } from 'picsur-shared/dist/util/parse-mime';

frontend/src/app/services/api/apikeys.service.ts

@@ -8,7 +8,7 @@ import {
   ApiKeyListRequest,
   ApiKeyListResponse,
   ApiKeyUpdateRequest,
-  ApiKeyUpdateResponse
+  ApiKeyUpdateResponse,
 } from 'picsur-shared/dist/dto/api/apikeys.dto';
 import { EApiKey } from 'picsur-shared/dist/entities/apikey.entity';
 import { AsyncFailable } from 'picsur-shared/dist/types';

frontend/src/app/services/api/roles.service.ts

@@ -8,7 +8,7 @@ import {
   RoleInfoResponse,
   RoleListResponse,
   RoleUpdateRequest,
-  RoleUpdateResponse
+  RoleUpdateResponse,
 } from 'picsur-shared/dist/dto/api/roles.dto';
 import { ERole } from 'picsur-shared/dist/entities/role.entity';
 import { AsyncFailable, Open } from 'picsur-shared/dist/types';

frontend/src/app/services/api/sys-pref.service.ts

@@ -4,19 +4,19 @@ import {
   GetPreferenceResponse,
   MultiplePreferencesResponse,
   UpdatePreferenceRequest,
-  UpdatePreferenceResponse
+  UpdatePreferenceResponse,
 } from 'picsur-shared/dist/dto/api/pref.dto';
 import { Permission } from 'picsur-shared/dist/dto/permissions.enum';
 import {
   DecodedPref,
-  PrefValueType
+  PrefValueType,
 } from 'picsur-shared/dist/dto/preferences.dto';
 import {
   AsyncFailable,
   Fail,
   FT,
   HasFailed,
-  Map
+  Map,
 } from 'picsur-shared/dist/types';
 import { BehaviorSubject } from 'rxjs';
 import { ErrorService } from 'src/app/util/error-manager/error.service';

frontend/src/app/services/api/user-manage.service.ts

@@ -9,7 +9,7 @@ import {
   UserListRequest,
   UserListResponse,
   UserUpdateRequest,
-  UserUpdateResponse
+  UserUpdateResponse,
 } from 'picsur-shared/dist/dto/api/user-manage.dto';
 import { EUser } from 'picsur-shared/dist/entities/user.entity';
 import { AsyncFailable } from 'picsur-shared/dist/types';

frontend/src/app/services/api/user.service.ts

@@ -7,7 +7,7 @@ import {
   UserLoginResponse,
   UserMeResponse,
   UserRegisterRequest,
-  UserRegisterResponse
+  UserRegisterResponse,
 } from 'picsur-shared/dist/dto/api/user.dto';
 import { JwtDataSchema } from 'picsur-shared/dist/dto/jwt.dto';
 import { EUser } from 'picsur-shared/dist/entities/user.entity';
@@ -16,7 +16,7 @@ import {
   Fail,
   FT,
   HasFailed,
-  Open
+  Open,
 } from 'picsur-shared/dist/types';
 import { BehaviorSubject } from 'rxjs';
 import { Logger } from '../logger/logger.service';

frontend/src/app/services/api/usr-pref.service.ts

@@ -4,19 +4,19 @@ import {
   GetPreferenceResponse,
   MultiplePreferencesResponse,
   UpdatePreferenceRequest,
-  UpdatePreferenceResponse
+  UpdatePreferenceResponse,
 } from 'picsur-shared/dist/dto/api/pref.dto';
 import { Permission } from 'picsur-shared/dist/dto/permissions.enum';
 import {
   DecodedPref,
-  PrefValueType
+  PrefValueType,
 } from 'picsur-shared/dist/dto/preferences.dto';
 import {
   AsyncFailable,
   Fail,
   FT,
   HasFailed,
-  Map
+  Map,
 } from 'picsur-shared/dist/types';
 import { BehaviorSubject } from 'rxjs';
 import { ErrorService } from 'src/app/util/error-manager/error.service';

frontend/src/app/util/download-manager/download.service.ts

@@ -35,14 +35,15 @@ export class DownloadService {
   }
 
   public async downloadFile(url: string) {
-
-
     const request = this.api.getBuffer(url);
-    const closeDialog = this.showDownloadDialog('image', request.downloadProgress);
+    const closeDialog = this.showDownloadDialog(
+      'image',
+      request.downloadProgress,
+    );
 
     const file = await request.result;
 
-    if (HasFailed(file)){
+    if (HasFailed(file)) {
       closeDialog();
       return this.errorService.showFailure(file, this.logger);
     }