Update deps

.vscode/tasks.json

@@ -55,7 +55,7 @@
     {
       "type": "shell",
       "label": "Start postgres",
-      "command": "yarn devdb:start",
+      "command": "yarn devdb:up",
       "options": {
         "cwd": "${cwd}",
         "shell": {

backend/.env.example

@@ -0,0 +1,4 @@
+PICSUR_FILESTORAGE_MODE="S3"
+PICSUR_FILESTORAGE_S3_ENDPOINT="http://localhost:8000"
+PICSUR_FILESTORAGE_S3_ACCESS_KEY="username"
+PICSUR_FILESTORAGE_S3_SECRET_KEY="password"

backend/.gitignore

@@ -396,3 +396,4 @@ Temporary Items
 # Local
 .env
 dist
+temp

backend/package.json

@@ -22,6 +22,7 @@
     "purge": "rm -rf dist && rm -rf node_modules"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.238.0",
     "@fastify/helmet": "^10.1.0",
     "@fastify/multipart": "^7.5.0",
     "@fastify/reply-from": "^9.0.1",
@@ -56,6 +57,7 @@
     "reflect-metadata": "^0.1.13",
     "rimraf": "^4.4.0",
     "rxjs": "^7.8.0",
+    "semver": "^7.3.8",
     "sharp": "^0.31.3",
     "stream-parser": "^0.3.1",
     "thunks": "^4.9.6",
@@ -73,12 +75,14 @@
     "@types/passport-jwt": "^3.0.8",
     "@types/passport-local": "^1.0.35",
     "@types/passport-strategy": "^0.2.35",
-    "@types/sharp": "^0.31.1",
+    "@types/semver": "^7.3.12",
+    "@types/sharp": "^0.31.0",
     "@types/supertest": "^2.0.12",
-    "@typescript-eslint/eslint-plugin": "^5.55.0",
-    "@typescript-eslint/parser": "^5.55.0",
-    "eslint": "^8.36.0",
-    "eslint-config-prettier": "^8.7.0",
+    "@types/uuid": "^9.0.0",
+    "@typescript-eslint/eslint-plugin": "^5.47.0",
+    "@typescript-eslint/parser": "^5.47.0",
+    "eslint": "^8.30.0",
+    "eslint-config-prettier": "^8.5.0",
     "eslint-plugin-prettier": "^4.2.1",
     "prettier": "^2.8.4",
     "source-map-support": "^0.5.21",

backend/src/app.module.ts

@@ -1,8 +1,10 @@
-import { MiddlewareConsumer, Module, NestModule } from '@nestjs/common';
+import { Logger, MiddlewareConsumer, Module, NestModule, OnApplicationBootstrap, OnApplicationShutdown } from '@nestjs/common';
 import { ScheduleModule } from '@nestjs/schedule';
 import { ServeStaticModule } from '@nestjs/serve-static';
 import cors from 'cors';
 import { IncomingMessage, ServerResponse } from 'http';
+import semver from 'semver';
+import { FileStorageDBModule } from './collections/filestorage-db/filestorage-db.module';
 import { EarlyConfigModule } from './config/early/early-config.module';
 import { ServeStaticConfigService } from './config/early/serve-static.config.service';
 import { DatabaseModule } from './database/database.module';
@@ -13,6 +15,8 @@ import { DemoManagerModule } from './managers/demo/demo.module';
 import { UsageManagerModule } from './managers/usage/usage.module';
 import { PicsurRoutesModule } from './routes/routes.module';
 
+const supportedNodeVersions = ['^16.17.0', '^18.6.0'];
+
 const mainCorsConfig = cors({
   origin: '<origin>',
 });
@@ -46,6 +50,7 @@ const imageCorsOverride = (
     }),
     ScheduleModule.forRoot(),
     DatabaseModule,
+    FileStorageDBModule,
     AuthManagerModule,
     UsageManagerModule,
     DemoManagerModule,
@@ -53,9 +58,28 @@ const imageCorsOverride = (
     PicsurLayersModule,
   ],
 })
-export class AppModule implements NestModule {
+export class AppModule implements NestModule, OnApplicationBootstrap, OnApplicationShutdown {
+  private readonly logger = new Logger(AppModule.name);
+
   configure(consumer: MiddlewareConsumer) {
     consumer.apply(mainCorsConfig).exclude('/i').forRoutes('/');
     consumer.apply(imageCorsConfig, imageCorsOverride).forRoutes('/i');
   }
+
+  onApplicationBootstrap() {
+    const nodeVersion = process.version;
+    if (!supportedNodeVersions.some((v) => semver.satisfies(nodeVersion, v))) {
+      this.logger.error(
+        `Unsupported Node version: ${nodeVersion}. Transcoding performance will be severely degraded.`,
+      );
+
+      this.logger.log(
+        `Supported Node versions: ${supportedNodeVersions.join(', ')}`,
+      );
+    }
+  }
+
+  onApplicationShutdown() {
+    this.logger.warn(`Shutting down`);
+  }
 }

backend/src/collections/filestorage-db/filestorage-db.module.ts

@@ -0,0 +1,12 @@
+import { Module } from '@nestjs/common';
+import { LateConfigModule } from '../../config/late/late-config.module';
+import { FileStorageGeneric } from './filestorage-generic.service';
+
+export const FSServiceToken = 'FileStorageService';
+
+@Module({
+  imports: [LateConfigModule],
+  providers: [FileStorageGeneric],
+  exports: [FileStorageGeneric],
+})
+export class FileStorageDBModule {}

backend/src/collections/filestorage-db/filestorage-generic.service.ts

@@ -0,0 +1,47 @@
+import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
+import { AsyncFailable } from 'picsur-shared/dist/types';
+import {
+  FileStorageMode
+} from '../../config/early/early-fs.config.service';
+import { FSConfigService } from '../../config/late/fs.config.service';
+import { FileStorageEmpty } from './services/filestorage-empty';
+import { FileStorageLocalService } from './services/filestorage-local';
+
+import { FileStorageS3Service } from './services/filestorage-s3';
+import { FileStorageService } from './services/filestorage-service';
+
+@Injectable()
+export class FileStorageGeneric implements OnModuleInit {
+  private readonly logger = new Logger(FileStorageGeneric.name);
+  private backingService: FileStorageService = new FileStorageEmpty();
+
+  constructor(private readonly fsConfig: FSConfigService) {}
+
+  async onModuleInit() {
+    const mode = this.fsConfig.getFileStorageMode();
+    if (mode === FileStorageMode.Local) {
+      this.backingService = new FileStorageLocalService(this.fsConfig);
+    } else if (mode === FileStorageMode.S3) {
+      this.backingService = new FileStorageS3Service(this.fsConfig);
+    } else {}
+
+    try {
+      await this.backingService.onStorageInit();
+    } catch (e) {
+      this.logger.error(e);
+    }
+  }
+
+  async putFile(key: string, data: Buffer): AsyncFailable<string> {
+    return this.backingService.putFile(key, data);
+  }
+  async getFile(key: string): AsyncFailable<Buffer> {
+    return this.backingService.getFile(key);
+  }
+  async deleteFile(key: string): AsyncFailable<true> {
+    return this.backingService.deleteFile(key);
+  }
+  async deleteFiles(keys: string[]): AsyncFailable<true> {
+    return this.backingService.deleteFiles(keys);
+  }
+}

backend/src/collections/filestorage-db/services/filestorage-empty.ts

@@ -0,0 +1,25 @@
+import { AsyncFailable, Fail, FT } from 'picsur-shared/dist/types';
+import { FileStorageService } from './filestorage-service';
+
+export class FileStorageEmpty extends FileStorageService {
+  private readonly errorMessage = 'No file storage configured';
+
+  constructor() {
+    super(undefined as any);
+  }
+
+  onStorageInit(): void {}
+
+  async putFile(key: string, data: Buffer): AsyncFailable<string> {
+    return Fail(FT.Internal, this.errorMessage);
+  }
+  async getFile(key: string): AsyncFailable<Buffer> {
+    return Fail(FT.Internal, this.errorMessage);
+  }
+  async deleteFile(key: string): AsyncFailable<true> {
+    return Fail(FT.Internal, this.errorMessage);
+  }
+  async deleteFiles(keys: string[]): AsyncFailable<true> {
+    return Fail(FT.Internal, this.errorMessage);
+  }
+}

backend/src/collections/filestorage-db/services/filestorage-local.ts

@@ -0,0 +1,84 @@
+import { Logger } from '@nestjs/common';
+import * as fs from 'fs';
+import afs from 'fs/promises';
+import pathlib from 'path';
+import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
+import { FSConfigService } from '../../../config/late/fs.config.service';
+import { FileStorageService } from './filestorage-service';
+
+export class FileStorageLocalService extends FileStorageService {
+  private readonly logger = new Logger(FileStorageLocalService.name);
+
+  private path = './temp';
+
+  constructor(config: FSConfigService) {
+    super(config);
+  }
+
+  async onStorageInit() {
+    this.path = await this.config.getLocalPath();
+    await this.ensureFileDir(this.path);
+  }
+
+  public async putFile(key: string, data: Buffer): AsyncFailable<string> {
+    const path = this.getKeyFilePath(key);
+    const result = await this.ensureFileDir(path);
+    if (HasFailed(result)) return result;
+
+    try {
+      await afs.writeFile(path, data);
+      return key;
+    } catch (e) {
+      return Fail(FT.FileStorage, e);
+    }
+  }
+
+  public async getFile(key: string): AsyncFailable<Buffer> {
+    const path = this.getKeyFilePath(key);
+    try {
+      const result = await afs.readFile(path);
+      return result;
+    } catch (e) {
+      return Fail(FT.FileStorage, e);
+    }
+  }
+
+  public async deleteFile(key: string): AsyncFailable<true> {
+    const path = this.getKeyFilePath(key);
+    try {
+      await afs.unlink(path);
+      return true;
+    } catch (e) {
+      return Fail(FT.FileStorage, e);
+    }
+  }
+
+  public async deleteFiles(keys: string[]): AsyncFailable<true> {
+    const paths = keys.map((key) => this.getKeyFilePath(key));
+    try {
+      await Promise.all(paths.map((path) => afs.unlink(path)));
+      return true;
+    } catch (e) {
+      return Fail(FT.FileStorage, e);
+    }
+  }
+
+  private getKeyFilePath(key: string): string {
+    const subfolder = key.slice(0, 4);
+    return pathlib.resolve(this.path, subfolder, key);
+  }
+
+  private async ensureFileDir(path: string): AsyncFailable<true> {
+    try {
+      const dir = path.split('/').slice(0, -1).join('/');
+      if (!fs.existsSync(dir)) {
+        await afs.mkdir(dir, {
+          recursive: true,
+        });
+      }
+      return true;
+    } catch (e) {
+      return Fail(FT.FileStorage, e);
+    }
+  }
+}

backend/src/collections/filestorage-db/services/filestorage-s3.ts

@@ -0,0 +1,130 @@
+import {
+  CreateBucketCommand,
+  DeleteObjectCommand,
+  DeleteObjectsCommand,
+  GetObjectCommand,
+  ListBucketsCommand,
+  PutObjectCommand,
+  S3Client
+} from '@aws-sdk/client-s3';
+import { Logger } from '@nestjs/common';
+import { buffer as streamToBuffer } from 'get-stream';
+import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
+import { Readable } from 'stream';
+import { FileStorageService } from './filestorage-service';
+
+export class FileStorageS3Service extends FileStorageService {
+  private readonly logger = new Logger(FileStorageS3Service.name);
+  private S3: S3Client | null = null;
+
+  onStorageInit() {
+    this.loadS3();
+  }
+
+  public async putFile(key: string, data: Buffer): AsyncFailable<string> {
+    const S3 = await this.getS3();
+    if (HasFailed(S3)) return S3;
+
+    const request = new PutObjectCommand({
+      Bucket: await this.config.getS3Bucket(),
+      Key: key,
+      Body: data,
+    });
+
+    try {
+      await S3.send(request);
+      return key;
+    } catch (e) {
+      return Fail(FT.FileStorage, e);
+    }
+  }
+
+  public async getFile(key: string): AsyncFailable<Buffer> {
+    const S3 = await this.getS3();
+    if (HasFailed(S3)) return S3;
+
+    const request = new GetObjectCommand({
+      Bucket: await this.config.getS3Bucket(),
+      Key: key,
+    });
+
+    try {
+      const result = await S3.send(request);
+      if (!result.Body) return Fail(FT.NotFound, 'File not found');
+
+      if (result.Body instanceof Blob) {
+        return Buffer.from(await result.Body.arrayBuffer());
+      }
+      return streamToBuffer(result.Body as Readable);
+    } catch (e) {
+      return Fail(FT.FileStorage, e);
+    }
+  }
+
+  public async deleteFile(key: string): AsyncFailable<true> {
+    const S3 = await this.getS3();
+    if (HasFailed(S3)) return S3;
+
+    const request = new DeleteObjectCommand({
+      Bucket: await this.config.getS3Bucket(),
+      Key: key,
+    });
+
+    try {
+      await S3.send(request);
+      return true;
+    } catch (e) {
+      return Fail(FT.FileStorage, e);
+    }
+  }
+
+  public async deleteFiles(keys: string[]): AsyncFailable<true> {
+    const S3 = await this.getS3();
+    if (HasFailed(S3)) return S3;
+
+    const request = new DeleteObjectsCommand({
+      Bucket: await this.config.getS3Bucket(),
+      Delete: {
+        Objects: keys.map((key) => ({ Key: key })),
+      },
+    });
+
+    try {
+      await S3.send(request);
+      return true;
+    } catch (e) {
+      return Fail(FT.FileStorage, e);
+    }
+  }
+
+  private async getS3(): AsyncFailable<S3Client> {
+    if (this.S3) return this.S3;
+    await this.loadS3();
+    if (this.S3) return this.S3;
+    return Fail(FT.FileStorage, 'S3 not loaded');
+  }
+
+  private async loadS3(): Promise<void> {
+    const S3 = new S3Client(await this.config.getS3Config());
+
+    try {
+      // Create bucket if it doesn't exist
+      const bucket = await this.config.getS3Bucket();
+
+      // List buckets
+      const listBuckets = await S3.send(new ListBucketsCommand({}));
+
+      const bucketExists = listBuckets.Buckets?.some((b) => b.Name === bucket);
+      if (!bucketExists) {
+        this.logger.verbose(`Creating S3 Bucket ${bucket}`);
+        await S3.send(new CreateBucketCommand({ Bucket: bucket }));
+      } else {
+        this.logger.verbose(`Using existing S3 Bucket ${bucket}`);
+      }
+
+      this.S3 = S3;
+    } catch (e) {
+      this.logger.error(e);
+    }
+  }
+}

backend/src/collections/filestorage-db/services/filestorage-service.ts

@@ -0,0 +1,12 @@
+import { AsyncFailable } from 'picsur-shared/dist/types';
+import { FSConfigService } from '../../../config/late/fs.config.service';
+
+export abstract class FileStorageService {
+  constructor(protected readonly config: FSConfigService) {}
+
+  public abstract onStorageInit(): Promise<void> | void;
+  public abstract putFile(key: string, data: Buffer): AsyncFailable<string>;
+  public abstract getFile(key: string): AsyncFailable<Buffer>;
+  public abstract deleteFile(key: string): AsyncFailable<true>;
+  public abstract deleteFiles(keys: string[]): AsyncFailable<true>;
+}

backend/src/collections/image-db/image-db.module.ts

@@ -3,6 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
 import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity';
 import { EImageFileBackend } from '../../database/entities/images/image-file.entity';
 import { EImageBackend } from '../../database/entities/images/image.entity';
+import { FileStorageDBModule } from '../filestorage-db/filestorage-db.module';
 import { ImageDBService } from './image-db.service';
 import { ImageFileDBService } from './image-file-db.service';
 
@@ -13,6 +14,7 @@ import { ImageFileDBService } from './image-file-db.service';
       EImageFileBackend,
       EImageDerivativeBackend,
     ]),
+    FileStorageDBModule
   ],
   providers: [ImageDBService, ImageFileDBService],
   exports: [ImageDBService, ImageFileDBService],

backend/src/collections/image-db/image-file-db.service.ts

@@ -2,9 +2,11 @@ import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum';
 import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
-import { LessThan, Repository } from 'typeorm';
+import { In, IsNull, LessThan, Not, Repository } from 'typeorm';
+import { v4 as uuidv4 } from 'uuid';
 import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity';
 import { EImageFileBackend } from '../../database/entities/images/image-file.entity';
+import { FileStorageGeneric } from '../filestorage-db/filestorage-generic.service';
 
 const A_DAY_IN_SECONDS = 24 * 60 * 60;
 
@@ -16,24 +18,61 @@ export class ImageFileDBService {
 
     @InjectRepository(EImageDerivativeBackend)
     private readonly imageDerivativeRepo: Repository<EImageDerivativeBackend>,
+
+    private readonly fsService: FileStorageGeneric,
   ) {}
 
+  public async getFileData(
+    file: EImageFileBackend | EImageDerivativeBackend,
+  ): AsyncFailable<Buffer> {
+    if (file.data !== null) {
+      // Migrate files from old format to s3
+      const data = file.data;
+
+      const s3result = await this.fsService.putFile(file.fileKey, data);
+      if (HasFailed(s3result)) return s3result;
+
+      file.data = null;
+      let repoResult: EImageFileBackend | EImageDerivativeBackend;
+      if (file instanceof EImageFileBackend) {
+        repoResult = await this.imageFileRepo.save(file);
+      } else if (file instanceof EImageDerivativeBackend) {
+        repoResult = await this.imageDerivativeRepo.save(file);
+      } else {
+        return Fail(FT.SysValidation, 'Invalid file type');
+      }
+      if (HasFailed(repoResult)) return repoResult;
+
+      return data;
+    }
+
+    const result = await this.fsService.getFile(file.fileKey);
+    if (HasFailed(result)) return result;
+
+    return result;
+  }
+
   public async setFile(
     imageId: string,
     variant: ImageEntryVariant,
     file: Buffer,
     filetype: string,
   ): AsyncFailable<true> {
+    const s3key = uuidv4();
+
     const imageFile = new EImageFileBackend();
     imageFile.image_id = imageId;
     imageFile.variant = variant;
     imageFile.filetype = filetype;
-    imageFile.data = file;
+    imageFile.fileKey = s3key;
 
     try {
       await this.imageFileRepo.upsert(imageFile, {
         conflictPaths: ['image_id', 'variant'],
       });
+
+      const s3result = await this.fsService.putFile(s3key, file);
+      if (HasFailed(s3result)) return s3result;
     } catch (e) {
       return Fail(FT.Database, e);
     }
@@ -84,6 +123,9 @@ export class ImageFileDBService {
 
       if (!found) return Fail(FT.NotFound, 'Image not found');
 
+      const s3result = await this.fsService.deleteFile(found.fileKey);
+      if (HasFailed(s3result)) return s3result;
+
       await this.imageFileRepo.delete({ image_id: imageId, variant: variant });
       return found;
     } catch (e) {
@@ -120,15 +162,22 @@ export class ImageFileDBService {
     filetype: string,
     file: Buffer,
   ): AsyncFailable<EImageDerivativeBackend> {
+    const s3key = uuidv4();
+
     const imageDerivative = new EImageDerivativeBackend();
     imageDerivative.image_id = imageId;
     imageDerivative.key = key;
     imageDerivative.filetype = filetype;
-    imageDerivative.data = file;
+    imageDerivative.fileKey = s3key;
     imageDerivative.last_read = new Date();
 
     try {
-      return await this.imageDerivativeRepo.save(imageDerivative);
+      const result = await this.imageDerivativeRepo.save(imageDerivative);
+
+      const s3result = await this.fsService.putFile(s3key, file);
+      if (HasFailed(s3result)) return s3result;
+
+      return result;
     } catch (e) {
       return Fail(FT.Database, e);
     }
@@ -171,4 +220,83 @@ export class ImageFileDBService {
       return Fail(FT.Database, e);
     }
   }
+
+  public async cleanupOrphanedDerivatives(): AsyncFailable<number> {
+    return this.cleanupRepoWithFilekey(this.imageDerivativeRepo);
+  }
+
+  public async cleanupOrphanedFiles(): AsyncFailable<number> {
+    return this.cleanupRepoWithFilekey(this.imageFileRepo);
+  }
+
+  // Go over all image files in the db, and any that are not linked to an image are deleted from s3 and the db
+  private async cleanupRepoWithFilekey(
+    repo: Repository<{ image_id: string | null; fileKey: string }>,
+  ): AsyncFailable<number> {
+    try {
+      let remaining = Infinity;
+      let processed = 0;
+
+      while (remaining > 0) {
+        const orphaned = await repo.findAndCount({
+          where: {
+            image_id: IsNull(),
+          },
+          select: ['fileKey'],
+          take: 100,
+        });
+        if (orphaned[1] === 0) break;
+        remaining = orphaned[1] - orphaned[0].length;
+
+        const keys = orphaned[0].map((d) => d.fileKey);
+
+        const s3result = await this.fsService.deleteFiles(keys);
+        if (HasFailed(s3result)) return s3result;
+
+        const result = await repo.delete({
+          fileKey: In(keys),
+        });
+
+        processed += result.affected ?? 0;
+      }
+
+      return processed;
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
+
+  public async migrateFilesToFilekey(): AsyncFailable<number> {
+    return this.migrateRepoToFilekey(this.imageFileRepo);
+  }
+
+  public async migrateDerivativesToFilekey(): AsyncFailable<number> {
+    return this.migrateRepoToFilekey(this.imageDerivativeRepo);
+  }
+
+  private async migrateRepoToFilekey(
+    repo: Repository<EImageFileBackend | EImageDerivativeBackend>,
+  ): AsyncFailable<number> {
+    let processed = 0;
+
+    try {
+      while (true) {
+        const current = await repo.findOne({
+          where: {
+            data: Not(IsNull()),
+          },
+        });
+        if (!current) break;
+
+        const result = await this.getFileData(current);
+        if (HasFailed(result)) return result;
+
+        processed++;
+      }
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+
+    return processed;
+  }
 }

backend/src/collections/preference-db/preference-defaults.service.ts

@@ -3,6 +3,7 @@ import { PrefValueType } from 'picsur-shared/dist/dto/preferences.dto';
 import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
 import { UsrPreference } from 'picsur-shared/dist/dto/usr-preferences.enum';
 import { generateRandomString } from 'picsur-shared/dist/util/random';
+import { EarlyFSConfigService } from '../../config/early/early-fs.config.service';
 import { EarlyJwtConfigService } from '../../config/early/early-jwt.config.service';
 
 // This specific service holds the default values for system and user preferences
@@ -13,7 +14,7 @@ import { EarlyJwtConfigService } from '../../config/early/early-jwt.config.servi
 export class PreferenceDefaultsService {
   private readonly logger = new Logger(PreferenceDefaultsService.name);
 
-  constructor(private readonly jwtConfigService: EarlyJwtConfigService) {}
+  constructor(private readonly jwtConfigService: EarlyJwtConfigService,private readonly fsConfigService: EarlyFSConfigService) {}
 
   private readonly usrDefaults: {
     [key in UsrPreference]: (() => PrefValueType) | PrefValueType;
@@ -47,6 +48,14 @@ export class PreferenceDefaultsService {
     [SysPreference.ConversionTimeLimit]: '15s',
     [SysPreference.ConversionMemoryLimit]: 512,
 
+    [SysPreference.FSLocalPath]: () =>  this.fsConfigService.getLocalPath(),
+    [SysPreference.FSS3Endpoint]: () => this.fsConfigService.getS3Endpoint() ?? '',
+    [SysPreference.FSS3Bucket]: () => this.fsConfigService.getS3Bucket(),
+    [SysPreference.FSS3Region]: () => this.fsConfigService.getS3Region(),
+    [SysPreference.FSS3AccessKey]: () => this.fsConfigService.getS3AccessKey(),
+    [SysPreference.FSS3SecretKey]: () => this.fsConfigService.getS3SecretKey(),
+
+
     [SysPreference.EnableTracking]: false,
     [SysPreference.TrackingUrl]: '',
     [SysPreference.TrackingId]: '',

backend/src/config/config.static.ts

@@ -1,6 +1,7 @@
 import { dirname, resolve } from 'path';
 import { fileURLToPath } from 'url';
 
+export const GithubUrl = "https://github.com/rubikscraft/picsur"
 export const ReportUrl = 'https://metrics.picsur.org';
 export const ReportInterval = 1000 * 60 * 60;
 export const EnvPrefix = 'PICSUR_';

backend/src/config/early/early-config.module.ts

@@ -1,6 +1,7 @@
 import { Module } from '@nestjs/common';
 import { ConfigModule } from '@nestjs/config';
 import { AuthConfigService } from './auth.config.service';
+import { EarlyFSConfigService } from './early-fs.config.service';
 import { EarlyJwtConfigService } from './early-jwt.config.service';
 import { HostConfigService } from './host.config.service';
 import { MultipartConfigService } from './multipart.config.service';
@@ -23,6 +24,7 @@ import { TypeOrmConfigService } from './type-orm.config.service';
     AuthConfigService,
     MultipartConfigService,
     RedisConfigService,
+    EarlyFSConfigService,
   ],
   exports: [
     ConfigModule,
@@ -33,6 +35,7 @@ import { TypeOrmConfigService } from './type-orm.config.service';
     AuthConfigService,
     MultipartConfigService,
     RedisConfigService,
+    EarlyFSConfigService,
   ],
 })
 export class EarlyConfigModule {}

backend/src/config/early/early-fs.config.service.ts

@@ -0,0 +1,70 @@
+import { Injectable } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { ParseString } from 'picsur-shared/dist/util/parse-simple';
+import { EnvPrefix } from '../config.static';
+
+export enum FileStorageMode {
+  None = 'none',
+  Local = 'local',
+  S3 = 's3',
+}
+
+const FSEnvPrefix = `${EnvPrefix}FILESTORAGE_`;
+
+@Injectable()
+export class EarlyFSConfigService {
+  constructor(private readonly configService: ConfigService) {}
+
+  public getFileStorageMode(): FileStorageMode {
+    const parsed = ParseString(
+      this.configService.get(`${FSEnvPrefix}MODE`),
+      FileStorageMode.None,
+    ).toLowerCase();
+    if (Object.values(FileStorageMode).includes(parsed as FileStorageMode)) {
+      return parsed as FileStorageMode;
+    }
+    return FileStorageMode.None;
+  }
+
+  public getLocalPath(): string {
+    return ParseString(
+      this.configService.get(`${FSEnvPrefix}LOCAL_PATH`),
+      '/data',
+    );
+  }
+
+  public getS3Endpoint(): string | null {
+    return ParseString(
+      this.configService.get(`${FSEnvPrefix}S3_ENDPOINT`),
+      null,
+    );
+  }
+
+  public getS3Bucket(): string {
+    return ParseString(
+      this.configService.get(`${FSEnvPrefix}S3_BUCKET`),
+      'picsur',
+    );
+  }
+
+  public getS3Region(): string {
+    return ParseString(
+      this.configService.get(`${FSEnvPrefix}S3_REGION`),
+      'us-east-1',
+    );
+  }
+
+  public getS3AccessKey(): string {
+    return ParseString(
+      this.configService.get(`${FSEnvPrefix}S3_ACCESS_KEY`),
+      'picsur',
+    );
+  }
+
+  public getS3SecretKey(): string {
+    return ParseString(
+      this.configService.get(`${FSEnvPrefix}S3_SECRET_KEY`),
+      'picsur',
+    );
+  }
+}

backend/src/config/early/type-orm.config.service.ts

@@ -6,6 +6,7 @@ import { EntityList } from '../../database/entities';
 import { MigrationList } from '../../database/migrations';
 import { DefaultName, EnvPrefix } from '../config.static';
 import { HostConfigService } from './host.config.service';
+import { RedisConfigService } from './redis.config.service';
 
 @Injectable()
 export class TypeOrmConfigService implements TypeOrmOptionsFactory {
@@ -13,6 +14,7 @@ export class TypeOrmConfigService implements TypeOrmOptionsFactory {
 
   constructor(
     private readonly configService: ConfigService,
+    private readonly redisConfig: RedisConfigService,
     private readonly hostService: HostConfigService,
   ) {
     const varOptions = this.getTypeOrmServerOptions();
@@ -66,6 +68,13 @@ export class TypeOrmConfigService implements TypeOrmOptionsFactory {
         entitiesDir: 'src/database/entities',
       },
 
+      // cache: {
+      //   duration: 60000,
+      //   type: 'ioredis',
+      //   alwaysEnabled: false,
+      //   options: this.redisConfig.getRedisUrl(),
+      // },
+
       ...varOptions,
     } as TypeOrmModuleOptions;
   }

backend/src/config/late/fs.config.service.ts

@@ -0,0 +1,117 @@
+import { S3ClientConfig } from '@aws-sdk/client-s3';
+import { Injectable, Logger } from '@nestjs/common';
+import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
+import { ThrowIfFailed } from 'picsur-shared/dist/types';
+import { SysPreferenceDbService } from '../../collections/preference-db/sys-preference-db.service';
+import { GithubUrl } from '../config.static';
+import { EarlyFSConfigService, FileStorageMode } from '../early/early-fs.config.service';
+
+@Injectable()
+export class FSConfigService {
+  private readonly logger = new Logger(FSConfigService.name);
+
+  constructor(
+    private readonly earlyFsConfigService: EarlyFSConfigService,
+    private readonly sysPrefService: SysPreferenceDbService,
+  ) {
+    this.printDebug().catch(this.logger.error);
+  }
+
+  private async printDebug() {
+    const mode = this.getFileStorageMode();
+
+    if (mode === FileStorageMode.Local) {
+      this.logger.log('File storage Mode: Local');
+
+      this.logger.log('Local Path: ' + (await this.getLocalPath()));
+    } else if (mode === FileStorageMode.S3) {
+      this.logger.log('File storage Mode: S3');
+
+      const [endpoint, region, bucket, accessKey, secretKey] =
+        await Promise.all([
+          this.getS3Endpoint(),
+          this.getS3Region(),
+          this.getS3Bucket(),
+          this.getS3AccessKey(),
+          this.getS3SecretKey(),
+        ]);
+
+      if (endpoint) this.logger.log('Custom S3 Endpoint: ' + endpoint);
+
+      this.logger.log('S3 Region: ' + region);
+      this.logger.log('S3 Bucket: ' + bucket);
+
+      this.logger.verbose('S3 Access Key: ' + accessKey);
+      this.logger.verbose('S3 Secret Key: ' + secretKey);
+    } else {
+      this.logger.error('File storage mode: None');
+      this.logger.warn(
+        `Please set the storage mode setting. Check ${GithubUrl} for more information.`,
+      );
+    }
+  }
+
+  public getFileStorageMode(): FileStorageMode {
+    return this.earlyFsConfigService.getFileStorageMode();
+  }
+
+  public async getLocalPath(): Promise<string> {
+    return ThrowIfFailed(
+      await this.sysPrefService.getStringPreference(SysPreference.FSLocalPath),
+    );
+  }
+
+  public async getS3Config(): Promise<S3ClientConfig> {
+    return {
+      credentials: {
+        accessKeyId: await this.getS3AccessKey(),
+        secretAccessKey: await this.getS3SecretKey(),
+      },
+      endpoint: (await this.getS3Endpoint()) ?? undefined,
+      region: await this.getS3Region(),
+      tls: await this.getS3TLS(),
+    };
+  }
+
+  public async getS3Endpoint(): Promise<string | null> {
+    return ThrowIfFailed(
+      await this.sysPrefService.getStringPreference(SysPreference.FSS3Endpoint),
+    );
+  }
+
+  public async getS3TLS(): Promise<boolean | undefined> {
+    const endpoint = await this.getS3Endpoint();
+    if (endpoint) {
+      return endpoint.startsWith('https');
+    }
+    return undefined;
+  }
+
+  public async getS3Bucket(): Promise<string> {
+    return ThrowIfFailed(
+      await this.sysPrefService.getStringPreference(SysPreference.FSS3Bucket),
+    );
+  }
+
+  public async getS3Region(): Promise<string> {
+    return ThrowIfFailed(
+      await this.sysPrefService.getStringPreference(SysPreference.FSS3Region),
+    );
+  }
+
+  public async getS3AccessKey(): Promise<string> {
+    return ThrowIfFailed(
+      await this.sysPrefService.getStringPreference(
+        SysPreference.FSS3AccessKey,
+      ),
+    );
+  }
+
+  public async getS3SecretKey(): Promise<string> {
+    return ThrowIfFailed(
+      await this.sysPrefService.getStringPreference(
+        SysPreference.FSS3SecretKey,
+      ),
+    );
+  }
+}

backend/src/config/late/late-config.module.ts

@@ -3,6 +3,7 @@ import { PreferenceDbModule } from '../../collections/preference-db/preference-d
 import { SysPreferenceDbService } from '../../collections/preference-db/sys-preference-db.service';
 import { EarlyConfigModule } from '../early/early-config.module';
 import { EarlyJwtConfigService } from '../early/early-jwt.config.service';
+import { FSConfigService } from './fs.config.service';
 import { InfoConfigService } from './info.config.service';
 import { JwtConfigService } from './jwt.config.service';
 import { UsageConfigService } from './usage.config.service';
@@ -14,10 +15,16 @@ import { UsageConfigService } from './usage.config.service';
 
 @Module({
   imports: [EarlyConfigModule, PreferenceDbModule],
-  providers: [JwtConfigService, InfoConfigService, UsageConfigService],
+  providers: [
+    JwtConfigService,
+    FSConfigService,
+    InfoConfigService,
+    UsageConfigService,
+  ],
   exports: [
     EarlyConfigModule,
     JwtConfigService,
+    FSConfigService,
     InfoConfigService,
     UsageConfigService,
   ],

backend/src/database/entities/images/image-derivative.entity.ts

@@ -3,39 +3,42 @@ import {
   Entity,
   Index,
   JoinColumn,
-  ManyToOne,
-  PrimaryGeneratedColumn,
-  Unique,
+  ManyToOne, PrimaryColumn, Unique
 } from 'typeorm';
 import { EImageBackend } from './image.entity';
 
 @Entity()
 @Unique(['image_id', 'key'])
 export class EImageDerivativeBackend {
-  @PrimaryGeneratedColumn('uuid')
-  private _id?: string;
+  @PrimaryColumn({ type: 'uuid', nullable: false, name: '_id' })
+  @Index()
+  fileKey: string;
 
-  // We do a little trickery
+  // == Reference to parent image
   @Index()
   @ManyToOne(() => EImageBackend, (image) => image.derivatives, {
-    nullable: false,
-    onDelete: 'CASCADE',
+    nullable: true,
+    onDelete: 'SET NULL',
   })
   @JoinColumn({ name: 'image_id' })
   private _image?: any;
 
   @Column({
     name: 'image_id',
+    nullable: true,
   })
-  image_id: string;
+  image_id: string | null;
 
+  // == Derivative options hash
   @Index()
   @Column({ nullable: false })
   key: string;
 
+  // == Filetype of the derivative
   @Column({ nullable: false })
   filetype: string;
 
+  // == Last time the derivative was read
   @Column({
     type: 'timestamptz',
     name: 'last_read',
@@ -43,7 +46,7 @@ export class EImageDerivativeBackend {
   })
   last_read: Date;
 
-  // Binary data
-  @Column({ type: 'bytea', nullable: false })
-  data: Buffer;
+  // == Binary data
+  @Column({ type: 'bytea', nullable: true })
+  data: Buffer | null;
 }

backend/src/database/entities/images/image-file.entity.ts

@@ -5,39 +5,42 @@ import {
   Index,
   JoinColumn,
   ManyToOne,
-  PrimaryGeneratedColumn,
-  Unique,
+  PrimaryColumn, Unique
 } from 'typeorm';
 import { EImageBackend } from './image.entity';
 
 @Entity()
 @Unique(['image_id', 'variant'])
 export class EImageFileBackend {
-  @PrimaryGeneratedColumn('uuid')
-  private _id?: string;
+  @PrimaryColumn({ type: 'uuid', nullable: false, name: '_id' })
+  @Index()
+  fileKey: string;
 
-  // We do a little trickery
+  // == Reference to parent image
   @Index()
   @ManyToOne(() => EImageBackend, (image) => image.files, {
-    nullable: false,
-    onDelete: 'CASCADE',
+    nullable: true,
+    onDelete: 'SET NULL',
   })
   @JoinColumn({ name: 'image_id' })
   private _image?: any;
 
   @Column({
     name: 'image_id',
+    nullable: true,
   })
-  image_id: string;
+  image_id: string | null;
 
+  // == File variant
   @Index()
   @Column({ nullable: false, enum: ImageEntryVariant })
   variant: ImageEntryVariant;
 
+  // == Filetype of the derivative
   @Column({ nullable: false })
   filetype: string;
 
-  // Binary data
-  @Column({ type: 'bytea', nullable: false })
-  data: Buffer;
+  // == Binary data
+  @Column({ type: 'bytea', nullable: true })
+  data: Buffer | null;
 }

backend/src/database/migrations/1672247794308-V_0_6_0_a.ts

@@ -0,0 +1,44 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class V060A1672247794308 implements MigrationInterface {
+    name = 'V060A1672247794308'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "e_image_file_backend" DROP CONSTRAINT "FK_8055f37d3b9f52f421b94ee84db"`);
+        await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" DROP CONSTRAINT "FK_37055605f39b3f8847232d604f8"`);
+        await queryRunner.query(`ALTER TABLE "e_image_file_backend" DROP CONSTRAINT "UQ_872384f20feaf7bfd27e28b8d4a"`);
+        await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "_id" DROP DEFAULT`);
+        await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "image_id" DROP NOT NULL`);
+        await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "data" DROP NOT NULL`);
+        await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" DROP CONSTRAINT "UQ_fa03f5333afd74c5cc5ff780d75"`);
+        await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "_id" DROP DEFAULT`);
+        await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "image_id" DROP NOT NULL`);
+        await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "data" DROP NOT NULL`);
+        await queryRunner.query(`CREATE INDEX "IDX_95953be58a506e5de46feec618" ON "e_image_file_backend" ("_id") `);
+        await queryRunner.query(`CREATE INDEX "IDX_ff1ecff935b8d7bdcea8908781" ON "e_image_derivative_backend" ("_id") `);
+        await queryRunner.query(`ALTER TABLE "e_image_file_backend" ADD CONSTRAINT "UQ_872384f20feaf7bfd27e28b8d4a" UNIQUE ("image_id", "variant")`);
+        await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ADD CONSTRAINT "UQ_fa03f5333afd74c5cc5ff780d75" UNIQUE ("image_id", "key")`);
+        await queryRunner.query(`ALTER TABLE "e_image_file_backend" ADD CONSTRAINT "FK_8055f37d3b9f52f421b94ee84db" FOREIGN KEY ("image_id") REFERENCES "e_image_backend"("id") ON DELETE SET NULL ON UPDATE NO ACTION`);
+        await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ADD CONSTRAINT "FK_37055605f39b3f8847232d604f8" FOREIGN KEY ("image_id") REFERENCES "e_image_backend"("id") ON DELETE SET NULL ON UPDATE NO ACTION`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" DROP CONSTRAINT "FK_37055605f39b3f8847232d604f8"`);
+        await queryRunner.query(`ALTER TABLE "e_image_file_backend" DROP CONSTRAINT "FK_8055f37d3b9f52f421b94ee84db"`);
+        await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" DROP CONSTRAINT "UQ_fa03f5333afd74c5cc5ff780d75"`);
+        await queryRunner.query(`ALTER TABLE "e_image_file_backend" DROP CONSTRAINT "UQ_872384f20feaf7bfd27e28b8d4a"`);
+        await queryRunner.query(`DROP INDEX "public"."IDX_ff1ecff935b8d7bdcea8908781"`);
+        await queryRunner.query(`DROP INDEX "public"."IDX_95953be58a506e5de46feec618"`);
+        await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "data" SET NOT NULL`);
+        await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "image_id" SET NOT NULL`);
+        await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "_id" SET DEFAULT uuid_generate_v4()`);
+        await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ADD CONSTRAINT "UQ_fa03f5333afd74c5cc5ff780d75" UNIQUE ("image_id", "key")`);
+        await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "data" SET NOT NULL`);
+        await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "image_id" SET NOT NULL`);
+        await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "_id" SET DEFAULT uuid_generate_v4()`);
+        await queryRunner.query(`ALTER TABLE "e_image_file_backend" ADD CONSTRAINT "UQ_872384f20feaf7bfd27e28b8d4a" UNIQUE ("image_id", "variant")`);
+        await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ADD CONSTRAINT "FK_37055605f39b3f8847232d604f8" FOREIGN KEY ("image_id") REFERENCES "e_image_backend"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
+        await queryRunner.query(`ALTER TABLE "e_image_file_backend" ADD CONSTRAINT "FK_8055f37d3b9f52f421b94ee84db" FOREIGN KEY ("image_id") REFERENCES "e_image_backend"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
+    }
+
+}

backend/src/database/migrations/index.ts

@@ -5,6 +5,7 @@ import { V040B1662485374471 } from './1662485374471-V_0_4_0_b';
 import { V040C1662535484200 } from './1662535484200-V_0_4_0_c';
 import { V040D1662728275448 } from './1662728275448-V_0_4_0_d';
 import { V050A1672154027079 } from './1672154027079-V_0_5_0_a';
+import { V060A1672247794308 } from './1672247794308-V_0_6_0_a';
 
 export const MigrationList: Function[] = [
   V030A1661692206479,
@@ -14,4 +15,5 @@ export const MigrationList: Function[] = [
   V040C1662535484200,
   V040D1662728275448,
   V050A1672154027079,
+  V060A1672247794308
 ];

backend/src/main.ts

@@ -4,7 +4,7 @@ import fastifyReplyFrom from '@fastify/reply-from';
 import { NestFactory } from '@nestjs/core';
 import {
   FastifyAdapter,
-  NestFastifyApplication,
+  NestFastifyApplication
 } from '@nestjs/platform-fastify';
 import { AppModule } from './app.module';
 import { HostConfigService } from './config/early/host.config.service';
@@ -43,6 +43,8 @@ async function bootstrap() {
     },
   );
 
+  app.enableShutdownHooks();
+
   // Configure logger
   app.useLogger(app.get(PicsurLoggerService));
   app.flushLogs();

backend/src/managers/image/image-converter.service.ts

@@ -3,7 +3,7 @@ import ms from 'ms';
 import { ImageRequestParams } from 'picsur-shared/dist/dto/api/image.dto';
 import {
   FileType,
-  SupportedFileTypeCategory,
+  SupportedFileTypeCategory
 } from 'picsur-shared/dist/dto/mimes.dto';
 import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
 import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
@@ -12,6 +12,13 @@ import { SysPreferenceDbService } from '../../collections/preference-db/sys-pref
 import { SharpWrapper } from '../../workers/sharp.wrapper';
 import { ImageResult } from './imageresult';
 
+interface InternalConvertOptions {
+  lossless?: boolean;
+  effort?: number;
+}
+
+export type ConvertOptions = ImageRequestParams & InternalConvertOptions;
+
 @Injectable()
 export class ImageConverterService {
   constructor(private readonly sysPref: SysPreferenceDbService) {}
@@ -20,7 +27,7 @@ export class ImageConverterService {
     image: Buffer,
     sourceFiletype: FileType,
     targetFiletype: FileType,
-    options: ImageRequestParams,
+    options: ConvertOptions,
   ): AsyncFailable<ImageResult> {
     if (
       sourceFiletype.identifier === targetFiletype.identifier &&
@@ -32,23 +39,22 @@ export class ImageConverterService {
       };
     }
 
-    if (targetFiletype.category === SupportedFileTypeCategory.Image) {
-      return this.convertStill(image, sourceFiletype, targetFiletype, options);
-    } else if (
+    if (
+      targetFiletype.category === SupportedFileTypeCategory.Image ||
       targetFiletype.category === SupportedFileTypeCategory.Animation
     ) {
-      return this.convertStill(image, sourceFiletype, targetFiletype, options);
+      return this.convertImage(image, sourceFiletype, targetFiletype, options);
       //return this.convertAnimation(image, targetmime, options);
     } else {
       return Fail(FT.SysValidation, 'Unsupported mime type');
     }
   }
 
-  private async convertStill(
+  private async convertImage(
     image: Buffer,
     sourceFiletype: FileType,
     targetFiletype: FileType,
-    options: ImageRequestParams,
+    options: ConvertOptions,
   ): AsyncFailable<ImageResult> {
     const [memLimit, timeLimit] = await Promise.all([
       this.sysPref.getNumberPreference(SysPreference.ConversionMemoryLimit),

backend/src/managers/image/image-manager.module.ts

@@ -8,12 +8,15 @@ import { ImageDBService } from '../../collections/image-db/image-db.service';
 import { ImageFileDBService } from '../../collections/image-db/image-file-db.service';
 import { PreferenceDbModule } from '../../collections/preference-db/preference-db.module';
 import { SysPreferenceDbService } from '../../collections/preference-db/sys-preference-db.service';
+import { FileStorageMode } from '../../config/early/early-fs.config.service';
+import { FSConfigService } from '../../config/late/fs.config.service';
+import { LateConfigModule } from '../../config/late/late-config.module';
 import { ImageConverterService } from './image-converter.service';
 import { ImageProcessorService } from './image-processor.service';
 import { ImageManagerService } from './image.service';
 
 @Module({
-  imports: [ImageDBModule, PreferenceDbModule],
+  imports: [ImageDBModule, PreferenceDbModule, LateConfigModule],
   providers: [
     ImageManagerService,
     ImageProcessorService,
@@ -26,6 +29,7 @@ export class ImageManagerModule implements OnModuleInit {
 
   constructor(
     private readonly prefManager: SysPreferenceDbService,
+    private readonly fsConfig: FSConfigService,
     private readonly imageFileDB: ImageFileDBService,
     private readonly imageDB: ImageDBService,
   ) {}
@@ -38,6 +42,8 @@ export class ImageManagerModule implements OnModuleInit {
   private async imageManagerCron() {
     await this.cleanupDerivatives();
     await this.cleanupExpired();
+    await this.cleanupOrphanedFiles();
+    await this.migrateFilesToFilekey();
   }
 
   private async cleanupDerivatives() {
@@ -75,4 +81,48 @@ export class ImageManagerModule implements OnModuleInit {
     if (cleanedUp > 0)
       this.logger.log(`Cleaned up ${cleanedUp} expired images`);
   }
+
+  private async cleanupOrphanedFiles() {
+    const cleanedUpDerivatives =
+      await this.imageFileDB.cleanupOrphanedDerivatives();
+
+    if (HasFailed(cleanedUpDerivatives)) {
+      cleanedUpDerivatives.print(this.logger);
+      return;
+    }
+
+    const cleanedUpFiles = await this.imageFileDB.cleanupOrphanedFiles();
+    if (HasFailed(cleanedUpFiles)) {
+      cleanedUpFiles.print(this.logger);
+      return;
+    }
+
+    if (cleanedUpDerivatives > 0 || cleanedUpFiles > 0)
+      this.logger.log(
+        `Cleaned up ${cleanedUpDerivatives} orphaned derivatives and ${cleanedUpFiles} orphaned files`,
+      );
+  }
+
+  private async migrateFilesToFilekey() {
+    if ((await this.fsConfig.getFileStorageMode()) === FileStorageMode.None)
+      return;
+
+    const filesMigrated = await this.imageFileDB.migrateFilesToFilekey();
+    if (HasFailed(filesMigrated)) {
+      filesMigrated.print(this.logger);
+      return;
+    }
+
+    const derivativesMigrated =
+      await this.imageFileDB.migrateDerivativesToFilekey();
+    if (HasFailed(derivativesMigrated)) {
+      derivativesMigrated.print(this.logger);
+      return;
+    }
+
+    if (filesMigrated > 0 || derivativesMigrated > 0)
+      this.logger.log(
+        `Migrated ${filesMigrated} files and ${derivativesMigrated} derivatives to filekey`,
+      );
+  }
 }

backend/src/managers/image/image-processor.service.ts

@@ -1,8 +1,9 @@
 import { Injectable } from '@nestjs/common';
 import {
+  AnimFileType,
   FileType,
   ImageFileType,
-  SupportedFileTypeCategory,
+  SupportedFileTypeCategory
 } from 'picsur-shared/dist/dto/mimes.dto';
 
 import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
@@ -41,10 +42,12 @@ export class ImageProcessorService {
     image: Buffer,
     filetype: FileType,
   ): AsyncFailable<ImageResult> {
-    // Webps and gifs are stored as is for now
-    return {
-      image: image,
-      filetype: filetype.identifier,
-    };
+    const outputFileType = ParseFileType(AnimFileType.WEBP);
+    if (HasFailed(outputFileType)) return outputFileType;
+
+    return this.imageConverter.convert(image, filetype, outputFileType, {
+      lossless: true,
+      effort: 0,
+    });
   }
 }

backend/src/managers/image/image.service.ts

@@ -7,7 +7,7 @@ import {
   AnimFileType,
   FileType,
   ImageFileType,
-  Mime2FileType,
+  Mime2FileType
 } from 'picsur-shared/dist/dto/mimes.dto';
 import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
 import { UsrPreference } from 'picsur-shared/dist/dto/usr-preferences.enum';
@@ -57,11 +57,13 @@ export class ImageManagerService {
     userid: string | undefined,
     options: Partial<Pick<EImageBackend, 'file_name' | 'expires_at'>>,
   ): AsyncFailable<EImageBackend> {
-    if (options.expires_at !== undefined && options.expires_at !== null) {
-      if (options.expires_at < new Date()) {
-        return Fail(FT.UsrValidation, 'Expiration date must be in the future');
-      }
-    }
+    if (
+      options.expires_at !== undefined &&
+      options.expires_at !== null &&
+      options.expires_at < new Date()
+    )
+      return Fail(FT.UsrValidation, 'Expiration date must be in the future');
+
     return await this.imagesService.update(id, userid, options);
   }
 
@@ -114,13 +116,24 @@ export class ImageManagerService {
     );
     if (HasFailed(imageEntity)) return imageEntity;
 
+    const onFail = async () => {
+      const result = await this.imagesService.delete(
+        [imageEntity.id],
+        undefined,
+      );
+      if (HasFailed(result)) result.print(this.logger);
+    };
+
     const imageFileEntity = await this.imageFilesService.setFile(
       imageEntity.id,
       ImageEntryVariant.MASTER,
       processResult.image,
       processResult.filetype,
     );
-    if (HasFailed(imageFileEntity)) return imageFileEntity;
+    if (HasFailed(imageFileEntity)) {
+      await onFail();
+      return imageFileEntity;
+    }
 
     if (keepOriginal) {
       const originalFileEntity = await this.imageFilesService.setFile(
@@ -129,7 +142,10 @@ export class ImageManagerService {
         image,
         fileType.identifier,
       );
-      if (HasFailed(originalFileEntity)) return originalFileEntity;
+      if (HasFailed(originalFileEntity)) {
+        await onFail();
+        return originalFileEntity;
+      }
     }
 
     return imageEntity;
@@ -162,9 +178,12 @@ export class ImageManagerService {
         const sourceFileType = ParseFileType(masterImage.filetype);
         if (HasFailed(sourceFileType)) return sourceFileType;
 
+        const data = await this.imageFilesService.getFileData(masterImage);
+        if (HasFailed(data)) return data;
+
         const startTime = Date.now();
         const convertResult = await this.convertService.convert(
-          masterImage.data,
+          data,
           sourceFileType,
           targetFileType,
           allow_editing ? options : {},
@@ -234,6 +253,12 @@ export class ImageManagerService {
     };
   }
 
+  public async getFileData(
+    file: EImageFileBackend | EImageDerivativeBackend,
+  ): AsyncFailable<Buffer> {
+    return this.imageFilesService.getFileData(file);
+  }
+
   // Util stuff ==================================================================
 
   private async getFileTypeFromBuffer(image: Buffer): AsyncFailable<FileType> {

backend/src/routes/image/image-manage.controller.ts

@@ -5,7 +5,7 @@ import {
   Logger,
   Param,
   Post,
-  Res,
+  Res
 } from '@nestjs/common';
 import { Throttle } from '@nestjs/throttler';
 import type { FastifyReply } from 'fastify';
@@ -18,7 +18,7 @@ import {
   ImageListResponse,
   ImageUpdateRequest,
   ImageUpdateResponse,
-  ImageUploadResponse,
+  ImageUploadResponse
 } from 'picsur-shared/dist/dto/api/image-manage.dto';
 import { Permission } from 'picsur-shared/dist/dto/permissions.enum';
 import { Fail, FT, HasFailed, ThrowIfFailed } from 'picsur-shared/dist/types';
@@ -26,7 +26,7 @@ import { PostFiles } from '../../decorators/multipart/multipart.decorator';
 import type { FileIterator } from '../../decorators/multipart/postfiles.pipe';
 import {
   HasPermission,
-  RequiredPermissions,
+  RequiredPermissions
 } from '../../decorators/permissions.decorator';
 import { ReqUserID } from '../../decorators/request-user.decorator';
 import { Returns } from '../../decorators/returns.decorator';
@@ -91,14 +91,14 @@ export class ImageManageController {
   @RequiredPermissions(Permission.ImageManage)
   @Returns(ImageUpdateResponse)
   async updateImage(
-    @Body() body: ImageUpdateRequest,
+    @Body() options: ImageUpdateRequest,
     @ReqUserID() userid: string,
     @HasPermission(Permission.ImageAdmin) isImageAdmin: boolean,
   ): Promise<ImageUpdateResponse> {
     const user_id = isImageAdmin ? undefined : userid;
 
     const image = ThrowIfFailed(
-      await this.imagesService.update(body.id, user_id, body),
+      await this.imagesService.update(options.id, user_id, options),
     );
 
     return image;

backend/src/routes/image/image.controller.ts

@@ -3,12 +3,14 @@ import { SkipThrottle } from '@nestjs/throttler';
 import type { FastifyReply } from 'fastify';
 import {
   ImageMetaResponse,
-  ImageRequestParams,
+  ImageRequestParams
 } from 'picsur-shared/dist/dto/api/image.dto';
 import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum';
 import { FileType2Mime } from 'picsur-shared/dist/dto/mimes.dto';
 import { FT, IsFailure, ThrowIfFailed } from 'picsur-shared/dist/types';
 import { UserDbService } from '../../collections/user-db/user-db.service';
+import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity';
+import { EImageFileBackend } from '../../database/entities/images/image-file.entity';
 import { ImageFullIdParam } from '../../decorators/image-id/image-full-id.decorator';
 import { ImageIdParam } from '../../decorators/image-id/image-id.decorator';
 import { RequiredPermissions } from '../../decorators/permissions.decorator';
@@ -57,25 +59,23 @@ export class ImageController {
     @Query() params: ImageRequestParams,
   ): Promise<Buffer> {
     try {
+      let image: EImageFileBackend | EImageDerivativeBackend;
       if (fullid.variant === ImageEntryVariant.ORIGINAL) {
-        const image = ThrowIfFailed(
-          await this.imagesService.getOriginal(fullid.id),
+        image = ThrowIfFailed(await this.imagesService.getOriginal(fullid.id));
+      } else {
+        image = ThrowIfFailed(
+          await this.imagesService.getConverted(
+            fullid.id,
+            fullid.filetype,
+            params,
+          ),
         );
-
-        res.type(ThrowIfFailed(FileType2Mime(image.filetype)));
-        return image.data;
       }
 
-      const image = ThrowIfFailed(
-        await this.imagesService.getConverted(
-          fullid.id,
-          fullid.filetype,
-          params,
-        ),
-      );
+      const data = ThrowIfFailed(await this.imagesService.getFileData(image));
 
       res.type(ThrowIfFailed(FileType2Mime(image.filetype)));
-      return image.data;
+      return data;
     } catch (e) {
       if (!IsFailure(e) || e.getType() !== FT.NotFound) throw e;
 

backend/src/workers/sharp/sharp.message.ts

@@ -26,6 +26,10 @@ export type SharpWorkerOperation =
 
 export interface SharpWorkerFinishOptions {
   quality?: number;
+
+  // Only for internal use
+  lossless?: boolean;
+  effort?: number;
 }
 
 // Messages

backend/src/workers/sharp/universal-sharp.ts

@@ -6,6 +6,7 @@ import {
 } from 'picsur-shared/dist/dto/mimes.dto';
 import { QOIdecode, QOIencode } from 'qoi-img';
 import sharp, { Sharp, SharpOptions } from 'sharp';
+import { SharpWorkerFinishOptions } from './sharp.message';
 
 export interface SharpResult {
   data: Buffer;
@@ -72,9 +73,7 @@ function qoiSharpIn(image: Buffer, options?: SharpOptions) {
 export async function UniversalSharpOut(
   image: Sharp,
   filetype: FileType,
-  options?: {
-    quality?: number;
-  },
+  options?: SharpWorkerFinishOptions,
 ): Promise<SharpResult> {
   let result: SharpResult | undefined;
 
@@ -103,7 +102,11 @@ export async function UniversalSharpOut(
     case ImageFileType.WEBP:
     case AnimFileType.WEBP:
       result = await image
-        .webp({ quality: options?.quality })
+        .webp({
+          quality: options?.quality,
+          lossless: options?.lossless,
+          effort: options?.effort,
+        })
         .toBuffer({ resolveWithObject: true });
       break;
     case AnimFileType.GIF:

frontend/src/app/components/partial-sys-pref/partial-sys-pref.component.html

@@ -0,0 +1,15 @@
+<ng-container *ngFor="let category of preferences | async">
+  <h2 *ngIf="category.category !== null && showTitles">{{ category.title }}</h2>
+  <div class="row">
+    <ng-container *ngFor="let pref of category.prefs">
+      <pref-option
+        class="col-md-6 col-12"
+        [pref]="pref"
+        [update]="sysPrefService.setPreference.bind(sysPrefService)"
+        [name]="getName(pref.key)"
+        [helpText]="getHelpText(pref.key)"
+        [validator]="getValidator(pref.key)"
+      ></pref-option>
+    </ng-container>
+  </div>
+</ng-container>

frontend/src/app/components/partial-sys-pref/partial-sys-pref.component.ts

@@ -0,0 +1,75 @@
+import { Component, Input } from '@angular/core';
+import { DecodedPref } from 'picsur-shared/dist/dto/preferences.dto';
+import {
+  SysPreference,
+  SysPreferenceValidators
+} from 'picsur-shared/dist/dto/sys-preferences.enum';
+import { map, Observable } from 'rxjs';
+import {
+  SysPreferenceCategories,
+  SysPreferenceCategory,
+  SysPreferenceUI
+} from 'src/app/i18n/sys-pref.i18n';
+
+import { SysPrefService } from 'src/app/services/api/sys-pref.service';
+import { z, ZodTypeAny } from 'zod';
+
+@Component({
+  selector: 'partial-sys-pref',
+  templateUrl: './partial-sys-pref.component.html',
+})
+export class PartialSysPrefComponent {
+  @Input('hidden-categories') public set hiddenCategories(
+    value: SysPreferenceCategory[],
+  ) {
+    this.categories = this.makeCategories(value);
+  }
+
+  @Input("show-titles") public showTitles = true;
+
+  private categories = this.makeCategories();
+
+  public getName(key: string) {
+    return SysPreferenceUI[key as SysPreference]?.name ?? key;
+  }
+
+  public getHelpText(key: string) {
+    return SysPreferenceUI[key as SysPreference]?.helpText ?? '';
+  }
+
+  public getCategory(key: string): null | string {
+    return SysPreferenceUI[key as SysPreference]?.category ?? null;
+  }
+
+  public getValidator(key: string): ZodTypeAny {
+    return SysPreferenceValidators[key as SysPreference] ?? z.any();
+  }
+
+  preferences: Observable<
+    Array<{
+      category: string;
+      title: string;
+      prefs: DecodedPref[];
+    }>
+  >;
+
+  constructor(public readonly sysPrefService: SysPrefService) {
+    this.preferences = sysPrefService.live.pipe(
+      map((prefs) => {
+        return this.categories.map((category) => ({
+          category,
+          title: SysPreferenceCategories[category],
+          prefs: prefs.filter(
+            (pref) => this.getCategory(pref.key) === category,
+          ),
+        }));
+      }),
+    );
+  }
+
+  private makeCategories(hiddenCategories: SysPreferenceCategory[] = []) {
+    return Object.values(SysPreferenceCategory).filter(
+      (category) => !hiddenCategories.includes(category),
+    );
+  }
+}

frontend/src/app/components/partial-sys-pref/partial-sys-pref.module.ts

@@ -0,0 +1,14 @@
+import { CommonModule } from '@angular/common';
+import { NgModule } from '@angular/core';
+import { PrefOptionModule } from '../pref-option/pref-option.module';
+import { PartialSysPrefComponent } from './partial-sys-pref.component';
+
+@NgModule({
+  imports: [
+    CommonModule,
+    PrefOptionModule
+  ],
+  declarations: [PartialSysPrefComponent],
+  exports: [PartialSysPrefComponent],
+})
+export class PartialSysPrefModule {}

frontend/src/app/i18n/sys-pref.i18n.ts

@@ -1,84 +1,133 @@
 import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
 
+export enum SysPreferenceCategory {
+  General = 'general',
+  Authentication = 'authentication',
+  ImageProcessing = 'image-processing',
+  FileStorage = 'file-storage',
+  Usage = 'usage',
+}
+
+export const SysPreferenceCategories: {
+  [key in SysPreferenceCategory]: string;
+} = {
+  [SysPreferenceCategory.General]: 'General',
+  [SysPreferenceCategory.Authentication]: 'Authentication',
+  [SysPreferenceCategory.ImageProcessing]: 'Image Processing',
+  [SysPreferenceCategory.FileStorage]: 'File Storage',
+  [SysPreferenceCategory.Usage]: 'Usage',
+};
+
 export const SysPreferenceUI: {
   [key in SysPreference]: {
     name: string;
     helpText: string;
-    category: string;
+    category: SysPreferenceCategory;
   };
 } = {
   [SysPreference.HostOverride]: {
     name: 'Host Override',
     helpText:
       'Override the hostname for the server, useful for when you are accessing the server from a different domain.',
-    category: 'General',
+    category: SysPreferenceCategory.General,
   },
 
   [SysPreference.RemoveDerivativesAfter]: {
     name: 'Cached Images Expiry Time',
     helpText:
       'Time before cached converted images are deleted. This does not affect the original image. A lower cache time will save on disk space but cost more cpu. Set to 0 to disable.',
-    category: 'Image Processing',
+    category: SysPreferenceCategory.ImageProcessing,
   },
   [SysPreference.AllowEditing]: {
     name: 'Allow images to be edited',
     helpText:
       'Allow images to be edited (e.g. resize, flip). Using these features will use more CPU power.',
 
-    category: 'Image Processing',
+    category: SysPreferenceCategory.ImageProcessing,
   },
   [SysPreference.ConversionTimeLimit]: {
     name: 'Convert/Edit Time Limit',
     helpText:
       'Time limit for converting/editing images. You may need to increase this on low powered devices.',
-    category: 'Image Processing',
+    category: SysPreferenceCategory.ImageProcessing,
   },
   [SysPreference.ConversionMemoryLimit]: {
     name: 'Convert/Edit Memory Limit MB',
     helpText:
       'Memory limit for converting/editing images. You only need to increase this if you are storing massive images.',
-    category: 'Image Processing',
+    category: SysPreferenceCategory.ImageProcessing,
   },
 
   [SysPreference.JwtSecret]: {
     name: 'JWT Secret',
     helpText: 'Secret used to sign JWT authentication tokens.',
-    category: 'Authentication',
+    category: SysPreferenceCategory.Authentication,
   },
   [SysPreference.JwtExpiresIn]: {
     name: 'JWT Expiry Time',
     helpText: 'Time before JWT authentication tokens expire.',
-    category: 'Authentication',
+    category: SysPreferenceCategory.Authentication,
   },
   [SysPreference.BCryptStrength]: {
     name: 'BCrypt Strength',
     helpText:
       'Strength of BCrypt hashing algorithm, 10 is recommended. Reduce this if running on a low powered device.',
-    category: 'Authentication',
+    category: SysPreferenceCategory.Authentication,
+  },
+
+  [SysPreference.FSLocalPath]: {
+    name: 'FS Local - Path',
+    helpText: 'Storage location of the local storage provider.',
+    category: SysPreferenceCategory.FileStorage,
+  },
+  [SysPreference.FSS3Endpoint]: {
+    name: 'FS S3 - Endpoint',
+    helpText: 'Custom endpoint of the S3 storage provider.',
+    category: SysPreferenceCategory.FileStorage,
+  },
+  [SysPreference.FSS3Bucket]: {
+    name: 'FS S3 - Bucket',
+    helpText: 'Bucket of the S3 storage provider.',
+    category: SysPreferenceCategory.FileStorage,
+  },
+  [SysPreference.FSS3Region]: {
+    name: 'FS S3 - Region',
+    helpText: 'Region of the S3 storage provider.',
+    category: SysPreferenceCategory.FileStorage,
+  },
+  [SysPreference.FSS3AccessKey]: {
+    name: 'FS S3 - Access Key',
+    helpText: 'Access key of the S3 storage provider.',
+    category: SysPreferenceCategory.FileStorage,
+  },
+  [SysPreference.FSS3SecretKey]: {
+    name: 'FS S3 - Secret Key',
+    helpText: 'Secret key of the S3 storage provider.',
+    category: SysPreferenceCategory.FileStorage,
   },
 
   [SysPreference.EnableTracking]: {
     name: 'Enable Ackee Web Tracking',
     helpText:
       'Enable tracking of the website usage using Ackee. You will need to set the tracking URL and ID.',
-    category: 'Usage',
+    category: SysPreferenceCategory.Usage,
   },
   [SysPreference.TrackingUrl]: {
     name: 'Ackee tracking URL',
     helpText:
       'URL of the Ackee tracking server. Requests are proxied, so ensure the X-Forwarded-For header is handled.',
-    category: 'Usage',
+    category: SysPreferenceCategory.Usage,
   },
   [SysPreference.TrackingId]: {
     name: 'Ackee trackign website ID',
     helpText: 'ID of the website to track.',
-    category: 'Usage',
+    category: SysPreferenceCategory.Usage,
   },
 
   [SysPreference.EnableTelemetry]: {
     name: 'Enable System Telemetry',
     helpText:
       'Enable system telemetry, this will send anonymous usage data to the developers.',
-    category: 'Usage',
+    category: SysPreferenceCategory.Usage,
   },
 };

frontend/src/app/routes/settings/filestorage/settings-filestorage.component.html

@@ -0,0 +1,6 @@
+<h1>File Storage</h1>
+
+
+<h2>Storage Provider Settings</h2>
+
+<partial-sys-pref [show-titles]="false" [hidden-categories]="HiddenCategories"></partial-sys-pref>

frontend/src/app/routes/settings/filestorage/settings-filestorage.component.ts

@@ -0,0 +1,11 @@
+import { Component } from '@angular/core';
+import { SysPreferenceCategory } from 'src/app/i18n/sys-pref.i18n';
+
+@Component({
+  templateUrl: './settings-filestorage.component.html',
+})
+export class SettingsFileStorageComponent {
+  public readonly HiddenCategories = Object.values(
+    SysPreferenceCategory,
+  ).filter((c) => c !== SysPreferenceCategory.FileStorage);
+}

frontend/src/app/routes/settings/filestorage/settings-filestorage.module.ts

@@ -0,0 +1,15 @@
+import { CommonModule } from '@angular/common';
+import { NgModule } from '@angular/core';
+import { PartialSysPrefModule } from 'src/app/components/partial-sys-pref/partial-sys-pref.module';
+import { SettingsFileStorageComponent } from './settings-filestorage.component';
+import { SettingsFileStorageRoutingModule } from './settings-filestorage.routing.module';
+
+@NgModule({
+  declarations: [SettingsFileStorageComponent],
+  imports: [
+    CommonModule,
+    SettingsFileStorageRoutingModule,
+    PartialSysPrefModule,
+  ],
+})
+export default class SettingsFileStorageRouteModule {}

frontend/src/app/routes/settings/filestorage/settings-filestorage.routing.module.ts

@@ -0,0 +1,17 @@
+import { NgModule } from '@angular/core';
+import { RouterModule } from '@angular/router';
+import { PRoutes } from 'src/app/models/dto/picsur-routes.dto';
+import { SettingsFileStorageComponent } from './settings-filestorage.component';
+
+const routes: PRoutes = [
+  {
+    path: '',
+    component: SettingsFileStorageComponent,
+  },
+];
+
+@NgModule({
+  imports: [RouterModule.forChild(routes)],
+  exports: [RouterModule],
+})
+export class SettingsFileStorageRoutingModule {}

frontend/src/app/routes/settings/settings.routing.module.ts

@@ -80,6 +80,19 @@ const SettingsRoutes: PRoutes = [
           },
         },
       },
+      {
+        path: 'filestorage',
+        loadChildren: () =>
+          import('./filestorage/settings-filestorage.module').then((m) => m.default),
+        data: {
+          permissions: [Permission.SysPrefAdmin],
+          page: {
+            title: 'Storage',
+            icon: 'storage',
+            category: 'system',
+          },
+        },
+      },
       {
         path: 'system',
         loadChildren: () =>
@@ -87,7 +100,7 @@ const SettingsRoutes: PRoutes = [
         data: {
           permissions: [Permission.SysPrefAdmin],
           page: {
-            title: 'System Settings',
+            title: 'Settings',
             icon: 'tune',
             category: 'system',
           },

frontend/src/app/routes/settings/sys-pref/settings-sys-pref.component.html

@@ -1,17 +1,3 @@
 <h1>System Settings</h1>
 
-<ng-container *ngFor="let category of preferences | async">
-  <h2 *ngIf="category.category !== null">{{ category.category }}</h2>
-  <div class="row">
-    <ng-container *ngFor="let pref of category.prefs">
-      <pref-option
-        class="col-md-6 col-12"
-        [pref]="pref"
-        [update]="sysPrefService.setPreference.bind(sysPrefService)"
-        [name]="getName(pref.key)"
-        [helpText]="getHelpText(pref.key)"
-        [validator]="getValidator(pref.key)"
-      ></pref-option>
-    </ng-container>
-  </div>
-</ng-container>
+<partial-sys-pref [hidden-categories]="HiddenCategories"></partial-sys-pref>

frontend/src/app/routes/settings/sys-pref/settings-sys-pref.component.ts

@@ -1,54 +1,9 @@
 import { Component } from '@angular/core';
-import { DecodedPref } from 'picsur-shared/dist/dto/preferences.dto';
-import {
-  SysPreference,
-  SysPreferenceValidators,
-} from 'picsur-shared/dist/dto/sys-preferences.enum';
-import { map, Observable } from 'rxjs';
-import { SysPreferenceUI } from 'src/app/i18n/sys-pref.i18n';
-
-import { makeUnique } from 'picsur-shared/dist/util/unique';
-import { SysPrefService } from 'src/app/services/api/sys-pref.service';
-import { z, ZodTypeAny } from 'zod';
+import { SysPreferenceCategory } from 'src/app/i18n/sys-pref.i18n';
 
 @Component({
   templateUrl: './settings-sys-pref.component.html',
-  styleUrls: ['./settings-sys-pref.component.scss'],
 })
 export class SettingsSysprefComponent {
-  public getName(key: string) {
-    return SysPreferenceUI[key as SysPreference]?.name ?? key;
-  }
-
-  public getHelpText(key: string) {
-    return SysPreferenceUI[key as SysPreference]?.helpText ?? '';
-  }
-
-  public getCategory(key: string): null | string {
-    return SysPreferenceUI[key as SysPreference]?.category ?? null;
-  }
-
-  public getValidator(key: string): ZodTypeAny {
-    return SysPreferenceValidators[key as SysPreference] ?? z.any();
-  }
-
-  preferences: Observable<
-    Array<{ category: string | null; prefs: DecodedPref[] }>
-  >;
-
-  constructor(public readonly sysPrefService: SysPrefService) {
-    this.preferences = sysPrefService.live.pipe(
-      map((prefs) => {
-        const categories = makeUnique(
-          prefs.map((pref) => this.getCategory(pref.key)),
-        );
-        return categories.map((category) => ({
-          category,
-          prefs: prefs.filter(
-            (pref) => this.getCategory(pref.key) === category,
-          ),
-        }));
-      }),
-    );
-  }
+  public readonly HiddenCategories = [SysPreferenceCategory.FileStorage];
 }

frontend/src/app/routes/settings/sys-pref/settings-sys-pref.module.ts

@@ -1,11 +1,11 @@
 import { CommonModule } from '@angular/common';
 import { NgModule } from '@angular/core';
-import { PrefOptionModule } from 'src/app/components/pref-option/pref-option.module';
+import { PartialSysPrefModule } from 'src/app/components/partial-sys-pref/partial-sys-pref.module';
 import { SettingsSysprefComponent } from './settings-sys-pref.component';
 import { SettingsSysprefRoutingModule } from './settings-sys-pref.routing.module';
 
 @NgModule({
   declarations: [SettingsSysprefComponent],
-  imports: [CommonModule, SettingsSysprefRoutingModule, PrefOptionModule],
+  imports: [CommonModule, SettingsSysprefRoutingModule, PartialSysPrefModule],
 })
 export default class SettingsSysprefRouteModule {}

frontend/src/app/services/api/api.service.ts

@@ -3,7 +3,7 @@ import { WINDOW } from '@ng-web-apis/common';
 import axios, {
   AxiosRequestConfig,
   AxiosResponse,
-  AxiosResponseHeaders,
+  AxiosResponseHeaders
 } from 'axios';
 import { ApiResponseSchema } from 'picsur-shared/dist/dto/api/api.dto';
 import { FileType2Ext } from 'picsur-shared/dist/dto/mimes.dto';
@@ -13,7 +13,7 @@ import {
   Failure,
   FT,
   HasFailed,
-  HasSuccess,
+  HasSuccess
 } from 'picsur-shared/dist/types';
 import { ZodDtoStatic } from 'picsur-shared/dist/util/create-zod-dto';
 import { ParseMime2FileType } from 'picsur-shared/dist/util/parse-mime';
@@ -243,15 +243,13 @@ export class ApiService {
             uploadProgress.next((e.loaded / (e.total ?? 1000000)) * 100);
           },
           signal: abortController.signal,
+          validateStatus: () => true,
           ...options,
         });
 
         uploadProgress.complete();
         downloadProgress.complete();
 
-        if (result.status < 200 || result.status >= 300) {
-          return Fail(FT.Network, 'Recieved a non-ok response');
-        }
         return result;
       } catch (e) {
         return Fail(FT.Network, e);

package.json

@@ -7,10 +7,11 @@
     "frontend"
   ],
   "scripts": {
-    "devdb:start": "docker-compose -f ./support/dev.docker-compose.yml up -d",
+    "devdb:up": "docker-compose -f ./support/dev.docker-compose.yml up -d",
     "devdb:stop": "docker-compose -f ./support/dev.docker-compose.yml down",
     "devdb:restart": "docker-compose -f ./support/dev.docker-compose.yml restart",
-    "devdb:remove": "docker-compose -f ./support/dev.docker-compose.yml down --volumes",
+    "devdb:down": "docker-compose -f ./support/dev.docker-compose.yml down --volumes",
+    "update": "./support/actual-update.sh",
     "build": "./support/build.sh",
     "setversion": "./support/setversion.sh",
     "purge": "rm -rf ./node_modules",

shared/src/dto/sys-preferences.enum.ts

@@ -19,6 +19,13 @@ export enum SysPreference {
   ConversionTimeLimit = 'conversion_time_limit',
   ConversionMemoryLimit = 'conversion_memory_limit',
 
+  FSLocalPath = 'fs_local_path',
+  FSS3Endpoint = 'fs_s3_endpoint',
+  FSS3Bucket = 'fs_s3_bucket',
+  FSS3Region = 'fs_s3_region',
+  FSS3AccessKey = 'fs_s3_access_key',
+  FSS3SecretKey = 'fs_s3_secret_key',
+
   EnableTracking = 'enable_tracking',
   TrackingUrl = 'tracking_url',
   TrackingId = 'tracking_id',
@@ -45,6 +52,13 @@ export const SysPreferenceValueTypes: {
   [SysPreference.ConversionTimeLimit]: 'string',
   [SysPreference.ConversionMemoryLimit]: 'number',
 
+  [SysPreference.FSLocalPath]: 'string',
+  [SysPreference.FSS3Endpoint]: 'string',
+  [SysPreference.FSS3Bucket]: 'string',
+  [SysPreference.FSS3Region]: 'string',
+  [SysPreference.FSS3AccessKey]: 'string',
+  [SysPreference.FSS3SecretKey]: 'string',
+
   [SysPreference.EnableTracking]: 'boolean',
   [SysPreference.TrackingUrl]: 'string',
   [SysPreference.TrackingId]: 'string',
@@ -67,6 +81,13 @@ export const SysPreferenceValidators: {
   [SysPreference.ConversionTimeLimit]: IsValidMS(),
   [SysPreference.ConversionMemoryLimit]: IsPosInt(),
 
+  [SysPreference.FSLocalPath]: z.string(),
+  [SysPreference.FSS3Endpoint]: z.string().regex(URLRegex).or(z.literal('')),
+  [SysPreference.FSS3Bucket]: z.string(),
+  [SysPreference.FSS3Region]: z.string(),
+  [SysPreference.FSS3AccessKey]: z.string(),
+  [SysPreference.FSS3SecretKey]: z.string(),
+
   [SysPreference.EnableTracking]: z.boolean(),
   [SysPreference.TrackingUrl]: z.string().regex(URLRegex).or(z.literal('')),
   [SysPreference.TrackingId]: IsEntityID().or(z.literal('')),

shared/src/types/failable.ts

@@ -7,6 +7,7 @@
 export enum FT {
   Unknown = 'unknown',
   Database = 'database',
+  FileStorage = 'filestorage',
   SysValidation = 'sysvalidation',
   UsrValidation = 'usrvalidation',
   BadRequest = 'badrequest',
@@ -51,6 +52,11 @@ const FTProps: {
     code: 500,
     message: 'A database error occurred',
   },
+  [FT.FileStorage]: {
+    important: true,
+    code: 500,
+    message: 'A filestorage error occurred',
+  },
   [FT.Network]: {
     important: true,
     code: 500,

support/dev.docker-compose.yml

@@ -1,6 +1,7 @@
 version: '3'
 services:
   devdb:
+    container_name: devdb
     image: postgres:14-alpine
     environment:
       POSTGRES_DB: picsur
@@ -11,6 +12,23 @@ services:
     restart: unless-stopped
     volumes:
       - db-data:/var/lib/postgresql/data
-
+  devs3:
+    image: zenko/cloudserver:latest
+    container_name: devs3
+    environment:
+      S3BACKEND: file
+      S3DATAPATH: /storage/data/
+      S3METADATAPATH: /storage/metadata/
+      SCALITY_ACCESS_KEY_ID: username
+      SCALITY_SECRET_ACCESS_KEY: password
+      REMOTE_MANAGEMENT_DISABLE: 'true'
+    ports:
+      - '8000:8000'
+    restart: unless-stopped
+    volumes:
+      - s3-data:/storage/data
+      - s3-metadata:/storage/metadata
 volumes:
   db-data:
+  s3-data:
+  s3-metadata: