move image storage to s3/local s3

Fix minor grammatical errors in the README

.vscode/settings.json

@@ -1,5 +1,14 @@
 {
   "vsicons.presets.angular": true,
   "angular.log": "verbose",
-  "discord.enabled": true
+  "discord.enabled": true,
+  "files.exclude": {
+    "**/.git": true,
+    "**/.svn": true,
+    "**/.hg": true,
+    "**/CVS": true,
+    "**/.DS_Store": true,
+    "**/Thumbs.db": true,
+    "**/node_modules": true
+  }
 }

.vscode/tasks.json

@@ -55,7 +55,7 @@
     {
       "type": "shell",
       "label": "Start postgres",
-      "command": "yarn devdb:start",
+      "command": "yarn devdb:up",
       "options": {
         "cwd": "${cwd}",
         "shell": {

.yarnrc.yml

@@ -2,8 +2,8 @@ nodeLinker: node-modules
 
 plugins:
   - path: .yarn/plugins/@yarnpkg/plugin-version.cjs
-    spec: "@yarnpkg/plugin-version"
+    spec: '@yarnpkg/plugin-version'
   - path: .yarn/plugins/@yarnpkg/plugin-workspace-tools.cjs
-    spec: "@yarnpkg/plugin-workspace-tools"
+    spec: '@yarnpkg/plugin-workspace-tools'
 
 yarnPath: .yarn/releases/yarn-berry.cjs

README.md

@@ -19,7 +19,7 @@ But it does function, so feel free to give it a try.
 
 ## Demo
 
-You can view a live demo here: <https://picsur.rubikscraft.nl/>
+You can view a live demo here: <https://picsur.org/>
 
 The images are deleted every five minutes, and the maximum filesize is 16MB. But it should give you an indication of how it works.
 
@@ -28,7 +28,7 @@ The images are deleted every five minutes, and the maximum filesize is 16MB. But
 Here is a list of done features, and what is planned.
 For a more detailed list, you can always visit [the project](https://github.com/rubikscraft/Picsur/projects/1).
 
-Every featured marked here should work in the latest release.
+Every feature marked here should work in the latest release.
 
 - [x] Uploading and viewing images
 - [x] Anonymous uploads
@@ -57,9 +57,9 @@ Every featured marked here should work in the latest release.
 - [x] Proper DB migrations
 - [x] Show own images in list
 - [x] Correct previews on chats
-- [X] Expiring images
-- [X] ShareX endpoint
-- [X] Arm64 image
+- [x] Expiring images
+- [x] ShareX endpoint
+- [x] ARM64 and AMD64 Docker image
 
 - [ ] White mode
 - [ ] Public gallery
@@ -74,7 +74,7 @@ If you like this project, don't forget to give it a star. It tells me that I'm n
 
 ## Faq
 
-### Why do my images dissapear of the public instance?
+### Why do my images disappear off the public instance?
 
 The public instance is only a demo, and therefore only keeps images for 5 minutes. This is to prevent the server from running out of disk space, and to prevent people from using it to host questionable images.
 
@@ -94,9 +94,9 @@ If you want to enable this however, you can do so by going to `settings -> gener
 
 Do keep in mind here, that the exif data will NOT be removed from the original image. So make sure you do not accidentally share sensitive data.
 
-### This service says its supports the QOI format, what is this?
+### This service says it supports the QOI format, what is it?
 
-QOI is a new lossless image format that is designed to be very fast to encode and decode. All while still offering good compression ratios. This is the primary format the server will store images in when uploaded.
+QOI is a new lossless image format that is designed to be very fast to encode and decode. All the while still offering good compression ratios. This is the primary format the server will store images in when uploaded.
 
 You can [read more about QOI here](https://qoiformat.org/).
 
@@ -150,6 +150,10 @@ volumes:
   picsur-data:
 ```
 
+## Thanks
+
+- @awg13 for donating 5$
+
 ## Api
 
 Here is a usually up to date documentation of the api:

backend/package.json

@@ -12,7 +12,7 @@
     "build": "nest build",
     "start": "nest start --exec \"node --es-module-specifier-resolution=node\"",
     "start:dev": "yarn clean && nest start --watch --exec \"node --es-module-specifier-resolution=node\"",
-    "start:debug": "nest start --debug --watch --exec \"node --es-module-specifier-resolution=node\"",
+    "start:debug": "yarn clean && nest start --debug --watch --exec \"node --es-module-specifier-resolution=node\"",
     "start:prod": "node --es-module-specifier-resolution=node dist/main",
     "typeorm": "typeorm-ts-node-esm",
     "migrate": "PICSUR_PRODUCTION=\"true\" yarn typeorm migration:generate -d ./src/datasource.ts",
@@ -22,22 +22,33 @@
     "purge": "rm -rf dist && rm -rf node_modules"
   },
   "dependencies": {
-    "@fastify/helmet": "^10.0.0",
-    "@fastify/multipart": "^7.1.2",
+    "@aws-sdk/client-s3": "^3.181.0",
+    "@fastify/helmet": "^10.0.1",
+    "@fastify/multipart": "^7.2.0",
+    "@fastify/reply-from": "^8.3.0",
     "@fastify/static": "^6.5.0",
-    "@nestjs/common": "^9.0.11",
+    "@nestjs/bull": "^0.6.1",
+    "@nestjs/common": "^9.1.2",
     "@nestjs/config": "^2.2.0",
-    "@nestjs/core": "^9.0.11",
+    "@nestjs/core": "^9.1.2",
     "@nestjs/jwt": "^9.0.0",
     "@nestjs/passport": "^9.0.0",
-    "@nestjs/platform-fastify": "^9.0.11",
+    "@nestjs/platform-fastify": "^9.1.2",
+    "@nestjs/platform-socket.io": "^9.1.2",
+    "@nestjs/schedule": "^2.1.0",
     "@nestjs/serve-static": "^3.0.0",
+    "@nestjs/throttler": "^3.0.0",
     "@nestjs/typeorm": "^9.0.1",
+    "@nestjs/websockets": "^9.1.2",
     "bcrypt": "^5.0.1",
     "bmp-img": "^1.2.1",
+    "bull": "^4.10.0",
     "cors": "^2.8.5",
     "file-type": "^18.0.0",
+    "get-stream": "^6.0.1",
+    "is-docker": "^3.0.0",
     "ms": "^2.1.3",
+    "node-fetch": "^3.2.10",
     "p-timeout": "^6.0.0",
     "passport": "^0.6.0",
     "passport-headerapikey": "^1.2.2",
@@ -47,40 +58,44 @@
     "pg": "^8.8.0",
     "picsur-shared": "*",
     "posix.js": "^0.1.1",
-    "qoi-img": "^1.2.1",
+    "qoi-img": "^2.1.0",
     "reflect-metadata": "^0.1.13",
     "rimraf": "^3.0.2",
-    "rxjs": "^7.5.6",
-    "sharp": "^0.31.0",
+    "rxjs": "^7.5.7",
+    "semver": "^7.3.7",
+    "sharp": "^0.31.1",
     "stream-parser": "^0.3.1",
     "thunks": "^4.9.6",
-    "typeorm": "0.3.9",
-    "zod": "^3.19.0"
+    "typeorm": "0.3.10",
+    "uuid": "^9.0.0",
+    "zod": "^3.19.1"
   },
   "devDependencies": {
-    "@nestjs/cli": "^9.1.2",
+    "@nestjs/cli": "^9.1.4",
     "@nestjs/schematics": "^9.0.3",
-    "@nestjs/testing": "^9.0.11",
+    "@nestjs/testing": "^9.1.2",
     "@types/bcrypt": "^5.0.0",
+    "@types/bull": "^3.15.9",
     "@types/cors": "^2.8.12",
-    "@types/ms": "^0.7.31",
     "@types/multer": "^1.4.7",
-    "@types/node": "^18.7.16",
-    "@types/passport-jwt": "^3.0.6",
+    "@types/node": "^18.7.23",
+    "@types/passport-jwt": "^3.0.7",
     "@types/passport-local": "^1.0.34",
     "@types/passport-strategy": "^0.2.35",
-    "@types/sharp": "^0.30.5",
+    "@types/semver": "^7.3.12",
+    "@types/sharp": "^0.31.0",
     "@types/supertest": "^2.0.12",
-    "@typescript-eslint/eslint-plugin": "^5.36.2",
-    "@typescript-eslint/parser": "^5.36.2",
-    "eslint": "^8.23.0",
+    "@types/uuid": "^8.3.4",
+    "@typescript-eslint/eslint-plugin": "^5.38.1",
+    "@typescript-eslint/parser": "^5.38.1",
+    "eslint": "^8.24.0",
     "eslint-config-prettier": "^8.5.0",
     "eslint-plugin-prettier": "^4.2.1",
     "prettier": "^2.7.1",
     "source-map-support": "^0.5.21",
-    "ts-loader": "^9.3.1",
+    "ts-loader": "^9.4.1",
     "ts-node": "^10.9.1",
     "tsconfig-paths": "^4.1.0",
-    "typescript": "4.8.3"
+    "typescript": "4.8.4"
   }
 }

backend/src/app.module.ts

@@ -1,15 +1,29 @@
-import { MiddlewareConsumer, Module, NestModule } from '@nestjs/common';
+import { BullModule } from '@nestjs/bull';
+import {
+  Logger,
+  MiddlewareConsumer,
+  Module,
+  NestModule,
+  OnModuleInit,
+} from '@nestjs/common';
+import { ScheduleModule } from '@nestjs/schedule';
 import { ServeStaticModule } from '@nestjs/serve-static';
 import cors from 'cors';
 import { IncomingMessage, ServerResponse } from 'http';
+import semver from 'semver';
+import { BullConfigService } from './config/early/bull.config.service';
 import { EarlyConfigModule } from './config/early/early-config.module';
 import { ServeStaticConfigService } from './config/early/serve-static.config.service';
 import { DatabaseModule } from './database/database.module';
+import { PicsurLayersModule } from './layers/PicsurLayers.module';
 import { PicsurLoggerModule } from './logger/logger.module';
 import { AuthManagerModule } from './managers/auth/auth.module';
 import { DemoManagerModule } from './managers/demo/demo.module';
+import { UsageManagerModule } from './managers/usage/usage.module';
 import { PicsurRoutesModule } from './routes/routes.module';
 
+const supportedNodeVersions = ['^16.17.0', '^18.6.0'];
+
 const mainCorsConfig = cors({
   origin: '<origin>',
 });
@@ -41,15 +55,38 @@ const imageCorsOverride = (
       useExisting: ServeStaticConfigService,
       imports: [EarlyConfigModule],
     }),
+    ScheduleModule.forRoot(),
+    BullModule.forRootAsync({
+      useExisting: BullConfigService,
+      imports: [EarlyConfigModule],
+    }),
+
     DatabaseModule,
     AuthManagerModule,
+    UsageManagerModule,
     DemoManagerModule,
     PicsurRoutesModule,
+    PicsurLayersModule,
   ],
 })
-export class AppModule implements NestModule {
+export class AppModule implements NestModule, OnModuleInit {
+  private readonly logger = new Logger(AppModule.name);
+
   configure(consumer: MiddlewareConsumer) {
     consumer.apply(mainCorsConfig).exclude('/i').forRoutes('/');
     consumer.apply(imageCorsConfig, imageCorsOverride).forRoutes('/i');
   }
+
+  onModuleInit() {
+    const nodeVersion = process.version;
+    if (!supportedNodeVersions.some((v) => semver.satisfies(nodeVersion, v))) {
+      this.logger.error(
+        `Unsupported Node version: ${nodeVersion}. Transcoding performance will be severely degraded.`,
+      );
+
+      this.logger.log(
+        `Supported Node versions: ${supportedNodeVersions.join(', ')}`,
+      );
+    }
+  }
 }

backend/src/collections/apikey-db/apikey-db.service.ts

@@ -5,7 +5,7 @@ import { FindResult } from 'picsur-shared/dist/types/find-result';
 import { generateRandomString } from 'picsur-shared/dist/util/random';
 import { Repository } from 'typeorm';
 import { EApiKeyBackend } from '../../database/entities/apikey.entity';
-import { EUserBackend } from '../../database/entities/user.entity';
+import { EUserBackend } from '../../database/entities/users/user.entity';
 
 @Injectable()
 export class ApiKeyDbService {

backend/src/collections/file-s3/file-s3.module.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+import { EarlyConfigModule } from '../../config/early/early-config.module';
+import { FileS3Service } from './file-s3.service';
+
+@Module({
+  imports: [EarlyConfigModule],
+  providers: [FileS3Service],
+  exports: [FileS3Service],
+})
+export class FileS3Module {}

backend/src/collections/file-s3/file-s3.service.ts

@@ -0,0 +1,122 @@
+import {
+  CreateBucketCommand,
+  DeleteObjectCommand,
+  DeleteObjectsCommand,
+  GetObjectCommand,
+  ListBucketsCommand,
+  PutObjectCommand,
+  S3Client,
+} from '@aws-sdk/client-s3';
+import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
+import { buffer as streamToBuffer } from 'get-stream';
+import { AsyncFailable, Fail, FT } from 'picsur-shared/dist/types';
+import { Readable } from 'stream';
+import { S3ConfigService } from '../../config/early/s3.config.service';
+
+@Injectable()
+export class FileS3Service implements OnModuleInit {
+  private readonly logger = new Logger(FileS3Service.name);
+
+  private S3: Promise<S3Client> = this.loadS3();
+
+  constructor(private readonly s3config: S3ConfigService) {}
+
+  onModuleInit() {
+    this.loadS3();
+  }
+
+  public async putFile(key: string, data: Buffer): AsyncFailable<string> {
+    const S3 = await this.S3;
+
+    const request = new PutObjectCommand({
+      Bucket: this.s3config.getS3Bucket(),
+      Key: key,
+      Body: data,
+    });
+
+    try {
+      await S3.send(request);
+      return key;
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
+
+  public async getFile(key: string): AsyncFailable<Buffer> {
+    const S3 = await this.S3;
+
+    const request = new GetObjectCommand({
+      Bucket: this.s3config.getS3Bucket(),
+      Key: key,
+    });
+
+    try {
+      const result = await S3.send(request);
+      if (!result.Body) return Fail(FT.NotFound, 'File not found');
+
+      if (result.Body instanceof Blob) {
+        return Buffer.from(await result.Body.arrayBuffer());
+      }
+      return streamToBuffer(result.Body as Readable);
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
+
+  public async deleteFile(key: string): AsyncFailable<true> {
+    const S3 = await this.S3;
+
+    const request = new DeleteObjectCommand({
+      Bucket: this.s3config.getS3Bucket(),
+      Key: key,
+    });
+
+    try {
+      await S3.send(request);
+      return true;
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
+
+  public async deleteFiles(keys: string[]): AsyncFailable<true> {
+    const S3 = await this.S3;
+
+    const request = new DeleteObjectsCommand({
+      Bucket: this.s3config.getS3Bucket(),
+      Delete: {
+        Objects: keys.map((key) => ({ Key: key })),
+      },
+    });
+
+    try {
+      await S3.send(request);
+      return true;
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
+
+  private async loadS3(): Promise<S3Client> {
+    const S3 = new S3Client(this.s3config.getS3Config());
+
+    try {
+      // Create bucket if it doesn't exist
+      const bucket = this.s3config.getS3Bucket();
+
+      // List buckets
+      const listBuckets = await S3.send(new ListBucketsCommand({}));
+
+      const bucketExists = listBuckets.Buckets?.some((b) => b.Name === bucket);
+      if (!bucketExists) {
+        this.logger.verbose(`Creating S3 Bucket ${bucket}`);
+        await S3.send(new CreateBucketCommand({ Bucket: bucket }));
+      } else {
+        this.logger.verbose(`Using existing S3 Bucket ${bucket}`);
+      }
+    } catch (e) {
+      this.logger.error(e);
+    }
+    return S3;
+  }
+}

backend/src/collections/image-db/image-db.module.ts

@@ -1,8 +1,9 @@
 import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
-import { EImageDerivativeBackend } from '../../database/entities/image-derivative.entity';
-import { EImageFileBackend } from '../../database/entities/image-file.entity';
-import { EImageBackend } from '../../database/entities/image.entity';
+import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity';
+import { EImageFileBackend } from '../../database/entities/images/image-file.entity';
+import { EImageBackend } from '../../database/entities/images/image.entity';
+import { FileS3Module } from '../file-s3/file-s3.module';
 import { ImageDBService } from './image-db.service';
 import { ImageFileDBService } from './image-file-db.service';
 
@@ -13,6 +14,7 @@ import { ImageFileDBService } from './image-file-db.service';
       EImageFileBackend,
       EImageDerivativeBackend,
     ]),
+    FileS3Module,
   ],
   providers: [ImageDBService, ImageFileDBService],
   exports: [ImageDBService, ImageFileDBService],

backend/src/collections/image-db/image-db.service.ts

@@ -4,7 +4,7 @@ import { AsyncFailable, Fail, FT } from 'picsur-shared/dist/types';
 import { FindResult } from 'picsur-shared/dist/types/find-result';
 import { generateRandomString } from 'picsur-shared/dist/util/random';
 import { In, LessThan, Repository } from 'typeorm';
-import { EImageBackend } from '../../database/entities/image.entity';
+import { EImageBackend } from '../../database/entities/images/image.entity';
 
 @Injectable()
 export class ImageDBService {
@@ -83,6 +83,14 @@ export class ImageDBService {
     }
   }
 
+  public async count(): AsyncFailable<number> {
+    try {
+      return await this.imageRepo.count();
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
+
   public async update(
     id: string,
     userid: string | undefined,

backend/src/collections/image-db/image-file-db.service.ts

@@ -1,39 +1,59 @@
-import { Injectable } from '@nestjs/common';
+import { Injectable, Logger } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum';
-import { AsyncFailable, Fail, FT } from 'picsur-shared/dist/types';
-import { LessThan, Repository } from 'typeorm';
-import { EImageDerivativeBackend } from '../../database/entities/image-derivative.entity';
-import { EImageFileBackend } from '../../database/entities/image-file.entity';
+import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
+import { In, IsNull, LessThan, Repository } from 'typeorm';
+import { v4 as uuidv4 } from 'uuid';
+import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity';
+import { EImageFileBackend } from '../../database/entities/images/image-file.entity';
+import { FileS3Service } from '../file-s3/file-s3.service';
 
 const A_DAY_IN_SECONDS = 24 * 60 * 60;
 
 @Injectable()
 export class ImageFileDBService {
+  private readonly logger = new Logger(ImageFileDBService.name);
+
   constructor(
     @InjectRepository(EImageFileBackend)
     private readonly imageFileRepo: Repository<EImageFileBackend>,
 
     @InjectRepository(EImageDerivativeBackend)
     private readonly imageDerivativeRepo: Repository<EImageDerivativeBackend>,
+
+    private readonly s3Service: FileS3Service,
   ) {}
 
+  public async getData(
+    file: EImageFileBackend | EImageDerivativeBackend,
+  ): AsyncFailable<Buffer> {
+    const result = await this.s3Service.getFile(file.s3key);
+    if (HasFailed(result)) return result;
+
+    return result;
+  }
+
   public async setFile(
     imageId: string,
     variant: ImageEntryVariant,
     file: Buffer,
     filetype: string,
   ): AsyncFailable<true> {
+    const s3key = uuidv4();
+
     const imageFile = new EImageFileBackend();
     imageFile.image_id = imageId;
     imageFile.variant = variant;
     imageFile.filetype = filetype;
-    imageFile.data = file;
+    imageFile.s3key = s3key;
 
     try {
       await this.imageFileRepo.upsert(imageFile, {
         conflictPaths: ['image_id', 'variant'],
       });
+
+      const s3result = await this.s3Service.putFile(s3key, file);
+      if (HasFailed(s3result)) return s3result;
     } catch (e) {
       return Fail(FT.Database, e);
     }
@@ -57,6 +77,41 @@ export class ImageFileDBService {
     }
   }
 
+  public async migrateFile(
+    imageId: string,
+    sourceVariant: ImageEntryVariant,
+    targetVariant: ImageEntryVariant,
+  ): AsyncFailable<EImageFileBackend> {
+    try {
+      const sourceFile = await this.getFile(imageId, sourceVariant);
+      if (HasFailed(sourceFile)) return sourceFile;
+
+      sourceFile.variant = targetVariant;
+      return await this.imageFileRepo.save(sourceFile);
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
+
+  public async orphanFile(
+    imageId: string,
+    variant: ImageEntryVariant,
+  ): AsyncFailable<EImageFileBackend> {
+    try {
+      const found = await this.imageFileRepo.findOne({
+        where: { image_id: imageId, variant: variant },
+      });
+
+      if (!found) return Fail(FT.NotFound, 'Image not found');
+
+      found.image_id = null;
+
+      return await this.imageFileRepo.save(found);
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
+
   // This is useful because you dont have to pull the whole image file
   public async getFileTypes(
     imageId: string,
@@ -86,15 +141,22 @@ export class ImageFileDBService {
     filetype: string,
     file: Buffer,
   ): AsyncFailable<EImageDerivativeBackend> {
+    const s3key = uuidv4();
+
     const imageDerivative = new EImageDerivativeBackend();
     imageDerivative.image_id = imageId;
     imageDerivative.key = key;
     imageDerivative.filetype = filetype;
-    imageDerivative.data = file;
+    imageDerivative.s3key = s3key;
     imageDerivative.last_read = new Date();
 
     try {
-      return await this.imageDerivativeRepo.save(imageDerivative);
+      const result = await this.imageDerivativeRepo.save(imageDerivative);
+
+      const s3result = await this.s3Service.putFile(s3key, file);
+      if (HasFailed(s3result)) return s3result;
+
+      return result;
     } catch (e) {
       return Fail(FT.Database, e);
     }
@@ -112,10 +174,12 @@ export class ImageFileDBService {
       if (!derivative) return null;
 
       // Ensure read time updated to within 1 day precision
-      const yesterday = new Date(Date.now() - A_DAY_IN_SECONDS * 1000);
-      if (derivative.last_read > yesterday) {
+      const aMinuteAgo = new Date(Date.now() - 60 * 1000);
+      if (derivative.last_read > aMinuteAgo) {
         derivative.last_read = new Date();
-        return await this.imageDerivativeRepo.save(derivative);
+        this.imageDerivativeRepo.save(derivative).then((r) => {
+          if (HasFailed(r)) r.print(this.logger);
+        });
       }
 
       return derivative;
@@ -129,7 +193,7 @@ export class ImageFileDBService {
   ): AsyncFailable<number> {
     try {
       const result = await this.imageDerivativeRepo.delete({
-        last_read: LessThan(new Date()),
+        last_read: LessThan(new Date(Date.now() - olderThanSeconds * 1000)),
       });
 
       return result.affected ?? 0;
@@ -137,4 +201,47 @@ export class ImageFileDBService {
       return Fail(FT.Database, e);
     }
   }
+
+  public async cleanupOrphanedDerivatives(): AsyncFailable<number> {
+    return this.cleanupRepoWithS3(this.imageDerivativeRepo);
+  }
+
+  public async cleanupOrphanedFiles(): AsyncFailable<number> {
+    return this.cleanupRepoWithS3(this.imageFileRepo);
+  }
+
+  private async cleanupRepoWithS3(
+    repo: Repository<{ image_id: string | null; s3key: string }>,
+  ): AsyncFailable<number> {
+    try {
+      let remaining = Infinity;
+      let processed = 0;
+      while (remaining > 0) {
+        const orphaned = await repo.findAndCount({
+          where: {
+            image_id: IsNull(),
+          },
+          select: ['s3key'],
+          take: 100,
+        });
+        if (orphaned[1] === 0) break;
+        remaining = orphaned[1] - orphaned[0].length;
+
+        const keys = orphaned[0].map((d) => d.s3key);
+
+        const s3result = await this.s3Service.deleteFiles(keys);
+        if (HasFailed(s3result)) return s3result;
+
+        const result = await repo.delete({
+          s3key: In(keys),
+        });
+
+        processed += result.affected ?? 0;
+      }
+
+      return processed;
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
 }

backend/src/collections/preference-db/preference-common.service.ts

@@ -17,8 +17,8 @@ type EnumValue<E> = E[keyof E];
 type PrefValueTypeType<E extends Enum> = {
   [key in EnumValue<E>]: PrefValueTypeStrings;
 };
-type EncodedPref = {
-  key: string;
+type EncodedPref<E extends Enum> = {
+  key: EnumValue<E>;
   value: string;
 };
 
@@ -32,7 +32,7 @@ export class PreferenceCommonService {
   // E is either the SysPreference or the UsrPreference enum
   // the pref value types is the object containing the type of each key in E
   public DecodePref<E extends Enum>(
-    preference: EncodedPref,
+    preference: EncodedPref<E>,
     prefType: E,
     prefValueTypes: PrefValueTypeType<E>,
   ): Failable<DecodedPref> {
@@ -69,7 +69,7 @@ export class PreferenceCommonService {
     value: PrefValueType,
     prefType: E,
     prefValueTypes: PrefValueTypeType<E>,
-  ): AsyncFailable<EncodedPref> {
+  ): AsyncFailable<EncodedPref<E>> {
     const validatedKey = this.validatePrefKey(key, prefType);
     if (HasFailed(validatedKey)) return validatedKey;
 

backend/src/collections/preference-db/preference-db.module.ts

@@ -1,8 +1,8 @@
 import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
 import { EarlyConfigModule } from '../../config/early/early-config.module';
-import { ESysPreferenceBackend } from '../../database/entities/sys-preference.entity';
-import { EUsrPreferenceBackend } from '../../database/entities/usr-preference.entity';
+import { ESysPreferenceBackend } from '../../database/entities/system/sys-preference.entity';
+import { EUsrPreferenceBackend } from '../../database/entities/system/usr-preference.entity';
 import { PreferenceCommonService } from './preference-common.service';
 import { PreferenceDefaultsService } from './preference-defaults.service';
 import { SysPreferenceDbService } from './sys-preference-db.service';

backend/src/collections/preference-db/preference-defaults.service.ts

@@ -15,15 +15,17 @@ export class PreferenceDefaultsService {
 
   constructor(private readonly jwtConfigService: EarlyJwtConfigService) {}
 
-  public readonly usrDefaults: {
-    [key in UsrPreference]: () => PrefValueType;
+  private readonly usrDefaults: {
+    [key in UsrPreference]: (() => PrefValueType) | PrefValueType;
   } = {
-    [UsrPreference.KeepOriginal]: () => false,
+    [UsrPreference.KeepOriginal]: false,
   };
 
-  public readonly sysDefaults: {
-    [key in SysPreference]: () => PrefValueType;
+  private readonly sysDefaults: {
+    [key in SysPreference]: (() => PrefValueType) | PrefValueType;
   } = {
+    [SysPreference.HostOverride]: '',
+
     [SysPreference.JwtSecret]: () => {
       const envSecret = this.jwtConfigService.getJwtSecret();
       if (envSecret) {
@@ -37,13 +39,36 @@ export class PreferenceDefaultsService {
     },
     [SysPreference.JwtExpiresIn]: () =>
       this.jwtConfigService.getJwtExpiresIn() ?? '7d',
-    [SysPreference.BCryptStrength]: () => 12,
+    [SysPreference.BCryptStrength]: 10,
 
-    [SysPreference.RemoveDerivativesAfter]: () => '7d',
-    [SysPreference.SaveDerivatives]: () => true,
-    [SysPreference.AllowEditing]: () => true,
+    [SysPreference.RemoveDerivativesAfter]: '7d',
+    [SysPreference.AllowEditing]: true,
 
-    [SysPreference.ConversionTimeLimit]: () => '10s',
-    [SysPreference.ConversionMemoryLimit]: () => 512,
+    [SysPreference.ConversionTimeLimit]: '15s',
+    [SysPreference.ConversionMemoryLimit]: 512,
+
+    [SysPreference.EnableTracking]: false,
+    [SysPreference.TrackingUrl]: '',
+    [SysPreference.TrackingId]: '',
+
+    [SysPreference.EnableTelemetry]: true,
   };
+
+  public getSysDefault(pref: SysPreference): PrefValueType {
+    const value = this.sysDefaults[pref];
+    if (typeof value === 'function') {
+      return value();
+    } else {
+      return value;
+    }
+  }
+
+  public getUsrDefault(pref: UsrPreference): PrefValueType {
+    const value = this.usrDefaults[pref];
+    if (typeof value === 'function') {
+      return value();
+    } else {
+      return value;
+    }
+  }
 }

backend/src/collections/preference-db/sys-preference-db.service.ts

@@ -5,17 +5,18 @@ import {
   PrefValueType,
   PrefValueTypeStrings,
 } from 'picsur-shared/dist/dto/preferences.dto';
-import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
+import {
+  SysPreference,
+  SysPreferenceList,
+  SysPreferenceValidators,
+  SysPreferenceValueTypes,
+} from 'picsur-shared/dist/dto/sys-preferences.enum';
 import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
 import { Repository } from 'typeorm';
 import {
   ESysPreferenceBackend,
   ESysPreferenceSchema,
-} from '../../database/entities/sys-preference.entity';
-import {
-  SysPreferenceList,
-  SysPreferenceValueTypes,
-} from '../../models/constants/syspreferences.const';
+} from '../../database/entities/system/sys-preference.entity';
 import { MutexFallBack } from '../../util/mutex-fallback';
 import { PreferenceCommonService } from './preference-common.service';
 import { PreferenceDefaultsService } from './preference-defaults.service';
@@ -70,7 +71,6 @@ export class SysPreferenceDbService {
         try {
           existing = await this.sysPreferenceRepository.findOne({
             where: { key: validatedKey as SysPreference },
-            cache: 60000,
           });
           if (!existing) return null;
         } catch (e) {
@@ -85,7 +85,7 @@ export class SysPreferenceDbService {
 
         // Return
         return this.prefCommon.DecodePref(
-          result.data,
+          result.data as any,
           SysPreference,
           SysPreferenceValueTypes,
         );
@@ -136,7 +136,7 @@ export class SysPreferenceDbService {
   private async saveDefault(
     key: SysPreference, // Force enum here because we dont validate
   ): AsyncFailable<DecodedSysPref> {
-    return this.setPreference(key, this.defaultsService.sysDefaults[key]());
+    return this.setPreference(key, this.defaultsService.getSysDefault(key));
   }
 
   private async encodeSysPref(
@@ -151,6 +151,12 @@ export class SysPreferenceDbService {
     );
     if (HasFailed(validated)) return validated;
 
+    const valueValidated =
+      SysPreferenceValidators[key as SysPreference].safeParse(value);
+    if (!valueValidated.success) {
+      return Fail(FT.UsrValidation, undefined, valueValidated.error);
+    }
+
     let verifySysPreference = new ESysPreferenceBackend();
     verifySysPreference.key = validated.key;
     verifySysPreference.value = validated.value;

backend/src/collections/preference-db/usr-preference-db.service.ts

@@ -5,17 +5,18 @@ import {
   PrefValueType,
   PrefValueTypeStrings,
 } from 'picsur-shared/dist/dto/preferences.dto';
-import { UsrPreference } from 'picsur-shared/dist/dto/usr-preferences.enum';
+import {
+  UsrPreference,
+  UsrPreferenceList,
+  UsrPreferenceValidators,
+  UsrPreferenceValueTypes,
+} from 'picsur-shared/dist/dto/usr-preferences.enum';
 import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
 import { Repository } from 'typeorm';
 import {
   EUsrPreferenceBackend,
   EUsrPreferenceSchema,
-} from '../../database/entities/usr-preference.entity';
-import {
-  UsrPreferenceList,
-  UsrPreferenceValueTypes,
-} from '../../models/constants/usrpreferences.const';
+} from '../../database/entities/system/usr-preference.entity';
 import { MutexFallBack } from '../../util/mutex-fallback';
 import { PreferenceCommonService } from './preference-common.service';
 import { PreferenceDefaultsService } from './preference-defaults.service';
@@ -76,7 +77,6 @@ export class UsrPreferenceDbService {
         try {
           existing = await this.usrPreferenceRepository.findOne({
             where: { key: validatedKey as UsrPreference, user_id: userid },
-            cache: 60000,
           });
           if (!existing) return null;
         } catch (e) {
@@ -91,7 +91,7 @@ export class UsrPreferenceDbService {
 
         // Return
         const unpacked = this.prefCommon.DecodePref(
-          result.data,
+          result.data as any,
           UsrPreference,
           UsrPreferenceValueTypes,
         );
@@ -175,7 +175,7 @@ export class UsrPreferenceDbService {
     return this.setPreference(
       userid,
       key,
-      this.defaultsService.usrDefaults[key](),
+      this.defaultsService.getUsrDefault(key),
     );
   }
 
@@ -192,6 +192,12 @@ export class UsrPreferenceDbService {
     );
     if (HasFailed(validated)) return validated;
 
+    const valueValidated =
+      UsrPreferenceValidators[key as UsrPreference].safeParse(value);
+    if (!valueValidated.success) {
+      return Fail(FT.UsrValidation, undefined, valueValidated.error);
+    }
+
     let verifySysPreference = new EUsrPreferenceBackend();
     verifySysPreference.key = validated.key;
     verifySysPreference.value = validated.value;

backend/src/collections/role-db/role-db.module.ts

@@ -3,7 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
 import { HasFailed } from 'picsur-shared/dist/types';
 import { EarlyConfigModule } from '../../config/early/early-config.module';
 import { HostConfigService } from '../../config/early/host.config.service';
-import { ERoleBackend } from '../../database/entities/role.entity';
+import { ERoleBackend } from '../../database/entities/users/role.entity';
 import {
   ImmutableRolesList,
   SystemRoleDefaults,

backend/src/collections/role-db/role-db.service.ts

@@ -10,7 +10,7 @@ import {
 } from 'picsur-shared/dist/types';
 import { makeUnique } from 'picsur-shared/dist/util/unique';
 import { In, Repository } from 'typeorm';
-import { ERoleBackend } from '../../database/entities/role.entity';
+import { ERoleBackend } from '../../database/entities/users/role.entity';
 import { Permissions } from '../../models/constants/permissions.const';
 import {
   ImmutableRolesList,

backend/src/collections/system-state-db/system-state-db.module.ts

@@ -0,0 +1,11 @@
+import { Module } from '@nestjs/common';
+import { TypeOrmModule } from '@nestjs/typeorm';
+import { ESystemStateBackend } from '../../database/entities/system/system-state.entity';
+import { SystemStateDbService } from './system-state-db.service';
+
+@Module({
+  imports: [TypeOrmModule.forFeature([ESystemStateBackend])],
+  providers: [SystemStateDbService],
+  exports: [SystemStateDbService],
+})
+export class SystemStateDbModule {}

backend/src/collections/system-state-db/system-state-db.service.ts

@@ -0,0 +1,42 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
+import { AsyncFailable, Fail, FT } from 'picsur-shared/dist/types';
+import { Repository } from 'typeorm';
+import { ESystemStateBackend } from '../../database/entities/system/system-state.entity';
+
+@Injectable()
+export class SystemStateDbService {
+  private readonly logger = new Logger(SystemStateDbService.name);
+
+  constructor(
+    @InjectRepository(ESystemStateBackend)
+    private readonly stateRepo: Repository<ESystemStateBackend>,
+  ) {}
+
+  async get(key: string): AsyncFailable<string | null> {
+    try {
+      const state = await this.stateRepo.findOne({ where: { key } });
+      return state?.value ?? null;
+    } catch (err) {
+      return Fail(FT.Database, err);
+    }
+  }
+
+  async set(key: string, value: string): AsyncFailable<true> {
+    try {
+      await this.stateRepo.save({ key, value });
+      return true;
+    } catch (err) {
+      return Fail(FT.Database, err);
+    }
+  }
+
+  async clear(key: string): AsyncFailable<true> {
+    try {
+      await this.stateRepo.delete({ key });
+      return true;
+    } catch (err) {
+      return Fail(FT.Database, err);
+    }
+  }
+}

backend/src/collections/user-db/user-db.module.ts

@@ -4,7 +4,7 @@ import { HasFailed } from 'picsur-shared/dist/types';
 import { generateRandomString } from 'picsur-shared/dist/util/random';
 import { AuthConfigService } from '../../config/early/auth.config.service';
 import { EarlyConfigModule } from '../../config/early/early-config.module';
-import { EUserBackend } from '../../database/entities/user.entity';
+import { EUserBackend } from '../../database/entities/users/user.entity';
 import { PreferenceDbModule } from '../preference-db/preference-db.module';
 import { RoleDbModule } from '../role-db/role-db.module';
 import { UserDbService } from './user-db.service';

backend/src/collections/user-db/user-db.service.ts

@@ -12,7 +12,7 @@ import {
 import { FindResult } from 'picsur-shared/dist/types/find-result';
 import { makeUnique } from 'picsur-shared/dist/util/unique';
 import { Repository } from 'typeorm';
-import { EUserBackend } from '../../database/entities/user.entity';
+import { EUserBackend } from '../../database/entities/users/user.entity';
 import { Permissions } from '../../models/constants/permissions.const';
 import {
   DefaultRolesList,
@@ -263,6 +263,14 @@ export class UserDbService {
     }
   }
 
+  public async count(): AsyncFailable<number> {
+    try {
+      return await this.usersRepository.count();
+    } catch (e) {
+      return Fail(FT.Database, e);
+    }
+  }
+
   public async exists(username: string): Promise<boolean> {
     return HasSuccess(await this.findByUsername(username));
   }

backend/src/config/config.static.ts

@@ -1,6 +1,8 @@
 import { dirname, resolve } from 'path';
 import { fileURLToPath } from 'url';
 
+export const ReportUrl = 'https://metrics.picsur.org';
+export const ReportInterval = 1000 * 60 * 60;
 export const EnvPrefix = 'PICSUR_';
 export const DefaultName = 'picsur';
 

backend/src/config/early/bull.config.service.ts

@@ -0,0 +1,34 @@
+import {
+  BullRootModuleOptions,
+  SharedBullConfigurationFactory,
+} from '@nestjs/bull';
+import { Injectable } from '@nestjs/common';
+import { RedisConfigService } from './redis.config.service';
+
+@Injectable()
+export class BullConfigService implements SharedBullConfigurationFactory {
+  constructor(private readonly redisConfig: RedisConfigService) {}
+
+  async createSharedConfiguration(): Promise<BullRootModuleOptions> {
+    const options: BullRootModuleOptions = {
+      url: this.redisConfig.getRedisUrl(),
+      redis: {
+        lazyConnect: false,
+      },
+      defaultJobOptions: {
+        attempts: 3,
+        backoff: {
+          delay: 500,
+          type: 'fixed',
+        },
+        removeOnFail: {
+          age: 1000 * 60 * 60 * 24 * 7, // 7 days
+        },
+        removeOnComplete: {
+          age: 1000 * 60 * 60 * 24 * 7, // 7 days
+        },
+      },
+    };
+    return options;
+  }
+}

backend/src/config/early/early-config.module.ts

@@ -1,9 +1,12 @@
 import { Module } from '@nestjs/common';
 import { ConfigModule } from '@nestjs/config';
 import { AuthConfigService } from './auth.config.service';
+import { BullConfigService } from './bull.config.service';
 import { EarlyJwtConfigService } from './early-jwt.config.service';
 import { HostConfigService } from './host.config.service';
 import { MultipartConfigService } from './multipart.config.service';
+import { RedisConfigService } from './redis.config.service';
+import { S3ConfigService } from './s3.config.service';
 import { ServeStaticConfigService } from './serve-static.config.service';
 import { TypeOrmConfigService } from './type-orm.config.service';
 
@@ -21,6 +24,9 @@ import { TypeOrmConfigService } from './type-orm.config.service';
     HostConfigService,
     AuthConfigService,
     MultipartConfigService,
+    RedisConfigService,
+    BullConfigService,
+    S3ConfigService,
   ],
   exports: [
     ConfigModule,
@@ -30,6 +36,9 @@ import { TypeOrmConfigService } from './type-orm.config.service';
     HostConfigService,
     AuthConfigService,
     MultipartConfigService,
+    RedisConfigService,
+    BullConfigService,
+    S3ConfigService,
   ],
 })
 export class EarlyConfigModule {}

backend/src/config/early/host.config.service.ts

@@ -16,8 +16,15 @@ export class HostConfigService {
     this.logger.log('Verbose: ' + this.isVerbose());
     this.logger.log('Host: ' + this.getHost());
     this.logger.log('Port: ' + this.getPort());
-    this.logger.log('Demo: ' + this.isDemo());
-    this.logger.log('Demo Interval: ' + this.getDemoInterval() / 1000 + 's');
+
+    if (this.isDemo()) {
+      this.logger.log('Running in demo mode');
+      this.logger.log('Demo Interval: ' + this.getDemoInterval() / 1000 + 's');
+    }
+
+    if (!this.isTelemetry()) {
+      this.logger.log('Telemetry disabled');
+    }
   }
 
   public getHost(): string {
@@ -47,6 +54,10 @@ export class HostConfigService {
     return ParseBool(this.configService.get(`${EnvPrefix}VERBOSE`), false);
   }
 
+  public isTelemetry() {
+    return ParseBool(this.configService.get(`${EnvPrefix}TELEMETRY`), true);
+  }
+
   public getVersion() {
     return ParseString(this.configService.get(`npm_package_version`), '0.0.0');
   }

backend/src/config/early/multipart.config.service.ts

@@ -18,12 +18,12 @@ export class MultipartConfigService {
     );
   }
 
-  public getLimits() {
+  public getLimits(fileLimit?: number) {
     return {
       fieldNameSize: 128,
       fieldSize: 1024,
-      fields: 16,
-      files: 16,
+      fields: 20,
+      files: fileLimit ?? 20,
       fileSize: this.getMaxFileSize(),
     };
   }

backend/src/config/early/redis.config.service.ts

@@ -0,0 +1,20 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { ParseString } from 'picsur-shared/dist/util/parse-simple';
+import { EnvPrefix } from '../config.static';
+
+@Injectable()
+export class RedisConfigService {
+  private readonly logger = new Logger(RedisConfigService.name);
+
+  constructor(private readonly configService: ConfigService) {
+    this.logger.log('Redis URL: ' + this.getRedisUrl());
+  }
+
+  public getRedisUrl(): string {
+    return ParseString(
+      this.configService.get(`${EnvPrefix}REDIS_URL`),
+      'redis://localhost:6379',
+    );
+  }
+}

backend/src/config/early/s3.config.service.ts

@@ -0,0 +1,73 @@
+import { S3ClientConfig } from '@aws-sdk/client-s3';
+import { Injectable, Logger } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { ParseString } from 'picsur-shared/dist/util/parse-simple';
+import { EnvPrefix } from '../config.static';
+
+@Injectable()
+export class S3ConfigService {
+  private readonly logger = new Logger(S3ConfigService.name);
+
+  constructor(private readonly configService: ConfigService) {
+    if (this.getS3Endpoint())
+      this.logger.log('Custom S3 Endpoint: ' + this.getS3Endpoint());
+
+    this.logger.log('S3 Region: ' + this.getS3Region());
+    this.logger.log('S3 Bucket: ' + this.getS3Bucket());
+
+    this.logger.verbose('S3 Access Key: ' + this.getS3AccessKey());
+    this.logger.verbose('S3 Secret Key: ' + this.getS3SecretKey());
+  }
+
+  public getS3Config(): S3ClientConfig {
+    return {
+      credentials: {
+        accessKeyId: this.getS3AccessKey(),
+        secretAccessKey: this.getS3SecretKey(),
+      },
+      endpoint: this.getS3Endpoint() ?? undefined,
+      region: this.getS3Region(),
+      tls: this.getS3TLS(),
+    };
+  }
+
+  public getS3Endpoint(): string | null {
+    return ParseString(this.configService.get(`${EnvPrefix}S3_ENDPOINT`), null);
+  }
+
+  public getS3TLS(): boolean | undefined {
+    const endpoint = this.getS3Endpoint();
+    if (endpoint) {
+      return endpoint.startsWith('https');
+    }
+    return undefined;
+  }
+
+  public getS3Bucket(): string {
+    return ParseString(
+      this.configService.get(`${EnvPrefix}S3_BUCKET`),
+      'picsur',
+    );
+  }
+
+  public getS3Region(): string {
+    return ParseString(
+      this.configService.get(`${EnvPrefix}S3_REGION`),
+      'us-east-1',
+    );
+  }
+
+  public getS3AccessKey(): string {
+    return ParseString(
+      this.configService.get(`${EnvPrefix}S3_ACCESS_KEY`),
+      'picsur',
+    );
+  }
+
+  public getS3SecretKey(): string {
+    return ParseString(
+      this.configService.get(`${EnvPrefix}S3_SECRET_KEY`),
+      'picsur',
+    );
+  }
+}

backend/src/config/early/type-orm.config.service.ts

@@ -6,6 +6,7 @@ import { EntityList } from '../../database/entities';
 import { MigrationList } from '../../database/migrations';
 import { DefaultName, EnvPrefix } from '../config.static';
 import { HostConfigService } from './host.config.service';
+import { RedisConfigService } from './redis.config.service';
 
 @Injectable()
 export class TypeOrmConfigService implements TypeOrmOptionsFactory {
@@ -14,6 +15,7 @@ export class TypeOrmConfigService implements TypeOrmOptionsFactory {
   constructor(
     private readonly configService: ConfigService,
     private readonly hostService: HostConfigService,
+    private readonly redisConfig: RedisConfigService,
   ) {
     const varOptions = this.getTypeOrmServerOptions();
 
@@ -66,6 +68,13 @@ export class TypeOrmConfigService implements TypeOrmOptionsFactory {
         entitiesDir: 'src/database/entities',
       },
 
+      cache: {
+        duration: 60000,
+        type: 'ioredis',
+        alwaysEnabled: false,
+        options: this.redisConfig.getRedisUrl(),
+      },
+
       ...varOptions,
     } as TypeOrmModuleOptions;
   }

backend/src/config/late/info.config.service.ts

@@ -0,0 +1,25 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
+import { HasFailed } from 'picsur-shared/dist/types';
+import { SysPreferenceDbService } from '../../collections/preference-db/sys-preference-db.service';
+
+@Injectable()
+export class InfoConfigService {
+  private readonly logger = new Logger(InfoConfigService.name);
+
+  constructor(private readonly prefService: SysPreferenceDbService) {}
+
+  public async getHostnameOverride(): Promise<string | undefined> {
+    const hostname = await this.prefService.getStringPreference(
+      SysPreference.HostOverride,
+    );
+    if (HasFailed(hostname)) {
+      hostname.print(this.logger);
+      return undefined;
+    }
+
+    if (hostname === '') return undefined;
+
+    return hostname;
+  }
+}

backend/src/config/late/jwt.config.service.ts

@@ -32,8 +32,8 @@ export class JwtConfigService implements JwtOptionsFactory {
       await this.prefService.getStringPreference('jwt_expires_in'),
     );
 
-    let milliseconds = ms(expiresIn);
-    if (milliseconds === undefined) {
+    let milliseconds = ms(expiresIn as any);
+    if (isNaN(milliseconds)) {
       milliseconds = 1000 * 60 * 60 * 24; // 1 day
     }
 

backend/src/config/late/late-config.module.ts

@@ -3,7 +3,9 @@ import { PreferenceDbModule } from '../../collections/preference-db/preference-d
 import { SysPreferenceDbService } from '../../collections/preference-db/sys-preference-db.service';
 import { EarlyConfigModule } from '../early/early-config.module';
 import { EarlyJwtConfigService } from '../early/early-jwt.config.service';
+import { InfoConfigService } from './info.config.service';
 import { JwtConfigService } from './jwt.config.service';
+import { UsageConfigService } from './usage.config.service';
 
 // This module contains all configservices that depend on the syspref module
 // The syspref module can only be used when connected to the database
@@ -11,9 +13,14 @@ import { JwtConfigService } from './jwt.config.service';
 // Otherwise we will create a circular depedency
 
 @Module({
-  imports: [PreferenceDbModule, EarlyConfigModule],
-  providers: [JwtConfigService],
-  exports: [JwtConfigService, EarlyConfigModule],
+  imports: [EarlyConfigModule, PreferenceDbModule],
+  providers: [JwtConfigService, InfoConfigService, UsageConfigService],
+  exports: [
+    EarlyConfigModule,
+    JwtConfigService,
+    InfoConfigService,
+    UsageConfigService,
+  ],
 })
 export class LateConfigModule implements OnModuleInit {
   private readonly logger = new Logger(LateConfigModule.name);

backend/src/config/late/usage.config.service.ts

@@ -0,0 +1,53 @@
+import { Injectable } from '@nestjs/common';
+import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
+import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
+import { URLRegex, UUIDRegex } from 'picsur-shared/dist/util/common-regex';
+import { SysPreferenceDbService } from '../../collections/preference-db/sys-preference-db.service';
+import { ReportInterval, ReportUrl } from '../config.static';
+
+@Injectable()
+export class UsageConfigService {
+  constructor(private readonly sysPref: SysPreferenceDbService) {}
+
+  async getTrackingUrl(): AsyncFailable<string | null> {
+    const trackingUrl = await this.sysPref.getStringPreference(
+      SysPreference.TrackingUrl,
+    );
+    if (HasFailed(trackingUrl)) return trackingUrl;
+
+    if (trackingUrl === '') return null;
+
+    if (!URLRegex.test(trackingUrl)) {
+      return Fail(FT.UsrValidation, undefined, 'Invalid tracking URL');
+    }
+
+    return trackingUrl;
+  }
+
+  async getTrackingID(): AsyncFailable<string | null> {
+    const trackingID = await this.sysPref.getStringPreference(
+      SysPreference.TrackingId,
+    );
+    if (HasFailed(trackingID)) return trackingID;
+
+    if (trackingID === '') return null;
+
+    if (!UUIDRegex.test(trackingID)) {
+      return Fail(FT.UsrValidation, undefined, 'Invalid tracking ID');
+    }
+
+    return trackingID;
+  }
+
+  async getMetricsEnabled(): AsyncFailable<boolean> {
+    return this.sysPref.getBooleanPreference(SysPreference.EnableTelemetry);
+  }
+
+  async getMetricsInterval(): Promise<number> {
+    return ReportInterval;
+  }
+
+  async getMetricsUrl(): Promise<string> {
+    return ReportUrl;
+  }
+}

backend/src/database/entities/apikey.entity.ts

@@ -7,7 +7,7 @@ import {
   PrimaryGeneratedColumn,
 } from 'typeorm';
 import { z } from 'zod';
-import { EUserBackend } from './user.entity';
+import { EUserBackend } from './users/user.entity';
 
 const OverriddenEApiKeySchema = EApiKeySchema.omit({ user: true }).merge(
   z.object({

backend/src/database/entities/images/image-derivative.entity.ts

@@ -4,7 +4,7 @@ import {
   Index,
   JoinColumn,
   ManyToOne,
-  PrimaryGeneratedColumn,
+  PrimaryColumn,
   Unique,
 } from 'typeorm';
 import { EImageBackend } from './image.entity';
@@ -12,22 +12,24 @@ import { EImageBackend } from './image.entity';
 @Entity()
 @Unique(['image_id', 'key'])
 export class EImageDerivativeBackend {
-  @PrimaryGeneratedColumn('uuid')
-  private _id?: string;
+  @PrimaryColumn({ type: 'uuid', nullable: false })
+  @Index()
+  s3key: string;
 
   // We do a little trickery
   @Index()
   @ManyToOne(() => EImageBackend, (image) => image.derivatives, {
-    nullable: false,
-    onDelete: 'CASCADE',
+    nullable: true,
+    onDelete: 'SET NULL',
   })
   @JoinColumn({ name: 'image_id' })
   private _image?: any;
 
   @Column({
     name: 'image_id',
+    nullable: true,
   })
-  image_id: string;
+  image_id: string | null;
 
   @Index()
   @Column({ nullable: false })
@@ -42,8 +44,4 @@ export class EImageDerivativeBackend {
     nullable: false,
   })
   last_read: Date;
-
-  // Binary data
-  @Column({ type: 'bytea', nullable: false })
-  data: Buffer;
 }

backend/src/database/entities/images/image-file.entity.ts

@@ -5,7 +5,7 @@ import {
   Index,
   JoinColumn,
   ManyToOne,
-  PrimaryGeneratedColumn,
+  PrimaryColumn,
   Unique,
 } from 'typeorm';
 import { EImageBackend } from './image.entity';
@@ -13,22 +13,24 @@ import { EImageBackend } from './image.entity';
 @Entity()
 @Unique(['image_id', 'variant'])
 export class EImageFileBackend {
-  @PrimaryGeneratedColumn('uuid')
-  private _id?: string;
+  @PrimaryColumn({ type: 'uuid', nullable: false })
+  @Index()
+  s3key: string;
 
   // We do a little trickery
   @Index()
   @ManyToOne(() => EImageBackend, (image) => image.files, {
-    nullable: false,
-    onDelete: 'CASCADE',
+    nullable: true,
+    onDelete: 'SET NULL',
   })
   @JoinColumn({ name: 'image_id' })
   private _image?: any;
 
   @Column({
     name: 'image_id',
+    nullable: true,
   })
-  image_id: string;
+  image_id: string | null;
 
   @Index()
   @Column({ nullable: false, enum: ImageEntryVariant })
@@ -36,8 +38,4 @@ export class EImageFileBackend {
 
   @Column({ nullable: false })
   filetype: string;
-
-  // Binary data
-  @Column({ type: 'bytea', nullable: false })
-  data: Buffer;
 }

backend/src/database/entities/index.ts

@@ -1,11 +1,12 @@
 import { EApiKeyBackend } from './apikey.entity';
-import { EImageDerivativeBackend } from './image-derivative.entity';
-import { EImageFileBackend } from './image-file.entity';
-import { EImageBackend } from './image.entity';
-import { ERoleBackend } from './role.entity';
-import { ESysPreferenceBackend } from './sys-preference.entity';
-import { EUserBackend } from './user.entity';
-import { EUsrPreferenceBackend } from './usr-preference.entity';
+import { EImageDerivativeBackend } from './images/image-derivative.entity';
+import { EImageFileBackend } from './images/image-file.entity';
+import { EImageBackend } from './images/image.entity';
+import { ESysPreferenceBackend } from './system/sys-preference.entity';
+import { ESystemStateBackend } from './system/system-state.entity';
+import { EUsrPreferenceBackend } from './system/usr-preference.entity';
+import { ERoleBackend } from './users/role.entity';
+import { EUserBackend } from './users/user.entity';
 
 export const EntityList = [
   EImageBackend,
@@ -16,4 +17,5 @@ export const EntityList = [
   ESysPreferenceBackend,
   EUsrPreferenceBackend,
   EApiKeyBackend,
+  ESystemStateBackend,
 ];

backend/src/database/entities/system/system-state.entity.ts

@@ -0,0 +1,14 @@
+import { Column, Entity, Index, PrimaryGeneratedColumn } from 'typeorm';
+
+@Entity()
+export class ESystemStateBackend {
+  @PrimaryGeneratedColumn('uuid')
+  id?: string;
+
+  @Index()
+  @Column({ nullable: false, unique: true })
+  key: string;
+
+  @Column({ nullable: false })
+  value: string;
+}

backend/src/database/entities/system/usr-preference.entity.ts

@@ -9,7 +9,7 @@ import {
   Unique,
 } from 'typeorm';
 import z from 'zod';
-import { EUserBackend } from './user.entity';
+import { EUserBackend } from '../users/user.entity';
 
 export const EUsrPreferenceSchema = z.object({
   id: IsEntityID().optional(),

backend/src/database/entities/users/role.entity.ts

@@ -1,6 +1,6 @@
 import { ERole } from 'picsur-shared/dist/entities/role.entity';
 import { Column, Entity, Index, PrimaryGeneratedColumn } from 'typeorm';
-import type { Permissions } from '../../models/constants/permissions.const';
+import type { Permissions } from '../../../models/constants/permissions.const';
 
 @Entity()
 export class ERoleBackend implements ERole {

backend/src/database/entities/users/user.entity.ts

@@ -7,8 +7,8 @@ import {
   PrimaryGeneratedColumn,
 } from 'typeorm';
 import { z } from 'zod';
-import { EApiKeyBackend } from './apikey.entity';
-import { EUsrPreferenceBackend } from './usr-preference.entity';
+import { EApiKeyBackend } from '../apikey.entity';
+import { EUsrPreferenceBackend } from '../system/usr-preference.entity';
 
 // Different data for public and private
 const OverriddenEUserSchema = EUserSchema.omit({ hashedPassword: true }).merge(

backend/src/decorators/decorators.module.ts

@@ -1,8 +1,8 @@
 import { Module } from '@nestjs/common';
 import { EarlyConfigModule } from '../config/early/early-config.module';
 import { ImageIdPipe } from './image-id/image-id.pipe';
-import { MultiPartPipe } from './multipart/multipart.pipe';
 import { PostFilePipe } from './multipart/postfile.pipe';
+import { MultiPartPipe } from './multipart/postfiles.pipe';
 
 @Module({
   imports: [EarlyConfigModule],

backend/src/decorators/multipart/inject-request.decorator.ts

@@ -3,6 +3,9 @@ import { createParamDecorator, ExecutionContext } from '@nestjs/common';
 // Since pipes dont have direct access to the request object, we need this decorator to inject it
 export const InjectRequest = createParamDecorator(
   async (data: any, ctx: ExecutionContext) => {
-    return ctx.switchToHttp().getRequest();
+    return {
+      data: data,
+      request: ctx.switchToHttp().getRequest(),
+    };
   },
 );

backend/src/decorators/multipart/multipart.decorator.ts

@@ -1,7 +1,8 @@
 import { InjectRequest } from './inject-request.decorator';
-import { MultiPartPipe } from './multipart.pipe';
 import { PostFilePipe } from './postfile.pipe';
+import { MultiPartPipe } from './postfiles.pipe';
 
 export const PostFile = () => InjectRequest(PostFilePipe);
 
-export const MultiPart = () => InjectRequest(MultiPartPipe);
+export const PostFiles = (maxFiles?: number) =>
+  InjectRequest(maxFiles, MultiPartPipe);

backend/src/decorators/multipart/multipart.pipe.ts

@@ -1,81 +0,0 @@
-import { MultipartFields, MultipartFile } from '@fastify/multipart';
-import {
-  ArgumentMetadata,
-  Injectable,
-  Logger,
-  PipeTransform,
-  Scope,
-} from '@nestjs/common';
-import { FastifyRequest } from 'fastify';
-import { Fail, FT, HasFailed } from 'picsur-shared/dist/types';
-import { ZodDtoStatic } from 'picsur-shared/dist/util/create-zod-dto';
-import { MultipartConfigService } from '../../config/early/multipart.config.service';
-import {
-  CreateMultiPartFieldDto,
-  CreateMultiPartFileDto,
-} from '../../models/dto/multipart.dto';
-
-@Injectable({ scope: Scope.REQUEST })
-export class MultiPartPipe implements PipeTransform {
-  private readonly logger = new Logger(MultiPartPipe.name);
-
-  constructor(
-    private readonly multipartConfigService: MultipartConfigService,
-  ) {}
-
-  async transform<T extends Object>(
-    req: FastifyRequest,
-    metadata: ArgumentMetadata,
-  ) {
-    let zodSchema = (metadata?.metatype as ZodDtoStatic)?.zodSchema;
-    if (!zodSchema) {
-      this.logger.error('Invalid scheme on multipart body');
-      throw Fail(FT.Internal, 'Invalid scheme on backend');
-    }
-
-    let multipartData = {};
-    if (!req.isMultipart()) throw Fail(FT.UsrValidation, 'Invalid file');
-
-    // Fetch all fields from the request
-    let fields: MultipartFields | null = null;
-    try {
-      fields = (
-        await req.file({
-          limits: this.multipartConfigService.getLimits(),
-        })
-      ).fields;
-    } catch (e) {
-      this.logger.warn(e);
-    }
-    if (!fields) throw Fail(FT.UsrValidation, 'Invalid file');
-
-    // Loop over every formfield that was sent
-    for (const key of Object.keys(fields)) {
-      // Ignore duplicate fields
-      if (Array.isArray(fields[key])) {
-        continue;
-      }
-
-      // Use the value property to differentiate between a field and a file
-      // And then put the value into the correct property on the validatable class
-      if ((fields[key] as any).value) {
-        (multipartData as any)[key] = CreateMultiPartFieldDto(
-          fields[key] as MultipartFile,
-        );
-      } else {
-        const file = await CreateMultiPartFileDto(fields[key] as MultipartFile);
-        if (HasFailed(file)) throw file;
-        (multipartData as any)[key] = file;
-      }
-    }
-
-    // Now validate the class we made, if any properties were invalid, it will error here
-    const result = zodSchema.safeParse(multipartData);
-    if (!result.success) {
-      this.logger.warn(result.error);
-      throw Fail(FT.UsrValidation, 'Invalid file');
-    }
-
-    return result.data;
-  }
-}

backend/src/decorators/multipart/postfile.pipe.ts

@@ -1,4 +1,4 @@
-import { Multipart } from '@fastify/multipart';
+import { Multipart, MultipartFile } from '@fastify/multipart';
 import { Injectable, Logger, PipeTransform, Scope } from '@nestjs/common';
 import { FastifyRequest } from 'fastify';
 import { Fail, FT } from 'picsur-shared/dist/types';
@@ -12,11 +12,11 @@ export class PostFilePipe implements PipeTransform {
     private readonly multipartConfigService: MultipartConfigService,
   ) {}
 
-  async transform({ req }: { req: FastifyRequest }) {
-    if (!req.isMultipart()) throw Fail(FT.UsrValidation, 'Invalid file');
+  async transform({ request, data }: { data: any; request: FastifyRequest }) {
+    if (!request.isMultipart()) throw Fail(FT.UsrValidation, 'Invalid file');
 
     // Only one file is allowed
-    const file = await req.file({
+    const file = await request.file({
       limits: {
         ...this.multipartConfigService.getLimits(),
         files: 1,
@@ -30,7 +30,9 @@ export class PostFilePipe implements PipeTransform {
     ) as any;
 
     // Remove non-file fields
-    const files = allFields.filter((entry) => entry.file !== undefined);
+    const files: MultipartFile[] = allFields.filter(
+      (entry) => (entry as any).file !== undefined,
+    ) as MultipartFile[];
 
     if (files.length !== 1) throw Fail(FT.UsrValidation, 'Invalid file');
 

backend/src/decorators/multipart/postfiles.pipe.ts

@@ -0,0 +1,37 @@
+import { MultipartFile } from '@fastify/multipart';
+import {
+  ArgumentMetadata,
+  Injectable,
+  Logger,
+  PipeTransform,
+  Scope,
+} from '@nestjs/common';
+import { FastifyRequest } from 'fastify';
+import { Fail, FT } from 'picsur-shared/dist/types';
+import { MultipartConfigService } from '../../config/early/multipart.config.service';
+
+export type FileIterator = AsyncIterableIterator<MultipartFile>;
+
+@Injectable({ scope: Scope.REQUEST })
+export class MultiPartPipe implements PipeTransform {
+  private readonly logger = new Logger(MultiPartPipe.name);
+
+  constructor(
+    private readonly multipartConfigService: MultipartConfigService,
+  ) {}
+
+  async transform<T extends Object>(
+    { request, data }: { data: any; request: FastifyRequest },
+    metadata: ArgumentMetadata,
+  ) {
+    const filesLimit = typeof data === 'number' ? data : undefined;
+
+    if (!request.isMultipart()) throw Fail(FT.UsrValidation, 'Invalid files');
+
+    const files = request.files({
+      limits: this.multipartConfigService.getLimits(filesLimit),
+    });
+
+    return files;
+  }
+}

backend/src/decorators/returns.decorator.ts

@@ -16,3 +16,7 @@ export function Returns<N extends Object>(
 ): ReturnsMethodDecorator<N> {
   return SetMetadata('returns', newable);
 }
+
+export function ReturnsAnything(): ReturnsMethodDecorator<any> {
+  return SetMetadata('noreturns', true);
+}

backend/src/layers/PicsurLayers.module.ts

@@ -0,0 +1,28 @@
+import { Module } from '@nestjs/common';
+import { ThrottlerModule } from '@nestjs/throttler';
+import { MainExceptionFilter } from './exception/exception.filter';
+import { SuccessInterceptor } from './success/success.interceptor';
+import { PicsurThrottlerGuard } from './throttler/PicsurThrottler.guard';
+import { ZodValidationPipe } from './validate/zod-validator.pipe';
+
+@Module({
+  imports: [
+    ThrottlerModule.forRoot({
+      ttl: 60,
+      limit: 60,
+    }),
+  ],
+  providers: [
+    PicsurThrottlerGuard,
+    MainExceptionFilter,
+    SuccessInterceptor,
+    ZodValidationPipe,
+  ],
+  exports: [
+    PicsurThrottlerGuard,
+    MainExceptionFilter,
+    SuccessInterceptor,
+    ZodValidationPipe,
+  ],
+})
+export class PicsurLayersModule {}

backend/src/layers/exception/exception.filter.ts

@@ -39,23 +39,7 @@ export class MainExceptionFilter implements ExceptionFilter {
     const status = exception.getCode();
     const type = exception.getType();
 
-    const message = exception.getReason();
-    const logmessage =
-      message +
-      (exception.getDebugMessage() ? ' - ' + exception.getDebugMessage() : '');
-
-    if (exception.isImportant()) {
-      MainExceptionFilter.logger.error(
-        `${traceString} ${exception.getName()}: ${logmessage}`,
-      );
-      if (exception.getStack()) {
-        MainExceptionFilter.logger.debug(exception.getStack());
-      }
-    } else {
-      MainExceptionFilter.logger.warn(
-        `${traceString} ${exception.getName()}: ${logmessage}`,
-      );
-    }
+    exception.print(MainExceptionFilter.logger, { prefix: traceString });
 
     const toSend: ApiErrorResponse = {
       success: false,
@@ -65,7 +49,7 @@ export class MainExceptionFilter implements ExceptionFilter {
 
       data: {
         type,
-        message,
+        message: exception.getReason(),
       },
     };
 

backend/src/layers/success/success.interceptor.ts

@@ -46,10 +46,29 @@ export class SuccessInterceptor<T> implements NestInterceptor {
           return data;
         }
       }),
+      map((data) => {
+        const request = context.switchToHttp().getRequest();
+        const response = context.switchToHttp().getResponse<FastifyReply>();
+        const traceString = `(${request.ip} -> ${request.method} ${request.url})`;
+
+        this.logger.verbose(
+          `Handled ${traceString} with ${response.statusCode} in ${Math.ceil(
+            response.getResponseTime(),
+          )}ms`,
+          SuccessInterceptor.name,
+        );
+
+        return data;
+      }),
     );
   }
 
   private validate(context: ExecutionContext, data: unknown): unknown {
+    const canReturnAnything =
+      (this.reflector.get('noreturns', context.getHandler()) ?? false) === true;
+
+    if (canReturnAnything) return data;
+
     const schemaStatic = this.reflector.get<ZodDtoStatic>(
       'returns',
       context.getHandler(),

backend/src/layers/throttler/PicsurThrottler.guard.ts

@@ -0,0 +1,10 @@
+import { ExecutionContext, Injectable } from '@nestjs/common';
+import { ThrottlerGuard } from '@nestjs/throttler';
+import { Fail, FT } from 'picsur-shared/dist/types';
+
+@Injectable()
+export class PicsurThrottlerGuard extends ThrottlerGuard {
+  protected override throwThrottlingException(context: ExecutionContext): void {
+    throw Fail(FT.RateLimit);
+  }
+}

backend/src/main.ts

@@ -1,15 +1,16 @@
 import fastifyHelmet from '@fastify/helmet';
 import multipart from '@fastify/multipart';
-import { NestFactory, Reflector } from '@nestjs/core';
+import fastifyReplyFrom from '@fastify/reply-from';
+import { NestFactory } from '@nestjs/core';
 import {
   FastifyAdapter,
   NestFastifyApplication,
 } from '@nestjs/platform-fastify';
 import { AppModule } from './app.module';
-import { UserDbService } from './collections/user-db/user-db.service';
 import { HostConfigService } from './config/early/host.config.service';
 import { MainExceptionFilter } from './layers/exception/exception.filter';
 import { SuccessInterceptor } from './layers/success/success.interceptor';
+import { PicsurThrottlerGuard } from './layers/throttler/PicsurThrottler.guard';
 import { ZodValidationPipe } from './layers/validate/zod-validator.pipe';
 import { PicsurLoggerService } from './logger/logger.service';
 import { MainAuthGuard } from './managers/auth/guards/main.guard';
@@ -19,10 +20,18 @@ async function bootstrap() {
   const isProduction = process.env['PICSUR_PRODUCTION'] !== undefined;
 
   // Create fasify
-  const fastifyAdapter = new FastifyAdapter();
+  const fastifyAdapter = new FastifyAdapter({
+    trustProxy: [
+      '127.0.0.0/8',
+      '10.0.0.0/8',
+      '172.16.0.0/12',
+      '192.168.0.0/16',
+    ],
+  });
   // TODO: generic error messages
   await fastifyAdapter.register(multipart as any);
   await fastifyAdapter.register(fastifyHelmet as any, HelmetOptions);
+  await fastifyAdapter.register(fastifyReplyFrom as any);
 
   // Create nest app
   const app = await NestFactory.create<NestFastifyApplication>(
@@ -30,20 +39,19 @@ async function bootstrap() {
     fastifyAdapter,
     {
       bufferLogs: isProduction,
+      autoFlushLogs: true,
     },
   );
 
   // Configure logger
   app.useLogger(app.get(PicsurLoggerService));
-
   app.flushLogs();
 
-  app.useGlobalFilters(new MainExceptionFilter());
-  app.useGlobalInterceptors(new SuccessInterceptor(app.get(Reflector)));
-  app.useGlobalPipes(new ZodValidationPipe());
-  app.useGlobalGuards(
-    new MainAuthGuard(app.get(Reflector), app.get(UserDbService)),
-  );
+  app.useGlobalFilters(app.get(MainExceptionFilter));
+  app.useGlobalInterceptors(app.get(SuccessInterceptor));
+  app.useGlobalPipes(app.get(ZodValidationPipe));
+
+  app.useGlobalGuards(app.get(PicsurThrottlerGuard), app.get(MainAuthGuard));
 
   // Start app
   const hostConfigService = app.get(HostConfigService);

backend/src/managers/auth/auth.module.ts

@@ -13,7 +13,9 @@ import { AuthManagerService } from './auth.service';
 import { ApiKeyStrategy } from './guards/apikey.strategy';
 import { GuestStrategy } from './guards/guest.strategy';
 import { JwtStrategy } from './guards/jwt.strategy';
+import { LocalAuthGuard } from './guards/local-auth.guard';
 import { LocalAuthStrategy } from './guards/local-auth.strategy';
+import { MainAuthGuard } from './guards/main.guard';
 import { GuestService } from './guest.service';
 
 @Module({
@@ -30,13 +32,15 @@ import { GuestService } from './guest.service';
   ],
   providers: [
     AuthManagerService,
+    GuestService,
+    JwtSecretProvider,
     LocalAuthStrategy,
     JwtStrategy,
     GuestStrategy,
-    JwtSecretProvider,
     ApiKeyStrategy,
-    GuestService,
+    LocalAuthGuard,
+    MainAuthGuard,
   ],
-  exports: [UserDbModule, AuthManagerService],
+  exports: [UserDbModule, AuthManagerService, LocalAuthGuard, MainAuthGuard],
 })
 export class AuthManagerModule {}

backend/src/managers/auth/guest.service.ts

@@ -1,7 +1,7 @@
 import { Injectable } from '@nestjs/common';
 import { HasFailed } from 'picsur-shared/dist/types';
 import { UserDbService } from '../../collections/user-db/user-db.service';
-import { EUserBackend } from '../../database/entities/user.entity';
+import { EUserBackend } from '../../database/entities/users/user.entity';
 
 @Injectable()
 export class GuestService {

backend/src/managers/demo/demo.module.ts

@@ -1,4 +1,5 @@
-import { Logger, Module, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
+import { Logger, Module, OnModuleInit } from '@nestjs/common';
+import { SchedulerRegistry } from '@nestjs/schedule';
 import { ImageDBModule } from '../../collections/image-db/image-db.module';
 import { RoleDbModule } from '../../collections/role-db/role-db.module';
 import { EarlyConfigModule } from '../../config/early/early-config.module';
@@ -9,13 +10,13 @@ import { DemoManagerService } from './demo.service';
   imports: [ImageDBModule, EarlyConfigModule, RoleDbModule],
   providers: [DemoManagerService],
 })
-export class DemoManagerModule implements OnModuleInit, OnModuleDestroy {
+export class DemoManagerModule implements OnModuleInit {
   private readonly logger = new Logger(DemoManagerModule.name);
-  private interval: NodeJS.Timeout;
 
   constructor(
     private readonly demoManagerService: DemoManagerService,
     private readonly hostConfigService: HostConfigService,
+    private readonly schedulerRegistry: SchedulerRegistry,
   ) {}
 
   async onModuleInit() {
@@ -27,14 +28,12 @@ export class DemoManagerModule implements OnModuleInit, OnModuleDestroy {
 
   private async setupDemoMode() {
     this.demoManagerService.setupRoles();
-    this.interval = setInterval(
+
+    const interval = setInterval(
       // Run demoManagerService.execute() every interval
       this.demoManagerService.execute.bind(this.demoManagerService),
       this.hostConfigService.getDemoInterval(),
     );
-  }
-
-  onModuleDestroy() {
-    if (this.interval) clearInterval(this.interval);
+    this.schedulerRegistry.addInterval('demo', interval);
   }
 }

backend/src/managers/image/convert.consumer.ts

@@ -0,0 +1,97 @@
+import { OnQueueError, OnQueueFailed, Process, Processor } from '@nestjs/bull';
+import { Logger } from '@nestjs/common';
+import type { Job } from 'bull';
+import { ImageRequestParams } from 'picsur-shared/dist/dto/api/image.dto';
+import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
+import { IsFailure, ThrowIfFailed } from 'picsur-shared/dist/types';
+import { ParseFileType } from 'picsur-shared/dist/util/parse-mime';
+import { ImageFileDBService } from '../../collections/image-db/image-file-db.service';
+import { SysPreferenceDbService } from '../../collections/preference-db/sys-preference-db.service';
+import { ImageConverterService } from './image-converter.service';
+import { ImageManagerService } from './image-manager.service';
+import { ImageConvertQueueID } from './image.queue';
+
+// This contains the job to convert an image to a derivative and store it
+
+export interface ImageConvertJobData {
+  uniqueKey: string;
+  imageId: string;
+  fileType: string;
+  options: ImageRequestParams;
+}
+export type ImageConvertJob = Job<ImageConvertJobData>;
+
+@Processor(ImageConvertQueueID)
+export class ConvertConsumer {
+  private readonly logger = new Logger(ConvertConsumer.name);
+
+  constructor(
+    private readonly imageFilesService: ImageFileDBService,
+    private readonly imageConverter: ImageConverterService,
+    private readonly sysPref: SysPreferenceDbService,
+    private readonly imageService: ImageManagerService,
+  ) {}
+
+  @Process()
+  async convertImage(job: ImageConvertJob): Promise<void> {
+    const { imageId, fileType, options, uniqueKey } = job.data;
+
+    // Get file type
+    const targetFileType = ThrowIfFailed(ParseFileType(fileType));
+
+    // Get preferences
+    const allow_editing = ThrowIfFailed(
+      await this.sysPref.getBooleanPreference(SysPreference.AllowEditing),
+    );
+
+    // Get master image
+    const masterImage = ThrowIfFailed(
+      await this.imageService.getMaster(imageId),
+    );
+    const masterImageData = ThrowIfFailed(
+      await this.imageService.getData(masterImage),
+    );
+    const sourceFileType = ThrowIfFailed(ParseFileType(masterImage.filetype));
+
+    // Conver timage
+    const startTime = Date.now();
+    const convertResult = ThrowIfFailed(
+      await this.imageConverter.convert(
+        masterImageData,
+        sourceFileType,
+        targetFileType,
+        allow_editing ? options : {},
+      ),
+    );
+
+    this.logger.verbose(
+      `Converted ${imageId} from ${sourceFileType.identifier} to ${
+        targetFileType.identifier
+      } in ${Date.now() - startTime}ms`,
+    );
+
+    ThrowIfFailed(
+      await this.imageFilesService.addDerivative(
+        imageId,
+        uniqueKey,
+        convertResult.filetype,
+        convertResult.image,
+      ),
+    );
+  }
+
+  @OnQueueError()
+  async handleError(error: any) {
+    if (IsFailure(error)) error.print(this.logger);
+    else this.logger.error(error);
+  }
+
+  @OnQueueFailed()
+  async handleFailed(job: Job, error: any) {
+    if (IsFailure(error))
+      error.print(this.logger, {
+        prefix: `[JOB ${job.id}]`,
+      });
+    else this.logger.error(error);
+  }
+}

backend/src/managers/image/convert.service.ts

@@ -0,0 +1,103 @@
+import { InjectQueue } from '@nestjs/bull';
+import { Injectable } from '@nestjs/common';
+import Crypto from 'crypto';
+import { ImageRequestParams } from 'picsur-shared/dist/dto/api/image.dto';
+import {
+  AsyncFailable,
+  Fail,
+  FT,
+  HasFailed,
+  ThrowIfFailed,
+} from 'picsur-shared/dist/types';
+import { ImageFileDBService } from '../../collections/image-db/image-file-db.service';
+import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity';
+import { ImageConvertJob } from './convert.consumer';
+import * as ImageQueue from './image.queue';
+
+@Injectable()
+export class ConvertService {
+  constructor(
+    @InjectQueue(ImageQueue.ImageConvertQueueID)
+    private readonly imageQueue: ImageQueue.ImageConvertQueue,
+    private readonly imageFilesService: ImageFileDBService,
+  ) {}
+
+  public async convertJob(
+    imageId: string,
+    fileType: string,
+    options: ImageRequestParams,
+  ): AsyncFailable<ImageConvertJob> {
+    const jobID = this.getConvertHash(imageId, { fileType, ...options });
+
+    /*
+    Jobs with the same ID don't get executed, we abuse this by passing it a hash of the input parameters.
+    This way, if the same image is requested with the same parameters, we don't have to convert it again.
+    Since it will always produce the same output with the same inputs
+    */
+
+    let job: ImageConvertJob;
+    try {
+      job = (await this.imageQueue.add(
+        {
+          imageId,
+          fileType,
+          options,
+          uniqueKey: jobID,
+        },
+        {
+          jobId: jobID,
+        },
+      )) as ImageConvertJob;
+    } catch (e) {
+      return Fail(FT.Internal, e);
+    }
+
+    if (!job.id) return Fail(FT.Internal, undefined, 'Failed to queue job');
+    return job;
+  }
+
+  public async convertPromise(
+    imageId: string,
+    fileType: string,
+    options: ImageRequestParams,
+  ): AsyncFailable<EImageDerivativeBackend> {
+    const uniqueKey = this.getConvertHash(imageId, { fileType, ...options });
+
+    const startime = Date.now();
+    const findExisting = ThrowIfFailed(
+      await this.imageFilesService.getDerivative(imageId, uniqueKey),
+    );
+    if (findExisting !== null) {
+      console.log('Found existing derivative in ' + (Date.now() - startime));
+      return findExisting;
+    }
+
+    const job = await this.convertJob(imageId, fileType, options);
+    if (HasFailed(job)) return job;
+
+    try {
+      await job.finished();
+    } catch (e) {
+      return Fail(FT.Internal, 'Failed to convert image', e);
+    }
+
+    const findResult = ThrowIfFailed(
+      await this.imageFilesService.getDerivative(imageId, uniqueKey),
+    );
+    if (findResult !== null) {
+      console.log('Found new derivative');
+      return findResult;
+    }
+
+    return Fail(FT.Internal, 'Failed to convert image');
+  }
+
+  private getConvertHash(imageID: string, options: object) {
+    // Return a sha256 hash of the stringified options
+    const stringified = JSON.stringify(options) + '-' + imageID;
+    const hash = Crypto.createHash('sha256');
+    hash.update(stringified);
+    const digest = hash.digest('hex');
+    return digest;
+  }
+}

backend/src/managers/image/image-converter.service.ts

@@ -57,7 +57,8 @@ export class ImageConverterService {
     if (HasFailed(memLimit) || HasFailed(timeLimit)) {
       return Fail(FT.Internal, 'Failed to get conversion limits');
     }
-    const timeLimitMS = ms(timeLimit);
+    let timeLimitMS = ms(timeLimit as any);
+    if (isNaN(timeLimitMS) || timeLimitMS === 0) timeLimitMS = 15 * 1000; // 15 seconds
 
     const sharpWrapper = new SharpWrapper(timeLimitMS, memLimit);
     const sharpOptions: SharpOptions = {
@@ -121,16 +122,4 @@ export class ImageConverterService {
       filetype: targetFiletype.identifier,
     };
   }
-
-  private async convertAnimation(
-    image: Buffer,
-    targetFiletype: FileType,
-    options: ImageRequestParams,
-  ): AsyncFailable<ImageResult> {
-    // Apng and gif are stored as is for now
-    return {
-      image: image,
-      filetype: targetFiletype.identifier,
-    };
-  }
 }

backend/src/managers/image/image-manager.module.ts

@@ -0,0 +1,126 @@
+import { BullModule } from '@nestjs/bull';
+import { Logger, Module, OnModuleInit } from '@nestjs/common';
+import { Interval } from '@nestjs/schedule';
+import ms from 'ms';
+import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
+import { HasFailed } from 'picsur-shared/dist/types';
+import { ImageDBModule } from '../../collections/image-db/image-db.module';
+import { ImageDBService } from '../../collections/image-db/image-db.service';
+import { ImageFileDBService } from '../../collections/image-db/image-file-db.service';
+import { PreferenceDbModule } from '../../collections/preference-db/preference-db.module';
+import { SysPreferenceDbService } from '../../collections/preference-db/sys-preference-db.service';
+import { ConvertConsumer } from './convert.consumer';
+import { ConvertService } from './convert.service';
+import { ImageConverterService } from './image-converter.service';
+import { ImageManagerService } from './image-manager.service';
+import { ImageConvertQueueID, ImageIngestQueueID } from './image.queue';
+import { IngestConsumer } from './ingest.consumer';
+import { IngestService } from './ingest.service';
+
+@Module({
+  imports: [
+    ImageDBModule,
+    PreferenceDbModule,
+    BullModule.registerQueue({
+      name: ImageConvertQueueID,
+      defaultJobOptions: {
+        removeOnComplete: true,
+        removeOnFail: true,
+      },
+    }),
+    BullModule.registerQueue({
+      name: ImageIngestQueueID,
+    }),
+  ],
+  providers: [
+    ImageManagerService,
+    ImageConverterService,
+    IngestConsumer,
+    ConvertConsumer,
+    IngestService,
+    ConvertService,
+  ],
+  exports: [
+    ImageManagerService,
+    ImageConverterService,
+    IngestService,
+    ConvertService,
+  ],
+})
+export class ImageManagerModule implements OnModuleInit {
+  private readonly logger = new Logger(ImageManagerModule.name);
+
+  constructor(
+    private readonly prefManager: SysPreferenceDbService,
+    private readonly imageFileDB: ImageFileDBService,
+    private readonly imageDB: ImageDBService,
+  ) {}
+
+  async onModuleInit() {
+    await this.imageManagerCron();
+  }
+
+  @Interval(1000 * 60)
+  private async imageManagerCron() {
+    await this.cleanupDerivatives();
+    await this.cleanupExpired();
+    await this.cleanupOrphanedFiles();
+  }
+
+  private async cleanupDerivatives() {
+    const remove_derivatives_after = await this.prefManager.getStringPreference(
+      SysPreference.RemoveDerivativesAfter,
+    );
+    if (HasFailed(remove_derivatives_after)) {
+      this.logger.warn('Failed to get remove_derivatives_after preference');
+      return;
+    }
+
+    let after_ms = ms(remove_derivatives_after as any);
+    if (isNaN(after_ms) || after_ms === 0) {
+      this.logger.log('remove_derivatives_after is 0, skipping cron');
+      return;
+    }
+    if (after_ms < 60000) after_ms = 60000;
+
+    const result = await this.imageFileDB.cleanupDerivatives(after_ms / 1000);
+    if (HasFailed(result)) {
+      result.print(this.logger);
+    }
+
+    if (result > 0) this.logger.log(`Cleaned up ${result} derivatives`);
+  }
+
+  private async cleanupExpired() {
+    const cleanedUp = await this.imageDB.cleanupExpired();
+
+    if (HasFailed(cleanedUp)) {
+      cleanedUp.print(this.logger);
+      return;
+    }
+
+    if (cleanedUp > 0)
+      this.logger.log(`Cleaned up ${cleanedUp} expired images`);
+  }
+
+  private async cleanupOrphanedFiles() {
+    const cleanedUpDerivatives =
+      await this.imageFileDB.cleanupOrphanedDerivatives();
+
+    if (HasFailed(cleanedUpDerivatives)) {
+      cleanedUpDerivatives.print(this.logger);
+      return;
+    }
+
+    const cleanedUpFiles = await this.imageFileDB.cleanupOrphanedFiles();
+    if (HasFailed(cleanedUpFiles)) {
+      cleanedUpFiles.print(this.logger);
+      return;
+    }
+
+    if (cleanedUpDerivatives > 0 || cleanedUpFiles > 0)
+      this.logger.log(
+        `Cleaned up ${cleanedUpDerivatives} orphaned derivatives and ${cleanedUpFiles} orphaned files`,
+      );
+  }
+}

backend/src/managers/image/image-manager.service.ts

@@ -0,0 +1,111 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum';
+import { FileType } from 'picsur-shared/dist/dto/mimes.dto';
+import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
+import { FindResult } from 'picsur-shared/dist/types/find-result';
+import { ParseFileType } from 'picsur-shared/dist/util/parse-mime';
+import { ImageDBService } from '../../collections/image-db/image-db.service';
+import { ImageFileDBService } from '../../collections/image-db/image-file-db.service';
+import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity';
+import { EImageFileBackend } from '../../database/entities/images/image-file.entity';
+import { EImageBackend } from '../../database/entities/images/image.entity';
+
+@Injectable()
+export class ImageManagerService {
+  private readonly logger = new Logger(ImageManagerService.name);
+
+  constructor(
+    private readonly imagesService: ImageDBService,
+    private readonly imageFilesService: ImageFileDBService,
+  ) {}
+
+  public async findOne(id: string): AsyncFailable<EImageBackend> {
+    return await this.imagesService.findOne(id, undefined);
+  }
+
+  public async findMany(
+    count: number,
+    page: number,
+    userid: string | undefined,
+  ): AsyncFailable<FindResult<EImageBackend>> {
+    return await this.imagesService.findMany(count, page, userid);
+  }
+
+  public async update(
+    id: string,
+    userid: string | undefined,
+    options: Partial<Pick<EImageBackend, 'file_name' | 'expires_at'>>,
+  ): AsyncFailable<EImageBackend> {
+    if (options.expires_at !== undefined && options.expires_at !== null) {
+      if (options.expires_at < new Date()) {
+        return Fail(FT.UsrValidation, 'Expiration date must be in the future');
+      }
+    }
+    return await this.imagesService.update(id, userid, options);
+  }
+
+  public async deleteMany(
+    ids: string[],
+    userid: string | undefined,
+  ): AsyncFailable<EImageBackend[]> {
+    return await this.imagesService.delete(ids, userid);
+  }
+
+  public async deleteWithKey(
+    imageId: string,
+    key: string,
+  ): AsyncFailable<EImageBackend> {
+    return await this.imagesService.deleteWithKey(imageId, key);
+  }
+
+  // File getters ==============================================================
+
+  public async getMaster(imageId: string): AsyncFailable<EImageFileBackend> {
+    return this.imageFilesService.getFile(imageId, ImageEntryVariant.MASTER);
+  }
+
+  public async getMasterFileType(imageId: string): AsyncFailable<FileType> {
+    const mime = await this.imageFilesService.getFileTypes(imageId);
+    if (HasFailed(mime)) return mime;
+
+    if (mime['master'] === undefined)
+      return Fail(FT.NotFound, 'No master file');
+
+    return ParseFileType(mime['master']);
+  }
+
+  public async getOriginal(imageId: string): AsyncFailable<EImageFileBackend> {
+    return this.imageFilesService.getFile(imageId, ImageEntryVariant.ORIGINAL);
+  }
+
+  public async getOriginalFileType(imageId: string): AsyncFailable<FileType> {
+    const filetypes = await this.imageFilesService.getFileTypes(imageId);
+    if (HasFailed(filetypes)) return filetypes;
+
+    if (filetypes['original'] === undefined)
+      return Fail(FT.NotFound, 'No original file');
+
+    return ParseFileType(filetypes['original']);
+  }
+
+  public async getData(image: EImageFileBackend | EImageDerivativeBackend) {
+    return await this.imageFilesService.getData(image);
+  }
+
+  public async getFileMimes(imageId: string): AsyncFailable<{
+    [ImageEntryVariant.MASTER]: string;
+    [ImageEntryVariant.ORIGINAL]: string | undefined;
+  }> {
+    const result = await this.imageFilesService.getFileTypes(imageId);
+    if (HasFailed(result)) return result;
+
+    if (result[ImageEntryVariant.MASTER] === undefined) {
+      return Fail(FT.NotFound, 'No master file found');
+    }
+
+    return {
+      [ImageEntryVariant.MASTER]: result[ImageEntryVariant.MASTER]!,
+      [ImageEntryVariant.ORIGINAL]: result[ImageEntryVariant.ORIGINAL],
+    };
+  }
+}

backend/src/managers/image/image-processor.service.ts

@@ -1,50 +0,0 @@
-import { Injectable } from '@nestjs/common';
-import {
-  FileType,
-  ImageFileType,
-  SupportedFileTypeCategory,
-} from 'picsur-shared/dist/dto/mimes.dto';
-
-import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
-import { ParseFileType } from 'picsur-shared/dist/util/parse-mime';
-import { ImageConverterService } from './image-converter.service';
-import { ImageResult } from './imageresult';
-
-@Injectable()
-export class ImageProcessorService {
-  constructor(private readonly imageConverter: ImageConverterService) {}
-
-  public async process(
-    image: Buffer,
-    filetype: FileType,
-  ): AsyncFailable<ImageResult> {
-    if (filetype.category === SupportedFileTypeCategory.Image) {
-      return await this.processStill(image, filetype);
-    } else if (filetype.category === SupportedFileTypeCategory.Animation) {
-      return await this.processAnimation(image, filetype);
-    } else {
-      return Fail(FT.SysValidation, 'Unsupported mime type');
-    }
-  }
-
-  private async processStill(
-    image: Buffer,
-    filetype: FileType,
-  ): AsyncFailable<ImageResult> {
-    const outputFileType = ParseFileType(ImageFileType.QOI);
-    if (HasFailed(outputFileType)) return outputFileType;
-
-    return this.imageConverter.convert(image, filetype, outputFileType, {});
-  }
-
-  private async processAnimation(
-    image: Buffer,
-    filetype: FileType,
-  ): AsyncFailable<ImageResult> {
-    // Webps and gifs are stored as is for now
-    return {
-      image: image,
-      filetype: filetype.identifier,
-    };
-  }
-}

backend/src/managers/image/image.module.ts

@@ -1,83 +0,0 @@
-import { Logger, Module, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
-import ms from 'ms';
-import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
-import { HasFailed } from 'picsur-shared/dist/types';
-import { ImageDBModule } from '../../collections/image-db/image-db.module';
-import { ImageDBService } from '../../collections/image-db/image-db.service';
-import { ImageFileDBService } from '../../collections/image-db/image-file-db.service';
-import { PreferenceDbModule } from '../../collections/preference-db/preference-db.module';
-import { SysPreferenceDbService } from '../../collections/preference-db/sys-preference-db.service';
-import { ImageConverterService } from './image-converter.service';
-import { ImageProcessorService } from './image-processor.service';
-import { ImageManagerService } from './image.service';
-
-@Module({
-  imports: [ImageDBModule, PreferenceDbModule],
-  providers: [
-    ImageManagerService,
-    ImageProcessorService,
-    ImageConverterService,
-  ],
-  exports: [ImageManagerService],
-})
-export class ImageManagerModule implements OnModuleInit, OnModuleDestroy {
-  private readonly logger = new Logger(ImageManagerModule.name);
-  private interval: NodeJS.Timeout;
-
-  constructor(
-    private readonly prefManager: SysPreferenceDbService,
-    private readonly imageFileDB: ImageFileDBService,
-    private readonly imageDB: ImageDBService,
-  ) {}
-
-  async onModuleInit() {
-    this.interval = setInterval(
-      // Run demoManagerService.execute() every interval
-      this.imageManagerCron.bind(this),
-      1000 * 60,
-    );
-    await this.imageManagerCron();
-  }
-
-  private async imageManagerCron() {
-    await this.cleanupDerivatives();
-    await this.cleanupExpired();
-  }
-
-  private async cleanupDerivatives() {
-    const remove_derivatives_after = await this.prefManager.getStringPreference(
-      SysPreference.RemoveDerivativesAfter,
-    );
-    if (HasFailed(remove_derivatives_after)) {
-      this.logger.warn('Failed to get remove_derivatives_after preference');
-      return;
-    }
-
-    const after_ms = ms(remove_derivatives_after);
-    if (after_ms === 0) {
-      this.logger.log('remove_derivatives_after is 0, skipping cron');
-      return;
-    }
-
-    const result = await this.imageFileDB.cleanupDerivatives(after_ms / 1000);
-    if (HasFailed(result)) {
-      this.logger.warn(result.print());
-    }
-
-    this.logger.log(`Cleaned up ${result} derivatives`);
-  }
-
-  private async cleanupExpired() {
-    const cleanedUp = await this.imageDB.cleanupExpired();
-
-    if (HasFailed(cleanedUp)) {
-      this.logger.warn(cleanedUp.print());
-    }
-
-    this.logger.log(`Cleaned up ${cleanedUp} expired images`);
-  }
-
-  onModuleDestroy() {
-    if (this.interval) clearInterval(this.interval);
-  }
-}

backend/src/managers/image/image.queue.ts

@@ -0,0 +1,9 @@
+import { Queue } from 'bull';
+import { ImageConvertJobData } from './convert.consumer';
+import { ImageIngestJobData } from './ingest.consumer';
+
+export const ImageConvertQueueID = 'image-convert-queue';
+export const ImageIngestQueueID = 'image-ingest-queue';
+
+export type ImageConvertQueue = Queue<ImageConvertJobData>;
+export type ImageIngestQueue = Queue<ImageIngestJobData>;

backend/src/managers/image/image.service.ts

@@ -1,289 +0,0 @@
-import { Injectable, Logger } from '@nestjs/common';
-import Crypto from 'crypto';
-import { fileTypeFromBuffer, FileTypeResult } from 'file-type';
-import { ImageRequestParams } from 'picsur-shared/dist/dto/api/image.dto';
-import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum';
-import {
-  AnimFileType,
-  FileType,
-  ImageFileType,
-  Mime2FileType,
-} from 'picsur-shared/dist/dto/mimes.dto';
-import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
-import { UsrPreference } from 'picsur-shared/dist/dto/usr-preferences.enum';
-import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
-import { FindResult } from 'picsur-shared/dist/types/find-result';
-import { ParseFileType } from 'picsur-shared/dist/util/parse-mime';
-import { IsQOI } from 'qoi-img';
-import { ImageDBService } from '../../collections/image-db/image-db.service';
-import { ImageFileDBService } from '../../collections/image-db/image-file-db.service';
-import { SysPreferenceDbService } from '../../collections/preference-db/sys-preference-db.service';
-import { UsrPreferenceDbService } from '../../collections/preference-db/usr-preference-db.service';
-import { EImageDerivativeBackend } from '../../database/entities/image-derivative.entity';
-import { EImageFileBackend } from '../../database/entities/image-file.entity';
-import { EImageBackend } from '../../database/entities/image.entity';
-import { MutexFallBack } from '../../util/mutex-fallback';
-import { ImageConverterService } from './image-converter.service';
-import { ImageProcessorService } from './image-processor.service';
-import { WebPInfo } from './webpinfo/webpinfo';
-
-@Injectable()
-export class ImageManagerService {
-  private readonly logger = new Logger(ImageManagerService.name);
-
-  constructor(
-    private readonly imagesService: ImageDBService,
-    private readonly imageFilesService: ImageFileDBService,
-    private readonly processService: ImageProcessorService,
-    private readonly convertService: ImageConverterService,
-    private readonly userPref: UsrPreferenceDbService,
-    private readonly sysPref: SysPreferenceDbService,
-  ) {}
-
-  public async findOne(id: string): AsyncFailable<EImageBackend> {
-    return await this.imagesService.findOne(id, undefined);
-  }
-
-  public async findMany(
-    count: number,
-    page: number,
-    userid: string | undefined,
-  ): AsyncFailable<FindResult<EImageBackend>> {
-    return await this.imagesService.findMany(count, page, userid);
-  }
-
-  public async update(
-    id: string,
-    userid: string | undefined,
-    options: Partial<Pick<EImageBackend, 'file_name' | 'expires_at'>>,
-  ): AsyncFailable<EImageBackend> {
-    if (options.expires_at !== undefined && options.expires_at !== null) {
-      if (options.expires_at < new Date()) {
-        return Fail(FT.UsrValidation, 'Expiration date must be in the future');
-      }
-    }
-    return await this.imagesService.update(id, userid, options);
-  }
-
-  public async deleteMany(
-    ids: string[],
-    userid: string | undefined,
-  ): AsyncFailable<EImageBackend[]> {
-    return await this.imagesService.delete(ids, userid);
-  }
-
-  public async deleteWithKey(
-    imageId: string,
-    key: string,
-  ): AsyncFailable<EImageBackend> {
-    return await this.imagesService.deleteWithKey(imageId, key);
-  }
-
-  public async upload(
-    userid: string,
-    filename: string,
-    image: Buffer,
-    withDeleteKey: boolean,
-  ): AsyncFailable<EImageBackend> {
-    const fileType = await this.getFileTypeFromBuffer(image);
-    if (HasFailed(fileType)) return fileType;
-
-    // Check if need to save orignal
-    const keepOriginal = await this.userPref.getBooleanPreference(
-      userid,
-      UsrPreference.KeepOriginal,
-    );
-    if (HasFailed(keepOriginal)) return keepOriginal;
-
-    // Process
-    const processResult = await this.processService.process(image, fileType);
-    if (HasFailed(processResult)) return processResult;
-
-    // Strip extension from filename
-    const name = (() => {
-      const index = filename.lastIndexOf('.');
-      if (index === -1) return filename;
-      return filename.substring(0, index);
-    })();
-
-    // Save processed to db
-    const imageEntity = await this.imagesService.create(
-      userid,
-      name,
-      withDeleteKey,
-    );
-    if (HasFailed(imageEntity)) return imageEntity;
-
-    const imageFileEntity = await this.imageFilesService.setFile(
-      imageEntity.id,
-      ImageEntryVariant.MASTER,
-      processResult.image,
-      processResult.filetype,
-    );
-    if (HasFailed(imageFileEntity)) return imageFileEntity;
-
-    if (keepOriginal) {
-      const originalFileEntity = await this.imageFilesService.setFile(
-        imageEntity.id,
-        ImageEntryVariant.ORIGINAL,
-        image,
-        fileType.identifier,
-      );
-      if (HasFailed(originalFileEntity)) return originalFileEntity;
-    }
-
-    return imageEntity;
-  }
-
-  public async getConverted(
-    imageId: string,
-    fileType: string,
-    options: ImageRequestParams,
-  ): AsyncFailable<EImageDerivativeBackend> {
-    const targetFileType = ParseFileType(fileType);
-    if (HasFailed(targetFileType)) return targetFileType;
-
-    const converted_key = this.getConvertHash({ mime: fileType, ...options });
-
-    const [save_derivatives, allow_editing] = await Promise.all([
-      this.sysPref.getBooleanPreference(SysPreference.SaveDerivatives),
-      this.sysPref.getBooleanPreference(SysPreference.AllowEditing),
-    ]);
-    if (HasFailed(save_derivatives)) return save_derivatives;
-    if (HasFailed(allow_editing)) return allow_editing;
-
-    return MutexFallBack(
-      converted_key,
-      () => {
-        if (save_derivatives)
-          return this.imageFilesService.getDerivative(imageId, converted_key);
-        else return Promise.resolve(null);
-      },
-      async () => {
-        const masterImage = await this.getMaster(imageId);
-        if (HasFailed(masterImage)) return masterImage;
-
-        const sourceFileType = ParseFileType(masterImage.filetype);
-        if (HasFailed(sourceFileType)) return sourceFileType;
-
-        const startTime = Date.now();
-        const convertResult = await this.convertService.convert(
-          masterImage.data,
-          sourceFileType,
-          targetFileType,
-          allow_editing ? options : {},
-        );
-        if (HasFailed(convertResult)) return convertResult;
-
-        this.logger.verbose(
-          `Converted ${imageId} from ${sourceFileType.identifier} to ${
-            targetFileType.identifier
-          } in ${Date.now() - startTime}ms`,
-        );
-
-        if (save_derivatives) {
-          return await this.imageFilesService.addDerivative(
-            imageId,
-            converted_key,
-            convertResult.filetype,
-            convertResult.image,
-          );
-        } else {
-          const derivative = new EImageDerivativeBackend();
-          derivative.filetype = convertResult.filetype;
-          derivative.data = convertResult.image;
-          derivative.image_id = imageId;
-          derivative.key = converted_key;
-          return derivative;
-        }
-      },
-    );
-  }
-
-  // File getters ==============================================================
-
-  public async getMaster(imageId: string): AsyncFailable<EImageFileBackend> {
-    return this.imageFilesService.getFile(imageId, ImageEntryVariant.MASTER);
-  }
-
-  public async getMasterFileType(imageId: string): AsyncFailable<FileType> {
-    const mime = await this.imageFilesService.getFileTypes(imageId);
-    if (HasFailed(mime)) return mime;
-
-    if (mime['master'] === undefined)
-      return Fail(FT.NotFound, 'No master file');
-
-    return ParseFileType(mime['master']);
-  }
-
-  public async getOriginal(imageId: string): AsyncFailable<EImageFileBackend> {
-    return this.imageFilesService.getFile(imageId, ImageEntryVariant.ORIGINAL);
-  }
-
-  public async getOriginalFileType(imageId: string): AsyncFailable<FileType> {
-    const filetypes = await this.imageFilesService.getFileTypes(imageId);
-    if (HasFailed(filetypes)) return filetypes;
-
-    if (filetypes['original'] === undefined)
-      return Fail(FT.NotFound, 'No original file');
-
-    return ParseFileType(filetypes['original']);
-  }
-
-  public async getFileMimes(imageId: string): AsyncFailable<{
-    [ImageEntryVariant.MASTER]: string;
-    [ImageEntryVariant.ORIGINAL]: string | undefined;
-  }> {
-    const result = await this.imageFilesService.getFileTypes(imageId);
-    if (HasFailed(result)) return result;
-
-    if (result[ImageEntryVariant.MASTER] === undefined) {
-      return Fail(FT.NotFound, 'No master file found');
-    }
-
-    return {
-      [ImageEntryVariant.MASTER]: result[ImageEntryVariant.MASTER]!,
-      [ImageEntryVariant.ORIGINAL]: result[ImageEntryVariant.ORIGINAL],
-    };
-  }
-
-  // Util stuff ==================================================================
-
-  private async getFileTypeFromBuffer(image: Buffer): AsyncFailable<FileType> {
-    const filetypeResult: FileTypeResult | undefined = await fileTypeFromBuffer(
-      image,
-    );
-
-    let mime: string | undefined;
-    if (filetypeResult === undefined) {
-      if (IsQOI(image)) mime = 'image/qoi';
-    } else {
-      mime = filetypeResult.mime;
-    }
-
-    if (mime === undefined) mime = 'other/unknown';
-
-    let filetype: string | undefined;
-    if (mime === 'image/webp') {
-      const header = await WebPInfo.from(image);
-      if (header.summary.isAnimated) filetype = AnimFileType.WEBP;
-      else filetype = ImageFileType.WEBP;
-    }
-    if (filetype === undefined) {
-      const parsed = Mime2FileType(mime);
-      if (HasFailed(parsed)) return parsed;
-      filetype = parsed;
-    }
-
-    return ParseFileType(filetype);
-  }
-
-  private getConvertHash(options: object) {
-    // Return a sha256 hash of the stringified options
-    const stringified = JSON.stringify(options);
-    const hash = Crypto.createHash('sha256');
-    hash.update(stringified);
-    const digest = hash.digest('hex');
-    return digest;
-  }
-}

backend/src/managers/image/ingest.consumer.ts

@@ -0,0 +1,140 @@
+import { OnQueueError, OnQueueFailed, Process, Processor } from '@nestjs/bull';
+import { Logger } from '@nestjs/common';
+import type { Job } from 'bull';
+import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum';
+import {
+  FileType,
+  ImageFileType,
+  SupportedFileTypeCategory,
+} from 'picsur-shared/dist/dto/mimes.dto';
+import {
+  AsyncFailable,
+  Fail,
+  FT,
+  HasFailed,
+  IsFailure,
+  ThrowIfFailed,
+} from 'picsur-shared/dist/types';
+import { ParseFileType } from 'picsur-shared/dist/util/parse-mime';
+import { ImageDBService } from '../../collections/image-db/image-db.service';
+import { ImageFileDBService } from '../../collections/image-db/image-file-db.service';
+import { EImageBackend } from '../../database/entities/images/image.entity';
+import { ImageConverterService } from '../image/image-converter.service';
+import { ImageResult } from '../image/imageresult';
+import { ImageIngestQueueID } from './image.queue';
+
+export interface ImageIngestJobData {
+  imageID: string;
+  storeOriginal: boolean;
+}
+export type ImageIngestJob = Job<ImageIngestJobData>;
+
+@Processor(ImageIngestQueueID)
+export class IngestConsumer {
+  private readonly logger = new Logger(IngestConsumer.name);
+
+  constructor(
+    private readonly imagesService: ImageDBService,
+    private readonly imageFilesService: ImageFileDBService,
+    private readonly imageConverter: ImageConverterService,
+  ) {}
+
+  @Process({
+    concurrency: 5,
+  })
+  async ingestImage(job: ImageIngestJob): Promise<EImageBackend> {
+    const { imageID, storeOriginal } = job.data;
+
+    // Already start the query for the image, we only need it when returning
+    const imagePromise = this.imagesService.findOne(imageID, undefined);
+
+    this.logger.verbose(
+      `Ingesting image ${imageID} and store original: ${storeOriginal}`,
+    );
+
+    const ingestFile = ThrowIfFailed(
+      await this.imageFilesService.getFile(imageID, ImageEntryVariant.INGEST),
+    );
+    const ingestFileData = ThrowIfFailed(
+      await this.imageFilesService.getData(ingestFile),
+    );
+    const ingestFiletype = ThrowIfFailed(ParseFileType(ingestFile.filetype));
+
+    const processed = ThrowIfFailed(
+      await this.process(ingestFileData, ingestFiletype),
+    );
+
+    const masterPromise = this.imageFilesService.setFile(
+      imageID,
+      ImageEntryVariant.MASTER,
+      processed.image,
+      processed.filetype,
+    );
+
+    const originalPromise = storeOriginal
+      ? this.imageFilesService.migrateFile(
+          imageID,
+          ImageEntryVariant.INGEST,
+          ImageEntryVariant.ORIGINAL,
+        )
+      : this.imageFilesService.orphanFile(imageID, ImageEntryVariant.INGEST);
+
+    const results = await Promise.all([masterPromise, originalPromise]);
+    results.map((r) => ThrowIfFailed(r));
+
+    const image = ThrowIfFailed(await imagePromise);
+
+    this.logger.verbose(`Ingested image ${imageID}`);
+
+    return image;
+  }
+
+  private async process(
+    image: Buffer,
+    filetype: FileType,
+  ): AsyncFailable<ImageResult> {
+    if (filetype.category === SupportedFileTypeCategory.Image) {
+      return await this.processStill(image, filetype);
+    } else if (filetype.category === SupportedFileTypeCategory.Animation) {
+      return await this.processAnimation(image, filetype);
+    } else {
+      return Fail(FT.SysValidation, 'Unsupported mime type');
+    }
+  }
+
+  private async processStill(
+    image: Buffer,
+    filetype: FileType,
+  ): AsyncFailable<ImageResult> {
+    const outputFileType = ParseFileType(ImageFileType.QOI);
+    if (HasFailed(outputFileType)) return outputFileType;
+
+    return this.imageConverter.convert(image, filetype, outputFileType, {});
+  }
+
+  private async processAnimation(
+    image: Buffer,
+    filetype: FileType,
+  ): AsyncFailable<ImageResult> {
+    // Webps and gifs are stored as is for now
+    return {
+      image: image,
+      filetype: filetype.identifier,
+    };
+  }
+
+  @OnQueueError()
+  async handleError(error: any) {
+    if (IsFailure(error)) error.print(this.logger);
+    else this.logger.error(error);
+  }
+
+  @OnQueueFailed()
+  async handleFailed(job: Job, error: any) {
+    if (IsFailure(error))
+      error.print(this.logger, {
+        prefix: `[JOB ${job.id}]`,
+      });
+    else this.logger.error(error);
+  }
+}

backend/src/managers/image/ingest.service.ts

@@ -0,0 +1,177 @@
+import { InjectQueue } from '@nestjs/bull';
+import { Injectable, Logger } from '@nestjs/common';
+import { fileTypeFromBuffer, FileTypeResult } from 'file-type';
+import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum';
+import {
+  AnimFileType,
+  FileType,
+  ImageFileType,
+  Mime2FileType,
+} from 'picsur-shared/dist/dto/mimes.dto';
+import { UsrPreference } from 'picsur-shared/dist/dto/usr-preferences.enum';
+import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
+import { ParseFileType } from 'picsur-shared/dist/util/parse-mime';
+import { IsQOI } from 'qoi-img';
+import { ImageDBService } from '../../collections/image-db/image-db.service';
+import { ImageFileDBService } from '../../collections/image-db/image-file-db.service';
+import { UsrPreferenceDbService } from '../../collections/preference-db/usr-preference-db.service';
+import { EImageBackend } from '../../database/entities/images/image.entity';
+import { WebPInfo } from '../image/webpinfo/webpinfo';
+import * as ImageQueue from './image.queue';
+import { ImageIngestJob } from './ingest.consumer';
+import { v4 as uuidv4 } from 'uuid';
+
+@Injectable()
+export class IngestService {
+  private readonly logger = new Logger(IngestService.name);
+
+  constructor(
+    @InjectQueue(ImageQueue.ImageIngestQueueID)
+    private readonly imageQueue: ImageQueue.ImageIngestQueue,
+    private readonly imagesService: ImageDBService,
+    private readonly imageFilesService: ImageFileDBService,
+    private readonly userPref: UsrPreferenceDbService,
+  ) {}
+
+  public async uploadJob(
+    userid: string,
+    filename: string,
+    image: Buffer,
+    withDeleteKey: boolean,
+  ): AsyncFailable<[ImageIngestJob, EImageBackend]> {
+    const fileType = await this.getFileTypeFromBuffer(image);
+    if (HasFailed(fileType)) return fileType;
+
+    // Check if need to save orignal
+    const keepOriginal = await this.userPref.getBooleanPreference(
+      userid,
+      UsrPreference.KeepOriginal,
+    );
+    if (HasFailed(keepOriginal)) return keepOriginal;
+
+    // Strip extension from filename
+    const name = (() => {
+      const index = filename.lastIndexOf('.');
+      if (index === -1) return filename;
+      return filename.substring(0, index);
+    })();
+
+    // Save unprocessed image to be processed by worker
+    const imageEntity = await this.imagesService.create(
+      userid,
+      name,
+      withDeleteKey,
+    );
+    if (HasFailed(imageEntity)) return imageEntity;
+
+    {
+      const imageFileEntity = await this.imageFilesService.setFile(
+        imageEntity.id,
+        ImageEntryVariant.INGEST,
+        image,
+        fileType.identifier,
+      );
+      if (HasFailed(imageFileEntity)) return imageFileEntity;
+    }
+
+    try {
+      const job = (await this.imageQueue.add(
+        {
+          imageID: imageEntity.id,
+          storeOriginal: keepOriginal,
+        },
+        {
+          jobId: uuidv4(),
+        },
+      )) as ImageIngestJob;
+      if (!job.id) return Fail(FT.Internal, undefined, 'Failed to queue job');
+
+      return [job, imageEntity];
+    } catch (e) {
+      return Fail(FT.Internal, e);
+    }
+  }
+
+  public async uploadPromise(
+    userid: string,
+    filename: string,
+    image: Buffer,
+    withDeleteKey: boolean,
+  ): AsyncFailable<EImageBackend> {
+    const result = await this.uploadJob(userid, filename, image, withDeleteKey);
+    if (HasFailed(result)) return result;
+
+    const [job, imageEntity] = result;
+
+    try {
+      await job.finished();
+      return imageEntity;
+    } catch (e) {
+      return Fail(FT.Internal, 'Failed to process image', e);
+    }
+  }
+
+  public async getProgress(jobsIds: string[]): AsyncFailable<{
+    progress: number;
+    failed: string[];
+  }> {
+    try {
+      const jobs = await Promise.all(
+        jobsIds.map((id) => this.imageQueue.getJob(id)),
+      );
+
+      const cleanJobs: ImageIngestJob[] = jobs.filter(
+        (job) => job !== null,
+      ) as ImageIngestJob[];
+
+      if (cleanJobs.length === 0) return { progress: 1, failed: [] };
+
+      const statefulJobs = await Promise.all(
+        cleanJobs.map(async (job) => ({ job, state: await job.getState() })),
+      );
+
+      const progress =
+        statefulJobs.filter(
+          (job) => job.state === 'completed' || job.state === 'failed',
+        ).length / cleanJobs.length;
+
+      return {
+        progress,
+        failed: statefulJobs
+          .filter((job) => job.state === 'failed')
+          .map((job) => job.job.id.toString()),
+      };
+    } catch (e) {
+      return Fail(FT.Internal, e);
+    }
+  }
+
+  private async getFileTypeFromBuffer(image: Buffer): AsyncFailable<FileType> {
+    const filetypeResult: FileTypeResult | undefined = await fileTypeFromBuffer(
+      image,
+    );
+
+    let mime: string | undefined;
+    if (filetypeResult === undefined) {
+      if (IsQOI(image)) mime = 'image/x-qoi';
+    } else {
+      mime = filetypeResult.mime;
+    }
+
+    if (mime === undefined) mime = 'other/unknown';
+
+    let filetype: string | undefined;
+    if (mime === 'image/webp') {
+      const header = await WebPInfo.from(image);
+      if (header.summary.isAnimated) filetype = AnimFileType.WEBP;
+      else filetype = ImageFileType.WEBP;
+    }
+    if (filetype === undefined) {
+      const parsed = Mime2FileType(mime);
+      if (HasFailed(parsed)) return parsed;
+      filetype = parsed;
+    }
+
+    return ParseFileType(filetype);
+  }
+}

backend/src/managers/usage/usage.module.ts

@@ -0,0 +1,43 @@
+import { Logger, Module, OnModuleInit } from '@nestjs/common';
+import { SchedulerRegistry } from '@nestjs/schedule';
+import { ImageDBModule } from '../../collections/image-db/image-db.module';
+import { SystemStateDbModule } from '../../collections/system-state-db/system-state-db.module';
+import { UserDbModule } from '../../collections/user-db/user-db.module';
+import { LateConfigModule } from '../../config/late/late-config.module';
+import { UsageConfigService } from '../../config/late/usage.config.service';
+import { UsageService } from './usage.service';
+
+@Module({
+  imports: [LateConfigModule, SystemStateDbModule, ImageDBModule, UserDbModule],
+  providers: [UsageService],
+  exports: [UsageService],
+})
+export class UsageManagerModule implements OnModuleInit {
+  private readonly logger = new Logger(UsageManagerModule.name);
+
+  constructor(
+    private readonly usageService: UsageService,
+    private readonly usageConfigService: UsageConfigService,
+    private readonly schedulerRegistry: SchedulerRegistry,
+  ) {}
+
+  async onModuleInit() {
+    if (!(await this.usageConfigService.getMetricsEnabled())) {
+      this.logger.log('Telemetry is disabled');
+    }
+
+    const interval = setInterval(
+      this.cronJob.bind(this),
+      await this.usageConfigService.getMetricsInterval(),
+    );
+    this.schedulerRegistry.addInterval('usage', interval);
+
+    this.cronJob();
+  }
+
+  private cronJob() {
+    this.usageService.execute().catch((err) => {
+      this.logger.warn(err);
+    });
+  }
+}

backend/src/managers/usage/usage.service.ts

@@ -0,0 +1,158 @@
+import { Injectable, Logger } from '@nestjs/common';
+import isDocker from 'is-docker';
+import fetch from 'node-fetch';
+import os from 'os';
+import { FallbackIfFailed, HasFailed } from 'picsur-shared/dist/types';
+import { UUIDRegex } from 'picsur-shared/dist/util/common-regex';
+import { ImageDBService } from '../../collections/image-db/image-db.service';
+import { SystemStateDbService } from '../../collections/system-state-db/system-state-db.service';
+import { UserDbService } from '../../collections/user-db/user-db.service';
+import { HostConfigService } from '../../config/early/host.config.service';
+import { UsageConfigService } from '../../config/late/usage.config.service';
+
+interface UsageData {
+  id?: string;
+
+  uptime: number;
+  version: string;
+  demo_active: boolean;
+
+  users: number;
+  images: number;
+
+  architecture: string;
+  cpu_count: number;
+  ram_total: number;
+
+  is_docker: boolean;
+  is_production: boolean;
+}
+
+@Injectable()
+export class UsageService {
+  private readonly logger = new Logger(UsageService.name);
+
+  constructor(
+    private readonly systemState: SystemStateDbService,
+    private readonly hostConfig: HostConfigService,
+    private readonly usageConfig: UsageConfigService,
+    private readonly userRepo: UserDbService,
+    private readonly imageRepo: ImageDBService,
+  ) {}
+
+  public async execute() {
+    if (!(await this.usageConfig.getMetricsEnabled())) return;
+
+    const id = await this.getSystemID();
+
+    if (id === null) {
+      await this.sendInitialData();
+    } else {
+      await this.sendUpdateData(id);
+    }
+  }
+
+  private async sendInitialData() {
+    const url =
+      (await this.usageConfig.getMetricsUrl()) + '/api/install/create';
+
+    const result: any = await fetch(url, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(await this.collectData()),
+    }).then((res) => res.json());
+
+    const id = result?.data?.id;
+    if (typeof id !== 'string')
+      return this.logger.warn(
+        'Invalid response when sending initial data: ' + JSON.stringify(result),
+      );
+    if (!UUIDRegex.test(id))
+      return this.logger.warn('Invalid system ID: ' + id);
+
+    await this.setSystemID(id);
+  }
+
+  private async sendUpdateData(id: string) {
+    const url =
+      (await this.usageConfig.getMetricsUrl()) + '/api/install/update';
+
+    const body = await this.collectData();
+    body.id = id;
+
+    const result = await fetch(url, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(body),
+    });
+
+    if (result.status < 200 || result.status >= 300) {
+      const data: any = await result.json();
+
+      if (data?.type === 'notfound') {
+        this.logger.warn('System ID not found, clearing');
+        await this.clearSystemID();
+      } else {
+        this.logger.warn(
+          'Failed to send update data: ' + JSON.stringify(await result.json()),
+        );
+      }
+    }
+  }
+
+  private async getSystemID(): Promise<string | null> {
+    const result = await this.systemState.get('systemID');
+    if (HasFailed(result)) {
+      this.logger.warn(result);
+      return null;
+    }
+    if (result === null) return null;
+    if (UUIDRegex.test(result)) return result;
+    this.logger.warn('Invalid system ID');
+    return null;
+  }
+
+  private async setSystemID(id: string) {
+    if (!UUIDRegex.test(id)) {
+      return this.logger.warn('Invalid system ID');
+    }
+    const result = await this.systemState.set('systemID', id);
+    if (HasFailed(result)) {
+      this.logger.warn(result);
+    }
+  }
+
+  private async clearSystemID() {
+    const result = await this.systemState.clear('systemID');
+    if (HasFailed(result)) {
+      this.logger.warn(result);
+    }
+  }
+
+  private async collectData(): Promise<UsageData> {
+    const users = FallbackIfFailed(await this.userRepo.count(), 0, this.logger);
+    const images = FallbackIfFailed(
+      await this.imageRepo.count(),
+      0,
+      this.logger,
+    );
+
+    const data: UsageData = {
+      uptime: Math.floor(process.uptime()),
+      version: this.hostConfig.getVersion(),
+      demo_active: this.hostConfig.isDemo(),
+      users,
+      images,
+      architecture: process.arch,
+      cpu_count: os.cpus().length,
+      ram_total: Math.floor(os.totalmem() / 1024 / 1024),
+      is_docker: isDocker(),
+      is_production: this.hostConfig.isProduction(),
+    };
+    return data;
+  }
+}

backend/src/models/constants/syspreferences.const.ts

@@ -1,21 +0,0 @@
-import { PrefValueTypeStrings } from 'picsur-shared/dist/dto/preferences.dto';
-import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
-
-export type SysPreferences = SysPreference[];
-export const SysPreferenceList: string[] = Object.values(SysPreference);
-
-// Syspref Value types
-export const SysPreferenceValueTypes: {
-  [key in SysPreference]: PrefValueTypeStrings;
-} = {
-  [SysPreference.JwtSecret]: 'string',
-  [SysPreference.JwtExpiresIn]: 'string',
-  [SysPreference.BCryptStrength]: 'number',
-
-  [SysPreference.RemoveDerivativesAfter]: 'string',
-  [SysPreference.SaveDerivatives]: 'boolean',
-  [SysPreference.AllowEditing]: 'boolean',
-
-  [SysPreference.ConversionTimeLimit]: 'string',
-  [SysPreference.ConversionMemoryLimit]: 'number',
-};

backend/src/models/constants/usrpreferences.const.ts

@@ -1,12 +0,0 @@
-import { PrefValueTypeStrings } from 'picsur-shared/dist/dto/preferences.dto';
-import { UsrPreference } from 'picsur-shared/dist/dto/usr-preferences.enum';
-
-export type UsrPreferences = UsrPreference[];
-export const UsrPreferenceList: string[] = Object.values(UsrPreference);
-
-// Syspref Value types
-export const UsrPreferenceValueTypes: {
-  [key in UsrPreference]: PrefValueTypeStrings;
-} = {
-  [UsrPreference.KeepOriginal]: 'boolean',
-};

backend/src/models/dto/image-upload.dto.ts

@@ -1,9 +0,0 @@
-import { createZodDto } from 'picsur-shared/dist/util/create-zod-dto';
-import { z } from 'zod';
-import { MultiPartFileDtoSchema } from './multipart.dto';
-
-// A validation class for form based file upload of an image
-export const ImageUploadDtoSchema = z.object({
-  image: MultiPartFileDtoSchema,
-});
-export class ImageUploadDto extends createZodDto(ImageUploadDtoSchema) {}

backend/src/models/dto/multipart.dto.ts

@@ -1,48 +0,0 @@
-import { MultipartFile } from '@fastify/multipart';
-import { AsyncFailable, Fail, FT } from 'picsur-shared/dist/types';
-import { z } from 'zod';
-
-export const MultiPartFileDtoSchema = z.object({
-  fieldname: z.string(),
-  encoding: z.string(),
-  filename: z.string(),
-  mimetype: z.string(),
-  buffer: z.any(),
-  file: z.any(),
-});
-export type MultiPartFileDto = z.infer<typeof MultiPartFileDtoSchema>;
-
-export async function CreateMultiPartFileDto(
-  file: MultipartFile,
-): AsyncFailable<MultiPartFileDto> {
-  try {
-    const buffer = await file.toBuffer();
-    return {
-      fieldname: file.fieldname,
-      encoding: file.encoding,
-      filename: file.filename,
-      mimetype: file.mimetype,
-      buffer,
-      file: file.file,
-    };
-  } catch (e) {
-    return Fail(FT.Internal, e);
-  }
-}
-
-export const MultiPartFieldDtoSchema = z.object({
-  fieldname: z.string(),
-  encoding: z.string(),
-  value: z.string(),
-});
-export type MultiPartFieldDto = z.infer<typeof MultiPartFieldDtoSchema>;
-
-export function CreateMultiPartFieldDto(
-  file: MultipartFile,
-): MultiPartFieldDto {
-  return {
-    fieldname: file.fieldname,
-    encoding: file.encoding,
-    value: (file as any).value,
-  };
-}

backend/src/models/transformers/user.transformer.ts

@@ -1,5 +1,5 @@
 import { EUser } from 'picsur-shared/dist/entities/user.entity';
-import { EUserBackend } from '../../database/entities/user.entity';
+import { EUserBackend } from '../../database/entities/users/user.entity';
 
 export function EUserBackend2EUser(eUser: EUserBackend): EUser {
   if (eUser.hashed_password === undefined) return eUser as EUser;

backend/src/routes/api/api.module.ts

@@ -4,6 +4,7 @@ import { ExperimentModule } from './experiment/experiment.module';
 import { InfoModule } from './info/info.module';
 import { PrefModule } from './pref/pref.module';
 import { RolesApiModule } from './roles/roles.module';
+import { UsageApiModule } from './usage/usage.module';
 import { UserApiModule } from './user/user.module';
 
 @Module({
@@ -14,6 +15,7 @@ import { UserApiModule } from './user/user.module';
     InfoModule,
     RolesApiModule,
     ApiKeysModule,
+    UsageApiModule,
   ],
 })
 export class PicsurApiModule {}

backend/src/routes/api/apikeys/apikeys.controller.ts

@@ -1,4 +1,5 @@
 import { Body, Controller, Post } from '@nestjs/common';
+import { Throttle } from '@nestjs/throttler';
 import {
   ApiKeyCreateResponse,
   ApiKeyDeleteRequest,
@@ -53,6 +54,7 @@ export class ApiKeysController {
 
   @Post('create')
   @Returns(ApiKeyCreateResponse)
+  @Throttle(10)
   async createApiKey(
     @ReqUserID() userID: string,
   ): Promise<ApiKeyCreateResponse> {

backend/src/routes/api/experiment/experiment.controller.ts

@@ -1,22 +1,17 @@
-import { Controller, Get, Request, Response } from '@nestjs/common';
-import type { FastifyReply } from 'fastify';
-import { UserInfoResponse } from 'picsur-shared/dist/dto/api/user-manage.dto';
+import { Controller, Get, Logger } from '@nestjs/common';
 import { NoPermissions } from '../../../decorators/permissions.decorator';
-import { Returns } from '../../../decorators/returns.decorator';
-import type AuthFastifyRequest from '../../../models/interfaces/authrequest.dto';
+import { ReturnsAnything } from '../../../decorators/returns.decorator';
+
 @Controller('api/experiment')
 @NoPermissions()
 export class ExperimentController {
+  private readonly logger = new Logger(ExperimentController.name);
+
   constructor() {}
 
   @Get()
-  @Returns(UserInfoResponse)
-  async testRoute(
-    @Request() req: AuthFastifyRequest,
-    @Response({ passthrough: true }) res: FastifyReply,
-  ): Promise<UserInfoResponse> {
-    res.header('Location', '/error/delete-success');
-    res.code(302);
-    return req.user;
+  @ReturnsAnything()
+  async testRoute(): Promise<any> {
+    return 'ok';
   }
 }

backend/src/routes/api/experiment/experiment.module.ts

@@ -1,11 +1,13 @@
 import { Module } from '@nestjs/common';
+import { PicsurLoggerModule } from '../../../logger/logger.module';
+import { ImageManagerModule } from '../../../managers/image/image-manager.module';
 import { ExperimentController } from './experiment.controller';
 
 // This is comletely useless module, but is used for testing
 // TODO: remove when out of beta
 
 @Module({
-  imports: [],
+  imports: [ImageManagerModule, PicsurLoggerModule],
   controllers: [ExperimentController],
 })
 export class ExperimentModule {}

backend/src/routes/api/info/info.controller.ts

@@ -10,7 +10,11 @@ import {
   SupportedAnimFileTypes,
   SupportedImageFileTypes,
 } from 'picsur-shared/dist/dto/mimes.dto';
+import { TrackingState } from 'picsur-shared/dist/dto/tracking-state.enum';
+import { FallbackIfFailed } from 'picsur-shared/dist/types';
 import { HostConfigService } from '../../../config/early/host.config.service';
+import { InfoConfigService } from '../../../config/late/info.config.service';
+import { UsageConfigService } from '../../../config/late/usage.config.service';
 import { NoPermissions } from '../../../decorators/permissions.decorator';
 import { Returns } from '../../../decorators/returns.decorator';
 import { PermissionsList } from '../../../models/constants/permissions.const';
@@ -18,15 +22,29 @@ import { PermissionsList } from '../../../models/constants/permissions.const';
 @Controller('api/info')
 @NoPermissions()
 export class InfoController {
-  constructor(private readonly hostConfig: HostConfigService) {}
+  constructor(
+    private readonly hostConfig: HostConfigService,
+    private readonly infoConfig: InfoConfigService,
+    private readonly usageService: UsageConfigService,
+  ) {}
 
   @Get()
   @Returns(InfoResponse)
   async getInfo(): Promise<InfoResponse> {
+    const trackingID =
+      FallbackIfFailed(await this.usageService.getTrackingID(), null) ??
+      undefined;
+    const hostOverride = await this.infoConfig.getHostnameOverride();
+
     return {
       demo: this.hostConfig.isDemo(),
       production: this.hostConfig.isProduction(),
       version: this.hostConfig.getVersion(),
+      host_override: hostOverride,
+      tracking: {
+        id: trackingID,
+        state: TrackingState.Detailed,
+      },
     };
   }
 

backend/src/routes/api/info/info.module.ts

@@ -1,9 +1,9 @@
 import { Module } from '@nestjs/common';
-import { EarlyConfigModule } from '../../../config/early/early-config.module';
+import { LateConfigModule } from '../../../config/late/late-config.module';
 import { InfoController } from './info.controller';
 
 @Module({
-  imports: [EarlyConfigModule],
+  imports: [LateConfigModule],
   controllers: [InfoController],
 })
 export class InfoModule {}

backend/src/routes/api/pref/sys-pref.controller.ts

@@ -1,4 +1,5 @@
 import { Body, Controller, Get, Logger, Param, Post } from '@nestjs/common';
+import { Throttle } from '@nestjs/throttler';
 import {
   GetPreferenceResponse,
   MultiplePreferencesResponse,
@@ -20,6 +21,7 @@ export class SysPrefController {
 
   @Get()
   @Returns(MultiplePreferencesResponse)
+  @Throttle(20)
   async getAllSysPrefs(): Promise<MultiplePreferencesResponse> {
     const prefs = ThrowIfFailed(await this.prefService.getAllPreferences());
 
@@ -39,6 +41,7 @@ export class SysPrefController {
 
   @Post(':key')
   @Returns(UpdatePreferenceResponse)
+  @Throttle(30)
   async setSysPref(
     @Param('key') key: string,
     @Body() body: UpdatePreferenceRequest,

backend/src/routes/api/pref/usr-pref.controller.ts

@@ -1,4 +1,5 @@
 import { Body, Controller, Get, Logger, Param, Post } from '@nestjs/common';
+import { Throttle } from '@nestjs/throttler';
 import {
   GetPreferenceResponse,
   MultiplePreferencesResponse,
@@ -21,7 +22,8 @@ export class UsrPrefController {
 
   @Get()
   @Returns(MultiplePreferencesResponse)
-  async getAllSysPrefs(
+  @Throttle(20)
+  async getAllUsrPrefs(
     @ReqUserID() userid: string,
   ): Promise<MultiplePreferencesResponse> {
     const prefs = ThrowIfFailed(
@@ -36,7 +38,7 @@ export class UsrPrefController {
 
   @Get(':key')
   @Returns(GetPreferenceResponse)
-  async getSysPref(
+  async getUsrPref(
     @Param('key') key: string,
     @ReqUserID() userid: string,
   ): Promise<GetPreferenceResponse> {
@@ -49,7 +51,8 @@ export class UsrPrefController {
 
   @Post(':key')
   @Returns(UpdatePreferenceResponse)
-  async setSysPref(
+  @Throttle(30)
+  async setUsrPref(
     @Param('key') key: string,
     @ReqUserID() userid: string,
     @Body() body: UpdatePreferenceRequest,

backend/src/routes/api/roles/roles.controller.ts

@@ -1,4 +1,5 @@
 import { Body, Controller, Get, Logger, Post } from '@nestjs/common';
+import { Throttle } from '@nestjs/throttler';
 import {
   RoleCreateRequest,
   RoleCreateResponse,
@@ -56,6 +57,7 @@ export class RolesController {
 
   @Post('update')
   @Returns(RoleUpdateResponse)
+  @Throttle(20)
   async updateRole(
     @Body() body: RoleUpdateRequest,
   ): Promise<RoleUpdateResponse> {
@@ -73,6 +75,7 @@ export class RolesController {
 
   @Post('create')
   @Returns(RoleCreateResponse)
+  @Throttle(10)
   async createRole(
     @Body() role: RoleCreateRequest,
   ): Promise<RoleCreateResponse> {

backend/src/routes/api/usage/usage.controller.ts

@@ -0,0 +1,47 @@
+import { Controller, Logger, Post, Req, Res } from '@nestjs/common';
+import { Throttle } from '@nestjs/throttler';
+import type { FastifyReply, FastifyRequest } from 'fastify';
+import { Fail, FT, ThrowIfFailed } from 'picsur-shared/dist/types';
+import { UsageConfigService } from '../../../config/late/usage.config.service';
+import { NoPermissions } from '../../../decorators/permissions.decorator';
+import { ReturnsAnything } from '../../../decorators/returns.decorator';
+
+@Controller('api/usage')
+@NoPermissions()
+export class UsageController {
+  private readonly logger = new Logger(UsageController.name);
+
+  constructor(private readonly usageService: UsageConfigService) {}
+
+  @Post(['report', 'report/*'])
+  @ReturnsAnything()
+  @Throttle(120)
+  async deleteRole(
+    @Req() req: FastifyRequest,
+    @Res({
+      passthrough: true,
+    })
+    res: FastifyReply,
+  ) {
+    const trackingUrl = ThrowIfFailed(await this.usageService.getTrackingUrl());
+
+    if (trackingUrl === null) {
+      throw Fail(FT.NotFound, undefined, 'Tracking URL not set');
+    }
+
+    await res.from(`${trackingUrl}/api`, {
+      rewriteRequestHeaders(request, headers) {
+        const req = request as any as FastifyRequest;
+
+        // remove cookies
+        delete headers.cookie;
+
+        // Add real ip, this should not work, but ackee uses a bad ip resolver
+        // So we might aswell use it
+        headers['X-Forwarded-For'] = req.ip;
+
+        return headers;
+      },
+    });
+  }
+}

backend/src/routes/api/usage/usage.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+import { LateConfigModule } from '../../../config/late/late-config.module';
+import { UsageController } from './usage.controller';
+
+@Module({
+  imports: [LateConfigModule],
+  controllers: [UsageController],
+})
+export class UsageApiModule {}

backend/src/routes/api/user/user-manage.controller.ts

@@ -1,4 +1,5 @@
 import { Body, Controller, Get, Logger, Post } from '@nestjs/common';
+import { Throttle } from '@nestjs/throttler';
 import {
   GetSpecialUsersResponse,
   UserCreateRequest,
@@ -46,6 +47,7 @@ export class UserAdminController {
 
   @Post('create')
   @Returns(UserCreateResponse)
+  @Throttle(10)
   async register(
     @Body() create: UserCreateRequest,
   ): Promise<UserCreateResponse> {
@@ -78,6 +80,7 @@ export class UserAdminController {
 
   @Post('update')
   @Returns(UserUpdateResponse)
+  @Throttle(20)
   async setPermissions(
     @Body() body: UserUpdateRequest,
   ): Promise<UserUpdateResponse> {

backend/src/routes/api/user/user.controller.ts

@@ -1,4 +1,5 @@
 import { Body, Controller, Get, Logger, Post } from '@nestjs/common';
+import { Throttle } from '@nestjs/throttler';
 import {
   UserCheckNameRequest,
   UserCheckNameResponse,
@@ -34,6 +35,7 @@ export class UserController {
   @Post('login')
   @Returns(UserLoginResponse)
   @UseLocalAuth(Permission.UserLogin)
+  @Throttle(30, 300)
   async login(@ReqUser() user: EUser): Promise<UserLoginResponse> {
     const jwt_token = ThrowIfFailed(await this.authService.createToken(user));
 
@@ -43,6 +45,7 @@ export class UserController {
   @Post('register')
   @Returns(UserRegisterResponse)
   @RequiredPermissions(Permission.UserRegister)
+  @Throttle(5, 300)
   async register(
     @Body() register: UserRegisterRequest,
   ): Promise<UserRegisterResponse> {
@@ -56,6 +59,7 @@ export class UserController {
   @Post('checkname')
   @Returns(UserCheckNameResponse)
   @RequiredPermissions(Permission.UserRegister)
+  @Throttle(20)
   async checkName(
     @Body() checkName: UserCheckNameRequest,
   ): Promise<UserCheckNameResponse> {
@@ -67,6 +71,7 @@ export class UserController {
   @Get('me')
   @Returns(UserMeResponse)
   @RequiredPermissions(Permission.UserKeepLogin)
+  @Throttle(10)
   async me(@ReqUserID() userid: string): Promise<UserMeResponse> {
     const backenduser = ThrowIfFailed(await this.usersService.findOne(userid));
 
@@ -81,6 +86,7 @@ export class UserController {
   @Get('me/permissions')
   @Returns(UserMePermissionsResponse)
   @NoPermissions()
+  @Throttle(20)
   async refresh(
     @ReqUserID() userid: string,
   ): Promise<UserMePermissionsResponse> {

backend/src/routes/image/image-manage.controller.ts

@@ -7,6 +7,7 @@ import {
   Post,
   Res,
 } from '@nestjs/common';
+import { Throttle } from '@nestjs/throttler';
 import type { FastifyReply } from 'fastify';
 import {
   ImageDeleteRequest,
@@ -15,40 +16,60 @@ import {
   ImageDeleteWithKeyResponse,
   ImageListRequest,
   ImageListResponse,
+  ImagesProgressRequest,
+  ImagesProgressResponse,
+  ImagesUploadResponse,
   ImageUpdateRequest,
   ImageUpdateResponse,
   ImageUploadResponse,
 } from 'picsur-shared/dist/dto/api/image-manage.dto';
 import { Permission } from 'picsur-shared/dist/dto/permissions.enum';
-import { HasFailed, ThrowIfFailed } from 'picsur-shared/dist/types';
-import { MultiPart } from '../../decorators/multipart/multipart.decorator';
+import { EImage } from 'picsur-shared/dist/entities/image.entity';
+import { Fail, FT, HasFailed, ThrowIfFailed } from 'picsur-shared/dist/types';
+import { EImageBackend } from '../../database/entities/images/image.entity';
+import { PostFiles } from '../../decorators/multipart/multipart.decorator';
+import type { FileIterator } from '../../decorators/multipart/postfiles.pipe';
 import {
   HasPermission,
   RequiredPermissions,
 } from '../../decorators/permissions.decorator';
 import { ReqUserID } from '../../decorators/request-user.decorator';
-import { Returns } from '../../decorators/returns.decorator';
-import { ImageManagerService } from '../../managers/image/image.service';
-import { ImageUploadDto } from '../../models/dto/image-upload.dto';
+import { Returns, ReturnsAnything } from '../../decorators/returns.decorator';
+import { ImageManagerService } from '../../managers/image/image-manager.service';
+import { IngestService } from '../../managers/image/ingest.service';
+import { GetNextAsync } from '../../util/iterator';
 @Controller('api/image')
 @RequiredPermissions(Permission.ImageUpload)
 export class ImageManageController {
   private readonly logger = new Logger(ImageManageController.name);
 
-  constructor(private readonly imagesService: ImageManagerService) {}
+  constructor(
+    private readonly imagesService: ImageManagerService,
+    private readonly ingestService: IngestService,
+  ) {}
 
   @Post('upload')
   @Returns(ImageUploadResponse)
+  @Throttle(20)
   async uploadImage(
-    @MultiPart() multipart: ImageUploadDto,
+    @PostFiles(1) multipart: FileIterator,
     @ReqUserID() userid: string,
     @HasPermission(Permission.ImageDeleteKey) withDeleteKey: boolean,
   ): Promise<ImageUploadResponse> {
+    const file = ThrowIfFailed(await GetNextAsync(multipart));
+
+    let buffer: Buffer;
+    try {
+      buffer = await file.toBuffer();
+    } catch (e) {
+      throw Fail(FT.Internal, e);
+    }
+
     const image = ThrowIfFailed(
-      await this.imagesService.upload(
+      await this.ingestService.uploadPromise(
         userid,
-        multipart.image.filename,
-        multipart.image.buffer,
+        file.filename,
+        buffer,
         withDeleteKey,
       ),
     );
@@ -56,6 +77,54 @@ export class ImageManageController {
     return image;
   }
 
+  @Post('upload/bulk')
+  @Returns(ImagesUploadResponse)
+  @Throttle(20)
+  async uploadImages(
+    @PostFiles() multipart: FileIterator,
+    @ReqUserID() userid: string,
+    @HasPermission(Permission.ImageDeleteKey) withDeleteKey: boolean,
+  ): Promise<ImagesUploadResponse> {
+    let jobs: {
+      job_id: string;
+      image: EImage;
+    }[] = [];
+    for await (const file of multipart) {
+      const buffer = await file.toBuffer();
+      const filename = file.filename;
+
+      const [job, image] = ThrowIfFailed(
+        await this.ingestService.uploadJob(
+          userid,
+          filename,
+          buffer,
+          withDeleteKey,
+        ),
+      );
+
+      jobs.push({
+        job_id: job.id.toString(),
+        image: image,
+      });
+    }
+    if (jobs.length === 0) {
+      throw Fail(FT.BadRequest, 'No files uploaded');
+    }
+
+    return {
+      count: jobs.length,
+      results: jobs,
+    };
+  }
+
+  @Post('upload/status')
+  @Returns(ImagesProgressResponse)
+  async getImagesProgress(
+    @Body() body: ImagesProgressRequest,
+  ): Promise<ImagesProgressResponse> {
+    return ThrowIfFailed(await this.ingestService.getProgress(body.job_ids));
+  }
+
   @Post('list')
   @RequiredPermissions(Permission.ImageManage)
   @Returns(ImageListResponse)

backend/src/routes/image/image.controller.ts

@@ -1,4 +1,5 @@
 import { Controller, Get, Head, Logger, Query, Res } from '@nestjs/common';
+import { SkipThrottle } from '@nestjs/throttler';
 import type { FastifyReply } from 'fastify';
 import {
   ImageMetaResponse,
@@ -12,7 +13,8 @@ import { ImageFullIdParam } from '../../decorators/image-id/image-full-id.decora
 import { ImageIdParam } from '../../decorators/image-id/image-id.decorator';
 import { RequiredPermissions } from '../../decorators/permissions.decorator';
 import { Returns } from '../../decorators/returns.decorator';
-import { ImageManagerService } from '../../managers/image/image.service';
+import { ConvertService } from '../../managers/image/convert.service';
+import { ImageManagerService } from '../../managers/image/image-manager.service';
 import type { ImageFullId } from '../../models/constants/image-full-id.const';
 import { Permission } from '../../models/constants/permissions.const';
 import { EUserBackend2EUser } from '../../models/transformers/user.transformer';
@@ -21,12 +23,14 @@ import { BrandMessageType, GetBrandMessage } from '../../util/branding';
 // This is the only controller with CORS enabled
 @Controller('i')
 @RequiredPermissions(Permission.ImageView)
+@SkipThrottle()
 export class ImageController {
   private readonly logger = new Logger(ImageController.name);
 
   constructor(
     private readonly imagesService: ImageManagerService,
     private readonly userService: UserDbService,
+    private readonly convertService: ConvertService,
   ) {}
 
   @Head(':id')
@@ -59,21 +63,23 @@ export class ImageController {
         const image = ThrowIfFailed(
           await this.imagesService.getOriginal(fullid.id),
         );
+        const data = ThrowIfFailed(await this.imagesService.getData(image));
 
         res.type(ThrowIfFailed(FileType2Mime(image.filetype)));
-        return image.data;
+        return data;
       }
 
       const image = ThrowIfFailed(
-        await this.imagesService.getConverted(
+        await this.convertService.convertPromise(
           fullid.id,
           fullid.filetype,
           params,
         ),
       );
+      const data = ThrowIfFailed(await this.imagesService.getData(image));
 
       res.type(ThrowIfFailed(FileType2Mime(image.filetype)));
-      return image.data;
+      return data;
     } catch (e) {
       if (!IsFailure(e) || e.getType() !== FT.NotFound) throw e;
 

backend/src/routes/image/image.module.ts

@@ -1,7 +1,7 @@
 import { Module } from '@nestjs/common';
 import { UserDbModule } from '../../collections/user-db/user-db.module';
 import { DecoratorsModule } from '../../decorators/decorators.module';
-import { ImageManagerModule } from '../../managers/image/image.module';
+import { ImageManagerModule } from '../../managers/image/image-manager.module';
 import { ImageManageController } from './image-manage.controller';
 import { ImageController } from './image.controller';
 

backend/src/util/iterator.ts

@@ -0,0 +1,9 @@
+import { AsyncFailable, Fail, FT } from 'picsur-shared/dist/types';
+
+export async function GetNextAsync<T>(
+  iterator: AsyncIterableIterator<T>,
+): AsyncFailable<T> {
+  const { done, value } = await iterator.next();
+  if (done) return Fail(FT.BadRequest);
+  return value;
+}

frontend/angular.json

@@ -41,7 +41,8 @@
             "scripts": [],
             "allowedCommonJsDependencies": [
               "ngx-auto-unsubscribe-decorator",
-              "moment"
+              "moment",
+              "platform"
             ],
             "optimization": true,
             "webWorkerTsConfig": "tsconfig.worker.json",
@@ -54,8 +55,8 @@
               "budgets": [
                 {
                   "type": "initial",
-                  "maximumWarning": "1500kb",
-                  "maximumError": "2mb"
+                  "maximumWarning": "500kb",
+                  "maximumError": "1mb"
                 },
                 {
                   "type": "anyComponentStyle",

frontend/package.json

@@ -15,28 +15,30 @@
   },
   "devDependencies": {
     "@angular-builders/custom-webpack": "^14.0.1",
-    "@angular-devkit/build-angular": "14.2.2",
-    "@angular/animations": "^14.2.1",
-    "@angular/cdk": "^14.2.1",
-    "@angular/cli": "^14.2.2",
-    "@angular/common": "^14.2.1",
-    "@angular/compiler": "^14.2.1",
-    "@angular/compiler-cli": "^14.2.1",
-    "@angular/core": "^14.2.1",
-    "@angular/forms": "^14.2.1",
-    "@angular/material": "^14.2.1",
-    "@angular/platform-browser": "^14.2.1",
-    "@angular/platform-browser-dynamic": "^14.2.1",
-    "@angular/router": "^14.2.1",
+    "@angular-devkit/build-angular": "14.2.4",
+    "@angular/animations": "^14.2.4",
+    "@angular/cdk": "^14.2.3",
+    "@angular/cli": "^14.2.4",
+    "@angular/common": "^14.2.4",
+    "@angular/compiler": "^14.2.4",
+    "@angular/compiler-cli": "^14.2.4",
+    "@angular/core": "^14.2.4",
+    "@angular/forms": "^14.2.4",
+    "@angular/material": "^14.2.3",
+    "@angular/platform-browser": "^14.2.4",
+    "@angular/platform-browser-dynamic": "^14.2.4",
+    "@angular/router": "^14.2.4",
     "@fontsource/material-icons": "^4.5.4",
     "@fontsource/material-icons-outlined": "^4.5.4",
     "@fontsource/roboto": "^4.5.8",
     "@ng-web-apis/common": "^2.0.1",
     "@ng-web-apis/resize-observer": "^2.0.0",
     "@ngui/common": "^1.0.0",
-    "@types/node": "^18.7.16",
+    "@types/ackee-tracker": "^5.0.2",
+    "@types/node": "^18.7.23",
     "@types/resize-observer-browser": "^0.1.7",
-    "@types/validator": "^13.7.6",
+    "@types/validator": "^13.7.7",
+    "ackee-tracker": "^5.1.0",
     "bootstrap": "^5.2.1",
     "fuse.js": "^6.6.2",
     "jwt-decode": "^3.1.2",
@@ -47,10 +49,13 @@
     "ngx-moment": "^6.0.2",
     "picsur-shared": "*",
     "reflect-metadata": "^0.1.13",
-    "rxjs": "~7.5.6",
+    "rxjs": "~7.5.7",
     "tslib": "^2.4.0",
-    "typescript": "4.8.3",
-    "zod": "^3.19.0",
+    "typescript": "4.8.4",
+    "zod": "^3.19.1",
     "zone.js": "~0.11.8"
+  },
+  "dependencies": {
+    "axios": "^0.27.2"
   }
 }

frontend/src/app/app.component.html

@@ -1,6 +1,7 @@
 <app-header
   [enableHamburger]="hasSidebar && !isDesktop"
   (onHamburgerClick)="sidebar.toggle()"
+  [loading]="loading"
 ></app-header>
 
 <mat-sidenav-container class="grow-full">

frontend/src/app/app.component.ts

@@ -5,11 +5,13 @@ import {
   ActivatedRoute,
   NavigationEnd,
   NavigationError,
+  NavigationStart,
   Router,
 } from '@angular/router';
 import { AutoUnsubscribe } from 'ngx-auto-unsubscribe-decorator';
 import { RouteTransitionAnimations } from './app.animation';
 import { PRouteData } from './models/dto/picsur-routes.dto';
+import { UsageService } from './services/usage/usage.service';
 import { BootstrapService } from './util/bootstrap.service';
 
 @Component({
@@ -23,6 +25,9 @@ export class AppComponent implements OnInit {
 
   @ViewChild(MatSidenav) sidebar: MatSidenav;
 
+  loading: boolean = false;
+  private loadingTimeout: number | null = null;
+
   wrapContentWithContainer: boolean = true;
   sidebarPortal: Portal<any> | undefined = undefined;
 
@@ -33,7 +38,10 @@ export class AppComponent implements OnInit {
     private readonly router: Router,
     private readonly activatedRoute: ActivatedRoute,
     private readonly bootstrapService: BootstrapService,
-  ) {}
+    usageService: UsageService,
+  ) {
+    usageService;
+  }
 
   public getRouteAnimData() {
     // Everyone is doing shit with the activated route
@@ -50,6 +58,12 @@ export class AppComponent implements OnInit {
   @AutoUnsubscribe()
   private subscribeRouter() {
     return this.router.events.subscribe((event) => {
+      if (event instanceof NavigationStart) {
+        this.loadingStart();
+      }
+      if (event instanceof NavigationEnd) {
+        this.loadingEnd();
+      }
       if (event instanceof NavigationEnd) this.onNavigationEnd(event);
       if (event instanceof NavigationError) this.onNavigationError(event);
     });
@@ -83,6 +97,21 @@ export class AppComponent implements OnInit {
     this.updateSidebar();
   }
 
+  private loadingStart() {
+    if (this.loadingTimeout !== null) clearTimeout(this.loadingTimeout);
+
+    this.loadingTimeout = window.setTimeout(() => {
+      this.loading = true;
+    }, 500);
+  }
+
+  private loadingEnd() {
+    if (this.loadingTimeout !== null) clearTimeout(this.loadingTimeout);
+    this.loadingTimeout = null;
+
+    this.loading = false;
+  }
+
   private updateSidebar() {
     if (!this.sidebar) return;
 

frontend/src/app/app.routing.module.ts

@@ -1,13 +1,6 @@
 import { NgModule } from '@angular/core';
 import { RouterModule } from '@angular/router';
 import { PRoutes } from './models/dto/picsur-routes.dto';
-import { ErrorsRouteModule } from './routes/errors/errors.module';
-import { ImagesRouteModule } from './routes/images/images.module';
-import { ProcessingRouteModule } from './routes/processing/processing.module';
-import { SettingsRouteModule } from './routes/settings/settings.module';
-import { UploadRouteModule } from './routes/upload/upload.module';
-import { UserRouteModule } from './routes/user/user.module';
-import { ViewRouteModule } from './routes/view/view.module';
 
 const routes: PRoutes = [
   {
@@ -17,31 +10,38 @@ const routes: PRoutes = [
   },
   {
     path: 'upload',
-    loadChildren: () => UploadRouteModule,
+    loadChildren: () =>
+      import('./routes/upload/upload.module').then((m) => m.default),
   },
   {
     path: 'processing',
-    loadChildren: () => ProcessingRouteModule,
+    loadChildren: () =>
+      import('./routes/processing/processing.module').then((m) => m.default),
   },
   {
     path: 'view',
-    loadChildren: () => ViewRouteModule,
+    loadChildren: () =>
+      import('./routes/view/view.module').then((m) => m.default),
   },
   {
     path: 'user',
-    loadChildren: () => UserRouteModule,
+    loadChildren: () =>
+      import('./routes/user/user.module').then((m) => m.default),
   },
   {
     path: 'images',
-    loadChildren: () => ImagesRouteModule,
+    loadChildren: () =>
+      import('./routes/images/images.module').then((m) => m.default),
   },
   {
     path: 'settings',
-    loadChildren: () => SettingsRouteModule,
+    loadChildren: () =>
+      import('./routes/settings/settings.module').then((m) => m.default),
   },
   {
     path: 'error',
-    loadChildren: () => ErrorsRouteModule,
+    loadChildren: () =>
+      import('./routes/errors/errors.module').then((m) => m.default),
   },
 ];