Compare commits
12 commits
Author | SHA1 | Date | |
---|---|---|---|
c3e691fed1 | |||
e509302e0b | |||
569f650881 | |||
6750ac17fa | |||
bd3ad9e480 | |||
838db4a8f5 | |||
3a265c62c1 | |||
a470c48d7d | |||
743bd56722 | |||
ada9fd8b4b | |||
6aa2550bdc | |||
1c19618eb2 |
2
.vscode/tasks.json
vendored
2
.vscode/tasks.json
vendored
|
@ -55,7 +55,7 @@
|
||||||
{
|
{
|
||||||
"type": "shell",
|
"type": "shell",
|
||||||
"label": "Start postgres",
|
"label": "Start postgres",
|
||||||
"command": "yarn devdb:start",
|
"command": "yarn devdb:up",
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${cwd}",
|
"cwd": "${cwd}",
|
||||||
"shell": {
|
"shell": {
|
||||||
|
|
4
backend/.env.example
Normal file
4
backend/.env.example
Normal file
|
@ -0,0 +1,4 @@
|
||||||
|
PICSUR_FILESTORAGE_MODE="S3"
|
||||||
|
PICSUR_FILESTORAGE_S3_ENDPOINT="http://localhost:8000"
|
||||||
|
PICSUR_FILESTORAGE_S3_ACCESS_KEY="username"
|
||||||
|
PICSUR_FILESTORAGE_S3_SECRET_KEY="password"
|
1
backend/.gitignore
vendored
1
backend/.gitignore
vendored
|
@ -396,3 +396,4 @@ Temporary Items
|
||||||
# Local
|
# Local
|
||||||
.env
|
.env
|
||||||
dist
|
dist
|
||||||
|
temp
|
||||||
|
|
|
@ -22,6 +22,7 @@
|
||||||
"purge": "rm -rf dist && rm -rf node_modules"
|
"purge": "rm -rf dist && rm -rf node_modules"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"@aws-sdk/client-s3": "^3.238.0",
|
||||||
"@fastify/helmet": "^10.1.0",
|
"@fastify/helmet": "^10.1.0",
|
||||||
"@fastify/multipart": "^7.5.0",
|
"@fastify/multipart": "^7.5.0",
|
||||||
"@fastify/reply-from": "^9.0.1",
|
"@fastify/reply-from": "^9.0.1",
|
||||||
|
@ -56,6 +57,7 @@
|
||||||
"reflect-metadata": "^0.1.13",
|
"reflect-metadata": "^0.1.13",
|
||||||
"rimraf": "^4.4.0",
|
"rimraf": "^4.4.0",
|
||||||
"rxjs": "^7.8.0",
|
"rxjs": "^7.8.0",
|
||||||
|
"semver": "^7.3.8",
|
||||||
"sharp": "^0.31.3",
|
"sharp": "^0.31.3",
|
||||||
"stream-parser": "^0.3.1",
|
"stream-parser": "^0.3.1",
|
||||||
"thunks": "^4.9.6",
|
"thunks": "^4.9.6",
|
||||||
|
@ -73,12 +75,14 @@
|
||||||
"@types/passport-jwt": "^3.0.8",
|
"@types/passport-jwt": "^3.0.8",
|
||||||
"@types/passport-local": "^1.0.35",
|
"@types/passport-local": "^1.0.35",
|
||||||
"@types/passport-strategy": "^0.2.35",
|
"@types/passport-strategy": "^0.2.35",
|
||||||
"@types/sharp": "^0.31.1",
|
"@types/semver": "^7.3.12",
|
||||||
|
"@types/sharp": "^0.31.0",
|
||||||
"@types/supertest": "^2.0.12",
|
"@types/supertest": "^2.0.12",
|
||||||
"@typescript-eslint/eslint-plugin": "^5.55.0",
|
"@types/uuid": "^9.0.0",
|
||||||
"@typescript-eslint/parser": "^5.55.0",
|
"@typescript-eslint/eslint-plugin": "^5.47.0",
|
||||||
"eslint": "^8.36.0",
|
"@typescript-eslint/parser": "^5.47.0",
|
||||||
"eslint-config-prettier": "^8.7.0",
|
"eslint": "^8.30.0",
|
||||||
|
"eslint-config-prettier": "^8.5.0",
|
||||||
"eslint-plugin-prettier": "^4.2.1",
|
"eslint-plugin-prettier": "^4.2.1",
|
||||||
"prettier": "^2.8.4",
|
"prettier": "^2.8.4",
|
||||||
"source-map-support": "^0.5.21",
|
"source-map-support": "^0.5.21",
|
||||||
|
|
|
@ -1,8 +1,10 @@
|
||||||
import { MiddlewareConsumer, Module, NestModule } from '@nestjs/common';
|
import { Logger, MiddlewareConsumer, Module, NestModule, OnApplicationBootstrap, OnApplicationShutdown } from '@nestjs/common';
|
||||||
import { ScheduleModule } from '@nestjs/schedule';
|
import { ScheduleModule } from '@nestjs/schedule';
|
||||||
import { ServeStaticModule } from '@nestjs/serve-static';
|
import { ServeStaticModule } from '@nestjs/serve-static';
|
||||||
import cors from 'cors';
|
import cors from 'cors';
|
||||||
import { IncomingMessage, ServerResponse } from 'http';
|
import { IncomingMessage, ServerResponse } from 'http';
|
||||||
|
import semver from 'semver';
|
||||||
|
import { FileStorageDBModule } from './collections/filestorage-db/filestorage-db.module';
|
||||||
import { EarlyConfigModule } from './config/early/early-config.module';
|
import { EarlyConfigModule } from './config/early/early-config.module';
|
||||||
import { ServeStaticConfigService } from './config/early/serve-static.config.service';
|
import { ServeStaticConfigService } from './config/early/serve-static.config.service';
|
||||||
import { DatabaseModule } from './database/database.module';
|
import { DatabaseModule } from './database/database.module';
|
||||||
|
@ -13,6 +15,8 @@ import { DemoManagerModule } from './managers/demo/demo.module';
|
||||||
import { UsageManagerModule } from './managers/usage/usage.module';
|
import { UsageManagerModule } from './managers/usage/usage.module';
|
||||||
import { PicsurRoutesModule } from './routes/routes.module';
|
import { PicsurRoutesModule } from './routes/routes.module';
|
||||||
|
|
||||||
|
const supportedNodeVersions = ['^16.17.0', '^18.6.0'];
|
||||||
|
|
||||||
const mainCorsConfig = cors({
|
const mainCorsConfig = cors({
|
||||||
origin: '<origin>',
|
origin: '<origin>',
|
||||||
});
|
});
|
||||||
|
@ -46,6 +50,7 @@ const imageCorsOverride = (
|
||||||
}),
|
}),
|
||||||
ScheduleModule.forRoot(),
|
ScheduleModule.forRoot(),
|
||||||
DatabaseModule,
|
DatabaseModule,
|
||||||
|
FileStorageDBModule,
|
||||||
AuthManagerModule,
|
AuthManagerModule,
|
||||||
UsageManagerModule,
|
UsageManagerModule,
|
||||||
DemoManagerModule,
|
DemoManagerModule,
|
||||||
|
@ -53,9 +58,28 @@ const imageCorsOverride = (
|
||||||
PicsurLayersModule,
|
PicsurLayersModule,
|
||||||
],
|
],
|
||||||
})
|
})
|
||||||
export class AppModule implements NestModule {
|
export class AppModule implements NestModule, OnApplicationBootstrap, OnApplicationShutdown {
|
||||||
|
private readonly logger = new Logger(AppModule.name);
|
||||||
|
|
||||||
configure(consumer: MiddlewareConsumer) {
|
configure(consumer: MiddlewareConsumer) {
|
||||||
consumer.apply(mainCorsConfig).exclude('/i').forRoutes('/');
|
consumer.apply(mainCorsConfig).exclude('/i').forRoutes('/');
|
||||||
consumer.apply(imageCorsConfig, imageCorsOverride).forRoutes('/i');
|
consumer.apply(imageCorsConfig, imageCorsOverride).forRoutes('/i');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
onApplicationBootstrap() {
|
||||||
|
const nodeVersion = process.version;
|
||||||
|
if (!supportedNodeVersions.some((v) => semver.satisfies(nodeVersion, v))) {
|
||||||
|
this.logger.error(
|
||||||
|
`Unsupported Node version: ${nodeVersion}. Transcoding performance will be severely degraded.`,
|
||||||
|
);
|
||||||
|
|
||||||
|
this.logger.log(
|
||||||
|
`Supported Node versions: ${supportedNodeVersions.join(', ')}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
onApplicationShutdown() {
|
||||||
|
this.logger.warn(`Shutting down`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,12 @@
|
||||||
|
import { Module } from '@nestjs/common';
|
||||||
|
import { LateConfigModule } from '../../config/late/late-config.module';
|
||||||
|
import { FileStorageGeneric } from './filestorage-generic.service';
|
||||||
|
|
||||||
|
export const FSServiceToken = 'FileStorageService';
|
||||||
|
|
||||||
|
@Module({
|
||||||
|
imports: [LateConfigModule],
|
||||||
|
providers: [FileStorageGeneric],
|
||||||
|
exports: [FileStorageGeneric],
|
||||||
|
})
|
||||||
|
export class FileStorageDBModule {}
|
|
@ -0,0 +1,47 @@
|
||||||
|
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
|
||||||
|
import { AsyncFailable } from 'picsur-shared/dist/types';
|
||||||
|
import {
|
||||||
|
FileStorageMode
|
||||||
|
} from '../../config/early/early-fs.config.service';
|
||||||
|
import { FSConfigService } from '../../config/late/fs.config.service';
|
||||||
|
import { FileStorageEmpty } from './services/filestorage-empty';
|
||||||
|
import { FileStorageLocalService } from './services/filestorage-local';
|
||||||
|
|
||||||
|
import { FileStorageS3Service } from './services/filestorage-s3';
|
||||||
|
import { FileStorageService } from './services/filestorage-service';
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class FileStorageGeneric implements OnModuleInit {
|
||||||
|
private readonly logger = new Logger(FileStorageGeneric.name);
|
||||||
|
private backingService: FileStorageService = new FileStorageEmpty();
|
||||||
|
|
||||||
|
constructor(private readonly fsConfig: FSConfigService) {}
|
||||||
|
|
||||||
|
async onModuleInit() {
|
||||||
|
const mode = this.fsConfig.getFileStorageMode();
|
||||||
|
if (mode === FileStorageMode.Local) {
|
||||||
|
this.backingService = new FileStorageLocalService(this.fsConfig);
|
||||||
|
} else if (mode === FileStorageMode.S3) {
|
||||||
|
this.backingService = new FileStorageS3Service(this.fsConfig);
|
||||||
|
} else {}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this.backingService.onStorageInit();
|
||||||
|
} catch (e) {
|
||||||
|
this.logger.error(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async putFile(key: string, data: Buffer): AsyncFailable<string> {
|
||||||
|
return this.backingService.putFile(key, data);
|
||||||
|
}
|
||||||
|
async getFile(key: string): AsyncFailable<Buffer> {
|
||||||
|
return this.backingService.getFile(key);
|
||||||
|
}
|
||||||
|
async deleteFile(key: string): AsyncFailable<true> {
|
||||||
|
return this.backingService.deleteFile(key);
|
||||||
|
}
|
||||||
|
async deleteFiles(keys: string[]): AsyncFailable<true> {
|
||||||
|
return this.backingService.deleteFiles(keys);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,25 @@
|
||||||
|
import { AsyncFailable, Fail, FT } from 'picsur-shared/dist/types';
|
||||||
|
import { FileStorageService } from './filestorage-service';
|
||||||
|
|
||||||
|
export class FileStorageEmpty extends FileStorageService {
|
||||||
|
private readonly errorMessage = 'No file storage configured';
|
||||||
|
|
||||||
|
constructor() {
|
||||||
|
super(undefined as any);
|
||||||
|
}
|
||||||
|
|
||||||
|
onStorageInit(): void {}
|
||||||
|
|
||||||
|
async putFile(key: string, data: Buffer): AsyncFailable<string> {
|
||||||
|
return Fail(FT.Internal, this.errorMessage);
|
||||||
|
}
|
||||||
|
async getFile(key: string): AsyncFailable<Buffer> {
|
||||||
|
return Fail(FT.Internal, this.errorMessage);
|
||||||
|
}
|
||||||
|
async deleteFile(key: string): AsyncFailable<true> {
|
||||||
|
return Fail(FT.Internal, this.errorMessage);
|
||||||
|
}
|
||||||
|
async deleteFiles(keys: string[]): AsyncFailable<true> {
|
||||||
|
return Fail(FT.Internal, this.errorMessage);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,84 @@
|
||||||
|
import { Logger } from '@nestjs/common';
|
||||||
|
import * as fs from 'fs';
|
||||||
|
import afs from 'fs/promises';
|
||||||
|
import pathlib from 'path';
|
||||||
|
import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
|
||||||
|
import { FSConfigService } from '../../../config/late/fs.config.service';
|
||||||
|
import { FileStorageService } from './filestorage-service';
|
||||||
|
|
||||||
|
export class FileStorageLocalService extends FileStorageService {
|
||||||
|
private readonly logger = new Logger(FileStorageLocalService.name);
|
||||||
|
|
||||||
|
private path = './temp';
|
||||||
|
|
||||||
|
constructor(config: FSConfigService) {
|
||||||
|
super(config);
|
||||||
|
}
|
||||||
|
|
||||||
|
async onStorageInit() {
|
||||||
|
this.path = await this.config.getLocalPath();
|
||||||
|
await this.ensureFileDir(this.path);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async putFile(key: string, data: Buffer): AsyncFailable<string> {
|
||||||
|
const path = this.getKeyFilePath(key);
|
||||||
|
const result = await this.ensureFileDir(path);
|
||||||
|
if (HasFailed(result)) return result;
|
||||||
|
|
||||||
|
try {
|
||||||
|
await afs.writeFile(path, data);
|
||||||
|
return key;
|
||||||
|
} catch (e) {
|
||||||
|
return Fail(FT.FileStorage, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async getFile(key: string): AsyncFailable<Buffer> {
|
||||||
|
const path = this.getKeyFilePath(key);
|
||||||
|
try {
|
||||||
|
const result = await afs.readFile(path);
|
||||||
|
return result;
|
||||||
|
} catch (e) {
|
||||||
|
return Fail(FT.FileStorage, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async deleteFile(key: string): AsyncFailable<true> {
|
||||||
|
const path = this.getKeyFilePath(key);
|
||||||
|
try {
|
||||||
|
await afs.unlink(path);
|
||||||
|
return true;
|
||||||
|
} catch (e) {
|
||||||
|
return Fail(FT.FileStorage, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async deleteFiles(keys: string[]): AsyncFailable<true> {
|
||||||
|
const paths = keys.map((key) => this.getKeyFilePath(key));
|
||||||
|
try {
|
||||||
|
await Promise.all(paths.map((path) => afs.unlink(path)));
|
||||||
|
return true;
|
||||||
|
} catch (e) {
|
||||||
|
return Fail(FT.FileStorage, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private getKeyFilePath(key: string): string {
|
||||||
|
const subfolder = key.slice(0, 4);
|
||||||
|
return pathlib.resolve(this.path, subfolder, key);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async ensureFileDir(path: string): AsyncFailable<true> {
|
||||||
|
try {
|
||||||
|
const dir = path.split('/').slice(0, -1).join('/');
|
||||||
|
if (!fs.existsSync(dir)) {
|
||||||
|
await afs.mkdir(dir, {
|
||||||
|
recursive: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
} catch (e) {
|
||||||
|
return Fail(FT.FileStorage, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,130 @@
|
||||||
|
import {
|
||||||
|
CreateBucketCommand,
|
||||||
|
DeleteObjectCommand,
|
||||||
|
DeleteObjectsCommand,
|
||||||
|
GetObjectCommand,
|
||||||
|
ListBucketsCommand,
|
||||||
|
PutObjectCommand,
|
||||||
|
S3Client
|
||||||
|
} from '@aws-sdk/client-s3';
|
||||||
|
import { Logger } from '@nestjs/common';
|
||||||
|
import { buffer as streamToBuffer } from 'get-stream';
|
||||||
|
import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
|
||||||
|
import { Readable } from 'stream';
|
||||||
|
import { FileStorageService } from './filestorage-service';
|
||||||
|
|
||||||
|
export class FileStorageS3Service extends FileStorageService {
|
||||||
|
private readonly logger = new Logger(FileStorageS3Service.name);
|
||||||
|
private S3: S3Client | null = null;
|
||||||
|
|
||||||
|
onStorageInit() {
|
||||||
|
this.loadS3();
|
||||||
|
}
|
||||||
|
|
||||||
|
public async putFile(key: string, data: Buffer): AsyncFailable<string> {
|
||||||
|
const S3 = await this.getS3();
|
||||||
|
if (HasFailed(S3)) return S3;
|
||||||
|
|
||||||
|
const request = new PutObjectCommand({
|
||||||
|
Bucket: await this.config.getS3Bucket(),
|
||||||
|
Key: key,
|
||||||
|
Body: data,
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
await S3.send(request);
|
||||||
|
return key;
|
||||||
|
} catch (e) {
|
||||||
|
return Fail(FT.FileStorage, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async getFile(key: string): AsyncFailable<Buffer> {
|
||||||
|
const S3 = await this.getS3();
|
||||||
|
if (HasFailed(S3)) return S3;
|
||||||
|
|
||||||
|
const request = new GetObjectCommand({
|
||||||
|
Bucket: await this.config.getS3Bucket(),
|
||||||
|
Key: key,
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await S3.send(request);
|
||||||
|
if (!result.Body) return Fail(FT.NotFound, 'File not found');
|
||||||
|
|
||||||
|
if (result.Body instanceof Blob) {
|
||||||
|
return Buffer.from(await result.Body.arrayBuffer());
|
||||||
|
}
|
||||||
|
return streamToBuffer(result.Body as Readable);
|
||||||
|
} catch (e) {
|
||||||
|
return Fail(FT.FileStorage, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async deleteFile(key: string): AsyncFailable<true> {
|
||||||
|
const S3 = await this.getS3();
|
||||||
|
if (HasFailed(S3)) return S3;
|
||||||
|
|
||||||
|
const request = new DeleteObjectCommand({
|
||||||
|
Bucket: await this.config.getS3Bucket(),
|
||||||
|
Key: key,
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
await S3.send(request);
|
||||||
|
return true;
|
||||||
|
} catch (e) {
|
||||||
|
return Fail(FT.FileStorage, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async deleteFiles(keys: string[]): AsyncFailable<true> {
|
||||||
|
const S3 = await this.getS3();
|
||||||
|
if (HasFailed(S3)) return S3;
|
||||||
|
|
||||||
|
const request = new DeleteObjectsCommand({
|
||||||
|
Bucket: await this.config.getS3Bucket(),
|
||||||
|
Delete: {
|
||||||
|
Objects: keys.map((key) => ({ Key: key })),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
await S3.send(request);
|
||||||
|
return true;
|
||||||
|
} catch (e) {
|
||||||
|
return Fail(FT.FileStorage, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async getS3(): AsyncFailable<S3Client> {
|
||||||
|
if (this.S3) return this.S3;
|
||||||
|
await this.loadS3();
|
||||||
|
if (this.S3) return this.S3;
|
||||||
|
return Fail(FT.FileStorage, 'S3 not loaded');
|
||||||
|
}
|
||||||
|
|
||||||
|
private async loadS3(): Promise<void> {
|
||||||
|
const S3 = new S3Client(await this.config.getS3Config());
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Create bucket if it doesn't exist
|
||||||
|
const bucket = await this.config.getS3Bucket();
|
||||||
|
|
||||||
|
// List buckets
|
||||||
|
const listBuckets = await S3.send(new ListBucketsCommand({}));
|
||||||
|
|
||||||
|
const bucketExists = listBuckets.Buckets?.some((b) => b.Name === bucket);
|
||||||
|
if (!bucketExists) {
|
||||||
|
this.logger.verbose(`Creating S3 Bucket ${bucket}`);
|
||||||
|
await S3.send(new CreateBucketCommand({ Bucket: bucket }));
|
||||||
|
} else {
|
||||||
|
this.logger.verbose(`Using existing S3 Bucket ${bucket}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
this.S3 = S3;
|
||||||
|
} catch (e) {
|
||||||
|
this.logger.error(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,12 @@
|
||||||
|
import { AsyncFailable } from 'picsur-shared/dist/types';
|
||||||
|
import { FSConfigService } from '../../../config/late/fs.config.service';
|
||||||
|
|
||||||
|
export abstract class FileStorageService {
|
||||||
|
constructor(protected readonly config: FSConfigService) {}
|
||||||
|
|
||||||
|
public abstract onStorageInit(): Promise<void> | void;
|
||||||
|
public abstract putFile(key: string, data: Buffer): AsyncFailable<string>;
|
||||||
|
public abstract getFile(key: string): AsyncFailable<Buffer>;
|
||||||
|
public abstract deleteFile(key: string): AsyncFailable<true>;
|
||||||
|
public abstract deleteFiles(keys: string[]): AsyncFailable<true>;
|
||||||
|
}
|
|
@ -3,6 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
|
||||||
import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity';
|
import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity';
|
||||||
import { EImageFileBackend } from '../../database/entities/images/image-file.entity';
|
import { EImageFileBackend } from '../../database/entities/images/image-file.entity';
|
||||||
import { EImageBackend } from '../../database/entities/images/image.entity';
|
import { EImageBackend } from '../../database/entities/images/image.entity';
|
||||||
|
import { FileStorageDBModule } from '../filestorage-db/filestorage-db.module';
|
||||||
import { ImageDBService } from './image-db.service';
|
import { ImageDBService } from './image-db.service';
|
||||||
import { ImageFileDBService } from './image-file-db.service';
|
import { ImageFileDBService } from './image-file-db.service';
|
||||||
|
|
||||||
|
@ -13,6 +14,7 @@ import { ImageFileDBService } from './image-file-db.service';
|
||||||
EImageFileBackend,
|
EImageFileBackend,
|
||||||
EImageDerivativeBackend,
|
EImageDerivativeBackend,
|
||||||
]),
|
]),
|
||||||
|
FileStorageDBModule
|
||||||
],
|
],
|
||||||
providers: [ImageDBService, ImageFileDBService],
|
providers: [ImageDBService, ImageFileDBService],
|
||||||
exports: [ImageDBService, ImageFileDBService],
|
exports: [ImageDBService, ImageFileDBService],
|
||||||
|
|
|
@ -2,9 +2,11 @@ import { Injectable } from '@nestjs/common';
|
||||||
import { InjectRepository } from '@nestjs/typeorm';
|
import { InjectRepository } from '@nestjs/typeorm';
|
||||||
import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum';
|
import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum';
|
||||||
import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
|
import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
|
||||||
import { LessThan, Repository } from 'typeorm';
|
import { In, IsNull, LessThan, Not, Repository } from 'typeorm';
|
||||||
|
import { v4 as uuidv4 } from 'uuid';
|
||||||
import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity';
|
import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity';
|
||||||
import { EImageFileBackend } from '../../database/entities/images/image-file.entity';
|
import { EImageFileBackend } from '../../database/entities/images/image-file.entity';
|
||||||
|
import { FileStorageGeneric } from '../filestorage-db/filestorage-generic.service';
|
||||||
|
|
||||||
const A_DAY_IN_SECONDS = 24 * 60 * 60;
|
const A_DAY_IN_SECONDS = 24 * 60 * 60;
|
||||||
|
|
||||||
|
@ -16,24 +18,61 @@ export class ImageFileDBService {
|
||||||
|
|
||||||
@InjectRepository(EImageDerivativeBackend)
|
@InjectRepository(EImageDerivativeBackend)
|
||||||
private readonly imageDerivativeRepo: Repository<EImageDerivativeBackend>,
|
private readonly imageDerivativeRepo: Repository<EImageDerivativeBackend>,
|
||||||
|
|
||||||
|
private readonly fsService: FileStorageGeneric,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
|
public async getFileData(
|
||||||
|
file: EImageFileBackend | EImageDerivativeBackend,
|
||||||
|
): AsyncFailable<Buffer> {
|
||||||
|
if (file.data !== null) {
|
||||||
|
// Migrate files from old format to s3
|
||||||
|
const data = file.data;
|
||||||
|
|
||||||
|
const s3result = await this.fsService.putFile(file.fileKey, data);
|
||||||
|
if (HasFailed(s3result)) return s3result;
|
||||||
|
|
||||||
|
file.data = null;
|
||||||
|
let repoResult: EImageFileBackend | EImageDerivativeBackend;
|
||||||
|
if (file instanceof EImageFileBackend) {
|
||||||
|
repoResult = await this.imageFileRepo.save(file);
|
||||||
|
} else if (file instanceof EImageDerivativeBackend) {
|
||||||
|
repoResult = await this.imageDerivativeRepo.save(file);
|
||||||
|
} else {
|
||||||
|
return Fail(FT.SysValidation, 'Invalid file type');
|
||||||
|
}
|
||||||
|
if (HasFailed(repoResult)) return repoResult;
|
||||||
|
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await this.fsService.getFile(file.fileKey);
|
||||||
|
if (HasFailed(result)) return result;
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
public async setFile(
|
public async setFile(
|
||||||
imageId: string,
|
imageId: string,
|
||||||
variant: ImageEntryVariant,
|
variant: ImageEntryVariant,
|
||||||
file: Buffer,
|
file: Buffer,
|
||||||
filetype: string,
|
filetype: string,
|
||||||
): AsyncFailable<true> {
|
): AsyncFailable<true> {
|
||||||
|
const s3key = uuidv4();
|
||||||
|
|
||||||
const imageFile = new EImageFileBackend();
|
const imageFile = new EImageFileBackend();
|
||||||
imageFile.image_id = imageId;
|
imageFile.image_id = imageId;
|
||||||
imageFile.variant = variant;
|
imageFile.variant = variant;
|
||||||
imageFile.filetype = filetype;
|
imageFile.filetype = filetype;
|
||||||
imageFile.data = file;
|
imageFile.fileKey = s3key;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await this.imageFileRepo.upsert(imageFile, {
|
await this.imageFileRepo.upsert(imageFile, {
|
||||||
conflictPaths: ['image_id', 'variant'],
|
conflictPaths: ['image_id', 'variant'],
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const s3result = await this.fsService.putFile(s3key, file);
|
||||||
|
if (HasFailed(s3result)) return s3result;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
return Fail(FT.Database, e);
|
return Fail(FT.Database, e);
|
||||||
}
|
}
|
||||||
|
@ -84,6 +123,9 @@ export class ImageFileDBService {
|
||||||
|
|
||||||
if (!found) return Fail(FT.NotFound, 'Image not found');
|
if (!found) return Fail(FT.NotFound, 'Image not found');
|
||||||
|
|
||||||
|
const s3result = await this.fsService.deleteFile(found.fileKey);
|
||||||
|
if (HasFailed(s3result)) return s3result;
|
||||||
|
|
||||||
await this.imageFileRepo.delete({ image_id: imageId, variant: variant });
|
await this.imageFileRepo.delete({ image_id: imageId, variant: variant });
|
||||||
return found;
|
return found;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
@ -120,15 +162,22 @@ export class ImageFileDBService {
|
||||||
filetype: string,
|
filetype: string,
|
||||||
file: Buffer,
|
file: Buffer,
|
||||||
): AsyncFailable<EImageDerivativeBackend> {
|
): AsyncFailable<EImageDerivativeBackend> {
|
||||||
|
const s3key = uuidv4();
|
||||||
|
|
||||||
const imageDerivative = new EImageDerivativeBackend();
|
const imageDerivative = new EImageDerivativeBackend();
|
||||||
imageDerivative.image_id = imageId;
|
imageDerivative.image_id = imageId;
|
||||||
imageDerivative.key = key;
|
imageDerivative.key = key;
|
||||||
imageDerivative.filetype = filetype;
|
imageDerivative.filetype = filetype;
|
||||||
imageDerivative.data = file;
|
imageDerivative.fileKey = s3key;
|
||||||
imageDerivative.last_read = new Date();
|
imageDerivative.last_read = new Date();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
return await this.imageDerivativeRepo.save(imageDerivative);
|
const result = await this.imageDerivativeRepo.save(imageDerivative);
|
||||||
|
|
||||||
|
const s3result = await this.fsService.putFile(s3key, file);
|
||||||
|
if (HasFailed(s3result)) return s3result;
|
||||||
|
|
||||||
|
return result;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
return Fail(FT.Database, e);
|
return Fail(FT.Database, e);
|
||||||
}
|
}
|
||||||
|
@ -171,4 +220,83 @@ export class ImageFileDBService {
|
||||||
return Fail(FT.Database, e);
|
return Fail(FT.Database, e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async cleanupOrphanedDerivatives(): AsyncFailable<number> {
|
||||||
|
return this.cleanupRepoWithFilekey(this.imageDerivativeRepo);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async cleanupOrphanedFiles(): AsyncFailable<number> {
|
||||||
|
return this.cleanupRepoWithFilekey(this.imageFileRepo);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Go over all image files in the db, and any that are not linked to an image are deleted from s3 and the db
|
||||||
|
private async cleanupRepoWithFilekey(
|
||||||
|
repo: Repository<{ image_id: string | null; fileKey: string }>,
|
||||||
|
): AsyncFailable<number> {
|
||||||
|
try {
|
||||||
|
let remaining = Infinity;
|
||||||
|
let processed = 0;
|
||||||
|
|
||||||
|
while (remaining > 0) {
|
||||||
|
const orphaned = await repo.findAndCount({
|
||||||
|
where: {
|
||||||
|
image_id: IsNull(),
|
||||||
|
},
|
||||||
|
select: ['fileKey'],
|
||||||
|
take: 100,
|
||||||
|
});
|
||||||
|
if (orphaned[1] === 0) break;
|
||||||
|
remaining = orphaned[1] - orphaned[0].length;
|
||||||
|
|
||||||
|
const keys = orphaned[0].map((d) => d.fileKey);
|
||||||
|
|
||||||
|
const s3result = await this.fsService.deleteFiles(keys);
|
||||||
|
if (HasFailed(s3result)) return s3result;
|
||||||
|
|
||||||
|
const result = await repo.delete({
|
||||||
|
fileKey: In(keys),
|
||||||
|
});
|
||||||
|
|
||||||
|
processed += result.affected ?? 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
return processed;
|
||||||
|
} catch (e) {
|
||||||
|
return Fail(FT.Database, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async migrateFilesToFilekey(): AsyncFailable<number> {
|
||||||
|
return this.migrateRepoToFilekey(this.imageFileRepo);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async migrateDerivativesToFilekey(): AsyncFailable<number> {
|
||||||
|
return this.migrateRepoToFilekey(this.imageDerivativeRepo);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async migrateRepoToFilekey(
|
||||||
|
repo: Repository<EImageFileBackend | EImageDerivativeBackend>,
|
||||||
|
): AsyncFailable<number> {
|
||||||
|
let processed = 0;
|
||||||
|
|
||||||
|
try {
|
||||||
|
while (true) {
|
||||||
|
const current = await repo.findOne({
|
||||||
|
where: {
|
||||||
|
data: Not(IsNull()),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (!current) break;
|
||||||
|
|
||||||
|
const result = await this.getFileData(current);
|
||||||
|
if (HasFailed(result)) return result;
|
||||||
|
|
||||||
|
processed++;
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
return Fail(FT.Database, e);
|
||||||
|
}
|
||||||
|
|
||||||
|
return processed;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,6 +3,7 @@ import { PrefValueType } from 'picsur-shared/dist/dto/preferences.dto';
|
||||||
import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
|
import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
|
||||||
import { UsrPreference } from 'picsur-shared/dist/dto/usr-preferences.enum';
|
import { UsrPreference } from 'picsur-shared/dist/dto/usr-preferences.enum';
|
||||||
import { generateRandomString } from 'picsur-shared/dist/util/random';
|
import { generateRandomString } from 'picsur-shared/dist/util/random';
|
||||||
|
import { EarlyFSConfigService } from '../../config/early/early-fs.config.service';
|
||||||
import { EarlyJwtConfigService } from '../../config/early/early-jwt.config.service';
|
import { EarlyJwtConfigService } from '../../config/early/early-jwt.config.service';
|
||||||
|
|
||||||
// This specific service holds the default values for system and user preferences
|
// This specific service holds the default values for system and user preferences
|
||||||
|
@ -13,7 +14,7 @@ import { EarlyJwtConfigService } from '../../config/early/early-jwt.config.servi
|
||||||
export class PreferenceDefaultsService {
|
export class PreferenceDefaultsService {
|
||||||
private readonly logger = new Logger(PreferenceDefaultsService.name);
|
private readonly logger = new Logger(PreferenceDefaultsService.name);
|
||||||
|
|
||||||
constructor(private readonly jwtConfigService: EarlyJwtConfigService) {}
|
constructor(private readonly jwtConfigService: EarlyJwtConfigService,private readonly fsConfigService: EarlyFSConfigService) {}
|
||||||
|
|
||||||
private readonly usrDefaults: {
|
private readonly usrDefaults: {
|
||||||
[key in UsrPreference]: (() => PrefValueType) | PrefValueType;
|
[key in UsrPreference]: (() => PrefValueType) | PrefValueType;
|
||||||
|
@ -47,6 +48,14 @@ export class PreferenceDefaultsService {
|
||||||
[SysPreference.ConversionTimeLimit]: '15s',
|
[SysPreference.ConversionTimeLimit]: '15s',
|
||||||
[SysPreference.ConversionMemoryLimit]: 512,
|
[SysPreference.ConversionMemoryLimit]: 512,
|
||||||
|
|
||||||
|
[SysPreference.FSLocalPath]: () => this.fsConfigService.getLocalPath(),
|
||||||
|
[SysPreference.FSS3Endpoint]: () => this.fsConfigService.getS3Endpoint() ?? '',
|
||||||
|
[SysPreference.FSS3Bucket]: () => this.fsConfigService.getS3Bucket(),
|
||||||
|
[SysPreference.FSS3Region]: () => this.fsConfigService.getS3Region(),
|
||||||
|
[SysPreference.FSS3AccessKey]: () => this.fsConfigService.getS3AccessKey(),
|
||||||
|
[SysPreference.FSS3SecretKey]: () => this.fsConfigService.getS3SecretKey(),
|
||||||
|
|
||||||
|
|
||||||
[SysPreference.EnableTracking]: false,
|
[SysPreference.EnableTracking]: false,
|
||||||
[SysPreference.TrackingUrl]: '',
|
[SysPreference.TrackingUrl]: '',
|
||||||
[SysPreference.TrackingId]: '',
|
[SysPreference.TrackingId]: '',
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
import { dirname, resolve } from 'path';
|
import { dirname, resolve } from 'path';
|
||||||
import { fileURLToPath } from 'url';
|
import { fileURLToPath } from 'url';
|
||||||
|
|
||||||
|
export const GithubUrl = "https://github.com/rubikscraft/picsur"
|
||||||
export const ReportUrl = 'https://metrics.picsur.org';
|
export const ReportUrl = 'https://metrics.picsur.org';
|
||||||
export const ReportInterval = 1000 * 60 * 60;
|
export const ReportInterval = 1000 * 60 * 60;
|
||||||
export const EnvPrefix = 'PICSUR_';
|
export const EnvPrefix = 'PICSUR_';
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
import { Module } from '@nestjs/common';
|
import { Module } from '@nestjs/common';
|
||||||
import { ConfigModule } from '@nestjs/config';
|
import { ConfigModule } from '@nestjs/config';
|
||||||
import { AuthConfigService } from './auth.config.service';
|
import { AuthConfigService } from './auth.config.service';
|
||||||
|
import { EarlyFSConfigService } from './early-fs.config.service';
|
||||||
import { EarlyJwtConfigService } from './early-jwt.config.service';
|
import { EarlyJwtConfigService } from './early-jwt.config.service';
|
||||||
import { HostConfigService } from './host.config.service';
|
import { HostConfigService } from './host.config.service';
|
||||||
import { MultipartConfigService } from './multipart.config.service';
|
import { MultipartConfigService } from './multipart.config.service';
|
||||||
|
@ -23,6 +24,7 @@ import { TypeOrmConfigService } from './type-orm.config.service';
|
||||||
AuthConfigService,
|
AuthConfigService,
|
||||||
MultipartConfigService,
|
MultipartConfigService,
|
||||||
RedisConfigService,
|
RedisConfigService,
|
||||||
|
EarlyFSConfigService,
|
||||||
],
|
],
|
||||||
exports: [
|
exports: [
|
||||||
ConfigModule,
|
ConfigModule,
|
||||||
|
@ -33,6 +35,7 @@ import { TypeOrmConfigService } from './type-orm.config.service';
|
||||||
AuthConfigService,
|
AuthConfigService,
|
||||||
MultipartConfigService,
|
MultipartConfigService,
|
||||||
RedisConfigService,
|
RedisConfigService,
|
||||||
|
EarlyFSConfigService,
|
||||||
],
|
],
|
||||||
})
|
})
|
||||||
export class EarlyConfigModule {}
|
export class EarlyConfigModule {}
|
||||||
|
|
70
backend/src/config/early/early-fs.config.service.ts
Normal file
70
backend/src/config/early/early-fs.config.service.ts
Normal file
|
@ -0,0 +1,70 @@
|
||||||
|
import { Injectable } from '@nestjs/common';
|
||||||
|
import { ConfigService } from '@nestjs/config';
|
||||||
|
import { ParseString } from 'picsur-shared/dist/util/parse-simple';
|
||||||
|
import { EnvPrefix } from '../config.static';
|
||||||
|
|
||||||
|
export enum FileStorageMode {
|
||||||
|
None = 'none',
|
||||||
|
Local = 'local',
|
||||||
|
S3 = 's3',
|
||||||
|
}
|
||||||
|
|
||||||
|
const FSEnvPrefix = `${EnvPrefix}FILESTORAGE_`;
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class EarlyFSConfigService {
|
||||||
|
constructor(private readonly configService: ConfigService) {}
|
||||||
|
|
||||||
|
public getFileStorageMode(): FileStorageMode {
|
||||||
|
const parsed = ParseString(
|
||||||
|
this.configService.get(`${FSEnvPrefix}MODE`),
|
||||||
|
FileStorageMode.None,
|
||||||
|
).toLowerCase();
|
||||||
|
if (Object.values(FileStorageMode).includes(parsed as FileStorageMode)) {
|
||||||
|
return parsed as FileStorageMode;
|
||||||
|
}
|
||||||
|
return FileStorageMode.None;
|
||||||
|
}
|
||||||
|
|
||||||
|
public getLocalPath(): string {
|
||||||
|
return ParseString(
|
||||||
|
this.configService.get(`${FSEnvPrefix}LOCAL_PATH`),
|
||||||
|
'/data',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public getS3Endpoint(): string | null {
|
||||||
|
return ParseString(
|
||||||
|
this.configService.get(`${FSEnvPrefix}S3_ENDPOINT`),
|
||||||
|
null,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public getS3Bucket(): string {
|
||||||
|
return ParseString(
|
||||||
|
this.configService.get(`${FSEnvPrefix}S3_BUCKET`),
|
||||||
|
'picsur',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public getS3Region(): string {
|
||||||
|
return ParseString(
|
||||||
|
this.configService.get(`${FSEnvPrefix}S3_REGION`),
|
||||||
|
'us-east-1',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public getS3AccessKey(): string {
|
||||||
|
return ParseString(
|
||||||
|
this.configService.get(`${FSEnvPrefix}S3_ACCESS_KEY`),
|
||||||
|
'picsur',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public getS3SecretKey(): string {
|
||||||
|
return ParseString(
|
||||||
|
this.configService.get(`${FSEnvPrefix}S3_SECRET_KEY`),
|
||||||
|
'picsur',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
|
@ -6,6 +6,7 @@ import { EntityList } from '../../database/entities';
|
||||||
import { MigrationList } from '../../database/migrations';
|
import { MigrationList } from '../../database/migrations';
|
||||||
import { DefaultName, EnvPrefix } from '../config.static';
|
import { DefaultName, EnvPrefix } from '../config.static';
|
||||||
import { HostConfigService } from './host.config.service';
|
import { HostConfigService } from './host.config.service';
|
||||||
|
import { RedisConfigService } from './redis.config.service';
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class TypeOrmConfigService implements TypeOrmOptionsFactory {
|
export class TypeOrmConfigService implements TypeOrmOptionsFactory {
|
||||||
|
@ -13,6 +14,7 @@ export class TypeOrmConfigService implements TypeOrmOptionsFactory {
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private readonly configService: ConfigService,
|
private readonly configService: ConfigService,
|
||||||
|
private readonly redisConfig: RedisConfigService,
|
||||||
private readonly hostService: HostConfigService,
|
private readonly hostService: HostConfigService,
|
||||||
) {
|
) {
|
||||||
const varOptions = this.getTypeOrmServerOptions();
|
const varOptions = this.getTypeOrmServerOptions();
|
||||||
|
@ -66,6 +68,13 @@ export class TypeOrmConfigService implements TypeOrmOptionsFactory {
|
||||||
entitiesDir: 'src/database/entities',
|
entitiesDir: 'src/database/entities',
|
||||||
},
|
},
|
||||||
|
|
||||||
|
// cache: {
|
||||||
|
// duration: 60000,
|
||||||
|
// type: 'ioredis',
|
||||||
|
// alwaysEnabled: false,
|
||||||
|
// options: this.redisConfig.getRedisUrl(),
|
||||||
|
// },
|
||||||
|
|
||||||
...varOptions,
|
...varOptions,
|
||||||
} as TypeOrmModuleOptions;
|
} as TypeOrmModuleOptions;
|
||||||
}
|
}
|
||||||
|
|
117
backend/src/config/late/fs.config.service.ts
Normal file
117
backend/src/config/late/fs.config.service.ts
Normal file
|
@ -0,0 +1,117 @@
|
||||||
|
import { S3ClientConfig } from '@aws-sdk/client-s3';
|
||||||
|
import { Injectable, Logger } from '@nestjs/common';
|
||||||
|
import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
|
||||||
|
import { ThrowIfFailed } from 'picsur-shared/dist/types';
|
||||||
|
import { SysPreferenceDbService } from '../../collections/preference-db/sys-preference-db.service';
|
||||||
|
import { GithubUrl } from '../config.static';
|
||||||
|
import { EarlyFSConfigService, FileStorageMode } from '../early/early-fs.config.service';
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class FSConfigService {
|
||||||
|
private readonly logger = new Logger(FSConfigService.name);
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly earlyFsConfigService: EarlyFSConfigService,
|
||||||
|
private readonly sysPrefService: SysPreferenceDbService,
|
||||||
|
) {
|
||||||
|
this.printDebug().catch(this.logger.error);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async printDebug() {
|
||||||
|
const mode = this.getFileStorageMode();
|
||||||
|
|
||||||
|
if (mode === FileStorageMode.Local) {
|
||||||
|
this.logger.log('File storage Mode: Local');
|
||||||
|
|
||||||
|
this.logger.log('Local Path: ' + (await this.getLocalPath()));
|
||||||
|
} else if (mode === FileStorageMode.S3) {
|
||||||
|
this.logger.log('File storage Mode: S3');
|
||||||
|
|
||||||
|
const [endpoint, region, bucket, accessKey, secretKey] =
|
||||||
|
await Promise.all([
|
||||||
|
this.getS3Endpoint(),
|
||||||
|
this.getS3Region(),
|
||||||
|
this.getS3Bucket(),
|
||||||
|
this.getS3AccessKey(),
|
||||||
|
this.getS3SecretKey(),
|
||||||
|
]);
|
||||||
|
|
||||||
|
if (endpoint) this.logger.log('Custom S3 Endpoint: ' + endpoint);
|
||||||
|
|
||||||
|
this.logger.log('S3 Region: ' + region);
|
||||||
|
this.logger.log('S3 Bucket: ' + bucket);
|
||||||
|
|
||||||
|
this.logger.verbose('S3 Access Key: ' + accessKey);
|
||||||
|
this.logger.verbose('S3 Secret Key: ' + secretKey);
|
||||||
|
} else {
|
||||||
|
this.logger.error('File storage mode: None');
|
||||||
|
this.logger.warn(
|
||||||
|
`Please set the storage mode setting. Check ${GithubUrl} for more information.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public getFileStorageMode(): FileStorageMode {
|
||||||
|
return this.earlyFsConfigService.getFileStorageMode();
|
||||||
|
}
|
||||||
|
|
||||||
|
public async getLocalPath(): Promise<string> {
|
||||||
|
return ThrowIfFailed(
|
||||||
|
await this.sysPrefService.getStringPreference(SysPreference.FSLocalPath),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async getS3Config(): Promise<S3ClientConfig> {
|
||||||
|
return {
|
||||||
|
credentials: {
|
||||||
|
accessKeyId: await this.getS3AccessKey(),
|
||||||
|
secretAccessKey: await this.getS3SecretKey(),
|
||||||
|
},
|
||||||
|
endpoint: (await this.getS3Endpoint()) ?? undefined,
|
||||||
|
region: await this.getS3Region(),
|
||||||
|
tls: await this.getS3TLS(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
public async getS3Endpoint(): Promise<string | null> {
|
||||||
|
return ThrowIfFailed(
|
||||||
|
await this.sysPrefService.getStringPreference(SysPreference.FSS3Endpoint),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async getS3TLS(): Promise<boolean | undefined> {
|
||||||
|
const endpoint = await this.getS3Endpoint();
|
||||||
|
if (endpoint) {
|
||||||
|
return endpoint.startsWith('https');
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async getS3Bucket(): Promise<string> {
|
||||||
|
return ThrowIfFailed(
|
||||||
|
await this.sysPrefService.getStringPreference(SysPreference.FSS3Bucket),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async getS3Region(): Promise<string> {
|
||||||
|
return ThrowIfFailed(
|
||||||
|
await this.sysPrefService.getStringPreference(SysPreference.FSS3Region),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async getS3AccessKey(): Promise<string> {
|
||||||
|
return ThrowIfFailed(
|
||||||
|
await this.sysPrefService.getStringPreference(
|
||||||
|
SysPreference.FSS3AccessKey,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async getS3SecretKey(): Promise<string> {
|
||||||
|
return ThrowIfFailed(
|
||||||
|
await this.sysPrefService.getStringPreference(
|
||||||
|
SysPreference.FSS3SecretKey,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
|
@ -3,6 +3,7 @@ import { PreferenceDbModule } from '../../collections/preference-db/preference-d
|
||||||
import { SysPreferenceDbService } from '../../collections/preference-db/sys-preference-db.service';
|
import { SysPreferenceDbService } from '../../collections/preference-db/sys-preference-db.service';
|
||||||
import { EarlyConfigModule } from '../early/early-config.module';
|
import { EarlyConfigModule } from '../early/early-config.module';
|
||||||
import { EarlyJwtConfigService } from '../early/early-jwt.config.service';
|
import { EarlyJwtConfigService } from '../early/early-jwt.config.service';
|
||||||
|
import { FSConfigService } from './fs.config.service';
|
||||||
import { InfoConfigService } from './info.config.service';
|
import { InfoConfigService } from './info.config.service';
|
||||||
import { JwtConfigService } from './jwt.config.service';
|
import { JwtConfigService } from './jwt.config.service';
|
||||||
import { UsageConfigService } from './usage.config.service';
|
import { UsageConfigService } from './usage.config.service';
|
||||||
|
@ -14,10 +15,16 @@ import { UsageConfigService } from './usage.config.service';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [EarlyConfigModule, PreferenceDbModule],
|
imports: [EarlyConfigModule, PreferenceDbModule],
|
||||||
providers: [JwtConfigService, InfoConfigService, UsageConfigService],
|
providers: [
|
||||||
|
JwtConfigService,
|
||||||
|
FSConfigService,
|
||||||
|
InfoConfigService,
|
||||||
|
UsageConfigService,
|
||||||
|
],
|
||||||
exports: [
|
exports: [
|
||||||
EarlyConfigModule,
|
EarlyConfigModule,
|
||||||
JwtConfigService,
|
JwtConfigService,
|
||||||
|
FSConfigService,
|
||||||
InfoConfigService,
|
InfoConfigService,
|
||||||
UsageConfigService,
|
UsageConfigService,
|
||||||
],
|
],
|
||||||
|
|
|
@ -3,39 +3,42 @@ import {
|
||||||
Entity,
|
Entity,
|
||||||
Index,
|
Index,
|
||||||
JoinColumn,
|
JoinColumn,
|
||||||
ManyToOne,
|
ManyToOne, PrimaryColumn, Unique
|
||||||
PrimaryGeneratedColumn,
|
|
||||||
Unique,
|
|
||||||
} from 'typeorm';
|
} from 'typeorm';
|
||||||
import { EImageBackend } from './image.entity';
|
import { EImageBackend } from './image.entity';
|
||||||
|
|
||||||
@Entity()
|
@Entity()
|
||||||
@Unique(['image_id', 'key'])
|
@Unique(['image_id', 'key'])
|
||||||
export class EImageDerivativeBackend {
|
export class EImageDerivativeBackend {
|
||||||
@PrimaryGeneratedColumn('uuid')
|
@PrimaryColumn({ type: 'uuid', nullable: false, name: '_id' })
|
||||||
private _id?: string;
|
@Index()
|
||||||
|
fileKey: string;
|
||||||
|
|
||||||
// We do a little trickery
|
// == Reference to parent image
|
||||||
@Index()
|
@Index()
|
||||||
@ManyToOne(() => EImageBackend, (image) => image.derivatives, {
|
@ManyToOne(() => EImageBackend, (image) => image.derivatives, {
|
||||||
nullable: false,
|
nullable: true,
|
||||||
onDelete: 'CASCADE',
|
onDelete: 'SET NULL',
|
||||||
})
|
})
|
||||||
@JoinColumn({ name: 'image_id' })
|
@JoinColumn({ name: 'image_id' })
|
||||||
private _image?: any;
|
private _image?: any;
|
||||||
|
|
||||||
@Column({
|
@Column({
|
||||||
name: 'image_id',
|
name: 'image_id',
|
||||||
|
nullable: true,
|
||||||
})
|
})
|
||||||
image_id: string;
|
image_id: string | null;
|
||||||
|
|
||||||
|
// == Derivative options hash
|
||||||
@Index()
|
@Index()
|
||||||
@Column({ nullable: false })
|
@Column({ nullable: false })
|
||||||
key: string;
|
key: string;
|
||||||
|
|
||||||
|
// == Filetype of the derivative
|
||||||
@Column({ nullable: false })
|
@Column({ nullable: false })
|
||||||
filetype: string;
|
filetype: string;
|
||||||
|
|
||||||
|
// == Last time the derivative was read
|
||||||
@Column({
|
@Column({
|
||||||
type: 'timestamptz',
|
type: 'timestamptz',
|
||||||
name: 'last_read',
|
name: 'last_read',
|
||||||
|
@ -43,7 +46,7 @@ export class EImageDerivativeBackend {
|
||||||
})
|
})
|
||||||
last_read: Date;
|
last_read: Date;
|
||||||
|
|
||||||
// Binary data
|
// == Binary data
|
||||||
@Column({ type: 'bytea', nullable: false })
|
@Column({ type: 'bytea', nullable: true })
|
||||||
data: Buffer;
|
data: Buffer | null;
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,39 +5,42 @@ import {
|
||||||
Index,
|
Index,
|
||||||
JoinColumn,
|
JoinColumn,
|
||||||
ManyToOne,
|
ManyToOne,
|
||||||
PrimaryGeneratedColumn,
|
PrimaryColumn, Unique
|
||||||
Unique,
|
|
||||||
} from 'typeorm';
|
} from 'typeorm';
|
||||||
import { EImageBackend } from './image.entity';
|
import { EImageBackend } from './image.entity';
|
||||||
|
|
||||||
@Entity()
|
@Entity()
|
||||||
@Unique(['image_id', 'variant'])
|
@Unique(['image_id', 'variant'])
|
||||||
export class EImageFileBackend {
|
export class EImageFileBackend {
|
||||||
@PrimaryGeneratedColumn('uuid')
|
@PrimaryColumn({ type: 'uuid', nullable: false, name: '_id' })
|
||||||
private _id?: string;
|
@Index()
|
||||||
|
fileKey: string;
|
||||||
|
|
||||||
// We do a little trickery
|
// == Reference to parent image
|
||||||
@Index()
|
@Index()
|
||||||
@ManyToOne(() => EImageBackend, (image) => image.files, {
|
@ManyToOne(() => EImageBackend, (image) => image.files, {
|
||||||
nullable: false,
|
nullable: true,
|
||||||
onDelete: 'CASCADE',
|
onDelete: 'SET NULL',
|
||||||
})
|
})
|
||||||
@JoinColumn({ name: 'image_id' })
|
@JoinColumn({ name: 'image_id' })
|
||||||
private _image?: any;
|
private _image?: any;
|
||||||
|
|
||||||
@Column({
|
@Column({
|
||||||
name: 'image_id',
|
name: 'image_id',
|
||||||
|
nullable: true,
|
||||||
})
|
})
|
||||||
image_id: string;
|
image_id: string | null;
|
||||||
|
|
||||||
|
// == File variant
|
||||||
@Index()
|
@Index()
|
||||||
@Column({ nullable: false, enum: ImageEntryVariant })
|
@Column({ nullable: false, enum: ImageEntryVariant })
|
||||||
variant: ImageEntryVariant;
|
variant: ImageEntryVariant;
|
||||||
|
|
||||||
|
// == Filetype of the derivative
|
||||||
@Column({ nullable: false })
|
@Column({ nullable: false })
|
||||||
filetype: string;
|
filetype: string;
|
||||||
|
|
||||||
// Binary data
|
// == Binary data
|
||||||
@Column({ type: 'bytea', nullable: false })
|
@Column({ type: 'bytea', nullable: true })
|
||||||
data: Buffer;
|
data: Buffer | null;
|
||||||
}
|
}
|
||||||
|
|
44
backend/src/database/migrations/1672247794308-V_0_6_0_a.ts
Normal file
44
backend/src/database/migrations/1672247794308-V_0_6_0_a.ts
Normal file
|
@ -0,0 +1,44 @@
|
||||||
|
import { MigrationInterface, QueryRunner } from "typeorm";
|
||||||
|
|
||||||
|
export class V060A1672247794308 implements MigrationInterface {
|
||||||
|
name = 'V060A1672247794308'
|
||||||
|
|
||||||
|
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_file_backend" DROP CONSTRAINT "FK_8055f37d3b9f52f421b94ee84db"`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" DROP CONSTRAINT "FK_37055605f39b3f8847232d604f8"`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_file_backend" DROP CONSTRAINT "UQ_872384f20feaf7bfd27e28b8d4a"`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "_id" DROP DEFAULT`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "image_id" DROP NOT NULL`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "data" DROP NOT NULL`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" DROP CONSTRAINT "UQ_fa03f5333afd74c5cc5ff780d75"`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "_id" DROP DEFAULT`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "image_id" DROP NOT NULL`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "data" DROP NOT NULL`);
|
||||||
|
await queryRunner.query(`CREATE INDEX "IDX_95953be58a506e5de46feec618" ON "e_image_file_backend" ("_id") `);
|
||||||
|
await queryRunner.query(`CREATE INDEX "IDX_ff1ecff935b8d7bdcea8908781" ON "e_image_derivative_backend" ("_id") `);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_file_backend" ADD CONSTRAINT "UQ_872384f20feaf7bfd27e28b8d4a" UNIQUE ("image_id", "variant")`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ADD CONSTRAINT "UQ_fa03f5333afd74c5cc5ff780d75" UNIQUE ("image_id", "key")`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_file_backend" ADD CONSTRAINT "FK_8055f37d3b9f52f421b94ee84db" FOREIGN KEY ("image_id") REFERENCES "e_image_backend"("id") ON DELETE SET NULL ON UPDATE NO ACTION`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ADD CONSTRAINT "FK_37055605f39b3f8847232d604f8" FOREIGN KEY ("image_id") REFERENCES "e_image_backend"("id") ON DELETE SET NULL ON UPDATE NO ACTION`);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" DROP CONSTRAINT "FK_37055605f39b3f8847232d604f8"`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_file_backend" DROP CONSTRAINT "FK_8055f37d3b9f52f421b94ee84db"`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" DROP CONSTRAINT "UQ_fa03f5333afd74c5cc5ff780d75"`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_file_backend" DROP CONSTRAINT "UQ_872384f20feaf7bfd27e28b8d4a"`);
|
||||||
|
await queryRunner.query(`DROP INDEX "public"."IDX_ff1ecff935b8d7bdcea8908781"`);
|
||||||
|
await queryRunner.query(`DROP INDEX "public"."IDX_95953be58a506e5de46feec618"`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "data" SET NOT NULL`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "image_id" SET NOT NULL`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ALTER COLUMN "_id" SET DEFAULT uuid_generate_v4()`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ADD CONSTRAINT "UQ_fa03f5333afd74c5cc5ff780d75" UNIQUE ("image_id", "key")`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "data" SET NOT NULL`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "image_id" SET NOT NULL`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_file_backend" ALTER COLUMN "_id" SET DEFAULT uuid_generate_v4()`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_file_backend" ADD CONSTRAINT "UQ_872384f20feaf7bfd27e28b8d4a" UNIQUE ("image_id", "variant")`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_derivative_backend" ADD CONSTRAINT "FK_37055605f39b3f8847232d604f8" FOREIGN KEY ("image_id") REFERENCES "e_image_backend"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
|
||||||
|
await queryRunner.query(`ALTER TABLE "e_image_file_backend" ADD CONSTRAINT "FK_8055f37d3b9f52f421b94ee84db" FOREIGN KEY ("image_id") REFERENCES "e_image_backend"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -5,6 +5,7 @@ import { V040B1662485374471 } from './1662485374471-V_0_4_0_b';
|
||||||
import { V040C1662535484200 } from './1662535484200-V_0_4_0_c';
|
import { V040C1662535484200 } from './1662535484200-V_0_4_0_c';
|
||||||
import { V040D1662728275448 } from './1662728275448-V_0_4_0_d';
|
import { V040D1662728275448 } from './1662728275448-V_0_4_0_d';
|
||||||
import { V050A1672154027079 } from './1672154027079-V_0_5_0_a';
|
import { V050A1672154027079 } from './1672154027079-V_0_5_0_a';
|
||||||
|
import { V060A1672247794308 } from './1672247794308-V_0_6_0_a';
|
||||||
|
|
||||||
export const MigrationList: Function[] = [
|
export const MigrationList: Function[] = [
|
||||||
V030A1661692206479,
|
V030A1661692206479,
|
||||||
|
@ -14,4 +15,5 @@ export const MigrationList: Function[] = [
|
||||||
V040C1662535484200,
|
V040C1662535484200,
|
||||||
V040D1662728275448,
|
V040D1662728275448,
|
||||||
V050A1672154027079,
|
V050A1672154027079,
|
||||||
|
V060A1672247794308
|
||||||
];
|
];
|
||||||
|
|
|
@ -4,7 +4,7 @@ import fastifyReplyFrom from '@fastify/reply-from';
|
||||||
import { NestFactory } from '@nestjs/core';
|
import { NestFactory } from '@nestjs/core';
|
||||||
import {
|
import {
|
||||||
FastifyAdapter,
|
FastifyAdapter,
|
||||||
NestFastifyApplication,
|
NestFastifyApplication
|
||||||
} from '@nestjs/platform-fastify';
|
} from '@nestjs/platform-fastify';
|
||||||
import { AppModule } from './app.module';
|
import { AppModule } from './app.module';
|
||||||
import { HostConfigService } from './config/early/host.config.service';
|
import { HostConfigService } from './config/early/host.config.service';
|
||||||
|
@ -43,6 +43,8 @@ async function bootstrap() {
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
app.enableShutdownHooks();
|
||||||
|
|
||||||
// Configure logger
|
// Configure logger
|
||||||
app.useLogger(app.get(PicsurLoggerService));
|
app.useLogger(app.get(PicsurLoggerService));
|
||||||
app.flushLogs();
|
app.flushLogs();
|
||||||
|
|
|
@ -3,7 +3,7 @@ import ms from 'ms';
|
||||||
import { ImageRequestParams } from 'picsur-shared/dist/dto/api/image.dto';
|
import { ImageRequestParams } from 'picsur-shared/dist/dto/api/image.dto';
|
||||||
import {
|
import {
|
||||||
FileType,
|
FileType,
|
||||||
SupportedFileTypeCategory,
|
SupportedFileTypeCategory
|
||||||
} from 'picsur-shared/dist/dto/mimes.dto';
|
} from 'picsur-shared/dist/dto/mimes.dto';
|
||||||
import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
|
import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
|
||||||
import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
|
import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
|
||||||
|
@ -12,6 +12,13 @@ import { SysPreferenceDbService } from '../../collections/preference-db/sys-pref
|
||||||
import { SharpWrapper } from '../../workers/sharp.wrapper';
|
import { SharpWrapper } from '../../workers/sharp.wrapper';
|
||||||
import { ImageResult } from './imageresult';
|
import { ImageResult } from './imageresult';
|
||||||
|
|
||||||
|
interface InternalConvertOptions {
|
||||||
|
lossless?: boolean;
|
||||||
|
effort?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ConvertOptions = ImageRequestParams & InternalConvertOptions;
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class ImageConverterService {
|
export class ImageConverterService {
|
||||||
constructor(private readonly sysPref: SysPreferenceDbService) {}
|
constructor(private readonly sysPref: SysPreferenceDbService) {}
|
||||||
|
@ -20,7 +27,7 @@ export class ImageConverterService {
|
||||||
image: Buffer,
|
image: Buffer,
|
||||||
sourceFiletype: FileType,
|
sourceFiletype: FileType,
|
||||||
targetFiletype: FileType,
|
targetFiletype: FileType,
|
||||||
options: ImageRequestParams,
|
options: ConvertOptions,
|
||||||
): AsyncFailable<ImageResult> {
|
): AsyncFailable<ImageResult> {
|
||||||
if (
|
if (
|
||||||
sourceFiletype.identifier === targetFiletype.identifier &&
|
sourceFiletype.identifier === targetFiletype.identifier &&
|
||||||
|
@ -32,23 +39,22 @@ export class ImageConverterService {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (targetFiletype.category === SupportedFileTypeCategory.Image) {
|
if (
|
||||||
return this.convertStill(image, sourceFiletype, targetFiletype, options);
|
targetFiletype.category === SupportedFileTypeCategory.Image ||
|
||||||
} else if (
|
|
||||||
targetFiletype.category === SupportedFileTypeCategory.Animation
|
targetFiletype.category === SupportedFileTypeCategory.Animation
|
||||||
) {
|
) {
|
||||||
return this.convertStill(image, sourceFiletype, targetFiletype, options);
|
return this.convertImage(image, sourceFiletype, targetFiletype, options);
|
||||||
//return this.convertAnimation(image, targetmime, options);
|
//return this.convertAnimation(image, targetmime, options);
|
||||||
} else {
|
} else {
|
||||||
return Fail(FT.SysValidation, 'Unsupported mime type');
|
return Fail(FT.SysValidation, 'Unsupported mime type');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async convertStill(
|
private async convertImage(
|
||||||
image: Buffer,
|
image: Buffer,
|
||||||
sourceFiletype: FileType,
|
sourceFiletype: FileType,
|
||||||
targetFiletype: FileType,
|
targetFiletype: FileType,
|
||||||
options: ImageRequestParams,
|
options: ConvertOptions,
|
||||||
): AsyncFailable<ImageResult> {
|
): AsyncFailable<ImageResult> {
|
||||||
const [memLimit, timeLimit] = await Promise.all([
|
const [memLimit, timeLimit] = await Promise.all([
|
||||||
this.sysPref.getNumberPreference(SysPreference.ConversionMemoryLimit),
|
this.sysPref.getNumberPreference(SysPreference.ConversionMemoryLimit),
|
||||||
|
|
|
@ -8,12 +8,15 @@ import { ImageDBService } from '../../collections/image-db/image-db.service';
|
||||||
import { ImageFileDBService } from '../../collections/image-db/image-file-db.service';
|
import { ImageFileDBService } from '../../collections/image-db/image-file-db.service';
|
||||||
import { PreferenceDbModule } from '../../collections/preference-db/preference-db.module';
|
import { PreferenceDbModule } from '../../collections/preference-db/preference-db.module';
|
||||||
import { SysPreferenceDbService } from '../../collections/preference-db/sys-preference-db.service';
|
import { SysPreferenceDbService } from '../../collections/preference-db/sys-preference-db.service';
|
||||||
|
import { FileStorageMode } from '../../config/early/early-fs.config.service';
|
||||||
|
import { FSConfigService } from '../../config/late/fs.config.service';
|
||||||
|
import { LateConfigModule } from '../../config/late/late-config.module';
|
||||||
import { ImageConverterService } from './image-converter.service';
|
import { ImageConverterService } from './image-converter.service';
|
||||||
import { ImageProcessorService } from './image-processor.service';
|
import { ImageProcessorService } from './image-processor.service';
|
||||||
import { ImageManagerService } from './image.service';
|
import { ImageManagerService } from './image.service';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [ImageDBModule, PreferenceDbModule],
|
imports: [ImageDBModule, PreferenceDbModule, LateConfigModule],
|
||||||
providers: [
|
providers: [
|
||||||
ImageManagerService,
|
ImageManagerService,
|
||||||
ImageProcessorService,
|
ImageProcessorService,
|
||||||
|
@ -26,6 +29,7 @@ export class ImageManagerModule implements OnModuleInit {
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private readonly prefManager: SysPreferenceDbService,
|
private readonly prefManager: SysPreferenceDbService,
|
||||||
|
private readonly fsConfig: FSConfigService,
|
||||||
private readonly imageFileDB: ImageFileDBService,
|
private readonly imageFileDB: ImageFileDBService,
|
||||||
private readonly imageDB: ImageDBService,
|
private readonly imageDB: ImageDBService,
|
||||||
) {}
|
) {}
|
||||||
|
@ -38,6 +42,8 @@ export class ImageManagerModule implements OnModuleInit {
|
||||||
private async imageManagerCron() {
|
private async imageManagerCron() {
|
||||||
await this.cleanupDerivatives();
|
await this.cleanupDerivatives();
|
||||||
await this.cleanupExpired();
|
await this.cleanupExpired();
|
||||||
|
await this.cleanupOrphanedFiles();
|
||||||
|
await this.migrateFilesToFilekey();
|
||||||
}
|
}
|
||||||
|
|
||||||
private async cleanupDerivatives() {
|
private async cleanupDerivatives() {
|
||||||
|
@ -75,4 +81,48 @@ export class ImageManagerModule implements OnModuleInit {
|
||||||
if (cleanedUp > 0)
|
if (cleanedUp > 0)
|
||||||
this.logger.log(`Cleaned up ${cleanedUp} expired images`);
|
this.logger.log(`Cleaned up ${cleanedUp} expired images`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async cleanupOrphanedFiles() {
|
||||||
|
const cleanedUpDerivatives =
|
||||||
|
await this.imageFileDB.cleanupOrphanedDerivatives();
|
||||||
|
|
||||||
|
if (HasFailed(cleanedUpDerivatives)) {
|
||||||
|
cleanedUpDerivatives.print(this.logger);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const cleanedUpFiles = await this.imageFileDB.cleanupOrphanedFiles();
|
||||||
|
if (HasFailed(cleanedUpFiles)) {
|
||||||
|
cleanedUpFiles.print(this.logger);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (cleanedUpDerivatives > 0 || cleanedUpFiles > 0)
|
||||||
|
this.logger.log(
|
||||||
|
`Cleaned up ${cleanedUpDerivatives} orphaned derivatives and ${cleanedUpFiles} orphaned files`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async migrateFilesToFilekey() {
|
||||||
|
if ((await this.fsConfig.getFileStorageMode()) === FileStorageMode.None)
|
||||||
|
return;
|
||||||
|
|
||||||
|
const filesMigrated = await this.imageFileDB.migrateFilesToFilekey();
|
||||||
|
if (HasFailed(filesMigrated)) {
|
||||||
|
filesMigrated.print(this.logger);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const derivativesMigrated =
|
||||||
|
await this.imageFileDB.migrateDerivativesToFilekey();
|
||||||
|
if (HasFailed(derivativesMigrated)) {
|
||||||
|
derivativesMigrated.print(this.logger);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (filesMigrated > 0 || derivativesMigrated > 0)
|
||||||
|
this.logger.log(
|
||||||
|
`Migrated ${filesMigrated} files and ${derivativesMigrated} derivatives to filekey`,
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,8 +1,9 @@
|
||||||
import { Injectable } from '@nestjs/common';
|
import { Injectable } from '@nestjs/common';
|
||||||
import {
|
import {
|
||||||
|
AnimFileType,
|
||||||
FileType,
|
FileType,
|
||||||
ImageFileType,
|
ImageFileType,
|
||||||
SupportedFileTypeCategory,
|
SupportedFileTypeCategory
|
||||||
} from 'picsur-shared/dist/dto/mimes.dto';
|
} from 'picsur-shared/dist/dto/mimes.dto';
|
||||||
|
|
||||||
import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
|
import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
|
||||||
|
@ -41,10 +42,12 @@ export class ImageProcessorService {
|
||||||
image: Buffer,
|
image: Buffer,
|
||||||
filetype: FileType,
|
filetype: FileType,
|
||||||
): AsyncFailable<ImageResult> {
|
): AsyncFailable<ImageResult> {
|
||||||
// Webps and gifs are stored as is for now
|
const outputFileType = ParseFileType(AnimFileType.WEBP);
|
||||||
return {
|
if (HasFailed(outputFileType)) return outputFileType;
|
||||||
image: image,
|
|
||||||
filetype: filetype.identifier,
|
return this.imageConverter.convert(image, filetype, outputFileType, {
|
||||||
};
|
lossless: true,
|
||||||
|
effort: 0,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,7 +7,7 @@ import {
|
||||||
AnimFileType,
|
AnimFileType,
|
||||||
FileType,
|
FileType,
|
||||||
ImageFileType,
|
ImageFileType,
|
||||||
Mime2FileType,
|
Mime2FileType
|
||||||
} from 'picsur-shared/dist/dto/mimes.dto';
|
} from 'picsur-shared/dist/dto/mimes.dto';
|
||||||
import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
|
import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
|
||||||
import { UsrPreference } from 'picsur-shared/dist/dto/usr-preferences.enum';
|
import { UsrPreference } from 'picsur-shared/dist/dto/usr-preferences.enum';
|
||||||
|
@ -57,11 +57,13 @@ export class ImageManagerService {
|
||||||
userid: string | undefined,
|
userid: string | undefined,
|
||||||
options: Partial<Pick<EImageBackend, 'file_name' | 'expires_at'>>,
|
options: Partial<Pick<EImageBackend, 'file_name' | 'expires_at'>>,
|
||||||
): AsyncFailable<EImageBackend> {
|
): AsyncFailable<EImageBackend> {
|
||||||
if (options.expires_at !== undefined && options.expires_at !== null) {
|
if (
|
||||||
if (options.expires_at < new Date()) {
|
options.expires_at !== undefined &&
|
||||||
return Fail(FT.UsrValidation, 'Expiration date must be in the future');
|
options.expires_at !== null &&
|
||||||
}
|
options.expires_at < new Date()
|
||||||
}
|
)
|
||||||
|
return Fail(FT.UsrValidation, 'Expiration date must be in the future');
|
||||||
|
|
||||||
return await this.imagesService.update(id, userid, options);
|
return await this.imagesService.update(id, userid, options);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -114,13 +116,24 @@ export class ImageManagerService {
|
||||||
);
|
);
|
||||||
if (HasFailed(imageEntity)) return imageEntity;
|
if (HasFailed(imageEntity)) return imageEntity;
|
||||||
|
|
||||||
|
const onFail = async () => {
|
||||||
|
const result = await this.imagesService.delete(
|
||||||
|
[imageEntity.id],
|
||||||
|
undefined,
|
||||||
|
);
|
||||||
|
if (HasFailed(result)) result.print(this.logger);
|
||||||
|
};
|
||||||
|
|
||||||
const imageFileEntity = await this.imageFilesService.setFile(
|
const imageFileEntity = await this.imageFilesService.setFile(
|
||||||
imageEntity.id,
|
imageEntity.id,
|
||||||
ImageEntryVariant.MASTER,
|
ImageEntryVariant.MASTER,
|
||||||
processResult.image,
|
processResult.image,
|
||||||
processResult.filetype,
|
processResult.filetype,
|
||||||
);
|
);
|
||||||
if (HasFailed(imageFileEntity)) return imageFileEntity;
|
if (HasFailed(imageFileEntity)) {
|
||||||
|
await onFail();
|
||||||
|
return imageFileEntity;
|
||||||
|
}
|
||||||
|
|
||||||
if (keepOriginal) {
|
if (keepOriginal) {
|
||||||
const originalFileEntity = await this.imageFilesService.setFile(
|
const originalFileEntity = await this.imageFilesService.setFile(
|
||||||
|
@ -129,7 +142,10 @@ export class ImageManagerService {
|
||||||
image,
|
image,
|
||||||
fileType.identifier,
|
fileType.identifier,
|
||||||
);
|
);
|
||||||
if (HasFailed(originalFileEntity)) return originalFileEntity;
|
if (HasFailed(originalFileEntity)) {
|
||||||
|
await onFail();
|
||||||
|
return originalFileEntity;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return imageEntity;
|
return imageEntity;
|
||||||
|
@ -162,9 +178,12 @@ export class ImageManagerService {
|
||||||
const sourceFileType = ParseFileType(masterImage.filetype);
|
const sourceFileType = ParseFileType(masterImage.filetype);
|
||||||
if (HasFailed(sourceFileType)) return sourceFileType;
|
if (HasFailed(sourceFileType)) return sourceFileType;
|
||||||
|
|
||||||
|
const data = await this.imageFilesService.getFileData(masterImage);
|
||||||
|
if (HasFailed(data)) return data;
|
||||||
|
|
||||||
const startTime = Date.now();
|
const startTime = Date.now();
|
||||||
const convertResult = await this.convertService.convert(
|
const convertResult = await this.convertService.convert(
|
||||||
masterImage.data,
|
data,
|
||||||
sourceFileType,
|
sourceFileType,
|
||||||
targetFileType,
|
targetFileType,
|
||||||
allow_editing ? options : {},
|
allow_editing ? options : {},
|
||||||
|
@ -234,6 +253,12 @@ export class ImageManagerService {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async getFileData(
|
||||||
|
file: EImageFileBackend | EImageDerivativeBackend,
|
||||||
|
): AsyncFailable<Buffer> {
|
||||||
|
return this.imageFilesService.getFileData(file);
|
||||||
|
}
|
||||||
|
|
||||||
// Util stuff ==================================================================
|
// Util stuff ==================================================================
|
||||||
|
|
||||||
private async getFileTypeFromBuffer(image: Buffer): AsyncFailable<FileType> {
|
private async getFileTypeFromBuffer(image: Buffer): AsyncFailable<FileType> {
|
||||||
|
|
|
@ -5,7 +5,7 @@ import {
|
||||||
Logger,
|
Logger,
|
||||||
Param,
|
Param,
|
||||||
Post,
|
Post,
|
||||||
Res,
|
Res
|
||||||
} from '@nestjs/common';
|
} from '@nestjs/common';
|
||||||
import { Throttle } from '@nestjs/throttler';
|
import { Throttle } from '@nestjs/throttler';
|
||||||
import type { FastifyReply } from 'fastify';
|
import type { FastifyReply } from 'fastify';
|
||||||
|
@ -18,7 +18,7 @@ import {
|
||||||
ImageListResponse,
|
ImageListResponse,
|
||||||
ImageUpdateRequest,
|
ImageUpdateRequest,
|
||||||
ImageUpdateResponse,
|
ImageUpdateResponse,
|
||||||
ImageUploadResponse,
|
ImageUploadResponse
|
||||||
} from 'picsur-shared/dist/dto/api/image-manage.dto';
|
} from 'picsur-shared/dist/dto/api/image-manage.dto';
|
||||||
import { Permission } from 'picsur-shared/dist/dto/permissions.enum';
|
import { Permission } from 'picsur-shared/dist/dto/permissions.enum';
|
||||||
import { Fail, FT, HasFailed, ThrowIfFailed } from 'picsur-shared/dist/types';
|
import { Fail, FT, HasFailed, ThrowIfFailed } from 'picsur-shared/dist/types';
|
||||||
|
@ -26,7 +26,7 @@ import { PostFiles } from '../../decorators/multipart/multipart.decorator';
|
||||||
import type { FileIterator } from '../../decorators/multipart/postfiles.pipe';
|
import type { FileIterator } from '../../decorators/multipart/postfiles.pipe';
|
||||||
import {
|
import {
|
||||||
HasPermission,
|
HasPermission,
|
||||||
RequiredPermissions,
|
RequiredPermissions
|
||||||
} from '../../decorators/permissions.decorator';
|
} from '../../decorators/permissions.decorator';
|
||||||
import { ReqUserID } from '../../decorators/request-user.decorator';
|
import { ReqUserID } from '../../decorators/request-user.decorator';
|
||||||
import { Returns } from '../../decorators/returns.decorator';
|
import { Returns } from '../../decorators/returns.decorator';
|
||||||
|
@ -91,14 +91,14 @@ export class ImageManageController {
|
||||||
@RequiredPermissions(Permission.ImageManage)
|
@RequiredPermissions(Permission.ImageManage)
|
||||||
@Returns(ImageUpdateResponse)
|
@Returns(ImageUpdateResponse)
|
||||||
async updateImage(
|
async updateImage(
|
||||||
@Body() body: ImageUpdateRequest,
|
@Body() options: ImageUpdateRequest,
|
||||||
@ReqUserID() userid: string,
|
@ReqUserID() userid: string,
|
||||||
@HasPermission(Permission.ImageAdmin) isImageAdmin: boolean,
|
@HasPermission(Permission.ImageAdmin) isImageAdmin: boolean,
|
||||||
): Promise<ImageUpdateResponse> {
|
): Promise<ImageUpdateResponse> {
|
||||||
const user_id = isImageAdmin ? undefined : userid;
|
const user_id = isImageAdmin ? undefined : userid;
|
||||||
|
|
||||||
const image = ThrowIfFailed(
|
const image = ThrowIfFailed(
|
||||||
await this.imagesService.update(body.id, user_id, body),
|
await this.imagesService.update(options.id, user_id, options),
|
||||||
);
|
);
|
||||||
|
|
||||||
return image;
|
return image;
|
||||||
|
|
|
@ -3,12 +3,14 @@ import { SkipThrottle } from '@nestjs/throttler';
|
||||||
import type { FastifyReply } from 'fastify';
|
import type { FastifyReply } from 'fastify';
|
||||||
import {
|
import {
|
||||||
ImageMetaResponse,
|
ImageMetaResponse,
|
||||||
ImageRequestParams,
|
ImageRequestParams
|
||||||
} from 'picsur-shared/dist/dto/api/image.dto';
|
} from 'picsur-shared/dist/dto/api/image.dto';
|
||||||
import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum';
|
import { ImageEntryVariant } from 'picsur-shared/dist/dto/image-entry-variant.enum';
|
||||||
import { FileType2Mime } from 'picsur-shared/dist/dto/mimes.dto';
|
import { FileType2Mime } from 'picsur-shared/dist/dto/mimes.dto';
|
||||||
import { FT, IsFailure, ThrowIfFailed } from 'picsur-shared/dist/types';
|
import { FT, IsFailure, ThrowIfFailed } from 'picsur-shared/dist/types';
|
||||||
import { UserDbService } from '../../collections/user-db/user-db.service';
|
import { UserDbService } from '../../collections/user-db/user-db.service';
|
||||||
|
import { EImageDerivativeBackend } from '../../database/entities/images/image-derivative.entity';
|
||||||
|
import { EImageFileBackend } from '../../database/entities/images/image-file.entity';
|
||||||
import { ImageFullIdParam } from '../../decorators/image-id/image-full-id.decorator';
|
import { ImageFullIdParam } from '../../decorators/image-id/image-full-id.decorator';
|
||||||
import { ImageIdParam } from '../../decorators/image-id/image-id.decorator';
|
import { ImageIdParam } from '../../decorators/image-id/image-id.decorator';
|
||||||
import { RequiredPermissions } from '../../decorators/permissions.decorator';
|
import { RequiredPermissions } from '../../decorators/permissions.decorator';
|
||||||
|
@ -57,25 +59,23 @@ export class ImageController {
|
||||||
@Query() params: ImageRequestParams,
|
@Query() params: ImageRequestParams,
|
||||||
): Promise<Buffer> {
|
): Promise<Buffer> {
|
||||||
try {
|
try {
|
||||||
|
let image: EImageFileBackend | EImageDerivativeBackend;
|
||||||
if (fullid.variant === ImageEntryVariant.ORIGINAL) {
|
if (fullid.variant === ImageEntryVariant.ORIGINAL) {
|
||||||
const image = ThrowIfFailed(
|
image = ThrowIfFailed(await this.imagesService.getOriginal(fullid.id));
|
||||||
await this.imagesService.getOriginal(fullid.id),
|
} else {
|
||||||
|
image = ThrowIfFailed(
|
||||||
|
await this.imagesService.getConverted(
|
||||||
|
fullid.id,
|
||||||
|
fullid.filetype,
|
||||||
|
params,
|
||||||
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
res.type(ThrowIfFailed(FileType2Mime(image.filetype)));
|
|
||||||
return image.data;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const image = ThrowIfFailed(
|
const data = ThrowIfFailed(await this.imagesService.getFileData(image));
|
||||||
await this.imagesService.getConverted(
|
|
||||||
fullid.id,
|
|
||||||
fullid.filetype,
|
|
||||||
params,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
|
|
||||||
res.type(ThrowIfFailed(FileType2Mime(image.filetype)));
|
res.type(ThrowIfFailed(FileType2Mime(image.filetype)));
|
||||||
return image.data;
|
return data;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
if (!IsFailure(e) || e.getType() !== FT.NotFound) throw e;
|
if (!IsFailure(e) || e.getType() !== FT.NotFound) throw e;
|
||||||
|
|
||||||
|
|
|
@ -26,6 +26,10 @@ export type SharpWorkerOperation =
|
||||||
|
|
||||||
export interface SharpWorkerFinishOptions {
|
export interface SharpWorkerFinishOptions {
|
||||||
quality?: number;
|
quality?: number;
|
||||||
|
|
||||||
|
// Only for internal use
|
||||||
|
lossless?: boolean;
|
||||||
|
effort?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Messages
|
// Messages
|
||||||
|
|
|
@ -6,6 +6,7 @@ import {
|
||||||
} from 'picsur-shared/dist/dto/mimes.dto';
|
} from 'picsur-shared/dist/dto/mimes.dto';
|
||||||
import { QOIdecode, QOIencode } from 'qoi-img';
|
import { QOIdecode, QOIencode } from 'qoi-img';
|
||||||
import sharp, { Sharp, SharpOptions } from 'sharp';
|
import sharp, { Sharp, SharpOptions } from 'sharp';
|
||||||
|
import { SharpWorkerFinishOptions } from './sharp.message';
|
||||||
|
|
||||||
export interface SharpResult {
|
export interface SharpResult {
|
||||||
data: Buffer;
|
data: Buffer;
|
||||||
|
@ -72,9 +73,7 @@ function qoiSharpIn(image: Buffer, options?: SharpOptions) {
|
||||||
export async function UniversalSharpOut(
|
export async function UniversalSharpOut(
|
||||||
image: Sharp,
|
image: Sharp,
|
||||||
filetype: FileType,
|
filetype: FileType,
|
||||||
options?: {
|
options?: SharpWorkerFinishOptions,
|
||||||
quality?: number;
|
|
||||||
},
|
|
||||||
): Promise<SharpResult> {
|
): Promise<SharpResult> {
|
||||||
let result: SharpResult | undefined;
|
let result: SharpResult | undefined;
|
||||||
|
|
||||||
|
@ -103,7 +102,11 @@ export async function UniversalSharpOut(
|
||||||
case ImageFileType.WEBP:
|
case ImageFileType.WEBP:
|
||||||
case AnimFileType.WEBP:
|
case AnimFileType.WEBP:
|
||||||
result = await image
|
result = await image
|
||||||
.webp({ quality: options?.quality })
|
.webp({
|
||||||
|
quality: options?.quality,
|
||||||
|
lossless: options?.lossless,
|
||||||
|
effort: options?.effort,
|
||||||
|
})
|
||||||
.toBuffer({ resolveWithObject: true });
|
.toBuffer({ resolveWithObject: true });
|
||||||
break;
|
break;
|
||||||
case AnimFileType.GIF:
|
case AnimFileType.GIF:
|
||||||
|
|
|
@ -0,0 +1,15 @@
|
||||||
|
<ng-container *ngFor="let category of preferences | async">
|
||||||
|
<h2 *ngIf="category.category !== null && showTitles">{{ category.title }}</h2>
|
||||||
|
<div class="row">
|
||||||
|
<ng-container *ngFor="let pref of category.prefs">
|
||||||
|
<pref-option
|
||||||
|
class="col-md-6 col-12"
|
||||||
|
[pref]="pref"
|
||||||
|
[update]="sysPrefService.setPreference.bind(sysPrefService)"
|
||||||
|
[name]="getName(pref.key)"
|
||||||
|
[helpText]="getHelpText(pref.key)"
|
||||||
|
[validator]="getValidator(pref.key)"
|
||||||
|
></pref-option>
|
||||||
|
</ng-container>
|
||||||
|
</div>
|
||||||
|
</ng-container>
|
|
@ -0,0 +1,75 @@
|
||||||
|
import { Component, Input } from '@angular/core';
|
||||||
|
import { DecodedPref } from 'picsur-shared/dist/dto/preferences.dto';
|
||||||
|
import {
|
||||||
|
SysPreference,
|
||||||
|
SysPreferenceValidators
|
||||||
|
} from 'picsur-shared/dist/dto/sys-preferences.enum';
|
||||||
|
import { map, Observable } from 'rxjs';
|
||||||
|
import {
|
||||||
|
SysPreferenceCategories,
|
||||||
|
SysPreferenceCategory,
|
||||||
|
SysPreferenceUI
|
||||||
|
} from 'src/app/i18n/sys-pref.i18n';
|
||||||
|
|
||||||
|
import { SysPrefService } from 'src/app/services/api/sys-pref.service';
|
||||||
|
import { z, ZodTypeAny } from 'zod';
|
||||||
|
|
||||||
|
@Component({
|
||||||
|
selector: 'partial-sys-pref',
|
||||||
|
templateUrl: './partial-sys-pref.component.html',
|
||||||
|
})
|
||||||
|
export class PartialSysPrefComponent {
|
||||||
|
@Input('hidden-categories') public set hiddenCategories(
|
||||||
|
value: SysPreferenceCategory[],
|
||||||
|
) {
|
||||||
|
this.categories = this.makeCategories(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Input("show-titles") public showTitles = true;
|
||||||
|
|
||||||
|
private categories = this.makeCategories();
|
||||||
|
|
||||||
|
public getName(key: string) {
|
||||||
|
return SysPreferenceUI[key as SysPreference]?.name ?? key;
|
||||||
|
}
|
||||||
|
|
||||||
|
public getHelpText(key: string) {
|
||||||
|
return SysPreferenceUI[key as SysPreference]?.helpText ?? '';
|
||||||
|
}
|
||||||
|
|
||||||
|
public getCategory(key: string): null | string {
|
||||||
|
return SysPreferenceUI[key as SysPreference]?.category ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public getValidator(key: string): ZodTypeAny {
|
||||||
|
return SysPreferenceValidators[key as SysPreference] ?? z.any();
|
||||||
|
}
|
||||||
|
|
||||||
|
preferences: Observable<
|
||||||
|
Array<{
|
||||||
|
category: string;
|
||||||
|
title: string;
|
||||||
|
prefs: DecodedPref[];
|
||||||
|
}>
|
||||||
|
>;
|
||||||
|
|
||||||
|
constructor(public readonly sysPrefService: SysPrefService) {
|
||||||
|
this.preferences = sysPrefService.live.pipe(
|
||||||
|
map((prefs) => {
|
||||||
|
return this.categories.map((category) => ({
|
||||||
|
category,
|
||||||
|
title: SysPreferenceCategories[category],
|
||||||
|
prefs: prefs.filter(
|
||||||
|
(pref) => this.getCategory(pref.key) === category,
|
||||||
|
),
|
||||||
|
}));
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private makeCategories(hiddenCategories: SysPreferenceCategory[] = []) {
|
||||||
|
return Object.values(SysPreferenceCategory).filter(
|
||||||
|
(category) => !hiddenCategories.includes(category),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,14 @@
|
||||||
|
import { CommonModule } from '@angular/common';
|
||||||
|
import { NgModule } from '@angular/core';
|
||||||
|
import { PrefOptionModule } from '../pref-option/pref-option.module';
|
||||||
|
import { PartialSysPrefComponent } from './partial-sys-pref.component';
|
||||||
|
|
||||||
|
@NgModule({
|
||||||
|
imports: [
|
||||||
|
CommonModule,
|
||||||
|
PrefOptionModule
|
||||||
|
],
|
||||||
|
declarations: [PartialSysPrefComponent],
|
||||||
|
exports: [PartialSysPrefComponent],
|
||||||
|
})
|
||||||
|
export class PartialSysPrefModule {}
|
|
@ -1,84 +1,133 @@
|
||||||
import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
|
import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
|
||||||
|
|
||||||
|
export enum SysPreferenceCategory {
|
||||||
|
General = 'general',
|
||||||
|
Authentication = 'authentication',
|
||||||
|
ImageProcessing = 'image-processing',
|
||||||
|
FileStorage = 'file-storage',
|
||||||
|
Usage = 'usage',
|
||||||
|
}
|
||||||
|
|
||||||
|
export const SysPreferenceCategories: {
|
||||||
|
[key in SysPreferenceCategory]: string;
|
||||||
|
} = {
|
||||||
|
[SysPreferenceCategory.General]: 'General',
|
||||||
|
[SysPreferenceCategory.Authentication]: 'Authentication',
|
||||||
|
[SysPreferenceCategory.ImageProcessing]: 'Image Processing',
|
||||||
|
[SysPreferenceCategory.FileStorage]: 'File Storage',
|
||||||
|
[SysPreferenceCategory.Usage]: 'Usage',
|
||||||
|
};
|
||||||
|
|
||||||
export const SysPreferenceUI: {
|
export const SysPreferenceUI: {
|
||||||
[key in SysPreference]: {
|
[key in SysPreference]: {
|
||||||
name: string;
|
name: string;
|
||||||
helpText: string;
|
helpText: string;
|
||||||
category: string;
|
category: SysPreferenceCategory;
|
||||||
};
|
};
|
||||||
} = {
|
} = {
|
||||||
[SysPreference.HostOverride]: {
|
[SysPreference.HostOverride]: {
|
||||||
name: 'Host Override',
|
name: 'Host Override',
|
||||||
helpText:
|
helpText:
|
||||||
'Override the hostname for the server, useful for when you are accessing the server from a different domain.',
|
'Override the hostname for the server, useful for when you are accessing the server from a different domain.',
|
||||||
category: 'General',
|
category: SysPreferenceCategory.General,
|
||||||
},
|
},
|
||||||
|
|
||||||
[SysPreference.RemoveDerivativesAfter]: {
|
[SysPreference.RemoveDerivativesAfter]: {
|
||||||
name: 'Cached Images Expiry Time',
|
name: 'Cached Images Expiry Time',
|
||||||
helpText:
|
helpText:
|
||||||
'Time before cached converted images are deleted. This does not affect the original image. A lower cache time will save on disk space but cost more cpu. Set to 0 to disable.',
|
'Time before cached converted images are deleted. This does not affect the original image. A lower cache time will save on disk space but cost more cpu. Set to 0 to disable.',
|
||||||
category: 'Image Processing',
|
category: SysPreferenceCategory.ImageProcessing,
|
||||||
},
|
},
|
||||||
[SysPreference.AllowEditing]: {
|
[SysPreference.AllowEditing]: {
|
||||||
name: 'Allow images to be edited',
|
name: 'Allow images to be edited',
|
||||||
helpText:
|
helpText:
|
||||||
'Allow images to be edited (e.g. resize, flip). Using these features will use more CPU power.',
|
'Allow images to be edited (e.g. resize, flip). Using these features will use more CPU power.',
|
||||||
|
|
||||||
category: 'Image Processing',
|
category: SysPreferenceCategory.ImageProcessing,
|
||||||
},
|
},
|
||||||
[SysPreference.ConversionTimeLimit]: {
|
[SysPreference.ConversionTimeLimit]: {
|
||||||
name: 'Convert/Edit Time Limit',
|
name: 'Convert/Edit Time Limit',
|
||||||
helpText:
|
helpText:
|
||||||
'Time limit for converting/editing images. You may need to increase this on low powered devices.',
|
'Time limit for converting/editing images. You may need to increase this on low powered devices.',
|
||||||
category: 'Image Processing',
|
category: SysPreferenceCategory.ImageProcessing,
|
||||||
},
|
},
|
||||||
[SysPreference.ConversionMemoryLimit]: {
|
[SysPreference.ConversionMemoryLimit]: {
|
||||||
name: 'Convert/Edit Memory Limit MB',
|
name: 'Convert/Edit Memory Limit MB',
|
||||||
helpText:
|
helpText:
|
||||||
'Memory limit for converting/editing images. You only need to increase this if you are storing massive images.',
|
'Memory limit for converting/editing images. You only need to increase this if you are storing massive images.',
|
||||||
category: 'Image Processing',
|
category: SysPreferenceCategory.ImageProcessing,
|
||||||
},
|
},
|
||||||
|
|
||||||
[SysPreference.JwtSecret]: {
|
[SysPreference.JwtSecret]: {
|
||||||
name: 'JWT Secret',
|
name: 'JWT Secret',
|
||||||
helpText: 'Secret used to sign JWT authentication tokens.',
|
helpText: 'Secret used to sign JWT authentication tokens.',
|
||||||
category: 'Authentication',
|
category: SysPreferenceCategory.Authentication,
|
||||||
},
|
},
|
||||||
[SysPreference.JwtExpiresIn]: {
|
[SysPreference.JwtExpiresIn]: {
|
||||||
name: 'JWT Expiry Time',
|
name: 'JWT Expiry Time',
|
||||||
helpText: 'Time before JWT authentication tokens expire.',
|
helpText: 'Time before JWT authentication tokens expire.',
|
||||||
category: 'Authentication',
|
category: SysPreferenceCategory.Authentication,
|
||||||
},
|
},
|
||||||
[SysPreference.BCryptStrength]: {
|
[SysPreference.BCryptStrength]: {
|
||||||
name: 'BCrypt Strength',
|
name: 'BCrypt Strength',
|
||||||
helpText:
|
helpText:
|
||||||
'Strength of BCrypt hashing algorithm, 10 is recommended. Reduce this if running on a low powered device.',
|
'Strength of BCrypt hashing algorithm, 10 is recommended. Reduce this if running on a low powered device.',
|
||||||
category: 'Authentication',
|
category: SysPreferenceCategory.Authentication,
|
||||||
|
},
|
||||||
|
|
||||||
|
[SysPreference.FSLocalPath]: {
|
||||||
|
name: 'FS Local - Path',
|
||||||
|
helpText: 'Storage location of the local storage provider.',
|
||||||
|
category: SysPreferenceCategory.FileStorage,
|
||||||
|
},
|
||||||
|
[SysPreference.FSS3Endpoint]: {
|
||||||
|
name: 'FS S3 - Endpoint',
|
||||||
|
helpText: 'Custom endpoint of the S3 storage provider.',
|
||||||
|
category: SysPreferenceCategory.FileStorage,
|
||||||
|
},
|
||||||
|
[SysPreference.FSS3Bucket]: {
|
||||||
|
name: 'FS S3 - Bucket',
|
||||||
|
helpText: 'Bucket of the S3 storage provider.',
|
||||||
|
category: SysPreferenceCategory.FileStorage,
|
||||||
|
},
|
||||||
|
[SysPreference.FSS3Region]: {
|
||||||
|
name: 'FS S3 - Region',
|
||||||
|
helpText: 'Region of the S3 storage provider.',
|
||||||
|
category: SysPreferenceCategory.FileStorage,
|
||||||
|
},
|
||||||
|
[SysPreference.FSS3AccessKey]: {
|
||||||
|
name: 'FS S3 - Access Key',
|
||||||
|
helpText: 'Access key of the S3 storage provider.',
|
||||||
|
category: SysPreferenceCategory.FileStorage,
|
||||||
|
},
|
||||||
|
[SysPreference.FSS3SecretKey]: {
|
||||||
|
name: 'FS S3 - Secret Key',
|
||||||
|
helpText: 'Secret key of the S3 storage provider.',
|
||||||
|
category: SysPreferenceCategory.FileStorage,
|
||||||
},
|
},
|
||||||
|
|
||||||
[SysPreference.EnableTracking]: {
|
[SysPreference.EnableTracking]: {
|
||||||
name: 'Enable Ackee Web Tracking',
|
name: 'Enable Ackee Web Tracking',
|
||||||
helpText:
|
helpText:
|
||||||
'Enable tracking of the website usage using Ackee. You will need to set the tracking URL and ID.',
|
'Enable tracking of the website usage using Ackee. You will need to set the tracking URL and ID.',
|
||||||
category: 'Usage',
|
category: SysPreferenceCategory.Usage,
|
||||||
},
|
},
|
||||||
[SysPreference.TrackingUrl]: {
|
[SysPreference.TrackingUrl]: {
|
||||||
name: 'Ackee tracking URL',
|
name: 'Ackee tracking URL',
|
||||||
helpText:
|
helpText:
|
||||||
'URL of the Ackee tracking server. Requests are proxied, so ensure the X-Forwarded-For header is handled.',
|
'URL of the Ackee tracking server. Requests are proxied, so ensure the X-Forwarded-For header is handled.',
|
||||||
category: 'Usage',
|
category: SysPreferenceCategory.Usage,
|
||||||
},
|
},
|
||||||
[SysPreference.TrackingId]: {
|
[SysPreference.TrackingId]: {
|
||||||
name: 'Ackee trackign website ID',
|
name: 'Ackee trackign website ID',
|
||||||
helpText: 'ID of the website to track.',
|
helpText: 'ID of the website to track.',
|
||||||
category: 'Usage',
|
category: SysPreferenceCategory.Usage,
|
||||||
},
|
},
|
||||||
|
|
||||||
[SysPreference.EnableTelemetry]: {
|
[SysPreference.EnableTelemetry]: {
|
||||||
name: 'Enable System Telemetry',
|
name: 'Enable System Telemetry',
|
||||||
helpText:
|
helpText:
|
||||||
'Enable system telemetry, this will send anonymous usage data to the developers.',
|
'Enable system telemetry, this will send anonymous usage data to the developers.',
|
||||||
category: 'Usage',
|
category: SysPreferenceCategory.Usage,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
|
@ -0,0 +1,6 @@
|
||||||
|
<h1>File Storage</h1>
|
||||||
|
|
||||||
|
|
||||||
|
<h2>Storage Provider Settings</h2>
|
||||||
|
|
||||||
|
<partial-sys-pref [show-titles]="false" [hidden-categories]="HiddenCategories"></partial-sys-pref>
|
|
@ -0,0 +1,11 @@
|
||||||
|
import { Component } from '@angular/core';
|
||||||
|
import { SysPreferenceCategory } from 'src/app/i18n/sys-pref.i18n';
|
||||||
|
|
||||||
|
@Component({
|
||||||
|
templateUrl: './settings-filestorage.component.html',
|
||||||
|
})
|
||||||
|
export class SettingsFileStorageComponent {
|
||||||
|
public readonly HiddenCategories = Object.values(
|
||||||
|
SysPreferenceCategory,
|
||||||
|
).filter((c) => c !== SysPreferenceCategory.FileStorage);
|
||||||
|
}
|
|
@ -0,0 +1,15 @@
|
||||||
|
import { CommonModule } from '@angular/common';
|
||||||
|
import { NgModule } from '@angular/core';
|
||||||
|
import { PartialSysPrefModule } from 'src/app/components/partial-sys-pref/partial-sys-pref.module';
|
||||||
|
import { SettingsFileStorageComponent } from './settings-filestorage.component';
|
||||||
|
import { SettingsFileStorageRoutingModule } from './settings-filestorage.routing.module';
|
||||||
|
|
||||||
|
@NgModule({
|
||||||
|
declarations: [SettingsFileStorageComponent],
|
||||||
|
imports: [
|
||||||
|
CommonModule,
|
||||||
|
SettingsFileStorageRoutingModule,
|
||||||
|
PartialSysPrefModule,
|
||||||
|
],
|
||||||
|
})
|
||||||
|
export default class SettingsFileStorageRouteModule {}
|
|
@ -0,0 +1,17 @@
|
||||||
|
import { NgModule } from '@angular/core';
|
||||||
|
import { RouterModule } from '@angular/router';
|
||||||
|
import { PRoutes } from 'src/app/models/dto/picsur-routes.dto';
|
||||||
|
import { SettingsFileStorageComponent } from './settings-filestorage.component';
|
||||||
|
|
||||||
|
const routes: PRoutes = [
|
||||||
|
{
|
||||||
|
path: '',
|
||||||
|
component: SettingsFileStorageComponent,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
@NgModule({
|
||||||
|
imports: [RouterModule.forChild(routes)],
|
||||||
|
exports: [RouterModule],
|
||||||
|
})
|
||||||
|
export class SettingsFileStorageRoutingModule {}
|
|
@ -80,6 +80,19 @@ const SettingsRoutes: PRoutes = [
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
path: 'filestorage',
|
||||||
|
loadChildren: () =>
|
||||||
|
import('./filestorage/settings-filestorage.module').then((m) => m.default),
|
||||||
|
data: {
|
||||||
|
permissions: [Permission.SysPrefAdmin],
|
||||||
|
page: {
|
||||||
|
title: 'Storage',
|
||||||
|
icon: 'storage',
|
||||||
|
category: 'system',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
path: 'system',
|
path: 'system',
|
||||||
loadChildren: () =>
|
loadChildren: () =>
|
||||||
|
@ -87,7 +100,7 @@ const SettingsRoutes: PRoutes = [
|
||||||
data: {
|
data: {
|
||||||
permissions: [Permission.SysPrefAdmin],
|
permissions: [Permission.SysPrefAdmin],
|
||||||
page: {
|
page: {
|
||||||
title: 'System Settings',
|
title: 'Settings',
|
||||||
icon: 'tune',
|
icon: 'tune',
|
||||||
category: 'system',
|
category: 'system',
|
||||||
},
|
},
|
||||||
|
|
|
@ -1,17 +1,3 @@
|
||||||
<h1>System Settings</h1>
|
<h1>System Settings</h1>
|
||||||
|
|
||||||
<ng-container *ngFor="let category of preferences | async">
|
<partial-sys-pref [hidden-categories]="HiddenCategories"></partial-sys-pref>
|
||||||
<h2 *ngIf="category.category !== null">{{ category.category }}</h2>
|
|
||||||
<div class="row">
|
|
||||||
<ng-container *ngFor="let pref of category.prefs">
|
|
||||||
<pref-option
|
|
||||||
class="col-md-6 col-12"
|
|
||||||
[pref]="pref"
|
|
||||||
[update]="sysPrefService.setPreference.bind(sysPrefService)"
|
|
||||||
[name]="getName(pref.key)"
|
|
||||||
[helpText]="getHelpText(pref.key)"
|
|
||||||
[validator]="getValidator(pref.key)"
|
|
||||||
></pref-option>
|
|
||||||
</ng-container>
|
|
||||||
</div>
|
|
||||||
</ng-container>
|
|
||||||
|
|
|
@ -1,54 +1,9 @@
|
||||||
import { Component } from '@angular/core';
|
import { Component } from '@angular/core';
|
||||||
import { DecodedPref } from 'picsur-shared/dist/dto/preferences.dto';
|
import { SysPreferenceCategory } from 'src/app/i18n/sys-pref.i18n';
|
||||||
import {
|
|
||||||
SysPreference,
|
|
||||||
SysPreferenceValidators,
|
|
||||||
} from 'picsur-shared/dist/dto/sys-preferences.enum';
|
|
||||||
import { map, Observable } from 'rxjs';
|
|
||||||
import { SysPreferenceUI } from 'src/app/i18n/sys-pref.i18n';
|
|
||||||
|
|
||||||
import { makeUnique } from 'picsur-shared/dist/util/unique';
|
|
||||||
import { SysPrefService } from 'src/app/services/api/sys-pref.service';
|
|
||||||
import { z, ZodTypeAny } from 'zod';
|
|
||||||
|
|
||||||
@Component({
|
@Component({
|
||||||
templateUrl: './settings-sys-pref.component.html',
|
templateUrl: './settings-sys-pref.component.html',
|
||||||
styleUrls: ['./settings-sys-pref.component.scss'],
|
|
||||||
})
|
})
|
||||||
export class SettingsSysprefComponent {
|
export class SettingsSysprefComponent {
|
||||||
public getName(key: string) {
|
public readonly HiddenCategories = [SysPreferenceCategory.FileStorage];
|
||||||
return SysPreferenceUI[key as SysPreference]?.name ?? key;
|
|
||||||
}
|
|
||||||
|
|
||||||
public getHelpText(key: string) {
|
|
||||||
return SysPreferenceUI[key as SysPreference]?.helpText ?? '';
|
|
||||||
}
|
|
||||||
|
|
||||||
public getCategory(key: string): null | string {
|
|
||||||
return SysPreferenceUI[key as SysPreference]?.category ?? null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public getValidator(key: string): ZodTypeAny {
|
|
||||||
return SysPreferenceValidators[key as SysPreference] ?? z.any();
|
|
||||||
}
|
|
||||||
|
|
||||||
preferences: Observable<
|
|
||||||
Array<{ category: string | null; prefs: DecodedPref[] }>
|
|
||||||
>;
|
|
||||||
|
|
||||||
constructor(public readonly sysPrefService: SysPrefService) {
|
|
||||||
this.preferences = sysPrefService.live.pipe(
|
|
||||||
map((prefs) => {
|
|
||||||
const categories = makeUnique(
|
|
||||||
prefs.map((pref) => this.getCategory(pref.key)),
|
|
||||||
);
|
|
||||||
return categories.map((category) => ({
|
|
||||||
category,
|
|
||||||
prefs: prefs.filter(
|
|
||||||
(pref) => this.getCategory(pref.key) === category,
|
|
||||||
),
|
|
||||||
}));
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,11 +1,11 @@
|
||||||
import { CommonModule } from '@angular/common';
|
import { CommonModule } from '@angular/common';
|
||||||
import { NgModule } from '@angular/core';
|
import { NgModule } from '@angular/core';
|
||||||
import { PrefOptionModule } from 'src/app/components/pref-option/pref-option.module';
|
import { PartialSysPrefModule } from 'src/app/components/partial-sys-pref/partial-sys-pref.module';
|
||||||
import { SettingsSysprefComponent } from './settings-sys-pref.component';
|
import { SettingsSysprefComponent } from './settings-sys-pref.component';
|
||||||
import { SettingsSysprefRoutingModule } from './settings-sys-pref.routing.module';
|
import { SettingsSysprefRoutingModule } from './settings-sys-pref.routing.module';
|
||||||
|
|
||||||
@NgModule({
|
@NgModule({
|
||||||
declarations: [SettingsSysprefComponent],
|
declarations: [SettingsSysprefComponent],
|
||||||
imports: [CommonModule, SettingsSysprefRoutingModule, PrefOptionModule],
|
imports: [CommonModule, SettingsSysprefRoutingModule, PartialSysPrefModule],
|
||||||
})
|
})
|
||||||
export default class SettingsSysprefRouteModule {}
|
export default class SettingsSysprefRouteModule {}
|
||||||
|
|
|
@ -3,7 +3,7 @@ import { WINDOW } from '@ng-web-apis/common';
|
||||||
import axios, {
|
import axios, {
|
||||||
AxiosRequestConfig,
|
AxiosRequestConfig,
|
||||||
AxiosResponse,
|
AxiosResponse,
|
||||||
AxiosResponseHeaders,
|
AxiosResponseHeaders
|
||||||
} from 'axios';
|
} from 'axios';
|
||||||
import { ApiResponseSchema } from 'picsur-shared/dist/dto/api/api.dto';
|
import { ApiResponseSchema } from 'picsur-shared/dist/dto/api/api.dto';
|
||||||
import { FileType2Ext } from 'picsur-shared/dist/dto/mimes.dto';
|
import { FileType2Ext } from 'picsur-shared/dist/dto/mimes.dto';
|
||||||
|
@ -13,7 +13,7 @@ import {
|
||||||
Failure,
|
Failure,
|
||||||
FT,
|
FT,
|
||||||
HasFailed,
|
HasFailed,
|
||||||
HasSuccess,
|
HasSuccess
|
||||||
} from 'picsur-shared/dist/types';
|
} from 'picsur-shared/dist/types';
|
||||||
import { ZodDtoStatic } from 'picsur-shared/dist/util/create-zod-dto';
|
import { ZodDtoStatic } from 'picsur-shared/dist/util/create-zod-dto';
|
||||||
import { ParseMime2FileType } from 'picsur-shared/dist/util/parse-mime';
|
import { ParseMime2FileType } from 'picsur-shared/dist/util/parse-mime';
|
||||||
|
@ -243,15 +243,13 @@ export class ApiService {
|
||||||
uploadProgress.next((e.loaded / (e.total ?? 1000000)) * 100);
|
uploadProgress.next((e.loaded / (e.total ?? 1000000)) * 100);
|
||||||
},
|
},
|
||||||
signal: abortController.signal,
|
signal: abortController.signal,
|
||||||
|
validateStatus: () => true,
|
||||||
...options,
|
...options,
|
||||||
});
|
});
|
||||||
|
|
||||||
uploadProgress.complete();
|
uploadProgress.complete();
|
||||||
downloadProgress.complete();
|
downloadProgress.complete();
|
||||||
|
|
||||||
if (result.status < 200 || result.status >= 300) {
|
|
||||||
return Fail(FT.Network, 'Recieved a non-ok response');
|
|
||||||
}
|
|
||||||
return result;
|
return result;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
return Fail(FT.Network, e);
|
return Fail(FT.Network, e);
|
||||||
|
|
|
@ -7,10 +7,11 @@
|
||||||
"frontend"
|
"frontend"
|
||||||
],
|
],
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"devdb:start": "docker-compose -f ./support/dev.docker-compose.yml up -d",
|
"devdb:up": "docker-compose -f ./support/dev.docker-compose.yml up -d",
|
||||||
"devdb:stop": "docker-compose -f ./support/dev.docker-compose.yml down",
|
"devdb:stop": "docker-compose -f ./support/dev.docker-compose.yml down",
|
||||||
"devdb:restart": "docker-compose -f ./support/dev.docker-compose.yml restart",
|
"devdb:restart": "docker-compose -f ./support/dev.docker-compose.yml restart",
|
||||||
"devdb:remove": "docker-compose -f ./support/dev.docker-compose.yml down --volumes",
|
"devdb:down": "docker-compose -f ./support/dev.docker-compose.yml down --volumes",
|
||||||
|
"update": "./support/actual-update.sh",
|
||||||
"build": "./support/build.sh",
|
"build": "./support/build.sh",
|
||||||
"setversion": "./support/setversion.sh",
|
"setversion": "./support/setversion.sh",
|
||||||
"purge": "rm -rf ./node_modules",
|
"purge": "rm -rf ./node_modules",
|
||||||
|
|
|
@ -19,6 +19,13 @@ export enum SysPreference {
|
||||||
ConversionTimeLimit = 'conversion_time_limit',
|
ConversionTimeLimit = 'conversion_time_limit',
|
||||||
ConversionMemoryLimit = 'conversion_memory_limit',
|
ConversionMemoryLimit = 'conversion_memory_limit',
|
||||||
|
|
||||||
|
FSLocalPath = 'fs_local_path',
|
||||||
|
FSS3Endpoint = 'fs_s3_endpoint',
|
||||||
|
FSS3Bucket = 'fs_s3_bucket',
|
||||||
|
FSS3Region = 'fs_s3_region',
|
||||||
|
FSS3AccessKey = 'fs_s3_access_key',
|
||||||
|
FSS3SecretKey = 'fs_s3_secret_key',
|
||||||
|
|
||||||
EnableTracking = 'enable_tracking',
|
EnableTracking = 'enable_tracking',
|
||||||
TrackingUrl = 'tracking_url',
|
TrackingUrl = 'tracking_url',
|
||||||
TrackingId = 'tracking_id',
|
TrackingId = 'tracking_id',
|
||||||
|
@ -45,6 +52,13 @@ export const SysPreferenceValueTypes: {
|
||||||
[SysPreference.ConversionTimeLimit]: 'string',
|
[SysPreference.ConversionTimeLimit]: 'string',
|
||||||
[SysPreference.ConversionMemoryLimit]: 'number',
|
[SysPreference.ConversionMemoryLimit]: 'number',
|
||||||
|
|
||||||
|
[SysPreference.FSLocalPath]: 'string',
|
||||||
|
[SysPreference.FSS3Endpoint]: 'string',
|
||||||
|
[SysPreference.FSS3Bucket]: 'string',
|
||||||
|
[SysPreference.FSS3Region]: 'string',
|
||||||
|
[SysPreference.FSS3AccessKey]: 'string',
|
||||||
|
[SysPreference.FSS3SecretKey]: 'string',
|
||||||
|
|
||||||
[SysPreference.EnableTracking]: 'boolean',
|
[SysPreference.EnableTracking]: 'boolean',
|
||||||
[SysPreference.TrackingUrl]: 'string',
|
[SysPreference.TrackingUrl]: 'string',
|
||||||
[SysPreference.TrackingId]: 'string',
|
[SysPreference.TrackingId]: 'string',
|
||||||
|
@ -67,6 +81,13 @@ export const SysPreferenceValidators: {
|
||||||
[SysPreference.ConversionTimeLimit]: IsValidMS(),
|
[SysPreference.ConversionTimeLimit]: IsValidMS(),
|
||||||
[SysPreference.ConversionMemoryLimit]: IsPosInt(),
|
[SysPreference.ConversionMemoryLimit]: IsPosInt(),
|
||||||
|
|
||||||
|
[SysPreference.FSLocalPath]: z.string(),
|
||||||
|
[SysPreference.FSS3Endpoint]: z.string().regex(URLRegex).or(z.literal('')),
|
||||||
|
[SysPreference.FSS3Bucket]: z.string(),
|
||||||
|
[SysPreference.FSS3Region]: z.string(),
|
||||||
|
[SysPreference.FSS3AccessKey]: z.string(),
|
||||||
|
[SysPreference.FSS3SecretKey]: z.string(),
|
||||||
|
|
||||||
[SysPreference.EnableTracking]: z.boolean(),
|
[SysPreference.EnableTracking]: z.boolean(),
|
||||||
[SysPreference.TrackingUrl]: z.string().regex(URLRegex).or(z.literal('')),
|
[SysPreference.TrackingUrl]: z.string().regex(URLRegex).or(z.literal('')),
|
||||||
[SysPreference.TrackingId]: IsEntityID().or(z.literal('')),
|
[SysPreference.TrackingId]: IsEntityID().or(z.literal('')),
|
||||||
|
|
|
@ -7,6 +7,7 @@
|
||||||
export enum FT {
|
export enum FT {
|
||||||
Unknown = 'unknown',
|
Unknown = 'unknown',
|
||||||
Database = 'database',
|
Database = 'database',
|
||||||
|
FileStorage = 'filestorage',
|
||||||
SysValidation = 'sysvalidation',
|
SysValidation = 'sysvalidation',
|
||||||
UsrValidation = 'usrvalidation',
|
UsrValidation = 'usrvalidation',
|
||||||
BadRequest = 'badrequest',
|
BadRequest = 'badrequest',
|
||||||
|
@ -51,6 +52,11 @@ const FTProps: {
|
||||||
code: 500,
|
code: 500,
|
||||||
message: 'A database error occurred',
|
message: 'A database error occurred',
|
||||||
},
|
},
|
||||||
|
[FT.FileStorage]: {
|
||||||
|
important: true,
|
||||||
|
code: 500,
|
||||||
|
message: 'A filestorage error occurred',
|
||||||
|
},
|
||||||
[FT.Network]: {
|
[FT.Network]: {
|
||||||
important: true,
|
important: true,
|
||||||
code: 500,
|
code: 500,
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
version: '3'
|
version: '3'
|
||||||
services:
|
services:
|
||||||
devdb:
|
devdb:
|
||||||
|
container_name: devdb
|
||||||
image: postgres:14-alpine
|
image: postgres:14-alpine
|
||||||
environment:
|
environment:
|
||||||
POSTGRES_DB: picsur
|
POSTGRES_DB: picsur
|
||||||
|
@ -11,6 +12,23 @@ services:
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
volumes:
|
volumes:
|
||||||
- db-data:/var/lib/postgresql/data
|
- db-data:/var/lib/postgresql/data
|
||||||
|
devs3:
|
||||||
|
image: zenko/cloudserver:latest
|
||||||
|
container_name: devs3
|
||||||
|
environment:
|
||||||
|
S3BACKEND: file
|
||||||
|
S3DATAPATH: /storage/data/
|
||||||
|
S3METADATAPATH: /storage/metadata/
|
||||||
|
SCALITY_ACCESS_KEY_ID: username
|
||||||
|
SCALITY_SECRET_ACCESS_KEY: password
|
||||||
|
REMOTE_MANAGEMENT_DISABLE: 'true'
|
||||||
|
ports:
|
||||||
|
- '8000:8000'
|
||||||
|
restart: unless-stopped
|
||||||
|
volumes:
|
||||||
|
- s3-data:/storage/data
|
||||||
|
- s3-metadata:/storage/metadata
|
||||||
volumes:
|
volumes:
|
||||||
db-data:
|
db-data:
|
||||||
|
s3-data:
|
||||||
|
s3-metadata:
|
||||||
|
|
Loading…
Reference in a new issue