Add usage reporting

This commit is contained in:
rubikscraft 2022-12-25 23:36:42 +01:00
parent 286333f598
commit a750d65baa
No known key found for this signature in database
GPG Key ID: 1463EBE9200A5CD4
21 changed files with 402 additions and 54 deletions

View File

@ -39,7 +39,9 @@
"bmp-img": "^1.2.1",
"cors": "^2.8.5",
"file-type": "^18.0.0",
"is-docker": "^3.0.0",
"ms": "^2.1.3",
"node-fetch": "^3.2.10",
"p-timeout": "^6.0.0",
"passport": "^0.6.0",
"passport-headerapikey": "^1.2.2",

View File

@ -9,6 +9,7 @@ import { PicsurLayersModule } from './layers/PicsurLayers.module';
import { PicsurLoggerModule } from './logger/logger.module';
import { AuthManagerModule } from './managers/auth/auth.module';
import { DemoManagerModule } from './managers/demo/demo.module';
import { UsageManagerModule } from './managers/usage/usage.module';
import { PicsurRoutesModule } from './routes/routes.module';
const mainCorsConfig = cors({
@ -44,6 +45,7 @@ const imageCorsOverride = (
}),
DatabaseModule,
AuthManagerModule,
UsageManagerModule,
DemoManagerModule,
PicsurRoutesModule,
PicsurLayersModule,

View File

@ -83,6 +83,14 @@ export class ImageDBService {
}
}
public async count(): AsyncFailable<number> {
try {
return await this.imageRepo.count();
} catch (e) {
return Fail(FT.Database, e);
}
}
public async update(
id: string,
userid: string | undefined,

View File

@ -0,0 +1,11 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { ESystemStateBackend } from '../../database/entities/system-state.entity';
import { SystemStateDbService } from './system-state-db.service';
@Module({
imports: [TypeOrmModule.forFeature([ESystemStateBackend])],
providers: [SystemStateDbService],
exports: [SystemStateDbService],
})
export class SystemStateDbModule {}

View File

@ -0,0 +1,42 @@
import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { AsyncFailable, Fail, FT } from 'picsur-shared/dist/types';
import { Repository } from 'typeorm';
import { ESystemStateBackend } from '../../database/entities/system-state.entity';
@Injectable()
export class SystemStateDbService {
private readonly logger = new Logger(SystemStateDbService.name);
constructor(
@InjectRepository(ESystemStateBackend)
private readonly stateRepo: Repository<ESystemStateBackend>,
) {}
async get(key: string): AsyncFailable<string | null> {
try {
const state = await this.stateRepo.findOne({ where: { key } });
return state?.value ?? null;
} catch (err) {
return Fail(FT.Database, err);
}
}
async set(key: string, value: string): AsyncFailable<true> {
try {
await this.stateRepo.save({ key, value });
return true;
} catch (err) {
return Fail(FT.Database, err);
}
}
async clear(key: string): AsyncFailable<true> {
try {
await this.stateRepo.delete({ key });
return true;
} catch (err) {
return Fail(FT.Database, err);
}
}
}

View File

@ -7,7 +7,7 @@ import {
Fail,
FT,
HasFailed,
HasSuccess,
HasSuccess
} from 'picsur-shared/dist/types';
import { FindResult } from 'picsur-shared/dist/types/find-result';
import { makeUnique } from 'picsur-shared/dist/util/unique';
@ -16,12 +16,12 @@ import { EUserBackend } from '../../database/entities/user.entity';
import { Permissions } from '../../models/constants/permissions.const';
import {
DefaultRolesList,
SoulBoundRolesList,
SoulBoundRolesList
} from '../../models/constants/roles.const';
import {
ImmutableUsersList,
LockedLoginUsersList,
UndeletableUsersList,
UndeletableUsersList
} from '../../models/constants/special-users.const';
import { GetCols } from '../../util/collection';
import { SysPreferenceDbService } from '../preference-db/sys-preference-db.service';
@ -263,6 +263,14 @@ export class UserDbService {
}
}
public async count(): AsyncFailable<number> {
try {
return await this.usersRepository.count();
} catch (e) {
return Fail(FT.Database, e);
}
}
public async exists(username: string): Promise<boolean> {
return HasSuccess(await this.findByUsername(username));
}

View File

@ -1,6 +1,8 @@
import { dirname, resolve } from 'path';
import { fileURLToPath } from 'url';
export const ReportUrl = 'https://metrics.picsur.org';
export const ReportInterval = 1000 * 60 * 60;
export const EnvPrefix = 'PICSUR_';
export const DefaultName = 'picsur';

View File

@ -5,6 +5,7 @@ import { EarlyConfigModule } from '../early/early-config.module';
import { EarlyJwtConfigService } from '../early/early-jwt.config.service';
import { InfoConfigService } from './info.config.service';
import { JwtConfigService } from './jwt.config.service';
import { UsageConfigService } from './usage.config.service';
// This module contains all configservices that depend on the syspref module
// The syspref module can only be used when connected to the database
@ -13,8 +14,8 @@ import { JwtConfigService } from './jwt.config.service';
@Module({
imports: [EarlyConfigModule, PreferenceDbModule],
providers: [JwtConfigService, InfoConfigService],
exports: [EarlyConfigModule, JwtConfigService, InfoConfigService],
providers: [JwtConfigService, InfoConfigService, UsageConfigService],
exports: [EarlyConfigModule, JwtConfigService, InfoConfigService, UsageConfigService],
})
export class LateConfigModule implements OnModuleInit {
private readonly logger = new Logger(LateConfigModule.name);

View File

@ -0,0 +1,53 @@
import { Injectable } from '@nestjs/common';
import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
import { AsyncFailable, Fail, FT, HasFailed } from 'picsur-shared/dist/types';
import { URLRegex, UUIDRegex } from 'picsur-shared/dist/util/common-regex';
import { SysPreferenceDbService } from '../../collections/preference-db/sys-preference-db.service';
import { ReportInterval, ReportUrl } from '../config.static';
@Injectable()
export class UsageConfigService {
constructor(private readonly sysPref: SysPreferenceDbService) {}
async getTrackingUrl(): AsyncFailable<string | null> {
const trackingUrl = await this.sysPref.getStringPreference(
SysPreference.TrackingUrl,
);
if (HasFailed(trackingUrl)) return trackingUrl;
if (trackingUrl === '') return null;
if (!URLRegex.test(trackingUrl)) {
return Fail(FT.UsrValidation, undefined, 'Invalid tracking URL');
}
return trackingUrl;
}
async getTrackingID(): AsyncFailable<string | null> {
const trackingID = await this.sysPref.getStringPreference(
SysPreference.TrackingId,
);
if (HasFailed(trackingID)) return trackingID;
if (trackingID === '') return null;
if (!UUIDRegex.test(trackingID)) {
return Fail(FT.UsrValidation, undefined, 'Invalid tracking ID');
}
return trackingID;
}
async getMetricsEnabled(): AsyncFailable<boolean> {
return this.sysPref.getBooleanPreference(SysPreference.EnableTelemetry);
}
async getMetricsInterval(): Promise<number> {
return ReportInterval;
}
async getMetricsUrl(): Promise<string> {
return ReportUrl;
}
}

View File

@ -4,6 +4,7 @@ import { EImageFileBackend } from './image-file.entity';
import { EImageBackend } from './image.entity';
import { ERoleBackend } from './role.entity';
import { ESysPreferenceBackend } from './sys-preference.entity';
import { ESystemStateBackend } from './system-state.entity';
import { EUserBackend } from './user.entity';
import { EUsrPreferenceBackend } from './usr-preference.entity';
@ -16,4 +17,5 @@ export const EntityList = [
ESysPreferenceBackend,
EUsrPreferenceBackend,
EApiKeyBackend,
ESystemStateBackend,
];

View File

@ -0,0 +1,14 @@
import { Column, Entity, Index, PrimaryGeneratedColumn } from 'typeorm';
@Entity()
export class ESystemStateBackend {
@PrimaryGeneratedColumn('uuid')
id?: string;
@Index()
@Column({ nullable: false, unique: true })
key: string;
@Column({ nullable: false })
value: string;
}

View File

@ -1,13 +1,12 @@
import fastifyHelmet from '@fastify/helmet';
import multipart from '@fastify/multipart';
import fastifyReplyFrom from '@fastify/reply-from';
import { NestFactory, Reflector } from '@nestjs/core';
import { NestFactory } from '@nestjs/core';
import {
FastifyAdapter,
NestFastifyApplication,
NestFastifyApplication
} from '@nestjs/platform-fastify';
import { AppModule } from './app.module';
import { UserDbService } from './collections/user-db/user-db.service';
import { HostConfigService } from './config/early/host.config.service';
import { MainExceptionFilter } from './layers/exception/exception.filter';
import { SuccessInterceptor } from './layers/success/success.interceptor';

View File

@ -1,10 +1,42 @@
import { Module } from '@nestjs/common';
import { PreferenceDbModule } from '../../collections/preference-db/preference-db.module';
import { Logger, Module, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import { ImageDBModule } from '../../collections/image-db/image-db.module';
import { SystemStateDbModule } from '../../collections/system-state-db/system-state-db.module';
import { UserDbModule } from '../../collections/user-db/user-db.module';
import { LateConfigModule } from '../../config/late/late-config.module';
import { UsageConfigService } from '../../config/late/usage.config.service';
import { UsageService } from './usage.service';
@Module({
imports: [PreferenceDbModule],
imports: [LateConfigModule, SystemStateDbModule, ImageDBModule, UserDbModule],
providers: [UsageService],
exports: [UsageService],
})
export class UsageManagerModule {}
export class UsageManagerModule implements OnModuleInit, OnModuleDestroy {
private readonly logger = new Logger(UsageManagerModule.name);
private interval: NodeJS.Timeout;
constructor(
private readonly usageService: UsageService,
private readonly usageConfigService: UsageConfigService,
) {}
async onModuleInit() {
if (!(await this.usageConfigService.getMetricsEnabled())) {
this.logger.log('Telemetry is disabled');
}
this.interval = setInterval(() => {
this.usageService.execute().catch((err) => {
this.logger.warn(err);
});
}, await this.usageConfigService.getMetricsInterval());
this.usageService.execute().catch((err) => {
this.logger.warn(err);
});
}
onModuleDestroy() {
if (this.interval) clearInterval(this.interval);
}
}

View File

@ -1,47 +1,158 @@
import { Inject, Injectable } from '@nestjs/common';
import exp from 'constants';
import { SysPreference } from 'picsur-shared/dist/dto/sys-preferences.enum';
import {
AsyncFailable,
Fail,
Failable,
FT,
HasFailed,
} from 'picsur-shared/dist/types';
import { URLRegex, UUIDRegex } from 'picsur-shared/dist/util/common-regex';
import { SysPreferenceDbService } from '../../collections/preference-db/sys-preference-db.service';
import { Injectable, Logger } from '@nestjs/common';
import isDocker from 'is-docker';
import fetch from 'node-fetch';
import os from 'os';
import { FallbackIfFailed, HasFailed } from 'picsur-shared/dist/types';
import { UUIDRegex } from 'picsur-shared/dist/util/common-regex';
import { ImageDBService } from '../../collections/image-db/image-db.service';
import { SystemStateDbService } from '../../collections/system-state-db/system-state-db.service';
import { UserDbService } from '../../collections/user-db/user-db.service';
import { HostConfigService } from '../../config/early/host.config.service';
import { UsageConfigService } from '../../config/late/usage.config.service';
interface UsageData {
id?: string;
uptime: number;
version: string;
demo_active: boolean;
users: number;
images: number;
architecture: string;
cpu_count: number;
ram_total: number;
is_docker: boolean;
is_production: boolean;
}
@Injectable()
export class UsageService {
constructor(private readonly sysPref: SysPreferenceDbService) {}
private readonly logger = new Logger(UsageService.name);
async getTrackingUrl(): AsyncFailable<string | null> {
const trackingUrl = await this.sysPref.getStringPreference(
SysPreference.TrackingUrl,
);
if (HasFailed(trackingUrl)) return trackingUrl;
constructor(
private readonly systemState: SystemStateDbService,
private readonly hostConfig: HostConfigService,
private readonly usageConfig: UsageConfigService,
private readonly userRepo: UserDbService,
private readonly imageRepo: ImageDBService,
) {}
if (trackingUrl === '') return null;
public async execute() {
if (!(await this.usageConfig.getMetricsEnabled())) return;
if (!URLRegex.test(trackingUrl)) {
return Fail(FT.UsrValidation, undefined, 'Invalid tracking URL');
const id = await this.getSystemID();
if (id === null) {
await this.sendInitialData();
} else {
await this.sendUpdateData(id);
}
return trackingUrl;
}
async getTrackingID(): AsyncFailable<string | null> {
const trackingID = await this.sysPref.getStringPreference(
SysPreference.TrackingId,
);
if (HasFailed(trackingID)) return trackingID;
private async sendInitialData() {
const url =
(await this.usageConfig.getMetricsUrl()) + '/api/install/create';
if (trackingID === '') return null;
const result: any = await fetch(url, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(await this.collectData()),
}).then((res) => res.json());
if (!UUIDRegex.test(trackingID)) {
return Fail(FT.UsrValidation, undefined, 'Invalid tracking ID');
const id = result?.data?.id;
if (typeof id !== 'string')
return this.logger.warn(
'Invalid response when sending initial data: ' + JSON.stringify(result),
);
if (!UUIDRegex.test(id))
return this.logger.warn('Invalid system ID: ' + id);
await this.setSystemID(id);
}
private async sendUpdateData(id: string) {
const url =
(await this.usageConfig.getMetricsUrl()) + '/api/install/update';
const body = await this.collectData();
body.id = id;
const result = await fetch(url, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(body),
});
if (result.status < 200 || result.status >= 300) {
const data: any = await result.json();
if (data?.type === 'notfound') {
this.logger.warn('System ID not found, clearing');
await this.clearSystemID();
} else {
this.logger.warn(
'Failed to send update data: ' + JSON.stringify(await result.json()),
);
}
}
}
return trackingID;
private async getSystemID(): Promise<string | null> {
const result = await this.systemState.get('systemID');
if (HasFailed(result)) {
this.logger.warn(result);
return null;
}
if (result === null) return null;
if (UUIDRegex.test(result)) return result;
this.logger.warn('Invalid system ID');
return null;
}
private async setSystemID(id: string) {
if (!UUIDRegex.test(id)) {
return this.logger.warn('Invalid system ID');
}
const result = await this.systemState.set('systemID', id);
if (HasFailed(result)) {
this.logger.warn(result);
}
}
private async clearSystemID() {
const result = await this.systemState.clear('systemID');
if (HasFailed(result)) {
this.logger.warn(result);
}
}
private async collectData(): Promise<UsageData> {
const users = FallbackIfFailed(await this.userRepo.count(), 0, this.logger);
const images = FallbackIfFailed(
await this.imageRepo.count(),
0,
this.logger,
);
const data: UsageData = {
uptime: Math.floor(process.uptime()),
version: this.hostConfig.getVersion(),
demo_active: this.hostConfig.isDemo(),
users,
images,
architecture: process.arch,
cpu_count: os.cpus().length,
ram_total: Math.floor(os.totalmem() / 1024 / 1024),
is_docker: isDocker(),
is_production: this.hostConfig.isProduction(),
};
return data;
}
}

View File

@ -14,9 +14,9 @@ import { TrackingState } from 'picsur-shared/dist/dto/tracking-state.enum';
import { FallbackIfFailed } from 'picsur-shared/dist/types';
import { HostConfigService } from '../../../config/early/host.config.service';
import { InfoConfigService } from '../../../config/late/info.config.service';
import { UsageConfigService } from '../../../config/late/usage.config.service';
import { NoPermissions } from '../../../decorators/permissions.decorator';
import { Returns } from '../../../decorators/returns.decorator';
import { UsageService } from '../../../managers/usage/usage.service';
import { PermissionsList } from '../../../models/constants/permissions.const';
@Controller('api/info')
@ -25,7 +25,7 @@ export class InfoController {
constructor(
private readonly hostConfig: HostConfigService,
private readonly infoConfig: InfoConfigService,
private readonly usageService: UsageService,
private readonly usageService: UsageConfigService,
) {}
@Get()

View File

@ -1,10 +1,9 @@
import { Module } from '@nestjs/common';
import { LateConfigModule } from '../../../config/late/late-config.module';
import { UsageManagerModule } from '../../../managers/usage/usage.module';
import { InfoController } from './info.controller';
@Module({
imports: [LateConfigModule, UsageManagerModule],
imports: [LateConfigModule],
controllers: [InfoController],
})
export class InfoModule {}

View File

@ -1,16 +1,16 @@
import { Controller, Logger, Post, Req, Res } from '@nestjs/common';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { Fail, FT, ThrowIfFailed } from 'picsur-shared/dist/types';
import { UsageConfigService } from '../../../config/late/usage.config.service';
import { NoPermissions } from '../../../decorators/permissions.decorator';
import { ReturnsAnything } from '../../../decorators/returns.decorator';
import { UsageService } from '../../../managers/usage/usage.service';
@Controller('api/usage')
@NoPermissions()
export class UsageController {
private readonly logger = new Logger(UsageController.name);
constructor(private readonly usageService: UsageService) {}
constructor(private readonly usageService: UsageConfigService) {}
@Post(['report', 'report/*'])
@ReturnsAnything()

View File

@ -1,9 +1,9 @@
import { Module } from '@nestjs/common';
import { UsageManagerModule } from '../../../managers/usage/usage.module';
import { LateConfigModule } from '../../../config/late/late-config.module';
import { UsageController } from './usage.controller';
@Module({
imports: [UsageManagerModule],
imports: [LateConfigModule],
controllers: [UsageController],
})
export class UsageApiModule {}

View File

@ -10,7 +10,7 @@
"devdb:start": "docker-compose -f ./support/dev.docker-compose.yml up -d",
"devdb:stop": "docker-compose -f ./support/dev.docker-compose.yml down",
"devdb:restart": "docker-compose -f ./support/dev.docker-compose.yml restart",
"devdb:remove": "docker-compose -f ./support/dev.docker-compose.yml down --rm all --volumes",
"devdb:remove": "docker-compose -f ./support/dev.docker-compose.yml down --volumes",
"build": "./support/build.sh",
"setversion": "./support/setversion.sh",
"purge": "rm -rf ./node_modules",

View File

@ -59,7 +59,7 @@ export const SysPreferenceValidators: {
} = {
[SysPreference.HostOverride]: z.string().regex(URLRegex).or(z.literal('')),
[SysPreference.JwtSecret]: z.boolean(),
[SysPreference.JwtSecret]: z.string(),
[SysPreference.JwtExpiresIn]: IsValidMS(),
[SysPreference.BCryptStrength]: IsPosInt(),

View File

@ -5289,6 +5289,13 @@ __metadata:
languageName: node
linkType: hard
"data-uri-to-buffer@npm:^4.0.0":
version: 4.0.0
resolution: "data-uri-to-buffer@npm:4.0.0"
checksum: a010653869abe8bb51259432894ac62c52bf79ad761d418d94396f48c346f2ae739c46b254e8bb5987bded8a653d467db1968db3a69bab1d33aa5567baa5cfc7
languageName: node
linkType: hard
"date-fns@npm:^2.28.0":
version: 2.28.0
resolution: "date-fns@npm:2.28.0"
@ -6377,6 +6384,16 @@ __metadata:
languageName: node
linkType: hard
"fetch-blob@npm:^3.1.2, fetch-blob@npm:^3.1.4":
version: 3.2.0
resolution: "fetch-blob@npm:3.2.0"
dependencies:
node-domexception: ^1.0.0
web-streams-polyfill: ^3.0.3
checksum: f19bc28a2a0b9626e69fd7cf3a05798706db7f6c7548da657cbf5026a570945f5eeaedff52007ea35c8bcd3d237c58a20bf1543bc568ab2422411d762dd3d5bf
languageName: node
linkType: hard
"figures@npm:^3.0.0":
version: 3.2.0
resolution: "figures@npm:3.2.0"
@ -6533,6 +6550,15 @@ __metadata:
languageName: node
linkType: hard
"formdata-polyfill@npm:^4.0.10":
version: 4.0.10
resolution: "formdata-polyfill@npm:4.0.10"
dependencies:
fetch-blob: ^3.1.2
checksum: 82a34df292afadd82b43d4a740ce387bc08541e0a534358425193017bf9fb3567875dc5f69564984b1da979979b70703aa73dee715a17b6c229752ae736dd9db
languageName: node
linkType: hard
"forwarded@npm:0.2.0":
version: 0.2.0
resolution: "forwarded@npm:0.2.0"
@ -7349,6 +7375,15 @@ __metadata:
languageName: node
linkType: hard
"is-docker@npm:^3.0.0":
version: 3.0.0
resolution: "is-docker@npm:3.0.0"
bin:
is-docker: cli.js
checksum: b698118f04feb7eaf3338922bd79cba064ea54a1c3db6ec8c0c8d8ee7613e7e5854d802d3ef646812a8a3ace81182a085dfa0a71cc68b06f3fa794b9783b3c90
languageName: node
linkType: hard
"is-extglob@npm:^2.1.1":
version: 2.1.1
resolution: "is-extglob@npm:2.1.1"
@ -8539,6 +8574,13 @@ __metadata:
languageName: node
linkType: hard
"node-domexception@npm:^1.0.0":
version: 1.0.0
resolution: "node-domexception@npm:1.0.0"
checksum: ee1d37dd2a4eb26a8a92cd6b64dfc29caec72bff5e1ed9aba80c294f57a31ba4895a60fd48347cf17dd6e766da0ae87d75657dfd1f384ebfa60462c2283f5c7f
languageName: node
linkType: hard
"node-emoji@npm:1.11.0":
version: 1.11.0
resolution: "node-emoji@npm:1.11.0"
@ -8562,6 +8604,17 @@ __metadata:
languageName: node
linkType: hard
"node-fetch@npm:^3.2.10":
version: 3.2.10
resolution: "node-fetch@npm:3.2.10"
dependencies:
data-uri-to-buffer: ^4.0.0
fetch-blob: ^3.1.4
formdata-polyfill: ^4.0.10
checksum: e65322431f4897ded04197aa5923eaec63a8d53e00432de4e70a4f7006625c8dc32629c5c35f4fe8ee719a4825544d07bf53f6e146a7265914262f493e8deac1
languageName: node
linkType: hard
"node-forge@npm:^1":
version: 1.3.1
resolution: "node-forge@npm:1.3.1"
@ -9390,7 +9443,9 @@ __metadata:
eslint-config-prettier: ^8.5.0
eslint-plugin-prettier: ^4.2.1
file-type: ^18.0.0
is-docker: ^3.0.0
ms: ^2.1.3
node-fetch: ^3.2.10
p-timeout: ^6.0.0
passport: ^0.6.0
passport-headerapikey: ^1.2.2
@ -11773,6 +11828,13 @@ __metadata:
languageName: node
linkType: hard
"web-streams-polyfill@npm:^3.0.3":
version: 3.2.1
resolution: "web-streams-polyfill@npm:3.2.1"
checksum: b119c78574b6d65935e35098c2afdcd752b84268e18746606af149e3c424e15621b6f1ff0b42b2676dc012fc4f0d313f964b41a4b5031e525faa03997457da02
languageName: node
linkType: hard
"webidl-conversions@npm:^3.0.0":
version: 3.0.1
resolution: "webidl-conversions@npm:3.0.1"