From 3fcdeeb22e69a94979e976f6bec18413184fc5e5 Mon Sep 17 00:00:00 2001 From: Derock Date: Sun, 12 Nov 2023 09:49:10 -0500 Subject: [PATCH] feat: stat graphs --- .gitignore | 1 + logs/combined.log | 95 -------------------------- logs/error.log | 0 src/app/home/StatCard.tsx | 4 +- src/app/home/SystemStatistics.tsx | 44 ++++++------ src/app/home/page.tsx | 10 ++- src/server/api/routers/system/index.ts | 6 ++ src/server/db/index.ts | 37 ++++++---- src/server/db/schema.ts | 18 +++++ src/server/modules/stats/index.ts | 72 +++++++++++++++---- src/server/server.ts | 6 +- src/server/utils/logger.ts | 2 + 12 files changed, 146 insertions(+), 149 deletions(-) delete mode 100644 logs/combined.log delete mode 100644 logs/error.log diff --git a/.gitignore b/.gitignore index 333130b..f30ffd7 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ # database /data/ +/logs/ # next.js /.next/ diff --git a/logs/combined.log b/logs/combined.log deleted file mode 100644 index 17329b7..0000000 --- a/logs/combined.log +++ /dev/null @@ -1,95 +0,0 @@ -2023-11-12T01:48:57.462Z info: Not running database migrations, use drizzle-kit push to migrate -2023-11-12T01:48:57.463Z info: 🚀 Hostforge v1.0.0 ready! -2023-11-12T01:48:57.469Z info: Server listening on port 3000 -2023-11-12T01:49:11.893Z warn: SIGTERM received, shutting down... -2023-11-12T01:49:17.625Z info: Not running database migrations, use drizzle-kit push to migrate -2023-11-12T01:49:17.627Z info: 🚀 Hostforge v1.0.0 ready! -2023-11-12T01:49:17.635Z info: Server listening on port 3000 -2023-11-12T01:50:31.777Z warn: SIGTERM received, shutting down... -{"level":"info","message":"Not running database migrations, use drizzle-kit push to migrate","module":"database"} -{"level":"info","message":"🚀 Hostforge v1.0.0 ready!"} -{"level":"info","message":"Server listening on port 3000"} -{"level":"warn","message":"SIGTERM received, shutting down..."} -{"level":"info","message":"Not running database migrations, use drizzle-kit push to migrate","module":"database"} -{"level":"info","message":"🚀 Hostforge v1.0.0 ready!"} -{"level":"info","message":"Server listening on port 3000"} -{"level":"warn","message":"SIGTERM received, shutting down..."} -{"level":"info","message":"Not running database migrations, use drizzle-kit push to migrate","module":"database"} -{"level":"info","message":"🚀 Hostforge v1.0.0 ready!"} -{"level":"info","message":"Server listening on port 3000"} -{"level":"warn","message":"SIGTERM received, shutting down..."} -{"level":"info","message":"Not running database migrations, use drizzle-kit push to migrate","module":"database"} -{"level":"info","message":"🚀 Hostforge v1.0.0 ready!"} -{"level":"info","message":"Server listening on port 3000"} -{"level":"warn","message":"SIGTERM received, shutting down..."} -{"level":"info","message":"Not running database migrations, use drizzle-kit push to migrate","module":"database"} -{"level":"info","message":"🚀 Hostforge v1.0.0 ready!"} -{"level":"info","message":"Server listening on port 3000"} -{"level":"warn","message":"SIGTERM received, shutting down..."} -{"level":"info","message":"Not running database migrations, use drizzle-kit push to migrate","module":"database"} -{"level":"info","message":"🚀 Hostforge v1.0.0 ready!"} -{"level":"info","message":"Server listening on port 3000"} -{"level":"warn","message":"SIGTERM received, shutting down..."} -{"level":"info","message":"Not running database migrations, use drizzle-kit push to migrate","module":"database"} -{"level":"info","message":"🚀 Hostforge v1.0.0 ready!"} -{"level":"info","message":"Server listening on port 3000"} -{"level":"warn","message":"SIGTERM received, shutting down..."} -{"level":"info","message":"Not running database migrations, use drizzle-kit push to migrate","module":"database"} -{"level":"info","message":"🚀 Hostforge v1.0.0 ready!"} -{"level":"info","message":"Server listening on port 3000"} -{"level":"warn","message":"SIGTERM received, shutting down..."} -{"level":"info","message":"Not running database migrations, use drizzle-kit push to migrate","module":"database"} -{"level":"info","message":"🚀 Hostforge v1.0.0 ready!"} -{"level":"info","message":"Server listening on port 3000"} -{"level":"warn","message":"SIGTERM received, shutting down..."} -{"level":"info","message":"Not running database migrations, use drizzle-kit push to migrate","module":"database"} -{"level":"info","message":"🚀 Hostforge v1.0.0 ready!"} -{"level":"info","message":"Server listening on port 3000"} -{"level":"warn","message":"SIGTERM received, shutting down..."} -{"level":"info","message":"Not running database migrations, use drizzle-kit push to migrate","module":"database"} -{"level":"info","message":"🚀 Hostforge v1.0.0 ready!"} -{"level":"info","message":"Server listening on port 3000"} -{"level":"warn","message":"SIGTERM received, shutting down..."} -{"level":"info","message":"Not running database migrations, use drizzle-kit push to migrate","module":"database"} -{"level":"info","message":"🚀 Hostforge v1.0.0 ready!"} -{"level":"info","message":"Server listening on port 3000"} -{"level":"warn","message":"SIGTERM received, shutting down..."} -{"level":"info","message":"Not running database migrations, use drizzle-kit push to migrate","module":"database"} -{"level":"info","message":"Not running database migrations, use drizzle-kit push to migrate","module":"database"} -{"level":"info","message":"🚀 Hostforge v1.0.0 ready!"} -{"level":"info","message":"Server listening on port 3000"} -{"level":"warn","message":"SIGTERM received, shutting down..."} -{"level":"info","message":"Not running database migrations, use drizzle-kit push to migrate","module":"database"} -{"level":"info","message":"Not running database migrations, use drizzle-kit push to migrate","module":"database"} -{"level":"info","message":"🚀 Hostforge"} -{"level":"info","message":"Server listening on localhost:3000"} -{"level":"info","message":"Version: 0.1.0"} -{"level":"info","message":"Environment: development"} -{"level":"info","message":"Build commit: unknown"} -{"level":"warn","message":"SIGTERM received, shutting down..."} -{"level":"info","message":"Not running database migrations, use drizzle-kit push to migrate","module":"database"} -{"level":"info","message":"🚀 Hostforge"} -{"level":"info","message":" │ Server listening on localhost:3000"} -{"level":"info","message":" │ Version: 0.1.0"} -{"level":"info","message":" │ Environment: development"} -{"level":"info","message":" ╰ Build commit: unknown"} -{"level":"warn","message":"SIGTERM received, shutting down..."} -{"level":"info","message":"Not running database migrations, use drizzle-kit push to migrate","module":"database"} -{"level":"info","message":"🚀 Hostforge"} -{"level":"info","message":"│ Server listening on localhost:3000"} -{"level":"info","message":"│ Version: 0.1.0"} -{"level":"info","message":"│ Environment: development"} -{"level":"info","message":"╰ Build commit: unknown"} -{"level":"warn","message":"SIGTERM received, shutting down..."} -{"level":"info","message":"Not running database migrations, use drizzle-kit push to migrate","module":"database"} -{"level":"info","message":"🚀 Hostforge"} -{"level":"info","message":"│ Server listening on localhost:3000"} -{"level":"info","message":"│ Version: 0.1.0"} -{"level":"info","message":"│ Environment: development"} -{"level":"info","message":"╰ Build commit: unknown"} -{"level":"info","message":"Not running database migrations, use drizzle-kit push to migrate","module":"database"} -{"level":"info","message":"🚀 Hostforge"} -{"level":"info","message":"│ Server listening on 0.0.0.0:3000"} -{"level":"info","message":"│ Version: 0.1.0"} -{"level":"info","message":"│ Environment: development"} -{"level":"info","message":"╰ Build commit: unknown"} diff --git a/logs/error.log b/logs/error.log deleted file mode 100644 index e69de29..0000000 diff --git a/src/app/home/StatCard.tsx b/src/app/home/StatCard.tsx index 71c07bb..73d5cdb 100644 --- a/src/app/home/StatCard.tsx +++ b/src/app/home/StatCard.tsx @@ -5,7 +5,7 @@ import { ResponsiveContainer, AreaChart, Area } from "recharts"; import styles from "./StatCard.module.css"; import { AnimatedNumber } from "~/components/AnimatedPercent"; -export function StatCard>(props: { +export function StatCard>(props: { title: string; icon: React.FC<{ className: string }>; @@ -35,7 +35,7 @@ export function StatCard>(props: {
(props.initialData); api.system.liveStats.useSubscription(undefined, { @@ -33,6 +26,15 @@ export function SystemStatistics(props: { initialData: StatData }) { }, }); + const historicalData = useMemo( + () => + props.historicalData.map((data) => ({ + ...data, + network: data.networkTx + data.networkRx, + })), + [props.historicalData], + ); + return (
); diff --git a/src/app/home/page.tsx b/src/app/home/page.tsx index 1c63e6b..c1058fd 100644 --- a/src/app/home/page.tsx +++ b/src/app/home/page.tsx @@ -4,12 +4,18 @@ import { StatCard } from "./StatCard"; import { SystemStatistics } from "./SystemStatistics"; export default async function DashboardHome() { - const initialStats = await api.system.currentStats.query(); + const [initialStats, historicalData] = await Promise.all([ + api.system.currentStats.query(), + api.system.history.query(), + ]); return (
- +
); } diff --git a/src/server/api/routers/system/index.ts b/src/server/api/routers/system/index.ts index ff0e0cc..c52644b 100644 --- a/src/server/api/routers/system/index.ts +++ b/src/server/api/routers/system/index.ts @@ -17,4 +17,10 @@ export const systemRouter = createTRPCRouter({ }; }); }), + + history: authenticatedProcedure.query(async ({ ctx }) => { + return await stats.getStatsInRange( + new Date(Date.now() - 1000 * 60 * 60 * 24), + ); + }), }); diff --git a/src/server/db/index.ts b/src/server/db/index.ts index 9ab0112..928f5da 100644 --- a/src/server/db/index.ts +++ b/src/server/db/index.ts @@ -2,21 +2,30 @@ import { drizzle } from "drizzle-orm/better-sqlite3"; import SQLite3 from "better-sqlite3"; import { join } from "path"; import { env } from "~/env"; +import logger from "../utils/logger"; -const sqlite = new SQLite3(env.DATABASE_PATH); +const globalForDB = globalThis as unknown as { + db: ReturnType; +}; -// enable WAL mode -sqlite.pragma("journal_mode = WAL"); +function createDatabaseInstance() { + logger.child({ module: "database" }).debug("Creating database client."); + const sqlite = new SQLite3(env.DATABASE_PATH); -// load uuidv7 extension -// built from https://github.com/craigpastro/sqlite-uuidv7 -sqlite.loadExtension( - env.SQLITE_UUIDV7_EXT_PATH ?? - join( - // cannot use __dirname since this file will change locations when compiled - process.cwd(), - "./exts/sqlite-uuidv7", - ), -); + // enable WAL mode + sqlite.pragma("journal_mode = WAL"); -export const db = drizzle(sqlite); + // load uuidv7 extension + // built from https://github.com/craigpastro/sqlite-uuidv7 + sqlite.loadExtension( + env.SQLITE_UUIDV7_EXT_PATH ?? + join( + // cannot use __dirname since this file will change locations when compiled + process.cwd(), + "./exts/sqlite-uuidv7", + ), + ); + return drizzle(sqlite); +} + +export const db = (globalForDB.db ??= createDatabaseInstance()); diff --git a/src/server/db/schema.ts b/src/server/db/schema.ts index 3f7d611..339f6c0 100644 --- a/src/server/db/schema.ts +++ b/src/server/db/schema.ts @@ -5,6 +5,7 @@ import { integer, sqliteTable, index, + real, } from "drizzle-orm/sqlite-core"; // util @@ -75,3 +76,20 @@ export const MFARequestSessionRelations = relations( }), }), ); + +/** + * System Statistics + * Historical data about the system's usage + */ +export const systemStats = sqliteTable("system_stats", { + timestamp: integer("id").primaryKey().default(now), + + // percent as decimal * 10_000 to keep 2 decimal places + cpuUsage: integer("cpu_usage"), + + // everything else is in megabytes + memoryUsage: integer("memory_usage").notNull(), + diskUsage: integer("disk_usage").notNull(), + networkTx: integer("network_tx").notNull(), + networkRx: integer("network_rx").notNull(), +}); diff --git a/src/server/modules/stats/index.ts b/src/server/modules/stats/index.ts index 95fbd42..8d008d9 100644 --- a/src/server/modules/stats/index.ts +++ b/src/server/modules/stats/index.ts @@ -3,6 +3,9 @@ import TypedEmitter from "typed-emitter"; import osu from "node-os-utils"; import os from "os"; import baseLogger from "../../utils/logger"; +import { db } from "~/server/db"; +import { systemStats } from "~/server/db/schema"; +import { between, lte } from "drizzle-orm"; export type BasicServerStats = { collectedAt: Date; @@ -86,7 +89,8 @@ type StatEvents = { * Manages the stats for the current server. */ export class StatManager { - private logger = baseLogger.child({ module: "StatManager" }); + static readonly DB_MAX_STORE_TIME = /* 1 week */ 7 * 24 * 60 * 60 * 1000; + private logger = baseLogger.child({ module: "stats" }); /** * The current stats for the server. @@ -126,16 +130,7 @@ export class StatManager { private liveInterval: NodeJS.Timeout | null = null; constructor() { - this.update(); - - // collect stats every hour - setInterval( - async () => { - await this.update(); - await this.updateDatabase(); - }, - 60 * 60 * 1000, - ); + this.update().then(() => this.updateDatabase()); // whenever a new listener is added, start the live interval this.events.on("newListener", (event) => { @@ -157,6 +152,18 @@ export class StatManager { }); } + start() { + this.logger.info("Starting hourly stat collection."); + // collect stats every hour + setInterval( + async () => { + await this.update(); + await this.updateDatabase(); + }, + 60 * 60 * 1000, + ); + } + /** * Gets the current stats for the server. */ @@ -219,7 +226,46 @@ export class StatManager { /** * Updates the database with the current stats. */ - async updateDatabase() {} + async updateDatabase() { + const startTime = Date.now(); + await db + .insert(systemStats) + .values({ + timestamp: this.currentStats.collectedAt.getTime(), + cpuUsage: this.currentStats.cpu.usage, + diskUsage: this.currentStats.storage.used * 1024, + memoryUsage: this.currentStats.memory.used * 1024, + networkTx: this.currentStats.network.tx / 1024 / 1024, + networkRx: this.currentStats.network.rx / 1024 / 1024, + }) + .execute(); + + this.logger.debug( + `Updated database with new stats. Took ${Date.now() - startTime}ms.`, + ); + + // delete old stats + const count = await db + .delete(systemStats) + .where( + lte(systemStats.timestamp, Date.now() - StatManager.DB_MAX_STORE_TIME), + ) + .execute(); + + this.logger.debug(`Deleted ${count.changes} old stats from the database.`); + } + + /** + * Fetches stats from the database in the given time range. + */ + async getStatsInRange(start: Date, end = Date.now()) { + return db + .select() + .from(systemStats) + .where(between(systemStats.timestamp, start.getTime(), end)); + } } -export const stats = new StatManager(); +export const stats = (((globalThis as any).statsManager as + | StatManager + | undefined) ??= new StatManager()); diff --git a/src/server/server.ts b/src/server/server.ts index 0f68951..ce62eb8 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -13,6 +13,7 @@ import { db } from "./db"; import { mkdir, stat } from "fs/promises"; import path from "path"; import { version } from "../../package.json"; +import { stats } from "./modules/stats"; // check if database folder exists try { @@ -34,14 +35,15 @@ if (env.NODE_ENV === "production") { .info("Not running database migrations, use drizzle-kit push to migrate"); } +// start statistics +stats.start(); + // initialize the next app const app = next({ dev: env.NODE_ENV !== "production", hostname: env.HOSTNAME, port: env.PORT, - // dir: path.join(__dirname, "../.."), customServer: true, - isNodeDebugging: true, }); await app.prepare(); diff --git a/src/server/utils/logger.ts b/src/server/utils/logger.ts index 4fa87e6..309bdee 100644 --- a/src/server/utils/logger.ts +++ b/src/server/utils/logger.ts @@ -19,6 +19,8 @@ const logger = createLogger({ )} ${message}`; }), ), + + level: "debug", }), new transports.File({ filename: "logs/error.log", level: "error" }), new transports.File({ filename: "logs/combined.log" }),