wip: generation-style migration

drizzle.config.ts

@@ -1,6 +1,7 @@
 import type { Config } from "drizzle-kit";
 
 export default {
+  out: "./drizzle",
   schema: "./src/server/db/schema/index.ts",
   driver: "better-sqlite",
   dbCredentials: {

drizzle/0000_tidy_vermin.sql

@@ -0,0 +1,163 @@
+CREATE TABLE `project_deployment` (
+	`id` text PRIMARY KEY DEFAULT (uuid_generate_v7()) NOT NULL,
+	`project_id` text NOT NULL,
+	`deployed_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
+	`status` integer NOT NULL,
+	FOREIGN KEY (`project_id`) REFERENCES `projects`(`id`) ON UPDATE no action ON DELETE cascade
+);
+--> statement-breakpoint
+CREATE TABLE `projects` (
+	`id` text PRIMARY KEY DEFAULT (uuid_generate_v7()) NOT NULL,
+	`friendly_name` text NOT NULL,
+	`internal_name` text NOT NULL,
+	`created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
+	`owner_id` text NOT NULL,
+	FOREIGN KEY (`owner_id`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE cascade
+);
+--> statement-breakpoint
+CREATE TABLE `service` (
+	`id` text PRIMARY KEY DEFAULT (uuid_generate_v7()) NOT NULL,
+	`name` text NOT NULL,
+	`project_id` text NOT NULL,
+	`latest_generation_id` text NOT NULL DEFERRABLE INITIALLY DEFERRED,
+	`redeploy_secret` text NOT NULL,
+	`deployed_generation_id` text,
+	`created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
+	FOREIGN KEY (`project_id`) REFERENCES `projects`(`id`) ON UPDATE no action ON DELETE cascade,
+	FOREIGN KEY (`latest_generation_id`) REFERENCES `service_generation`(`id`) ON UPDATE no action ON DELETE no action,
+	FOREIGN KEY (`deployed_generation_id`) REFERENCES `service_generation`(`id`) ON UPDATE no action ON DELETE no action
+);
+--> statement-breakpoint
+CREATE TABLE `service_deployment` (
+	`id` text PRIMARY KEY DEFAULT (uuid_generate_v7()) NOT NULL,
+	`project_deployment_id` text NOT NULL,
+	`service_id` text NOT NULL,
+	`created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
+	`deployed_by` text,
+	`build_logs` blob,
+	`status` integer NOT NULL,
+	FOREIGN KEY (`project_deployment_id`) REFERENCES `project_deployment`(`id`) ON UPDATE no action ON DELETE cascade,
+	FOREIGN KEY (`service_id`) REFERENCES `service_generation`(`id`) ON UPDATE no action ON DELETE cascade,
+	FOREIGN KEY (`deployed_by`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE no action
+);
+--> statement-breakpoint
+CREATE TABLE `service_domain` (
+	`id` text PRIMARY KEY DEFAULT (uuid_generate_v7()) NOT NULL,
+	`service_id` text NOT NULL,
+	`domain` text NOT NULL,
+	`internal_port` integer NOT NULL,
+	`https` integer DEFAULT false NOT NULL,
+	`force_ssl` integer DEFAULT false NOT NULL,
+	FOREIGN KEY (`service_id`) REFERENCES `service_generation`(`id`) ON UPDATE no action ON DELETE no action
+);
+--> statement-breakpoint
+CREATE TABLE `service_generation` (
+	`id` text PRIMARY KEY DEFAULT (uuid_generate_v7()) NOT NULL,
+	`service_id` text NOT NULL,
+	`deployment_id` text,
+	`source` integer NOT NULL,
+	`environment` text,
+	`docker_image` text,
+	`docker_registry_username` text,
+	`docker_registry_password` text,
+	`github_username` text,
+	`github_repository` text,
+	`github_branch` text,
+	`git_url` text,
+	`git_branch` text,
+	`build_method` integer DEFAULT 2 NOT NULL,
+	`build_path` text DEFAULT '/' NOT NULL,
+	`command` text,
+	`entrypoint` text,
+	`replicas` integer DEFAULT 1 NOT NULL,
+	`max_replicas_per_node` integer,
+	`deploy_mode` integer DEFAULT 1 NOT NULL,
+	`zero_downtime` integer DEFAULT false NOT NULL,
+	`max_cpu` real DEFAULT 0 NOT NULL,
+	`max_memory` text DEFAULT '0' NOT NULL,
+	`max_pids` integer DEFAULT false NOT NULL,
+	`restart` integer DEFAULT 2 NOT NULL,
+	`restart_delay` text DEFAULT '5s',
+	`restart_max_attempts` integer,
+	`healthcheck_enabled` integer DEFAULT false NOT NULL,
+	`healthcheck_command` text,
+	`healthcheck_interval` text DEFAULT '30s' NOT NULL,
+	`healthcheck_timeout` text DEFAULT '30s' NOT NULL,
+	`healthcheck_retries` integer DEFAULT 3 NOT NULL,
+	`healthcheck_start_period` text DEFAULT '0s' NOT NULL,
+	`logging_max_size` text DEFAULT '-1' NOT NULL,
+	`logging_max_files` integer DEFAULT 1 NOT NULL,
+	`created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
+	FOREIGN KEY (`service_id`) REFERENCES `service`(`id`) ON UPDATE no action ON DELETE cascade,
+	FOREIGN KEY (`deployment_id`) REFERENCES `service_deployment`(`id`) ON UPDATE no action ON DELETE no action
+);
+--> statement-breakpoint
+CREATE TABLE `service_port` (
+	`id` text PRIMARY KEY DEFAULT (uuid_generate_v7()) NOT NULL,
+	`service_id` text NOT NULL,
+	`internal_port` integer NOT NULL,
+	`external_port` integer NOT NULL,
+	`port_type` integer NOT NULL,
+	`type` integer NOT NULL,
+	FOREIGN KEY (`service_id`) REFERENCES `service_generation`(`id`) ON UPDATE no action ON DELETE no action
+);
+--> statement-breakpoint
+CREATE TABLE `service_sysctl` (
+	`id` text PRIMARY KEY DEFAULT (uuid_generate_v7()) NOT NULL,
+	`service_id` text NOT NULL,
+	`key` text NOT NULL,
+	`value` text NOT NULL,
+	FOREIGN KEY (`service_id`) REFERENCES `service_generation`(`id`) ON UPDATE no action ON DELETE no action
+);
+--> statement-breakpoint
+CREATE TABLE `service_ulimit` (
+	`id` text PRIMARY KEY DEFAULT (uuid_generate_v7()) NOT NULL,
+	`service_id` text NOT NULL,
+	`name` text NOT NULL,
+	`soft` integer NOT NULL,
+	`hard` integer NOT NULL,
+	FOREIGN KEY (`service_id`) REFERENCES `service_generation`(`id`) ON UPDATE no action ON DELETE no action
+);
+--> statement-breakpoint
+CREATE TABLE `service_volume` (
+	`id` text PRIMARY KEY DEFAULT (uuid_generate_v7()) NOT NULL,
+	`service_id` text NOT NULL,
+	`source` text,
+	`target` text NOT NULL,
+	`type` text NOT NULL,
+	FOREIGN KEY (`service_id`) REFERENCES `service_generation`(`id`) ON UPDATE no action ON DELETE no action
+);
+--> statement-breakpoint
+CREATE TABLE `session` (
+	`token` text PRIMARY KEY NOT NULL,
+	`last_useragent` text,
+	`last_ip` text,
+	`last_accessed` integer,
+	`created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
+	`id` text NOT NULL,
+	FOREIGN KEY (`id`) REFERENCES `users`(`id`) ON UPDATE no action ON DELETE no action
+);
+--> statement-breakpoint
+CREATE TABLE `system_stats` (
+	`id` integer PRIMARY KEY DEFAULT CURRENT_TIMESTAMP NOT NULL,
+	`cpu_usage` integer,
+	`memory_usage` integer NOT NULL,
+	`disk_usage` integer NOT NULL,
+	`network_tx` integer NOT NULL,
+	`network_rx` integer NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE `users` (
+	`id` text PRIMARY KEY DEFAULT (uuid_generate_v7()) NOT NULL,
+	`username` text NOT NULL,
+	`password` text,
+	`mfa_token` blob
+);
+--> statement-breakpoint
+CREATE INDEX `proj_deployment_idx` ON `project_deployment` (`id`,`project_id`);--> statement-breakpoint
+CREATE UNIQUE INDEX `projects_internal_name_unique` ON `projects` (`internal_name`);--> statement-breakpoint
+CREATE INDEX `name_project_idx` ON `service` (`name`,`project_id`);--> statement-breakpoint
+CREATE UNIQUE INDEX `name_project_unq` ON `service` (`name`,`project_id`);--> statement-breakpoint
+CREATE INDEX `proj_generation_idx` ON `service_generation` (`id`,`service_id`);--> statement-breakpoint
+CREATE UNIQUE INDEX `users_username_unique` ON `users` (`username`);--> statement-breakpoint
+CREATE INDEX `username_idx` ON `users` (`username`);

drizzle/meta/_journal.json

@@ -0,0 +1,13 @@
+{
+  "version": "5",
+  "dialect": "sqlite",
+  "entries": [
+    {
+      "idx": 0,
+      "version": "5",
+      "when": 1715730366620,
+      "tag": "0000_tidy_vermin",
+      "breakpoints": true
+    }
+  ]
+}

package.json

@@ -9,6 +9,7 @@
     "build:server": "tsup",
     "clean": "rm -rf .next dist",
     "db:push": "drizzle-kit push:sqlite",
+    "db:generate": "drizzle-kit generate:sqlite",
     "dev": "tsup --watch --onSuccess \"npm run dev:run\" --clean",
     "dev:run": "node --enable-source-maps dist/server.js",
     "lint": "next lint",

src/server/api/middleware/project.ts

@@ -12,7 +12,7 @@ export type BasicProjectDetails = {
 export const projectMiddleware = experimental_standaloneMiddleware<{
   ctx: { db: typeof db };
   input: { projectId: string };
-}>().create(async ({ ctx, input, next }) => {
+}>().create(async ({ input, next }) => {
   if (typeof input.projectId != "string") {
     throw new TRPCError({
       code: "INTERNAL_SERVER_ERROR",

src/server/api/routers/projects/index.ts

@@ -1,7 +1,7 @@
 import assert from "assert";
-import { eq } from "drizzle-orm";
 import { z } from "zod";
-import { projects, serviceGeneration } from "~/server/db/schema";
+import { projects } from "~/server/db/schema";
+import ProjectManager from "~/server/managers/Project";
 import { authenticatedProcedure, createTRPCRouter } from "../../trpc";
 import { deployProject } from "./deploy";
 import { getProject } from "./project";
@@ -21,28 +21,21 @@ export const projectRouter = createTRPCRouter({
     .input(z.void())
     .output(z.unknown())
     .query(async ({ ctx }) => {
-      const userProjects = await ctx.db
-        .select({
-          id: projects.id,
-          friendlyName: projects.friendlyName,
-          internalName: projects.internalName,
-          createdAt: projects.createdAt,
-        })
-        .from(projects);
-
-      return await Promise.all(
-        userProjects.map(async (project) => {
-          const projServices = await ctx.db
-            .select({
-              id: serviceGeneration.id,
-              name: serviceGeneration.name,
-            })
-            .from(serviceGeneration)
-            .where(eq(serviceGeneration.serviceId, project.id));
+      const projects = await ProjectManager.listForUser(
+        ctx.session.data.userId,
+      );
 
+      // we love the nested Promise.all's
+      // TODO: refactor
+      return Promise.all(
+        projects.map(async (project) => {
           return {
-            ...project,
-            services: projServices,
+            ...project.getData(),
+            services: await Promise.all(
+              (await project.getServices()).map((service) =>
+                service.getDataWithGenerations(),
+              ),
+            ),
           };
         }),
       );

src/server/api/routers/projects/project.ts

@@ -1,6 +1,4 @@
-import { eq } from "drizzle-orm";
 import { z } from "zod";
-import { serviceGeneration } from "~/server/db/schema";
 import { projectMiddleware } from "../../middleware/project";
 import { authenticatedProcedure } from "../../trpc";
 
@@ -16,26 +14,22 @@ export const getProject = authenticatedProcedure
   .use(projectMiddleware)
   .output(z.unknown())
   .query(async ({ ctx }) => {
-    const projServices = await ctx.db
-      .select({
-        id: serviceGeneration.id,
-        name: serviceGeneration.name,
-      })
-      .from(serviceGeneration)
-      .where(eq(serviceGeneration.serviceId, ctx.project.id));
+    const projServices = await ctx.project.getServices();
 
     // get docker stats
     const stats = await ctx.docker.listServices({
       filters: {
-        label: [`com.docker.stack.namespace=${ctx.project.internalName}`],
+        label: [
+          `com.docker.stack.namespace=${ctx.project.getData().internalName}`,
+        ],
       },
     });
 
     return {
-      ...ctx.project,
+      ...ctx.project.getData(),
       services: projServices.map((service) => ({
-        ...service,
-        stats: stats.find((stat) => stat.Spec?.Name === service.name),
+        ...service.getData(),
+        stats: stats.find((stat) => stat.Spec?.Name === service.getData().name),
       })),
     };
   });

src/server/api/routers/projects/service/index.ts

@@ -1,5 +1,5 @@
 import assert from "assert";
-import { eq } from "drizzle-orm";
+import { eq, sql } from "drizzle-orm";
 import { randomBytes } from "node:crypto";
 import { z } from "zod";
 import { env } from "~/env";
@@ -14,6 +14,7 @@ import {
   updateServiceDomainsProcedure,
   updateServiceProcedure,
 } from "./update";
+import { ServiceSource } from "~/server/db/types";
 
 export const serviceRouter = createTRPCRouter({
   containers: getServiceContainers,
@@ -33,27 +34,9 @@ export const serviceRouter = createTRPCRouter({
     .use(projectMiddleware)
     .use(serviceMiddleware)
     .query(async ({ ctx }) => {
-      // const fullServiceData = await ctx.db.query.service.findFirst({
-      //   where: eq(serviceGeneration.id, ctx.service.getData().id),
-      //   with: {
-      //     domains: true,
-      //     ports: true,
-      //     volumes: true,
-      //     sysctls: true,
-      //     ulimits: true,
-      //   },
-      // });
-
-      // assert(fullServiceData);
-
-      // return {
-      //   ...fullServiceData,
-      //   deployMode: DOCKER_DEPLOY_MODE_MAP[fullServiceData.deployMode],
-      // };
-
       return {
         ...ctx.service.getData(),
-        latestGeneration: ctx.service.getData().latestGeneration,
+        latestGeneration: await ctx.service.fetchFullLatestGeneration(),
       };
     }),
 
@@ -75,42 +58,61 @@ export const serviceRouter = createTRPCRouter({
     .use(projectMiddleware)
     .mutation(async ({ ctx, input }) => {
       // create a generation for the service
-      // const [defaultGeneration] = await ctx.db
-      //   .insert(serviceGeneration)
-      //   .values({
-      //     : ctx.project.getData().id,
+      const trxResult = await ctx.db.transaction(
+        async (trx) => {
+          // create the service
+          const [data] = await trx
+            .insert(service)
+            .values({
+              name: input.name,
+              projectId: ctx.project.getData().id,
+              latestGenerationId: "",
+              redeploySecret: randomBytes(env.REDEPLOY_SECRET_BYTES).toString(
+                "hex",
+              ),
+            })
+            .returning({
+              id: serviceGeneration.id,
+            })
+            .execute()
+            .catch((err) => {
+              console.error(err);
+              throw err;
+            });
 
-      //     status: "pending",
-      //   })
-      //   .returning({
-      //     id: serviceGeneration.id,
-      //   })
-      //   .execute();
+          assert(data?.id, "Expected service data to be returned");
 
-      const [data] = await ctx.db
-        .insert(service)
-        .values({
-          name: input.name,
-          projectId: ctx.project.getData().id,
-          latestGenerationId: "",
-          redeploySecret: randomBytes(env.REDEPLOY_SECRET_BYTES).toString(
-            "hex",
-          ),
-          // source: ServiceSource.Docker,
-          // dockerImage: "traefik/whoami",
-        })
-        .returning({
-          id: serviceGeneration.id,
-        })
-        .execute()
-        .catch((err) => {
-          console.error(err);
-          throw err;
-        });
+          // create initial generation
+          const [generation] = await trx
+            .insert(serviceGeneration)
+            .values({
+              serviceId: data.id,
+              source: ServiceSource.Docker,
+              dockerImage: "traefik/whoami",
+            })
+            .returning({
+              id: serviceGeneration.id,
+            });
 
-      assert(data?.id);
+          assert(generation?.id, "Expected generation data to be returned");
 
-      return data.id;
+          // update the service with the generation id
+          await trx
+            .update(service)
+            .set({
+              latestGenerationId: generation.id,
+            })
+            .where(eq(service.id, data.id))
+            .execute();
+
+          return data.id;
+        },
+        {
+          behavior: "deferred",
+        },
+      );
+
+      return trxResult;
     }),
 
   delete: authenticatedProcedure

src/server/api/routers/projects/service/update.ts

@@ -73,6 +73,7 @@ export const updateServiceProcedure = authenticatedProcedure
             id: true,
             projectId: true,
             name: true,
+            serviceId: true,
           })
           .partial(),
       )
@@ -95,7 +96,7 @@ export const updateServiceProcedure = authenticatedProcedure
     await ctx.db
       .update(serviceGeneration)
       .set(queryUpdate)
-      .where(eq(serviceGeneration.id, ctx.service.id))
+      .where(eq(serviceGeneration.id, ctx.service.getData().latestGenerationId))
       .execute();
 
     return true;

src/server/db/schema/schema.ts

@@ -140,10 +140,14 @@ export const service = sqliteTable(
         onDelete: "cascade",
       }),
 
+    // https://github.com/drizzle-team/drizzle-orm/issues/2252
+    // Must manually add `DEFERRABLE INITIALLY DEFERRED`
     latestGenerationId: text("latest_generation_id")
       .notNull()
       .references(() => serviceGeneration.id),
 
+    // latestGenerationId: sql<string>`service_generation_id REFERENCES service_generation(id) NOT NULL `,
+
     redeploySecret: text("redeploy_secret").notNull(),
     deployedGenerationId: text("deployed_generation_id").references(
       () => serviceGeneration.id,

src/server/managers/Project.ts

@@ -32,6 +32,17 @@ export default class ProjectManager {
     return data ? new ProjectManager(data) : null;
   }
 
+  /**
+   * Lists all projects for a user.
+   */
+  static async listForUser(userId: string) {
+    const data = await db.query.projects.findMany({
+      where: eq(projects.ownerId, userId),
+    });
+
+    return data.map((data) => new ProjectManager(data));
+  }
+
   /**
    * Returns the project data.
    */
@@ -143,7 +154,7 @@ export default class ProjectManager {
     const composeStack = await buildDockerStackFile(allServiceData);
 
     return await deployOptions.docker.cli(
-      ["stack", "deploy", "--compose-file", "-", this.projectData.internalName],
+      ["stack", "deploy", "--compose-file", "-", this.projectData.internalName, deployOptions.force ? "--force-recreate" : ""],
       {
         stdin: JSON.stringify(composeStack),
       },

src/server/server.ts

@@ -29,15 +29,9 @@ try {
 }
 
 // migrate the database
-if (env.NODE_ENV === "production") {
-  logger.child({ module: "database" }).info("⚙️ Migrating database");
-  migrate(db, { migrationsFolder: "./migrations" });
-  logger.child({ module: "database" }).info("✅ Database migrated");
-} else {
-  logger
-    .child({ module: "database" })
-    .info("Not running database migrations, use drizzle-kit push to migrate");
-}
+logger.child({ module: "database" }).info("⚙️ Starting database migrations...");
+migrate(db, { migrationsFolder: "./drizzle" });
+logger.child({ module: "database" }).info("✅ Migrations finished!");
 
 // start statistics
 void stats.start();