wip

drizzle.config.ts

@@ -1,7 +1,7 @@
 import type { Config } from "drizzle-kit";
 
 export default {
-  schema: "./src/server/db/schema.ts",
+  schema: "./src/server/db/schema/index.ts",
   driver: "better-sqlite",
   dbCredentials: {
     url: process.env.DATABASE_PATH ?? "./data/db.sqlite",

src/server/api/middleware/project.ts

@@ -1,6 +1,6 @@
 import { TRPCError, experimental_standaloneMiddleware } from "@trpc/server";
 import { type db } from "~/server/db";
-import ProjectManager from "~/server/managers/ProjectManager";
+import ProjectManager from "~/server/managers/Project";
 
 export type BasicProjectDetails = {
   id: string;

src/server/api/middleware/service.ts

@@ -1,11 +1,10 @@
 import { TRPCError, experimental_standaloneMiddleware } from "@trpc/server";
-import { and, eq, or } from "drizzle-orm";
 import { type db } from "~/server/db";
-import { service } from "~/server/db/schema";
+import type ProjectManager from "~/server/managers/Project";
-import { type BasicProjectDetails } from "./project";
+import ServiceManager from "~/server/managers/Service";
 
 export const serviceMiddleware = experimental_standaloneMiddleware<{
-  ctx: { db: typeof db; project: BasicProjectDetails };
+  ctx: { db: typeof db; project: ProjectManager };
   input: { serviceId: string };
 }>().create(async ({ ctx, input, next }) => {
   if (typeof input.serviceId != "string") {
@@ -15,7 +14,7 @@ export const serviceMiddleware = experimental_standaloneMiddleware<{
     });
   }
 
-  if (typeof ctx.project?.id != "string") {
+  if (typeof ctx.project != "object") {
     throw new TRPCError({
       code: "INTERNAL_SERVER_ERROR",
       message:
@@ -23,16 +22,10 @@ export const serviceMiddleware = experimental_standaloneMiddleware<{
     });
   }
 
-  const serviceDetails = await ctx.db.query.service.findFirst({
+  const serviceDetails = await ServiceManager.findByNameOrId(
-    where: and(
+    input.serviceId,
-      eq(service.projectId, ctx.project.id),
+    ctx.project.getData().id,
-      or(eq(service.name, input.serviceId), eq(service.id, input.serviceId)),
+  );
-    ),
-
-    with: {
-      latestGeneration: true,
-    },
-  });
 
   if (!serviceDetails)
     throw new TRPCError({

src/server/api/routers/projects/deploy.ts

@@ -1,9 +1,4 @@
-import { eq } from "drizzle-orm";
 import { z } from "zod";
-import { BuildManager } from "~/server/build/BuildManager";
-import { serviceGeneration } from "~/server/db/schema";
-import { buildDockerStackFile } from "~/server/docker/stack";
-import logger from "~/server/utils/logger";
 import { projectMiddleware } from "../../middleware/project";
 import { authenticatedProcedure } from "../../trpc";
 
@@ -22,32 +17,10 @@ export const deployProject = authenticatedProcedure
   )
   .use(projectMiddleware)
   .mutation(async ({ ctx, input }) => {
-    const services = await ctx.db.query.service.findMany({
+    const response = await ctx.project.deploy({
-      where: eq(serviceGeneration.serviceId, input.projectId),
+      docker: ctx.docker,
-
-      with: {
-        domains: true,
-        ports: true,
-        sysctls: true,
-        volumes: true,
-        ulimits: true,
-      },
     });
 
-    // run builds
-    // TODO: run only if needed
-    await BuildManager.getInstance().runBuilds(services);
-
-    const dockerStackFile = await buildDockerStackFile(services);
-    logger.debug("deploying stack", { dockerStackFile });
-
-    const response = await ctx.docker.cli(
-      ["stack", "deploy", "--compose-file", "-", ctx.project.internalName],
-      {
-        stdin: JSON.stringify(dockerStackFile),
-      },
-    );
-
     // TODO: stream progress to client
     // https://github.com/trpc/trpc/issues/4477
     // could do with ws, but very complicated

src/server/api/routers/projects/service/containers.ts

@@ -94,7 +94,9 @@ export const getServiceContainers = authenticatedProcedure
   .query(async ({ ctx }) => {
     // get docker service stats
     const service = (await ctx.docker
-      .getService(`${ctx.project.internalName}_${ctx.service.name}`)
+      .getService(
+        `${ctx.project.getData().internalName}_${ctx.service.getData().name}`,
+      )
       .inspect()
       .catch(docker404ToNull)) as
       | DockerAPITypes["/services/{id}"]["get"]["responses"]["200"]["schema"]

src/server/api/routers/projects/service/index.ts

@@ -1,13 +1,12 @@
 import assert from "assert";
-import { randomBytes } from "crypto";
 import { eq } from "drizzle-orm";
+import { randomBytes } from "node:crypto";
 import { z } from "zod";
 import { env } from "~/env";
 import { projectMiddleware } from "~/server/api/middleware/project";
 import { serviceMiddleware } from "~/server/api/middleware/service";
 import { authenticatedProcedure, createTRPCRouter } from "~/server/api/trpc";
-import { serviceGeneration } from "~/server/db/schema";
+import { service, serviceGeneration } from "~/server/db/schema";
-import { DOCKER_DEPLOY_MODE_MAP, ServiceSource } from "~/server/db/types";
 import { zDockerName } from "~/server/utils/zod";
 import { getServiceContainers } from "./containers";
 import {
@@ -34,22 +33,27 @@ export const serviceRouter = createTRPCRouter({
     .use(projectMiddleware)
     .use(serviceMiddleware)
     .query(async ({ ctx }) => {
-      const fullServiceData = await ctx.db.query.service.findFirst({
+      // const fullServiceData = await ctx.db.query.service.findFirst({
-        where: eq(serviceGeneration.id, ctx.service.id),
+      //   where: eq(serviceGeneration.id, ctx.service.getData().id),
-        with: {
+      //   with: {
-          domains: true,
+      //     domains: true,
-          ports: true,
+      //     ports: true,
-          volumes: true,
+      //     volumes: true,
-          sysctls: true,
+      //     sysctls: true,
-          ulimits: true,
+      //     ulimits: true,
-        },
+      //   },
-      });
+      // });
 
-      assert(fullServiceData);
+      // assert(fullServiceData);
+
+      // return {
+      //   ...fullServiceData,
+      //   deployMode: DOCKER_DEPLOY_MODE_MAP[fullServiceData.deployMode],
+      // };
 
       return {
-        ...fullServiceData,
+        ...ctx.service.getData(),
-        deployMode: DOCKER_DEPLOY_MODE_MAP[fullServiceData.deployMode],
+        latestGeneration: ctx.service.getData().latestGeneration,
       };
     }),
 
@@ -70,17 +74,30 @@ export const serviceRouter = createTRPCRouter({
     .output(z.string({ description: "Service ID" }))
     .use(projectMiddleware)
     .mutation(async ({ ctx, input }) => {
+      // create a generation for the service
+      // const [defaultGeneration] = await ctx.db
+      //   .insert(serviceGeneration)
+      //   .values({
+      //     : ctx.project.getData().id,
+
+      //     status: "pending",
+      //   })
+      //   .returning({
+      //     id: serviceGeneration.id,
+      //   })
+      //   .execute();
+
       const [data] = await ctx.db
-        .insert(serviceGeneration)
+        .insert(service)
         .values({
           name: input.name,
-          projectId: ctx.project.id,
+          projectId: ctx.project.getData().id,
+          latestGenerationId: "",
           redeploySecret: randomBytes(env.REDEPLOY_SECRET_BYTES).toString(
             "hex",
           ),
-          source: ServiceSource.Docker,
+          // source: ServiceSource.Docker,
-
+          // dockerImage: "traefik/whoami",
-          dockerImage: "traefik/whoami",
         })
         .returning({
           id: serviceGeneration.id,

src/server/build/BuildManager.ts

@@ -3,7 +3,7 @@ import { Queue } from "datastructures-js";
 import { db } from "../db";
 import { serviceDeployment } from "../db/schema/schema";
 import { ServiceDeploymentStatus, ServiceSource } from "../db/types";
-import { type Service } from "../docker/stack";
+import { type FullServiceGeneration } from "../docker/stack";
 import logger from "../utils/logger";
 import BuildTask from "./BuildTask";
 
@@ -35,7 +35,7 @@ export class BuildManager {
     });
   }
 
-  public async runBuilds(services: Service[]) {
+  public async runBuilds(services: FullServiceGeneration[]) {
     await Promise.all(
       services.map(async (service) => {
         if (service.source !== ServiceSource.Docker) {

src/server/db/schema/schema.ts

@@ -144,9 +144,10 @@ export const service = sqliteTable(
       .notNull()
       .references(() => serviceGeneration.id),
 
-    deployedGenerationId: text("deployed_generation_id")
+    redeploySecret: text("redeploy_secret").notNull(),
-      .notNull()
+    deployedGenerationId: text("deployed_generation_id").references(
-      .references(() => serviceGeneration.id),
+      () => serviceGeneration.id,
+    ),
 
     createdAt: integer("created_at").default(now).notNull(),
   },
@@ -181,7 +182,6 @@ export const serviceGeneration = sqliteTable(
 
     // service configuration
     source: integer("source").$type<ServiceSource>().notNull(),
-    redeploySecret: text("redeploy_secret").notNull(),
     environment: text("environment"),
 
     // for docker source

src/server/docker/stack.ts

@@ -19,7 +19,7 @@ import {
   type Ulimits,
 } from "./compose";
 
-export type Service = typeof serviceGeneration.$inferSelect & {
+export type FullServiceGeneration = typeof serviceGeneration.$inferSelect & {
   domains: (typeof serviceDomain.$inferSelect)[];
   ports: (typeof servicePort.$inferSelect)[];
   sysctls: (typeof serviceSysctl.$inferSelect)[];
@@ -38,7 +38,7 @@ export type Service = typeof serviceGeneration.$inferSelect & {
  */
 // eslint-disable-next-line @typescript-eslint/require-await
 export async function buildDockerStackFile(
-  services: Service[],
+  services: FullServiceGeneration[],
 ): Promise<ComposeSpecification> {
   // create services
   const swarmServices: Record<string, DefinitionsService> = {};

src/server/managers/Project.ts

@@ -1,13 +1,21 @@
 import assert from "assert";
 import { eq, or } from "drizzle-orm";
+import { BuildManager } from "../build/BuildManager";
 import { db } from "../db";
 import {
   projectDeployment,
   projects,
   service,
   serviceDeployment,
+  serviceGeneration,
 } from "../db/schema";
 import { ServiceDeploymentStatus } from "../db/types";
+import { type Docker } from "../docker/docker";
+import {
+  buildDockerStackFile,
+  type FullServiceGeneration,
+} from "../docker/stack";
+import logger from "../utils/logger";
 import ServiceManager from "./Service";
 
 export default class ProjectManager {
@@ -45,12 +53,9 @@ export default class ProjectManager {
   /**
    * Deploys the project.
    */
-  public async deploy(deployOptions?: { force?: boolean }) {
+  public async deploy(deployOptions: { docker: Docker; force?: boolean }) {
     // 1. get all services that have pending updates
     const services = await this.getServicesWithPendingUpdates();
-    const serviceData = await Promise.all(
-      services.map((service) => service.getDataWithGenerations()),
-    );
 
     // 2. Create a deployment entry
     const [deployment] = await db
@@ -66,13 +71,35 @@ export default class ProjectManager {
     assert(deployment, "deploymentId is missing");
 
     // 2. for each service, create a new deployment and run builds if needed
-    await Promise.all(
+    const allServiceData = await Promise.all(
-      serviceData.map(async (service) => {
+      services.map(async (service): Promise<FullServiceGeneration> => {
+        // fetch latest generation with all children
+        const fullGenerationData = await db.query.serviceGeneration.findFirst({
+          where: eq(serviceGeneration.id, service.getData().latestGenerationId),
+          with: {
+            deployment: true,
+            domains: true,
+            ports: true,
+            service: true,
+            sysctls: true,
+            ulimits: true,
+            volumes: true,
+          },
+        });
+
+        assert(fullGenerationData, "Generation data is missing!");
+
+        // if no deployment required, just return
+        if (await service.hasPendingChanges()) return fullGenerationData;
+
+        // fetch latest generation data
+        const serviceData = service.getData();
+
         // create a new deployment
         const [sDeployment] = await db
           .insert(serviceDeployment)
           .values({
-            serviceId: service.id,
+            serviceId: serviceData.id,
             status: ServiceDeploymentStatus.BuildPending,
             projectDeploymentId: deployment.id,
           })
@@ -80,12 +107,47 @@ export default class ProjectManager {
             id: serviceDeployment.id,
           });
 
+        assert(sDeployment, "serviceDeploymentId is missing");
+
         // link the new deployment to the service
+        if (fullGenerationData.deploymentId) {
+          logger.warn(
+            `Service "${serviceData.name}" already has a deployment linked.`,
+            {
+              serviceId: serviceData.id,
+              deploymentId: fullGenerationData.deploymentId,
+            },
+          );
+        }
+
+        await service.deriveNewGeneration(sDeployment.id);
+
+        // run builds if needed
+        if (await service.requiresImageBuild()) {
+          const image = await BuildManager.getInstance().startBuild(
+            serviceData.id,
+            sDeployment.id,
+          );
+
+          return {
+            ...fullGenerationData,
+            finalizedDockerImage: image,
+          };
+        }
+
+        return fullGenerationData;
       }),
     );
 
-    // create a new generation for each service
+    // now build the dockerfile
-    // await Promise.all(services.map((service) => service.deriveNewGeneration()));
+    const composeStack = await buildDockerStackFile(allServiceData);
+
+    return await deployOptions.docker.cli(
+      ["stack", "deploy", "--compose-file", "-", this.projectData.internalName],
+      {
+        stdin: JSON.stringify(composeStack),
+      },
+    );
   }
 
   /**

src/server/managers/Service.ts

@@ -4,6 +4,7 @@ import { create } from "jsondiffpatch";
 import assert from "node:assert";
 import { db } from "../db";
 import { service, serviceGeneration } from "../db/schema";
+import { ServiceSource } from "../db/types";
 import logger from "../utils/logger";
 
 export default class ServiceManager {
@@ -84,10 +85,39 @@ export default class ServiceManager {
     return ServiceManager.JSON_DIFF.diff(deployed, latest);
   }
 
+  /**
+   * Returns true if there has been a configuration change
+   */
+  public async hasPendingChanges() {
+    const diff = await this.buildDeployDiff();
+
+    if (typeof diff === "object") return Object.keys(diff).length !== 0;
+    return true;
+  }
+
+  /**
+   * Checks if the source has changed between the deployed and latest generation.
+   * If true, then a image build is required.
+   */
+  public async requiresImageBuild() {
+    const diff = await this.buildDeployDiff();
+    const latestGen = await this.fetchLatestGeneration();
+
+    // if latest gen has dockerimage source, no build required
+    if (latestGen.source === ServiceSource.Docker) {
+      return false;
+    }
+
+    // TODO: implement
+    logger.debug("Service diff", { diff });
+
+    return Object.keys(diff).length > 0;
+  }
+
   /**
    * Clones the latest generation and sets the original generation as the deployed generation.
    */
-  public async deriveNewGeneration() {
+  public async deriveNewGeneration(setDeploymentId?: string) {
     // do as much as possible on the database
     await db.transaction(async (trx) => {
       // clone the latest generation
@@ -98,8 +128,10 @@ export default class ServiceManager {
 
       assert(originalLatestGeneration, "Could not find latest generation??");
 
+      // update deployment id
+      originalLatestGeneration.deploymentId = setDeploymentId ?? null;
+
       // delete the ID so we can insert it again
-      originalLatestGeneration.deploymentId = null;
       // @ts-expect-error i dont feel like typing it as optional
       delete originalLatestGeneration.id;
 
@@ -122,6 +154,21 @@ export default class ServiceManager {
     await this.refetch();
   }
 
+  public async fetchFullLatestGeneration() {
+    return await db.query.serviceGeneration.findFirst({
+      where: eq(serviceGeneration.id, this.serviceData.latestGenerationId),
+      with: {
+        deployment: true,
+        domains: true,
+        ports: true,
+        service: true,
+        sysctls: true,
+        ulimits: true,
+        volumes: true,
+      },
+    });
+  }
+
   /**
    * Fetches the latest generation of the service.
    * @param forceRefetch if true, will ignore what's cached internally
@@ -132,10 +179,17 @@ export default class ServiceManager {
       return this.serviceData.latestGeneration;
     }
 
-    return (this.serviceData.latestGeneration =
+    this.serviceData.latestGeneration =
       await db.query.serviceGeneration.findFirst({
         where: eq(serviceGeneration.id, this.serviceData.latestGenerationId),
-      }));
+      });
+
+    assert(
+      this.serviceData.latestGeneration,
+      "Service is missing latest generation!",
+    );
+
+    return this.serviceData.latestGeneration;
   }
 
   /**
@@ -147,10 +201,14 @@ export default class ServiceManager {
       return this.serviceData.deployedGeneration;
     }
 
-    return (this.serviceData.deployedGeneration =
+    this.serviceData.deployedGeneration =
       await db.query.serviceGeneration.findFirst({
         where: eq(serviceGeneration.id, this.serviceData.deployedGenerationId),
-      }));
+      });
+
+    assert(this.serviceData.deployedGeneration, "Could not find deployed gen");
+
+    return this.serviceData.deployedGeneration;
   }
 
   /**