feat: log fixes, deploy change confirmation

package.json

@@ -61,7 +61,7 @@
     "extensionless": "^1.9.6",
     "framer-motion": "^11.0.3",
     "ipaddr.js": "^2.1.0",
-    "jsondiffpatch": "^0.6.0",
+    "json-diff-ts": "^4.0.1",
     "lucide-react": "^0.298.0",
     "next": "14.0.4",
     "next-nprogress-bar": "^2.1.2",

pnpm-lock.yaml

@@ -134,9 +134,9 @@ dependencies:
   ipaddr.js:
     specifier: ^2.1.0
     version: 2.1.0
-  jsondiffpatch:
-    specifier: ^0.6.0
-    version: 0.6.0
+  json-diff-ts:
+    specifier: ^4.0.1
+    version: 4.0.1
   lucide-react:
     specifier: ^0.298.0
     version: 0.298.0(react@18.2.0)
@@ -2883,10 +2883,6 @@ packages:
     resolution: {integrity: sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==}
     dev: false
 
-  /@types/diff-match-patch@1.0.36:
-    resolution: {integrity: sha512-xFdR6tkm0MWvBfO8xXCSsinYxHcqkQUlcHeSpMC2ukzOb6lwQAfDmW+Qt0AvlGd8HpsS28qKsB+oPeJn9I39jg==}
-    dev: false
-
   /@types/docker-modem@3.0.6:
     resolution: {integrity: sha512-yKpAGEuKRSS8wwx0joknWxsmLha78wNMe9R2S3UNsVOkZded8UqOrV8KoeDXoXsjndxwyF3eIhyClGbO1SEhEg==}
     dependencies:
@@ -4321,10 +4317,6 @@ packages:
   /didyoumean@1.2.2:
     resolution: {integrity: sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==}
 
-  /diff-match-patch@1.0.5:
-    resolution: {integrity: sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==}
-    dev: false
-
   /difflib@0.2.4:
     resolution: {integrity: sha512-9YVwmMb0wQHQNr5J9m6BSj6fk4pfGITGQOOs+D9Fl+INODWFOfvhIU1hNv6GgR1RBoC/9NJcwu77zShxV0kT7w==}
     dependencies:
@@ -6070,6 +6062,12 @@ packages:
     resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==}
     dev: true
 
+  /json-diff-ts@4.0.1:
+    resolution: {integrity: sha512-FEuq+gv4DXI+nL7oF/trBQIBQYDVcJe5Kqe3mYUZAkDtHBY/cdWmVZpV9BLZFp+wThMClajYp0fmNVmB81FsmA==}
+    dependencies:
+      lodash: 4.17.21
+    dev: false
+
   /json-diff@0.9.0:
     resolution: {integrity: sha512-cVnggDrVkAAA3OvFfHpFEhOnmcsUpleEKq4d4O8sQWWSH40MBrWstKigVB1kGrgLWzuom+7rRdaCsnBD6VyObQ==}
     hasBin: true
@@ -6119,16 +6117,6 @@ packages:
       minimist: 1.2.8
     dev: true
 
-  /jsondiffpatch@0.6.0:
-    resolution: {integrity: sha512-3QItJOXp2AP1uv7waBkao5nCvhEv+QmJAd38Ybq7wNI74Q+BBmnLn4EDKz6yI9xGAIQoUF87qHt+kc1IVxB4zQ==}
-    engines: {node: ^18.0.0 || >=20.0.0}
-    hasBin: true
-    dependencies:
-      '@types/diff-match-patch': 1.0.36
-      chalk: 5.3.0
-      diff-match-patch: 1.0.5
-    dev: false
-
   /jsx-ast-utils@3.3.5:
     resolution: {integrity: sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ==}
     engines: {node: '>=4.0'}

src/app/(dashboard)/project/[projectId]/_components/DeployChanges.tsx

@@ -5,6 +5,16 @@ import { toast } from "sonner";
 import { Button } from "~/components/ui/button";
 import { api } from "~/trpc/react";
 import { useProject } from "../_context/ProjectContext";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogHeader,
+  DialogTitle,
+  DialogTrigger,
+} from "~/components/ui/dialog";
+import { LoadingSpinner } from "~/components/LoadingSpinner";
+import { ServiceDiff } from "~/components/service/ServiceDiff";
 
 export function DeployChanges() {
   const project = useProject();
@@ -14,18 +24,39 @@ export function DeployChanges() {
     },
   });
 
+  const diff = api.projects.deployDiff.useQuery({
+    projectId: project.id,
+  });
+
   return (
-    <Button
-      variant="outline"
-      icon={UploadCloud}
-      onClick={() =>
-        mutation.mutate({
-          projectId: project.id,
-        })
-      }
-      isLoading={mutation.isPending}
-    >
-      Deploy Changes
-    </Button>
+    <Dialog>
+      <DialogTrigger>
+        <Button
+          variant="outline"
+          icon={UploadCloud}
+          onClick={() => diff.refetch()}
+          isLoading={mutation.isPending}
+        >
+          Deploy Changes
+        </Button>
+      </DialogTrigger>
+      <DialogContent>
+        <DialogHeader>
+          <DialogTitle>Deployment Confirmation</DialogTitle>
+        </DialogHeader>
+        <DialogDescription className="max-w-full">
+          {diff.isFetching && <LoadingSpinner />}
+
+          {diff.data?.map(({ service, diff }) => (
+            <div key={service.id} className="mb-4">
+              <h2 className="text-lg font-bold">{service.name}</h2>
+              <pre className="text-sm text-gray-400">
+                <ServiceDiff diff={diff} />
+              </pre>
+            </div>
+          ))}
+        </DialogDescription>
+      </DialogContent>
+    </Dialog>
   );
 }

src/app/(dashboard)/project/[projectId]/service/[serviceId]/deployments/_components/DeploymentLogs.tsx

@@ -1,14 +1,18 @@
 "use client";
 
 import { api } from "~/trpc/react";
-import { RouterOutputs } from "~/trpc/shared";
-import { useService } from "../../_hooks/service";
 import { useProject } from "../../../../_context/ProjectContext";
 import { useState } from "react";
 import { toast } from "sonner";
-import { Drawer, DrawerContent } from "~/components/ui/drawer";
 import { LogWindow, type LogLine } from "~/components/LogWindow";
 import { StringParam, useQueryParam } from "use-query-params";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogHeader,
+  DialogTitle,
+} from "~/components/ui/dialog";
 
 export function DeploymentLogs() {
   const project = useProject();
@@ -39,15 +43,25 @@ export function DeploymentLogs() {
   );
 
   return (
-    <Drawer open={!!deploymentId} onClose={() => setDeploymentId(null)}>
-      <DrawerContent className="max-h-full min-h-[80vh]">
-        <div className="mx-auto h-full min-w-[40vw] max-w-4xl">
-          <h2>Logs</h2>
-          <div className="mb-8 max-h-[80vh] overflow-scroll whitespace-nowrap">
+    <Dialog
+      open={!!deploymentId}
+      onOpenChange={(ev) => {
+        if (!ev) {
+          setLogs(null);
+          setDeploymentId(undefined);
+        }
+      }}
+    >
+      <DialogContent className="max-w-6xl">
+        <DialogHeader>
+          <DialogTitle>Deployment Logs</DialogTitle>
+        </DialogHeader>
+        <DialogDescription>
+          <div className="mx-auto h-full max-h-[80vh] max-w-full overflow-scroll overflow-y-scroll text-white">
             {logs && <LogWindow logs={logs} />}
           </div>
-        </div>
-      </DrawerContent>
-    </Drawer>
+        </DialogDescription>
+      </DialogContent>
+    </Dialog>
   );
 }

src/components/LogWindow.tsx

@@ -1,6 +1,7 @@
 "use client";
 
 import Ansi from "ansi-to-react";
+import { useMemo } from "react";
 
 export enum LogLevel {
   /**
@@ -31,22 +32,41 @@ const LOG_LEVEL_TO_CLASS = {
   [LogLevel.Notice]: "bg-blue-950/40",
 };
 
-export function LogWindow({ logs }: { logs: LogLine[] }) {
+export function LogWindow({
+  logs,
+  supressStderr,
+}: {
+  logs: LogLine[];
+  supressStderr?: boolean;
+}) {
+  const withTimestamp = useMemo(() => logs.some((log) => log.t), [logs]);
+
   return (
-    <div className="h-full w-full select-text">
-      {logs.map((log, i) => (
-        <div
-          key={i}
-          className={`flex gap-2 py-1 text-sm ${LOG_LEVEL_TO_CLASS[log.l]}`}
-        >
-          <div className="mt-1 text-xs text-gray-400">
-            {log.t ? new Date(log.t).toLocaleTimeString() : ""}
-          </div>
-          <div className="flex-1 whitespace-pre">
-            <Ansi>{log.m}</Ansi>
-          </div>
-        </div>
-      ))}
-    </div>
+    <table className="h-full w-full text-sm">
+      <tbody>
+        {logs.map((log, i) => (
+          <tr
+            key={i}
+            className={`${
+              supressStderr && log.l === LogLevel.Stderr
+                ? LOG_LEVEL_TO_CLASS[LogLevel.Stdout]
+                : LOG_LEVEL_TO_CLASS[log.l]
+            } hover:bg-black/20`}
+          >
+            {withTimestamp && (
+              <td className="select-none whitespace-nowrap align-top text-gray-300">
+                {log.t ? new Date(log.t).toLocaleTimeString() : ""}
+              </td>
+            )}
+
+            {
+              <td className="w-full whitespace-pre-wrap pl-2">
+                <Ansi>{log.m}</Ansi>
+              </td>
+            }
+          </tr>
+        ))}
+      </tbody>
+    </table>
   );
 }

src/components/service/ServiceDiff.tsx

@@ -0,0 +1,44 @@
+// import { type RouterOutputs } from "~/trpc/shared";
+import type { IChange } from "json-diff-ts";
+import { Badge } from "../ui/badge";
+import { ArrowDown } from "lucide-react";
+
+function Formatted({ children }: { children: React.ReactNode }) {
+  return (
+    <div className="rounded-md bg-muted px-2 py-1 font-mono text-sm text-muted-foreground">
+      {children}
+    </div>
+  );
+}
+
+export function ServiceDiff({ diff }: { diff: IChange[] | IChange }) {
+  if (Array.isArray(diff)) {
+    return (
+      <div>
+        {diff.map((change, i) => (
+          <ServiceDiff key={i} diff={change} />
+        ))}
+      </div>
+    );
+  }
+
+  return (
+    <div className="mt-2 items-center gap-4 rounded-md border border-border bg-card p-4 shadow-sm">
+      <div className="mb-1 flex flex-row font-medium text-white">
+        <span>{diff.key}</span>
+
+        <Badge
+          className="ml-auto mr-0 box-content bg-green-100 text-green-900 dark:bg-green-900/20 dark:text-green-400"
+          variant="outline"
+        >
+          Updated
+        </Badge>
+      </div>
+      <div className="whitespace-pre-wrap text-center text-muted-foreground">
+        <Formatted>{diff.oldValue}</Formatted>
+        <p className="my-1">Updated to</p>
+        <Formatted>{diff.value}</Formatted>
+      </div>
+    </div>
+  );
+}

src/components/ui/badge.tsx

@@ -0,0 +1,36 @@
+import * as React from "react"
+import { cva, type VariantProps } from "class-variance-authority"
+
+import { cn } from "~/utils/utils.ts"
+
+const badgeVariants = cva(
+  "inline-flex items-center rounded-md border px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2",
+  {
+    variants: {
+      variant: {
+        default:
+          "border-transparent bg-primary text-primary-foreground shadow hover:bg-primary/80",
+        secondary:
+          "border-transparent bg-secondary text-secondary-foreground hover:bg-secondary/80",
+        destructive:
+          "border-transparent bg-destructive text-destructive-foreground shadow hover:bg-destructive/80",
+        outline: "text-foreground",
+      },
+    },
+    defaultVariants: {
+      variant: "default",
+    },
+  }
+)
+
+export interface BadgeProps
+  extends React.HTMLAttributes<HTMLDivElement>,
+    VariantProps<typeof badgeVariants> {}
+
+function Badge({ className, variant, ...props }: BadgeProps) {
+  return (
+    <div className={cn(badgeVariants({ variant }), className)} {...props} />
+  )
+}
+
+export { Badge, badgeVariants }

src/server/api/routers/projects/deploy.ts

@@ -28,3 +28,31 @@ export const deployProject = authenticatedProcedure
       response,
     };
   });
+
+export const getDeployDiff = authenticatedProcedure
+  .meta({
+    openapi: {
+      method: "GET",
+      path: "/api/projects/:projectId/deploy/diff",
+      summary:
+        "Get changes from deployed generation and current undeployed generation.",
+    },
+  })
+  .input(
+    z.object({
+      projectId: z.string(),
+    }),
+  )
+  .use(projectMiddleware)
+  .query(async ({ ctx }) => {
+    const services = await ctx.project.getServicesWithPendingUpdates();
+    return await Promise.all(
+      services.map(async (service) => ({
+        service: {
+          name: service.getData().name,
+          id: service.getData().id,
+        },
+        diff: await service.buildDeployDiff(),
+      })),
+    );
+  });

src/server/api/routers/projects/index.ts

@@ -3,7 +3,7 @@ import { z } from "zod";
 import { projects } from "~/server/db/schema";
 import ProjectManager from "~/server/managers/Project";
 import { authenticatedProcedure, createTRPCRouter } from "../../trpc";
-import { deployProject } from "./deploy";
+import { deployProject, getDeployDiff } from "./deploy";
 import { getProject } from "./project";
 import { serviceRouter } from "./service";
 
@@ -73,4 +73,5 @@ export const projectRouter = createTRPCRouter({
 
   get: getProject,
   deploy: deployProject,
+  deployDiff: getDeployDiff,
 });

src/server/api/routers/projects/service/logs.ts

@@ -4,12 +4,9 @@ import { serviceMiddleware } from "~/server/api/middleware/service";
 import { authenticatedProcedure } from "~/server/api/trpc";
 import { observable } from "@trpc/server/observable";
 import assert from "node:assert";
-import { docker404ToNull, streamSort } from "~/server/utils/serverUtils";
-import { PassThrough, Transform } from "node:stream";
-import type DockerModem from "docker-modem";
+import { docker404ToNull } from "~/server/utils/serverUtils";
 import { LogLevel } from "~/server/build/utils/BuilderLogger";
 import type { LogLine } from "~/components/LogWindow";
-import { Queue } from "datastructures-js";
 import { Docker } from "~/server/docker/docker";
 
 // trpc doesn't have stream support yet, so websockets it is
@@ -48,6 +45,10 @@ export const getDeploymentLogsSubscription = authenticatedProcedure
         emit.complete();
       });
 
+      logs.on("error", (err) => {
+        emit.error(err);
+      });
+
       return () => {
         abort.abort();
         logs.destroy();
@@ -88,18 +89,10 @@ export const getServiceLogsSubscription = authenticatedProcedure
       "Unable to retrieve service logs. Maybe the service is not running",
     );
 
-    const unsorted = logs.pipe(Docker.demuxStream());
-    const final = new PassThrough();
-
-    streamSort(unsorted, final, (a, b) => {
-      const aTime = JSON.parse(a.toString()).t;
-      const bTime = JSON.parse(b.toString()).t;
-
-      return aTime - bTime;
-    });
+    const final = logs.pipe(Docker.demuxStream());
 
     function cleanup() {
-      unsorted.end();
+      final.end();
     }
 
     // tail logs now

src/server/build/BuildManager.ts

@@ -35,10 +35,16 @@ export class BuildManager {
     });
   }
 
+  /**
+   * Runs multiple builds in parallel.
+   * Returns a list of deployment IDs.
+   */
   public async runBuilds(
     services: FullServiceGeneration[],
     projectDeploymentId?: string,
-  ) {
+  ): Promise<string[]> {
+    const deploymentIds: string[] = [];
+
     await Promise.all(
       services.map(async (service) => {
         if (service.source !== ServiceSource.Docker) {
@@ -52,6 +58,7 @@ export class BuildManager {
             .returning();
 
           assert(deployment);
+          deploymentIds.push(deployment.id);
 
           service.finalizedDockerImage =
             await BuildManager.getInstance().startBuild(
@@ -61,6 +68,8 @@ export class BuildManager {
         }
       }),
     );
+
+    return deploymentIds;
   }
 
   public getTask(deploymentId: string) {

src/server/build/BuildTask.ts

@@ -18,6 +18,8 @@ import Nixpacks from "./builders/Nixpacks";
 import GitHubSource from "./sources/GitHub";
 import BuilderLogger from "./utils/BuilderLogger";
 import zlib from "node:zlib";
+import type winston from "winston";
+import mainLogger from "../utils/logger";
 
 export default class BuildTask {
   static BASE_BUILD_PATH = "/var/tmp";
@@ -31,12 +33,21 @@ export default class BuildTask {
   // prevents race conditions when updating the status
   private pendingStatusUpdatePromise: Promise<unknown> | null = null;
 
+  // private logging
+  private consoleLogger: winston.Logger;
+
   constructor(
     private readonly serviceId: string,
     private readonly deploymentId: string,
     private readonly finishCallback: (imageTag: string) => void,
     private readonly errorCallback: (error: unknown) => void,
   ) {
+    this.consoleLogger = mainLogger.child({
+      module: "buildTask",
+      deploymentId: this.deploymentId,
+    });
+    this.consoleLogger.debug(`Build dispatched for ${this.deploymentId}`);
+
     this.workingDirectory = path.join(
       BuildTask.BASE_BUILD_PATH,
       "hostforgebuild-" + this.deploymentId,

src/server/build/sources/GitHub.ts

@@ -57,6 +57,5 @@ export default class GitHubSource extends BaseSource {
 
     // wait for exit
     await waitForExit(git);
-    console.log("Downloaded code from GitHub");
   }
 }

src/server/build/utils/utils.ts

@@ -19,7 +19,6 @@ export function waitForExit(child: ChildProcess) {
   return new Promise<void>((resolve, reject) => {
     child.on("exit", (code) => {
       if (code === 0) {
-        console.log("Child process exited successfully");
         resolve();
       } else {
         reject(new Error(`Child process exited with code ${code}`));

src/server/docker/docker.ts

@@ -21,7 +21,6 @@ export class Docker extends Dockerode {
   static demuxStream() {
     return new Transform({
       transform(chunk: Buffer, encoding?: unknown, callback: () => void) {
-        console.log("chunk", chunk.toString());
         if (chunk.length < 8) {
           this.push(chunk);
           callback();

src/server/managers/Deployment.ts

@@ -56,6 +56,7 @@ export default class Deployment {
         const task = BuildManager.getInstance().getTask(this.deploymentData.id);
         if (!task) {
           this.logger.error("Task not found for deployment");
+          returnStream.destroy(new Error("Task not found for deployment"));
           return;
         }
 

src/server/managers/Project.ts

@@ -163,7 +163,7 @@ export default class ProjectManager {
     // now build the dockerfile
     const composeStack = await buildDockerStackFile(allServiceData);
 
-    return await deployOptions.docker.cli(
+    const output = await deployOptions.docker.cli(
       [
         "stack",
         "deploy",
@@ -176,6 +176,16 @@ export default class ProjectManager {
         stdin: JSON.stringify(composeStack),
       },
     );
+
+    // update all deployment statuses to success
+    await db
+      .update(serviceDeployment)
+      .set({
+        status: ServiceDeploymentStatus.Success,
+      })
+      .where(eq(serviceDeployment.projectDeploymentId, deployment.id));
+
+    return output;
   }
 
   /**
@@ -186,9 +196,9 @@ export default class ProjectManager {
       (await this.getServices())
         .map((service) => ({
           service,
-          pendingUpdate: service.buildDeployDiff(),
+          pendingUpdate: service.hasPendingChanges(),
         }))
-        .filter(({ pendingUpdate }) => Object.keys(pendingUpdate).length > 0),
+        .filter(({ pendingUpdate }) => pendingUpdate),
     );
 
     return services.map(({ service }) => service);

src/server/managers/Service.ts

@@ -1,37 +1,19 @@
 import { deterministicString } from "deterministic-object-hash";
 import { and, eq, or } from "drizzle-orm";
-import { create } from "jsondiffpatch";
 import assert from "node:assert";
 import { db } from "../db";
 import { service, serviceGeneration } from "../db/schema";
 import { ServiceSource } from "../db/types";
 import logger from "../utils/logger";
 import Deployment from "./Deployment";
-import type { Docker } from "../docker/docker";
-import { docker404ToNull } from "../utils/serverUtils";
 import type ProjectManager from "./Project";
-import { type paths as DockerAPITypes } from "~/server/docker/types";
+import { diff } from "json-diff-ts";
 
 export default class ServiceManager {
   private static LOGGER = logger.child({
     module: "ServiceManager",
   });
 
-  private static JSON_DIFF = create({
-    objectHash: (obj: unknown) => {
-      if (typeof obj !== "object" || obj === null) {
-        return deterministicString(obj);
-      }
-
-      if ("id" in obj && typeof obj.id === "string") {
-        return obj.id;
-      }
-
-      this.LOGGER.warn("Unexpected object in JSON diff.", { obj });
-      return deterministicString(obj);
-    },
-  });
-
   constructor(
     private serviceData: typeof service.$inferSelect & {
       latestGeneration?: typeof serviceGeneration.$inferSelect;
@@ -78,7 +60,7 @@ export default class ServiceManager {
       this.serviceData.latestGenerationId ===
       this.serviceData.deployedGenerationId
     ) {
-      return {};
+      return [];
     }
 
     // fetch the generations
@@ -88,7 +70,7 @@ export default class ServiceManager {
     ]);
 
     // compare the two
-    return ServiceManager.JSON_DIFF.diff(deployed, latest);
+    return diff(deployed, latest);
   }
 
   /**
@@ -96,9 +78,7 @@ export default class ServiceManager {
    */
   public async hasPendingChanges() {
     const diff = await this.buildDeployDiff();
-
-    if (typeof diff === "object") return Object.keys(diff).length !== 0;
-    return true;
+    return diff.length > 0;
   }
 
   /**