From f6a7f0ef5a2716c918a48e70d5e237539db4c2be Mon Sep 17 00:00:00 2001 From: twotalesanimation <80506065+twotalesanimation@users.noreply.github.com> Date: Thu, 10 Sep 2026 05:40:40 +0200 Subject: [PATCH] feat(dashboard): show what is blocked and how the farm is doing The dashboard was a generic task-manager view of a studio that does not work that way. Two of its five headline numbers counted Task.dueDate, which no pipeline code writes - the only writer is a manual field on the task detail page, so those cards read zero by construction. The rest were unscoped and cumulative ("approved" across every project ever finished), the activity feed was the notification table, and the shot queue sampled 15 rows by updatedAt, a column the farm touches constantly. Nothing about failed exports, QC depth or the render farm appeared at all, so a RENDER_FAILED was only visible if someone thought to open the queue and filter for it. Replaced with pipeline state, which the system maintains on its own: - Blocked queue: the four failed statuses plus exports that stopped progressing, each with the reason read off the actual cause - the failing validation check, the QC note, the aerender error, or the machine that claimed it and went quiet. - Farm strip: machines, queue depth by job type, oldest unclaimed job, and worker-version drift, since a worker's version decides which fixes it has. - Five counters that can go down, scoped to live productions, each linking to the rows it counted. The queue page ignored ?status=, so the counters had nowhere to link; it now takes an initial filter from the URL. Stat cards are hidden from CLIENT users - the old ones were generic version counts, these are internal pipeline state. Thresholds live in SystemConfig (dashboard.qcStaleHours, .stalledJobHours) so they can be tuned without a redeploy. Co-Authored-By: Claude Opus 5 --- app/(dashboard)/dashboard/page.tsx | 157 ++++----- .../pipeline/PipelineQueueClient.tsx | 6 +- app/(dashboard)/pipeline/page.tsx | 16 +- components/dashboard/BlockedQueue.tsx | 110 ++++++ components/dashboard/FarmStrip.tsx | 111 ++++++ components/dashboard/StatsCards.tsx | 118 ++++--- lib/dashboard/ops.test.ts | 33 ++ lib/dashboard/ops.ts | 316 ++++++++++++++++++ lib/render-pipeline/config.ts | 7 + types/index.ts | 18 +- 10 files changed, 730 insertions(+), 162 deletions(-) create mode 100644 components/dashboard/BlockedQueue.tsx create mode 100644 components/dashboard/FarmStrip.tsx create mode 100644 lib/dashboard/ops.test.ts create mode 100644 lib/dashboard/ops.ts diff --git a/app/(dashboard)/dashboard/page.tsx b/app/(dashboard)/dashboard/page.tsx index 2a5a4be..05eca0d 100644 --- a/app/(dashboard)/dashboard/page.tsx +++ b/app/(dashboard)/dashboard/page.tsx @@ -1,58 +1,43 @@ import { auth } from "@/auth"; import { db } from "@/lib/db"; import { StatsCards } from "@/components/dashboard/StatsCards"; +import { BlockedQueue } from "@/components/dashboard/BlockedQueue"; +import { FarmStrip } from "@/components/dashboard/FarmStrip"; import { ShotQueue } from "@/components/dashboard/ShotQueue"; -import { RecentActivity } from "@/components/dashboard/RecentActivity"; import { TaskWidgets } from "@/components/dashboard/TaskWidgets"; import { ScheduleWidgets } from "@/components/dashboard/ScheduleWidgets"; +import { getBlockedExports, getFarmStatus, getOpsCounts } from "@/lib/dashboard/ops"; import type { DashboardStats } from "@/types"; export const metadata = { title: "Dashboard" }; +// Queue depth and machine state are stale the moment they are cached. +export const dynamic = "force-dynamic"; + +/** + * The dashboard answers one question: can we ship what is due, and what is + * stopping us? Everything on it is pipeline state — exports, QC, deliveries, + * worker heartbeats — rather than anything a person has to maintain by hand. + */ async function getDashboardData(userId: string, role: string) { const isArtist = role === "ARTIST"; const isClient = role === "CLIENT"; - const now = new Date(); - const todayStart = new Date(now.getFullYear(), now.getMonth(), now.getDate()); - const todayEnd = new Date(todayStart.getTime() + 86400000); - const [awaitingReview, needsRevisions, approved, activeProjects, - tasksDueToday, tasksInReview, tasksOverdue, myTasksCount] = - await Promise.all([ - db.version.count({ where: { approvalStatus: "PENDING_REVIEW" } }), - db.version.count({ where: { approvalStatus: "NEEDS_CHANGES" } }), - db.version.count({ where: { approvalStatus: "APPROVED" } }), - db.project.count({ where: { status: "ACTIVE" } }), - db.task.count({ - where: { - dueDate: { gte: todayStart, lt: todayEnd }, - status: { not: "DONE" }, - ...(isArtist ? { assignedArtistId: userId } : {}), - }, - }), - db.task.count({ - where: { - status: { in: ["INTERNAL_REVIEW", "CLIENT_REVIEW"] }, - ...(isArtist ? { assignedArtistId: userId } : {}), - }, - }), - db.task.count({ - where: { - dueDate: { lt: todayStart }, - status: { not: "DONE" }, - ...(isArtist ? { assignedArtistId: userId } : {}), - }, - }), - db.task.count({ - where: { assignedArtistId: userId, status: { not: "DONE" } }, - }), - ]); + const [counts, blocked, farm] = await Promise.all([ + getOpsCounts(), + getBlockedExports(), + getFarmStatus(), + ]); - const shotsWhere = isArtist ? { artistId: userId } : {}; + // An artist's own shots come first; everyone else sees the floor. const shots = await db.shot.findMany({ - where: { ...shotsWhere, status: { not: "COMPLETE" } }, + where: { + ...(isArtist ? { artistId: userId } : {}), + status: { not: "COMPLETE" }, + project: { status: { in: ["ACTIVE", "ON_HOLD"] } }, + }, take: 15, - orderBy: { updatedAt: "desc" }, + orderBy: [{ dueDate: { sort: "asc", nulls: "last" } }, { updatedAt: "desc" }], include: { artist: { select: { id: true, name: true, image: true, email: true } }, versions: { @@ -63,12 +48,8 @@ async function getDashboardData(userId: string, role: string) { }, }); - // My tasks (for widget) const myTasks = await db.task.findMany({ - where: { - assignedArtistId: userId, - status: { not: "DONE" }, - }, + where: { assignedArtistId: userId, status: { not: "DONE" } }, take: 8, orderBy: [{ dueDate: "asc" }, { priority: "desc" }], include: { @@ -78,7 +59,6 @@ async function getDashboardData(userId: string, role: string) { }, }); - // Tasks in review (for supervisors/producers) const reviewTasks = isArtist || isClient ? [] : await db.task.findMany({ where: { status: { in: ["INTERNAL_REVIEW", "CLIENT_REVIEW"] } }, take: 8, @@ -91,14 +71,8 @@ async function getDashboardData(userId: string, role: string) { }, }); - const activity = await db.notification.findMany({ - where: isClient ? { userId } : {}, - take: 20, - orderBy: { createdAt: "desc" }, - include: { user: { select: { name: true, image: true } } }, - }); - - // Schedule data (for producers/supervisors/admins) + // Schedule overview (producers/supervisors/admins) — unchanged; it reads the + // schedule page's own data rather than anything the pipeline owns. const scheduledTasks = !isArtist && !isClient ? await db.task.findMany({ @@ -119,43 +93,24 @@ async function getDashboardData(userId: string, role: string) { !isArtist && !isClient ? await db.user.findMany({ where: { isActive: true, role: { not: "CLIENT" } }, - select: { - id: true, - name: true, - email: true, - image: true, - role: true, - }, + select: { id: true, name: true, email: true, image: true, role: true }, orderBy: { name: "asc" }, }) : []; - const stats: DashboardStats = { - awaitingReview, - needsRevisions, - approved, - overdue: tasksOverdue, - activeProjects, - tasksDueToday, - tasksInReview, - tasksOverdue, - myTasksCount, - }; + const stats: DashboardStats = counts; - return { stats, shots, activity, myTasks, reviewTasks, scheduledTasks, scheduleArtists }; + return { stats, blocked, farm, shots, myTasks, reviewTasks, scheduledTasks, scheduleArtists }; } export default async function DashboardPage() { const session = await auth(); if (!session?.user) return null; - const { stats, shots, activity, myTasks, reviewTasks, scheduledTasks, scheduleArtists } = - await getDashboardData(session.user.id, session.user.role); - const isClient = session.user.role === "CLIENT"; - const canSeeSchedule = ["ADMIN", "PRODUCER", "SUPERVISOR"].includes( - session.user.role - ); + const canSeeSchedule = ["ADMIN", "PRODUCER", "SUPERVISOR"].includes(session.user.role); + const { stats, blocked, farm, shots, myTasks, reviewTasks, scheduledTasks, scheduleArtists } = + await getDashboardData(session.user.id, session.user.role); return (
@@ -166,7 +121,25 @@ export default async function DashboardPage() {

- + {!isClient && } + + {!isClient && ( +
+
+

+ Blocked — needs a human +

+ +
+ +
+

+ Render farm +

+ +
+
+ )} {!isClient && ( Schedule Overview - + )} -
-
-

- Shot Queue -

-
- -
-
- -
-

- Recent Activity -

-
- -
+
+

+ {session.user.role === "ARTIST" ? "My shots" : "Shots in flight"} +

+
+
diff --git a/app/(dashboard)/pipeline/PipelineQueueClient.tsx b/app/(dashboard)/pipeline/PipelineQueueClient.tsx index 5a03a8b..13111b4 100644 --- a/app/(dashboard)/pipeline/PipelineQueueClient.tsx +++ b/app/(dashboard)/pipeline/PipelineQueueClient.tsx @@ -51,14 +51,16 @@ interface QueueExport { interface Props { projects: { id: string; name: string; code: string }[]; + /** Status to open filtered to, from `?status=` (see page.tsx). */ + initialStatus?: string; } -export function PipelineQueueClient({ projects }: Props) { +export function PipelineQueueClient({ projects, initialStatus = "all" }: Props) { const { data: session } = useSession(); const { toast } = useToast(); const queryClient = useQueryClient(); const [projectId, setProjectId] = useState("all"); - const [statusFilter, setStatusFilter] = useState("all"); + const [statusFilter, setStatusFilter] = useState(initialStatus); const isAdmin = ["ADMIN", "PRODUCER", "SUPERVISOR"].includes(session?.user?.role ?? ""); const { data, isLoading, refetch, isFetching } = useQuery({ diff --git a/app/(dashboard)/pipeline/page.tsx b/app/(dashboard)/pipeline/page.tsx index 4ea24e6..e37af09 100644 --- a/app/(dashboard)/pipeline/page.tsx +++ b/app/(dashboard)/pipeline/page.tsx @@ -1,20 +1,32 @@ import { auth } from "@/auth"; import { redirect } from "next/navigation"; +import { ExportStatus } from "@prisma/client"; import { db } from "@/lib/db"; import { PipelineQueueClient } from "./PipelineQueueClient"; export const dynamic = "force-dynamic"; -export default async function PipelinePage() { +export default async function PipelinePage({ + searchParams, +}: { + searchParams: Promise<{ status?: string }>; +}) { const session = await auth(); if (!session?.user) redirect("/login"); if (session.user.role === "CLIENT") redirect("/dashboard"); + // Lets other pages link straight to a filtered queue — the dashboard's "In + // QC" figure is only useful if it leads to the rows it counted. Anything + // that is not a real status falls back to the unfiltered queue. + const requested = (await searchParams).status; + const initialStatus = + requested && (Object.values(ExportStatus) as string[]).includes(requested) ? requested : "all"; + const projects = await db.project.findMany({ where: { status: { in: ["ACTIVE", "ON_HOLD"] } }, select: { id: true, name: true, code: true }, orderBy: { name: "asc" }, }); - return ; + return ; } diff --git a/components/dashboard/BlockedQueue.tsx b/components/dashboard/BlockedQueue.tsx new file mode 100644 index 0000000..3d5ee8d --- /dev/null +++ b/components/dashboard/BlockedQueue.tsx @@ -0,0 +1,110 @@ +import Link from "next/link"; +import { formatDistanceToNowStrict } from "date-fns"; +import { cn } from "@/lib/utils"; +import { EXPORT_STATUS_STYLES, statusLabel } from "@/app/(dashboard)/pipeline/status-colors"; +import type { BlockedExport } from "@/lib/dashboard/ops"; + +/** + * The exception list: exports that have stopped moving and need a person. + * + * Read by scanning down the left edge — the severity rail says which rows are + * dead in the water (a failed state, which nothing retries) versus merely + * overdue (still nominally in flight, or waiting on a QC pass). The reason + * column exists so the answer to "what happened" doesn't require opening + * anything. + */ +export function BlockedQueue({ blocked }: { blocked: BlockedExport[] }) { + const critical = blocked.filter((b) => b.severity === "critical").length; + + return ( +
+
+

+ {blocked.length === 0 + ? "Nothing blocked" + : `${blocked.length} export${blocked.length === 1 ? "" : "s"} stalled`} + {critical > 0 && ( + {critical} failed + )} +

+ + Open render queue → + +
+ + {blocked.length === 0 ? ( +

+ Every export is either moving or done. +

+ ) : ( +
+ + + + + + + + + + + + {blocked.map((b) => ( + + + + + + {/* The reason can be an aerender error several lines long; + it is clamped here and given in full on hover. */} + + + + ))} + +
+ ShotStatusStuckWhyArtist
+
+
+ + {b.shotCode}{" "} + {b.versionString} + +
+ {b.projectCode} + {b.episode ? ` · ep ${b.episode}` : ""} +
+ 		+ + {statusLabel(b.status)} + + + {formatDistanceToNowStrict(b.since)} + + {b.reason} + + {b.artistName ?? "—"} +
+
+ )} +
+ ); +} diff --git a/components/dashboard/FarmStrip.tsx b/components/dashboard/FarmStrip.tsx new file mode 100644 index 0000000..511c02c --- /dev/null +++ b/components/dashboard/FarmStrip.tsx @@ -0,0 +1,111 @@ +import Link from "next/link"; +import { formatDistanceToNowStrict } from "date-fns"; +import { cn } from "@/lib/utils"; +import { formatEta } from "@/app/(dashboard)/pipeline/status-colors"; +import type { FarmStatus } from "@/lib/dashboard/ops"; + +/** + * Farm health: who is up, what is waiting, and who is running an old worker. + * + * The version row is not trivia — a worker's version is what decides whether + * it has a given fix, and a machine that quietly stayed behind keeps producing + * the bug everyone else has stopped seeing. + */ +export function FarmStrip({ farm }: { farm: FarmStatus }) { + return ( +
+
+

+ {farm.online} of {farm.total} online +

+ + Machines → + +
+ + {farm.machines.length === 0 ? ( +

+ No machines have registered yet. +

+ ) : ( + farm.machines.map((m) => ( +
+ + {m.name} + + {m.currentJob + ? `${m.currentJob.shotCode ?? "job"} ${m.currentJob.versionString ?? ""} · ${Math.round( + (m.currentJob.progress ?? 0) * 100 + )}% · ETA ${formatEta(m.currentJob.etaSeconds)}` + : m.status === "ONLINE" + ? "Idle" + : m.lastSeenAt + ? `Offline · last seen ${formatDistanceToNowStrict(new Date(m.lastSeenAt))} ago` + : "Never seen"} + + + {m.workerVersion ?? "—"} + +
+ )) + )} + +
+
+ + {farm.queued.render} + + renders queued +
+
+ + {farm.queued.preview} + + previews queued +
+
+ + {farm.oldestQueuedMs == null + ? "—" + : formatDistanceToNowStrict(new Date(Date.now() - farm.oldestQueuedMs))} + + oldest wait +
+
+ + {farm.behindMachines.length > 0 && ( +
+ + + {farm.behindMachines.length} machine + {farm.behindMachines.length === 1 ? " is" : "s are"} behind{" "} + {farm.latestWorkerVersion} —{" "} + {farm.behindMachines + .map((m) => `${m.name} (${m.workerVersion ?? "unknown"})`) + .join(", ")} + +
+ )} +
+ ); +} diff --git a/components/dashboard/StatsCards.tsx b/components/dashboard/StatsCards.tsx index 4be4823..6b8a1f4 100644 --- a/components/dashboard/StatsCards.tsx +++ b/components/dashboard/StatsCards.tsx @@ -1,12 +1,11 @@ -"use client"; - +import Link from "next/link"; import { Card, CardContent } from "@/components/ui/card"; import { cn } from "@/lib/utils"; import { - Clock, - AlertCircle, - CheckCircle2, - XCircle, + AlertTriangle, + ClipboardCheck, + PackageCheck, + Eye, TrendingUp, } from "lucide-react"; import type { DashboardStats } from "@/types"; @@ -15,47 +14,54 @@ interface StatsCardsProps { stats: DashboardStats; } +/** + * Five numbers that can go down. Each one links to the page that lists what it + * counts — a figure with no way through to the rows behind it is wallpaper. + */ export function StatsCards({ stats }: StatsCardsProps) { const cards = [ { - label: "Awaiting Review", - value: stats.awaitingReview, - icon: Clock, - color: "text-amber-400", - bg: "bg-amber-500/10", - ring: "ring-amber-500/20", - }, - { - label: "Needs Revisions", - value: stats.needsRevisions, - icon: AlertCircle, - color: "text-orange-400", - bg: "bg-orange-500/10", - ring: "ring-orange-500/20", - }, - { - label: "Approved", - value: stats.approved, - icon: CheckCircle2, - color: "text-emerald-400", - bg: "bg-emerald-500/10", - ring: "ring-emerald-500/20", - }, - { - label: "Overdue Tasks", - value: stats.tasksOverdue ?? stats.overdue, - icon: XCircle, + label: "Blocked", + value: stats.blocked, + href: "#blocked", + icon: AlertTriangle, color: "text-red-400", bg: "bg-red-500/10", - ring: "ring-red-500/20", + // The only card that earns emphasis when it is non-zero: everything else + // here is a queue depth, this one is work that has stopped. + emphasise: stats.blocked > 0, }, { - label: "Active Projects", - value: stats.activeProjects, - icon: TrendingUp, + label: "In QC", + value: stats.inQc, + href: "/pipeline?status=READY_FOR_QC", + icon: ClipboardCheck, + color: "text-amber-400", + bg: "bg-amber-500/10", + }, + { + label: "Ready to deliver", + value: stats.readyToDeliver, + href: "/pipeline/deliveries", + icon: PackageCheck, + color: "text-green-400", + bg: "bg-green-500/10", + }, + { + label: "With the client", + value: stats.awaitingClientReview, + href: "/playlist", + icon: Eye, color: "text-blue-400", bg: "bg-blue-500/10", - ring: "ring-blue-500/20", + }, + { + label: "Active projects", + value: stats.activeProjects, + href: "/projects", + icon: TrendingUp, + color: "text-zinc-400", + bg: "bg-zinc-500/10", }, ]; @@ -64,19 +70,31 @@ export function StatsCards({ stats }: StatsCardsProps) { {cards.map((card) => { const Icon = card.icon; return ( - - -
-
- + + + +
+
+ +
-
-

- {card.value} -

-

{card.label}

- - +

+ {card.value} +

+

{card.label}

+ + + ); })}
diff --git a/lib/dashboard/ops.test.ts b/lib/dashboard/ops.test.ts new file mode 100644 index 0000000..5411fce --- /dev/null +++ b/lib/dashboard/ops.test.ts @@ -0,0 +1,33 @@ +import { describe, it, expect } from "vitest"; +import { compareVersions } from "./ops"; + +/** + * The farm strip flags machines running an older worker than the newest one + * out there, which is only as reliable as this comparison: a plain string + * compare ranks "1.10.0" below "1.9.0" and would quietly stop reporting drift + * on the day the minor version reaches double digits. + */ +describe("compareVersions", () => { + it("orders by numeric component, not lexically", () => { + expect(compareVersions("1.10.0", "1.9.0")).toBeGreaterThan(0); + expect(compareVersions("1.9.0", "1.10.0")).toBeLessThan(0); + }); + + it("ignores the build stamp publish.ps1 appends", () => { + // The exe reports "1.6.1+fe420b9"; two machines on the same release built + // from different commits are not drifting. + expect(compareVersions("1.6.1+fe420b9", "1.6.1")).toBe(0); + expect(compareVersions("1.6.1+fe420b9", "1.6.0+2a88880")).toBeGreaterThan(0); + }); + + it("treats a missing component as zero", () => { + expect(compareVersions("1.6", "1.6.0")).toBe(0); + expect(compareVersions("1.6", "1.6.1")).toBeLessThan(0); + }); + + it("is transitive enough to pick a maximum", () => { + const versions = ["1.5.2", "1.6.1", "1.6.0", "1.10.0"]; + const latest = versions.reduce((a, b) => (compareVersions(a, b) >= 0 ? a : b)); + expect(latest).toBe("1.10.0"); + }); +}); diff --git a/lib/dashboard/ops.ts b/lib/dashboard/ops.ts new file mode 100644 index 0000000..47c3937 --- /dev/null +++ b/lib/dashboard/ops.ts @@ -0,0 +1,316 @@ +import { db } from "@/lib/db"; +import { getConfigNumber } from "@/lib/render-pipeline/config"; +import { listMachines } from "@/lib/render-pipeline/machines"; +import type { ExportStatus, Prisma } from "@prisma/client"; + +/** + * Data for the operations dashboard. + * + * Everything here is derived from state the pipeline maintains on its own — + * export transitions, validation results, QC reviews, worker heartbeats — and + * never from a field a human has to remember to fill in. The dashboard this + * replaced counted `Task.dueDate`, which nothing in the pipeline writes, so + * its headline numbers were structurally always zero. + */ + +/** Terminal-until-someone-acts: these never move on their own. */ +const FAILED_STATUSES: ExportStatus[] = [ + "RENDER_FAILED", + "VALIDATION_FAILED", + "PREVIEW_FAILED", + "QC_FAILED", +]; + +/** In flight: fine until they stop progressing, then they are stuck. */ +const IN_FLIGHT_STATUSES: ExportStatus[] = [ + "QUEUED", + "RENDERING", + "VALIDATING", + "GENERATING_PREVIEW", +]; + +export type BlockedSeverity = "critical" | "warning"; + +export interface BlockedExport { + exportId: string; + shotId: string; + shotCode: string; + versionString: string; + projectCode: string; + episode: string | null; + status: ExportStatus; + severity: BlockedSeverity; + /** When it entered the status it is stuck in. */ + since: Date; + /** What a human has to do something about, in one line. */ + reason: string; + artistName: string | null; +} + +/** + * Everything that has stopped moving and needs a person. + * + * Two kinds: exports in a failed state (which stay there forever — nothing + * retries a VALIDATION_FAILED on its own), and exports sitting in a live state + * far longer than that state should take, which is how a dead worker or a + * forgotten QC pass shows up. A failed export is currently only visible if + * someone thinks to open the render queue and filter for it. + */ +export async function getBlockedExports(limit = 25): Promise { + const [qcStaleHours, stalledHours] = await Promise.all([ + getConfigNumber("dashboard.qcStaleHours"), + getConfigNumber("dashboard.stalledJobHours"), + ]); + const now = Date.now(); + const qcCutoff = new Date(now - qcStaleHours * 3600_000); + const stalledCutoff = new Date(now - stalledHours * 3600_000); + + const rows = await db.export.findMany({ + where: { + // Finished shows would otherwise fill the list with failures nobody + // intends to fix. Export carries a projectId but no project relation, so + // the scope is reached through the shot. + shot: { project: { status: { in: ["ACTIVE", "ON_HOLD"] } } }, + OR: [ + { status: { in: FAILED_STATUSES } }, + { status: "READY_FOR_QC", statusChangedAt: { lt: qcCutoff } }, + { status: { in: IN_FLIGHT_STATUSES }, statusChangedAt: { lt: stalledCutoff } }, + ], + }, + orderBy: { statusChangedAt: "asc" }, + take: limit, + select: { + id: true, + shotId: true, + versionString: true, + status: true, + statusChangedAt: true, + shot: { + select: { + shotCode: true, + episode: true, + artist: { select: { name: true, email: true } }, + project: { select: { code: true } }, + }, + }, + // Only the failures matter for the reason line, and only a couple of + // them: the point is to say what to look at, not to reproduce the + // validation report. + validations: { + where: { status: "FAIL" }, + orderBy: { createdAt: "desc" }, + take: 3, + select: { checkName: true, expected: true, actual: true, message: true }, + }, + qcReviews: { + orderBy: { createdAt: "desc" }, + take: 1, + select: { result: true, notes: true, reviewerName: true }, + }, + renderJobs: { + orderBy: { createdAt: "desc" }, + take: 1, + select: { + errorMessage: true, + status: true, + attempt: true, + maxAttempts: true, + machine: { select: { name: true } }, + }, + }, + }, + }); + + const blocked = rows.map((row): BlockedExport => { + const job = row.renderJobs[0]; + const qc = row.qcReviews[0]; + let reason: string; + + switch (row.status) { + case "VALIDATION_FAILED": { + const first = row.validations[0]; + const extra = row.validations.length > 1 ? ` (+${row.validations.length - 1} more)` : ""; + reason = first + ? `${first.checkName} — ${first.message ?? `expected ${first.expected ?? "?"}, got ${first.actual ?? "?"}`}${extra}` + : "Validation failed"; + break; + } + case "QC_FAILED": + reason = qc?.notes?.trim() + ? `${qc.notes.trim()}${qc.reviewerName ? ` — ${qc.reviewerName}` : ""}` + : "Failed technical QC"; + break; + case "RENDER_FAILED": + case "PREVIEW_FAILED": + reason = + job?.errorMessage?.trim() || + (row.status === "RENDER_FAILED" ? "Render failed" : "Preview build failed"); + if (job && job.attempt >= job.maxAttempts) reason += ` (attempt ${job.attempt}/${job.maxAttempts})`; + break; + case "READY_FOR_QC": + reason = "Waiting on a QC pass"; + break; + default: + // Still nominally in flight, but past the point where that is normal. + reason = job?.machine?.name + ? `No progress since claim on ${job.machine.name}` + : "Queued with no machine picking it up"; + } + + return { + exportId: row.id, + shotId: row.shotId, + shotCode: row.shot.shotCode, + versionString: row.versionString, + projectCode: row.shot.project.code, + episode: row.shot.episode, + status: row.status, + severity: FAILED_STATUSES.includes(row.status) ? "critical" : "warning", + since: row.statusChangedAt, + reason, + artistName: row.shot.artist?.name ?? row.shot.artist?.email ?? null, + }; + }); + + // Failures first, then whatever has been waiting longest. Both orderings + // matter: severity decides what to fix, age decides what to fix first. + return blocked.sort((a, b) => { + if (a.severity !== b.severity) return a.severity === "critical" ? -1 : 1; + return a.since.getTime() - b.since.getTime(); + }); +} + +export interface OpsCounts { + blocked: number; + inQc: number; + readyToDeliver: number; + awaitingClientReview: number; + activeProjects: number; +} + +/** + * The headline numbers, all scoped to live productions. + * + * The counters these replaced were unscoped and cumulative — "approved" across + * every project the studio has ever finished only ever goes up, so it never + * told anyone anything. Each of these can go down, which is what makes it + * worth looking at twice in a day. + */ +export async function getOpsCounts(): Promise { + // Export has a projectId column but no project relation, so live-production + // scope is reached through the shot. + const liveProject: Prisma.ExportWhereInput = { + shot: { project: { status: { in: ["ACTIVE", "ON_HOLD"] } } }, + }; + const [qcStaleHours, stalledHours] = await Promise.all([ + getConfigNumber("dashboard.qcStaleHours"), + getConfigNumber("dashboard.stalledJobHours"), + ]); + const now = Date.now(); + + const [blocked, inQc, readyToDeliver, awaitingClientReview, activeProjects] = await Promise.all([ + db.export.count({ + where: { + ...liveProject, + OR: [ + { status: { in: FAILED_STATUSES } }, + { status: "READY_FOR_QC", statusChangedAt: { lt: new Date(now - qcStaleHours * 3600_000) } }, + { + status: { in: IN_FLIGHT_STATUSES }, + statusChangedAt: { lt: new Date(now - stalledHours * 3600_000) }, + }, + ], + }, + }), + db.export.count({ where: { ...liveProject, status: "READY_FOR_QC" } }), + db.export.count({ where: { ...liveProject, status: "READY_FOR_DELIVERY" } }), + // Sent to the client and still unanswered — the wait that is not ours to + // end, as opposed to notes we have not acted on yet. + db.version.count({ + where: { + approvalStatus: "PENDING_REVIEW", + isClientVisible: true, + isLatest: true, + shot: { project: { status: { in: ["ACTIVE", "ON_HOLD"] } } }, + }, + }), + db.project.count({ where: { status: "ACTIVE" } }), + ]); + + return { blocked, inQc, readyToDeliver, awaitingClientReview, activeProjects }; +} + +export interface FarmStatus { + machines: Awaited>; + online: number; + total: number; + queued: { render: number; preview: number; delivery: number; blender: number; total: number }; + running: number; + /** Age of the oldest job still waiting to be claimed, in ms. */ + oldestQueuedMs: number | null; + /** Highest worker version any machine reports, and who is behind it. */ + latestWorkerVersion: string | null; + behindMachines: { name: string; workerVersion: string | null }[]; +} + +/** + * Farm health in one strip: who is up, what is waiting, and — because a + * worker's version is what decides whether it has a given fix — which + * machines are running something older than the newest one out there. + */ +export async function getFarmStatus(): Promise { + const [machines, queuedByType, running, oldest] = await Promise.all([ + listMachines(), + db.renderJob.groupBy({ by: ["type"], where: { status: "QUEUED" }, _count: { _all: true } }), + db.renderJob.count({ where: { status: { in: ["CLAIMED", "RUNNING"] } } }), + db.renderJob.findFirst({ + where: { status: "QUEUED" }, + orderBy: { createdAt: "asc" }, + select: { createdAt: true }, + }), + ]); + + const countOf = (type: string) => + queuedByType.find((q) => q.type === type)?._count._all ?? 0; + + const versions = machines + .map((m) => m.workerVersion) + .filter((v): v is string => Boolean(v)); + const latest = versions.length ? versions.reduce((a, b) => (compareVersions(a, b) >= 0 ? a : b)) : null; + + return { + machines, + online: machines.filter((m) => m.status === "ONLINE").length, + total: machines.length, + queued: { + render: countOf("AE_RENDER"), + preview: countOf("PREVIEW_ONLY"), + delivery: countOf("DELIVERY_BUILD"), + blender: countOf("BLENDER_RENDER"), + total: queuedByType.reduce((sum, q) => sum + q._count._all, 0), + }, + running, + oldestQueuedMs: oldest ? Date.now() - oldest.createdAt.getTime() : null, + latestWorkerVersion: latest, + behindMachines: latest + ? machines + .filter((m) => m.workerVersion !== latest) + .map((m) => ({ name: m.name, workerVersion: m.workerVersion })) + : [], + }; +} + +/** + * Compares "1.6.1+fe420b9" style versions numerically, ignoring the build + * stamp publish.ps1 appends. A string compare would rank 1.10.0 below 1.9.0. + */ +export function compareVersions(a: string, b: string): number { + const parts = (v: string) => + v.split("+")[0].split(".").map((n) => Number.parseInt(n, 10) || 0); + const [pa, pb] = [parts(a), parts(b)]; + for (let i = 0; i < Math.max(pa.length, pb.length); i++) { + const diff = (pa[i] ?? 0) - (pb[i] ?? 0); + if (diff !== 0) return diff; + } + return 0; +} diff --git a/lib/render-pipeline/config.ts b/lib/render-pipeline/config.ts index 5cbadf8..79544f5 100644 --- a/lib/render-pipeline/config.ts +++ b/lib/render-pipeline/config.ts @@ -48,6 +48,13 @@ export const PIPELINE_CONFIG_DEFAULTS = { // names as a warning, so it is a config fix rather than a code change. "preview.slateScopeProp": "Scope", "preview.slateSubmissionProp": "Notes", + /** + * When the dashboard starts calling something stuck. An export waiting on QC + * is normal for a day and suspicious after two; an export in a live render + * state has either progressed or died long before four hours are up. + */ + "dashboard.qcStaleHours": 48, + "dashboard.stalledJobHours": 4, } as const; export type PipelineConfigKey = keyof typeof PIPELINE_CONFIG_DEFAULTS; diff --git a/types/index.ts b/types/index.ts index 99c6e7e..5ece385 100644 --- a/types/index.ts +++ b/types/index.ts @@ -276,17 +276,17 @@ export interface AssetWithDetails { // ── Dashboard Types ────────────────────────────────────────────────────────── +/** + * Dashboard headline numbers. Every one is derived from pipeline state and + * scoped to live productions — see lib/dashboard/ops.ts. The previous shape + * carried task due/overdue counts, which read a column nothing writes. + */ export interface DashboardStats { - awaitingReview: number; - needsRevisions: number; - approved: number; - overdue: number; + blocked: number; + inQc: number; + readyToDeliver: number; + awaitingClientReview: number; activeProjects: number; - // Task widgets - tasksDueToday: number; - tasksInReview: number; - tasksOverdue: number; - myTasksCount: number; } // ── Notification Badge ────────────────────────────────────────────────────────