import { db } from "@/lib/db"; import { getConfigNumber } from "@/lib/render-pipeline/config"; import { listMachines } from "@/lib/render-pipeline/machines"; import type { ExportStatus, Prisma } from "@prisma/client"; /** * Data for the operations dashboard. * * Everything here is derived from state the pipeline maintains on its own — * export transitions, validation results, QC reviews, worker heartbeats — and * never from a field a human has to remember to fill in. The dashboard this * replaced counted `Task.dueDate`, which nothing in the pipeline writes, so * its headline numbers were structurally always zero. */ /** Terminal-until-someone-acts: these never move on their own. */ const FAILED_STATUSES: ExportStatus[] = [ "RENDER_FAILED", "VALIDATION_FAILED", "PREVIEW_FAILED", "QC_FAILED", ]; /** In flight: fine until they stop progressing, then they are stuck. */ const IN_FLIGHT_STATUSES: ExportStatus[] = [ "QUEUED", "RENDERING", "VALIDATING", "GENERATING_PREVIEW", ]; export type BlockedSeverity = "critical" | "warning"; export interface BlockedExport { exportId: string; shotId: string; shotCode: string; versionString: string; projectCode: string; episode: string | null; status: ExportStatus; severity: BlockedSeverity; /** When it entered the status it is stuck in. */ since: Date; /** What a human has to do something about, in one line. */ reason: string; artistName: string | null; } /** * Everything that has stopped moving and needs a person. * * Two kinds: exports in a failed state (which stay there forever — nothing * retries a VALIDATION_FAILED on its own), and exports sitting in a live state * far longer than that state should take, which is how a dead worker or a * forgotten QC pass shows up. A failed export is currently only visible if * someone thinks to open the render queue and filter for it. */ export async function getBlockedExports(limit = 25): Promise { const [qcStaleHours, stalledHours] = await Promise.all([ getConfigNumber("dashboard.qcStaleHours"), getConfigNumber("dashboard.stalledJobHours"), ]); const now = Date.now(); const qcCutoff = new Date(now - qcStaleHours * 3600_000); const stalledCutoff = new Date(now - stalledHours * 3600_000); const rows = await db.export.findMany({ where: { // Finished shows would otherwise fill the list with failures nobody // intends to fix. Export carries a projectId but no project relation, so // the scope is reached through the shot. shot: { project: { status: { in: ["ACTIVE", "ON_HOLD"] } } }, OR: [ { status: { in: FAILED_STATUSES } }, { status: "READY_FOR_QC", statusChangedAt: { lt: qcCutoff } }, { status: { in: IN_FLIGHT_STATUSES }, statusChangedAt: { lt: stalledCutoff } }, ], }, orderBy: { statusChangedAt: "asc" }, take: limit, select: { id: true, shotId: true, versionString: true, status: true, statusChangedAt: true, shot: { select: { shotCode: true, episode: true, artist: { select: { name: true, email: true } }, project: { select: { code: true } }, }, }, // Only the failures matter for the reason line, and only a couple of // them: the point is to say what to look at, not to reproduce the // validation report. validations: { where: { status: "FAIL" }, orderBy: { createdAt: "desc" }, take: 3, select: { checkName: true, expected: true, actual: true, message: true }, }, qcReviews: { orderBy: { createdAt: "desc" }, take: 1, select: { result: true, notes: true, reviewerName: true }, }, renderJobs: { orderBy: { createdAt: "desc" }, take: 1, select: { errorMessage: true, status: true, attempt: true, maxAttempts: true, machine: { select: { name: true } }, }, }, }, }); const blocked = rows.map((row): BlockedExport => { const job = row.renderJobs[0]; const qc = row.qcReviews[0]; let reason: string; switch (row.status) { case "VALIDATION_FAILED": { const first = row.validations[0]; const extra = row.validations.length > 1 ? ` (+${row.validations.length - 1} more)` : ""; reason = first ? `${first.checkName} — ${first.message ?? `expected ${first.expected ?? "?"}, got ${first.actual ?? "?"}`}${extra}` : "Validation failed"; break; } case "QC_FAILED": reason = qc?.notes?.trim() ? `${qc.notes.trim()}${qc.reviewerName ? ` — ${qc.reviewerName}` : ""}` : "Failed technical QC"; break; case "RENDER_FAILED": case "PREVIEW_FAILED": reason = job?.errorMessage?.trim() || (row.status === "RENDER_FAILED" ? "Render failed" : "Preview build failed"); if (job && job.attempt >= job.maxAttempts) reason += ` (attempt ${job.attempt}/${job.maxAttempts})`; break; case "READY_FOR_QC": reason = "Waiting on a QC pass"; break; default: // Still nominally in flight, but past the point where that is normal. reason = job?.machine?.name ? `No progress since claim on ${job.machine.name}` : "Queued with no machine picking it up"; } return { exportId: row.id, shotId: row.shotId, shotCode: row.shot.shotCode, versionString: row.versionString, projectCode: row.shot.project.code, episode: row.shot.episode, status: row.status, severity: FAILED_STATUSES.includes(row.status) ? "critical" : "warning", since: row.statusChangedAt, reason, artistName: row.shot.artist?.name ?? row.shot.artist?.email ?? null, }; }); // Failures first, then whatever has been waiting longest. Both orderings // matter: severity decides what to fix, age decides what to fix first. return blocked.sort((a, b) => { if (a.severity !== b.severity) return a.severity === "critical" ? -1 : 1; return a.since.getTime() - b.since.getTime(); }); } export interface OpsCounts { blocked: number; inQc: number; readyToDeliver: number; awaitingClientReview: number; activeProjects: number; } /** * The headline numbers, all scoped to live productions. * * The counters these replaced were unscoped and cumulative — "approved" across * every project the studio has ever finished only ever goes up, so it never * told anyone anything. Each of these can go down, which is what makes it * worth looking at twice in a day. */ export async function getOpsCounts(): Promise { // Export has a projectId column but no project relation, so live-production // scope is reached through the shot. const liveProject: Prisma.ExportWhereInput = { shot: { project: { status: { in: ["ACTIVE", "ON_HOLD"] } } }, }; const [qcStaleHours, stalledHours] = await Promise.all([ getConfigNumber("dashboard.qcStaleHours"), getConfigNumber("dashboard.stalledJobHours"), ]); const now = Date.now(); const [blocked, inQc, readyToDeliver, awaitingClientReview, activeProjects] = await Promise.all([ db.export.count({ where: { ...liveProject, OR: [ { status: { in: FAILED_STATUSES } }, { status: "READY_FOR_QC", statusChangedAt: { lt: new Date(now - qcStaleHours * 3600_000) } }, { status: { in: IN_FLIGHT_STATUSES }, statusChangedAt: { lt: new Date(now - stalledHours * 3600_000) }, }, ], }, }), db.export.count({ where: { ...liveProject, status: "READY_FOR_QC" } }), db.export.count({ where: { ...liveProject, status: "READY_FOR_DELIVERY" } }), // Sent to the client and still unanswered — the wait that is not ours to // end, as opposed to notes we have not acted on yet. db.version.count({ where: { approvalStatus: "PENDING_REVIEW", isClientVisible: true, isLatest: true, shot: { project: { status: { in: ["ACTIVE", "ON_HOLD"] } } }, }, }), db.project.count({ where: { status: "ACTIVE" } }), ]); return { blocked, inQc, readyToDeliver, awaitingClientReview, activeProjects }; } export interface FarmStatus { machines: Awaited>; online: number; total: number; queued: { render: number; preview: number; delivery: number; blender: number; total: number }; running: number; /** Age of the oldest job still waiting to be claimed, in ms. */ oldestQueuedMs: number | null; /** Highest worker version any machine reports, and who is behind it. */ latestWorkerVersion: string | null; behindMachines: { name: string; workerVersion: string | null }[]; } /** * Farm health in one strip: who is up, what is waiting, and — because a * worker's version is what decides whether it has a given fix — which * machines are running something older than the newest one out there. */ export async function getFarmStatus(): Promise { const [machines, queuedByType, running, oldest] = await Promise.all([ listMachines(), db.renderJob.groupBy({ by: ["type"], where: { status: "QUEUED" }, _count: { _all: true } }), db.renderJob.count({ where: { status: { in: ["CLAIMED", "RUNNING"] } } }), db.renderJob.findFirst({ where: { status: "QUEUED" }, orderBy: { createdAt: "asc" }, select: { createdAt: true }, }), ]); const countOf = (type: string) => queuedByType.find((q) => q.type === type)?._count._all ?? 0; const versions = machines .map((m) => m.workerVersion) .filter((v): v is string => Boolean(v)); const latest = versions.length ? versions.reduce((a, b) => (compareVersions(a, b) >= 0 ? a : b)) : null; return { machines, online: machines.filter((m) => m.status === "ONLINE").length, total: machines.length, queued: { render: countOf("AE_RENDER"), preview: countOf("PREVIEW_ONLY"), delivery: countOf("DELIVERY_BUILD"), blender: countOf("BLENDER_RENDER"), total: queuedByType.reduce((sum, q) => sum + q._count._all, 0), }, running, oldestQueuedMs: oldest ? Date.now() - oldest.createdAt.getTime() : null, latestWorkerVersion: latest, behindMachines: latest ? machines .filter((m) => m.workerVersion !== latest) .map((m) => ({ name: m.name, workerVersion: m.workerVersion })) : [], }; } /** * Compares "1.6.1+fe420b9" style versions numerically, ignoring the build * stamp publish.ps1 appends. A string compare would rank 1.10.0 below 1.9.0. */ export function compareVersions(a: string, b: string): number { const parts = (v: string) => v.split("+")[0].split(".").map((n) => Number.parseInt(n, 10) || 0); const [pa, pb] = [parts(a), parts(b)]; for (let i = 0; i < Math.max(pa.length, pb.length); i++) { const diff = (pa[i] ?? 0) - (pb[i] ?? 0); if (diff !== 0) return diff; } return 0; }