import { db } from "@/lib/db"; import { getConfigNumber } from "@/lib/render-pipeline/config"; import { listMachines } from "@/lib/render-pipeline/machines"; import { handoffStage, type HandoffStage } from "@/lib/render-pipeline/delivery-handoff"; import type { ExportStatus, Prisma } from "@prisma/client"; /** * Data for the operations dashboard. * * Everything here is derived from state the pipeline maintains on its own — * export transitions, validation results, QC reviews, worker heartbeats — and * never from a field a human has to remember to fill in. The dashboard this * replaced counted `Task.dueDate`, which nothing in the pipeline writes, so * its headline numbers were structurally always zero. */ /** Terminal-until-someone-acts: these never move on their own. */ const FAILED_STATUSES: ExportStatus[] = [ "RENDER_FAILED", "VALIDATION_FAILED", "PREVIEW_FAILED", "QC_FAILED", ]; /** In flight: fine until they stop progressing, then they are stuck. */ const IN_FLIGHT_STATUSES: ExportStatus[] = [ "QUEUED", "RENDERING", "VALIDATING", "GENERATING_PREVIEW", ]; export type BlockedSeverity = "critical" | "warning"; export interface BlockedExport { exportId: string; shotId: string; shotCode: string; versionString: string; projectCode: string; episode: string | null; status: ExportStatus; severity: BlockedSeverity; /** When it entered the status it is stuck in. */ since: Date; /** What a human has to do something about, in one line. */ reason: string; artistName: string | null; } /** * Everything that has stopped moving and needs a person. * * Two kinds: exports in a failed state (which stay there forever — nothing * retries a VALIDATION_FAILED on its own), and exports sitting in a live state * far longer than that state should take, which is how a dead worker or a * forgotten QC pass shows up. A failed export is currently only visible if * someone thinks to open the render queue and filter for it. */ export async function getBlockedExports( opts: { artistId?: string; limit?: number } = {} ): Promise { const { artistId, limit = 25 } = opts; const [qcStaleHours, stalledHours] = await Promise.all([ getConfigNumber("dashboard.qcStaleHours"), getConfigNumber("dashboard.stalledJobHours"), ]); const now = Date.now(); const qcCutoff = new Date(now - qcStaleHours * 3600_000); const stalledCutoff = new Date(now - stalledHours * 3600_000); const rows = await db.export.findMany({ where: { // Finished shows would otherwise fill the list with failures nobody // intends to fix. Export carries a projectId but no project relation, so // the scope is reached through the shot. shot: { project: { status: { in: ["ACTIVE", "ON_HOLD"] } }, // An artist's dashboard shows the failures that are theirs to fix. ...(artistId ? { artistId } : {}), }, OR: [ { status: { in: FAILED_STATUSES } }, { status: "READY_FOR_QC", statusChangedAt: { lt: qcCutoff } }, { status: { in: IN_FLIGHT_STATUSES }, statusChangedAt: { lt: stalledCutoff } }, ], }, orderBy: { statusChangedAt: "asc" }, take: limit, select: { id: true, shotId: true, versionString: true, status: true, statusChangedAt: true, shot: { select: { shotCode: true, episode: true, artist: { select: { name: true, email: true } }, project: { select: { code: true } }, }, }, // Only the failures matter for the reason line, and only a couple of // them: the point is to say what to look at, not to reproduce the // validation report. validations: { where: { status: "FAIL" }, orderBy: { createdAt: "desc" }, take: 3, select: { checkName: true, expected: true, actual: true, message: true }, }, qcReviews: { orderBy: { createdAt: "desc" }, take: 1, select: { result: true, notes: true, reviewerName: true }, }, renderJobs: { orderBy: { createdAt: "desc" }, take: 1, select: { errorMessage: true, status: true, attempt: true, maxAttempts: true, machine: { select: { name: true } }, }, }, }, }); const blocked = rows.map((row): BlockedExport => { const job = row.renderJobs[0]; const qc = row.qcReviews[0]; let reason: string; switch (row.status) { case "VALIDATION_FAILED": { const first = row.validations[0]; const extra = row.validations.length > 1 ? ` (+${row.validations.length - 1} more)` : ""; reason = first ? `${first.checkName} — ${first.message ?? `expected ${first.expected ?? "?"}, got ${first.actual ?? "?"}`}${extra}` : "Validation failed"; break; } case "QC_FAILED": reason = qc?.notes?.trim() ? `${qc.notes.trim()}${qc.reviewerName ? ` — ${qc.reviewerName}` : ""}` : "Failed technical QC"; break; case "RENDER_FAILED": case "PREVIEW_FAILED": reason = job?.errorMessage?.trim() || (row.status === "RENDER_FAILED" ? "Render failed" : "Preview build failed"); if (job && job.attempt >= job.maxAttempts) reason += ` (attempt ${job.attempt}/${job.maxAttempts})`; break; case "READY_FOR_QC": reason = "Waiting on a QC pass"; break; default: // Still nominally in flight, but past the point where that is normal. reason = job?.machine?.name ? `No progress since claim on ${job.machine.name}` : "Queued with no machine picking it up"; } return { exportId: row.id, shotId: row.shotId, shotCode: row.shot.shotCode, versionString: row.versionString, projectCode: row.shot.project.code, episode: row.shot.episode, status: row.status, severity: FAILED_STATUSES.includes(row.status) ? "critical" : "warning", since: row.statusChangedAt, reason, artistName: row.shot.artist?.name ?? row.shot.artist?.email ?? null, }; }); // Failures first, then whatever has been waiting longest. Both orderings // matter: severity decides what to fix, age decides what to fix first. return blocked.sort((a, b) => { if (a.severity !== b.severity) return a.severity === "critical" ? -1 : 1; return a.since.getTime() - b.since.getTime(); }); } export type CountdownVerdict = "complete" | "overdue" | "stalled" | "behind" | "tight" | "ahead"; export interface EpisodeCountdown { projectId: string; projectCode: string; projectName: string; /** Null on a standard (non-episodic) show, which has one date of its own. */ episode: string | null; dueDate: Date; /** Calendar days to the due date; negative once it has passed. */ daysLeft: number; total: number; delivered: number; inQc: number; inProgress: number; notStarted: number; /** Shots per day still needed to land the rest by the due date. */ requiredPerDay: number; /** Shots per day actually delivered over the last week. */ actualPerDay: number; verdict: CountdownVerdict; /** * How late this is: the days the current rate overruns by when `verdict` is * "behind", or the days the date has already passed when it is "overdue". */ daysBehind: number | null; } const DAY_MS = 86_400_000; /** Shots past QC but not yet in a delivered package. */ const POST_QC_STATUSES: ExportStatus[] = ["READY_FOR_QC", "READY_FOR_DELIVERY", "PACKAGED"]; /** * Turns "23 shots left, 14 days, 8 delivered last week" into a verdict. * * Kept free of the database so the arithmetic that decides whether a producer * panics can be tested directly. The trailing week is the rate we have * evidence for; anything else is a hope. A run that only just lands on the due * date is reported as tight rather than fine — finishing on the last available * day is luck, not schedule. */ export function projectPace( remaining: number, daysLeft: number, deliveredThisWeek: number ): Pick { const actualPerDay = deliveredThisWeek / 7; const requiredPerDay = remaining <= 0 ? 0 : remaining / Math.max(daysLeft, 1); if (remaining <= 0) { return { requiredPerDay, actualPerDay, verdict: "complete", daysBehind: null }; } if (daysLeft < 0) { return { requiredPerDay, actualPerDay, verdict: "overdue", daysBehind: -daysLeft }; } if (actualPerDay === 0) { return { requiredPerDay, actualPerDay, verdict: "stalled", daysBehind: null }; } const projectedDays = remaining / actualPerDay; if (projectedDays > daysLeft) { return { requiredPerDay, actualPerDay, verdict: "behind", daysBehind: Math.ceil(projectedDays - daysLeft), }; } return { requiredPerDay, actualPerDay, verdict: projectedDays > daysLeft * 0.9 ? "tight" : "ahead", daysBehind: null, }; } /** * Are we going to make the date, per episode? * * A shot counts as delivered when it appears in a package that has actually * gone out — a DeliveryItem in a DELIVERED EXR_DELIVERY package — or when it * was recorded as delivered by hand before the pipeline existed * (Shot.manualDeliveredAt), since those shots have no Export to package. Shot.status * is deliberately not the measure: it says where the work is, not whether the * client has the frames, and a review package (MOV_REVIEW) ships nothing. * * The run rate compares what is left per remaining day against what the studio * actually managed per day over the last week, which is the only honest way to * turn "23 shots to go" into "you are not going to make it". */ export async function getDeliveryCountdown(limit = 6): Promise { const projects = await db.project.findMany({ where: { status: "ACTIVE" }, select: { id: true, code: true, name: true, dueDate: true, projectType: true, episodeDueDates: { select: { episode: true, dueDate: true } }, }, }); if (projects.length === 0) return []; const projectIds = projects.map((p) => p.id); const weekAgo = new Date(Date.now() - 7 * DAY_MS); const [shots, postQc, deliveredItems] = await Promise.all([ db.shot.findMany({ where: { projectId: { in: projectIds } }, select: { id: true, projectId: true, episode: true, status: true, manualDeliveredAt: true }, }), db.export.findMany({ where: { status: { in: POST_QC_STATUSES }, shot: { projectId: { in: projectIds } } }, select: { shotId: true }, }), db.deliveryItem.findMany({ where: { package: { projectId: { in: projectIds }, kind: "EXR_DELIVERY", status: "DELIVERED", }, }, select: { shotId: true, package: { select: { deliveredAt: true } } }, }), ]); const postQcShots = new Set(postQc.map((e) => e.shotId)); // A shot can be in several packages (a redelivery); the first one out is the // one that counts towards the rate. const deliveredAt = new Map(); for (const item of deliveredItems) { const when = item.package.deliveredAt; if (!when) continue; const seen = deliveredAt.get(item.shotId); if (!seen || when < seen) deliveredAt.set(item.shotId, when); } for (const shot of shots) { const when = shot.manualDeliveredAt; if (!when) continue; const seen = deliveredAt.get(shot.id); if (!seen || when < seen) deliveredAt.set(shot.id, when); } const now = Date.now(); const rows: EpisodeCountdown[] = []; for (const project of projects) { // Episodic shows carry a date per episode; a standard show has one date on // the project itself, and is treated as a single unnamed target. const targets: { episode: string | null; dueDate: Date }[] = project.projectType === "EPISODIC" ? project.episodeDueDates.map((e) => ({ episode: e.episode, dueDate: e.dueDate })) : project.dueDate ? [{ episode: null, dueDate: project.dueDate }] : []; for (const target of targets) { const targetShots = shots.filter( (s) => s.projectId === project.id && (target.episode === null || (s.episode ?? "") === target.episode) ); if (targetShots.length === 0) continue; let delivered = 0; let inQc = 0; let inProgress = 0; let notStarted = 0; let deliveredThisWeek = 0; for (const shot of targetShots) { const when = deliveredAt.get(shot.id); if (when) { delivered++; if (when >= weekAgo) deliveredThisWeek++; } else if (postQcShots.has(shot.id)) { inQc++; } else if (shot.status === "WAITING") { notStarted++; } else { inProgress++; } } const total = targetShots.length; const daysLeft = Math.ceil((target.dueDate.getTime() - now) / DAY_MS); const pace = projectPace(total - delivered, daysLeft, deliveredThisWeek); // A finished episode whose date has passed is history, not a countdown. if (pace.verdict === "complete" && daysLeft < 0) continue; rows.push({ projectId: project.id, projectCode: project.code, projectName: project.name, episode: target.episode, dueDate: target.dueDate, daysLeft, total, delivered, inQc, inProgress, notStarted, ...pace, }); } } return rows.sort((a, b) => a.dueDate.getTime() - b.dueDate.getTime()).slice(0, limit); } export interface OpsCounts { blocked: number; inQc: number; readyToDeliver: number; awaitingClientReview: number; activeProjects: number; } /** * The headline numbers, all scoped to live productions. * * The counters these replaced were unscoped and cumulative — "approved" across * every project the studio has ever finished only ever goes up, so it never * told anyone anything. Each of these can go down, which is what makes it * worth looking at twice in a day. */ export async function getOpsCounts(artistId?: string): Promise { // Export has a projectId column but no project relation, so live-production // scope is reached through the shot. An artist gets their own figures, so the // cards agree with the blocked list underneath them rather than quoting a // studio-wide number beside a personal one. const liveProject: Prisma.ExportWhereInput = { shot: { project: { status: { in: ["ACTIVE", "ON_HOLD"] } }, ...(artistId ? { artistId } : {}), }, }; const [qcStaleHours, stalledHours] = await Promise.all([ getConfigNumber("dashboard.qcStaleHours"), getConfigNumber("dashboard.stalledJobHours"), ]); const now = Date.now(); const [blocked, inQc, readyToDeliver, awaitingClientReview, activeProjects] = await Promise.all([ db.export.count({ where: { ...liveProject, OR: [ { status: { in: FAILED_STATUSES } }, { status: "READY_FOR_QC", statusChangedAt: { lt: new Date(now - qcStaleHours * 3600_000) } }, { status: { in: IN_FLIGHT_STATUSES }, statusChangedAt: { lt: new Date(now - stalledHours * 3600_000) }, }, ], }, }), db.export.count({ where: { ...liveProject, status: "READY_FOR_QC" } }), db.export.count({ where: { ...liveProject, status: "READY_FOR_DELIVERY" } }), // Sent to the client and still unanswered — the wait that is not ours to // end, as opposed to notes we have not acted on yet. db.version.count({ where: { approvalStatus: "PENDING_REVIEW", isClientVisible: true, isLatest: true, shot: { project: { status: { in: ["ACTIVE", "ON_HOLD"] } }, ...(artistId ? { artistId } : {}), }, }, }), db.project.count({ where: { status: "ACTIVE" } }), ]); return { blocked, inQc, readyToDeliver, awaitingClientReview, activeProjects }; } /** How long an export may sit in READY_FOR_QC before the UI calls it stale. */ export function getQcStaleHours(): Promise { return getConfigNumber("dashboard.qcStaleHours"); } export interface QcQueueEntry { exportId: string; shotCode: string; versionString: string; projectCode: string; episode: string | null; since: Date; frameStart: number; frameEnd: number; submittedBy: string | null; } /** * What is waiting on a human to look at the frames. * * QC is the last gate before anything can be packaged, and it is the only * stage in the pipeline that cannot drain itself — everything before it is * worked by the farm. Depth here is the single best predictor of a delivery * slipping, and until now it had no home outside the render queue's filter. */ export async function getQcQueue(limit = 8): Promise { const rows = await db.export.findMany({ where: { status: "READY_FOR_QC", shot: { project: { status: { in: ["ACTIVE", "ON_HOLD"] } } }, }, orderBy: { statusChangedAt: "asc" }, take: limit, select: { id: true, versionString: true, statusChangedAt: true, frameStart: true, frameEnd: true, submittedByName: true, shot: { select: { shotCode: true, episode: true, project: { select: { code: true } } }, }, }, }); return rows.map((row) => ({ exportId: row.id, shotCode: row.shot.shotCode, versionString: row.versionString, projectCode: row.shot.project.code, episode: row.shot.episode, since: row.statusChangedAt, frameStart: row.frameStart, frameEnd: row.frameEnd, // Free text from the panel's config: Export keeps submittedById for the // resolved account but carries no relation to it. submittedBy: row.submittedByName, })); } export interface DeliverySummary { id: string; packageNumber: number; name: string; projectCode: string; episode: string; kind: string; status: string; itemCount: number; errorMessage: string | null; /** Most recent of delivered/built/created — what the timestamp column means. */ at: Date; atLabel: "delivered" | "built" | "created"; /** BUILDING for longer than a build plausibly takes. */ stuck: boolean; } /** * Recent packages, and whether any of them went wrong. * * A failed delivery build is otherwise silent: the export stays * READY_FOR_DELIVERY, nothing is on the delivery volume, and the only trace is * a status on a page nobody opens between deliveries. */ export async function getRecentDeliveries(limit = 5): Promise { const rows = await db.deliveryPackage.findMany({ where: { project: { status: { in: ["ACTIVE", "ON_HOLD"] } } }, orderBy: { createdAt: "desc" }, take: limit, select: { id: true, packageNumber: true, name: true, episode: true, kind: true, status: true, errorMessage: true, createdAt: true, builtAt: true, deliveredAt: true, project: { select: { code: true } }, _count: { select: { items: true } }, }, }); const buildStuckAfter = 2 * 3600_000; return rows.map((row) => { const at = row.deliveredAt ?? row.builtAt ?? row.createdAt; return { id: row.id, packageNumber: row.packageNumber, name: row.name, projectCode: row.project.code, episode: row.episode, kind: row.kind, status: row.status, itemCount: row._count.items, errorMessage: row.errorMessage, at, atLabel: row.deliveredAt ? "delivered" : row.builtAt ? "built" : "created", // A build that never finished leaves the package sitting in BUILDING // forever; nothing times it out. stuck: row.status === "BUILDING" && Date.now() - row.createdAt.getTime() > buildStuckAfter, }; }); } export interface AwaitingDelivery { id: string; packageNumber: number; name: string; projectCode: string; episode: string; itemCount: number; stage: HandoffStage; /** When the folder was ready to go — the clock the client is waiting on. */ builtAt: Date; severity: "normal" | "warning" | "critical"; } /** * Built packages the client does not have yet. * * A READY package is the easiest thing in the pipeline to believe is done: the * folder exists and every file verified, but nothing has left the building. * It stays listed until the upload location and the person it was shared with * are both on record, and gets louder the longer that takes. */ export async function getAwaitingDelivery(): Promise { const rows = await db.deliveryPackage.findMany({ where: { kind: "EXR_DELIVERY", status: "READY", project: { status: { in: ["ACTIVE", "ON_HOLD"] } }, }, orderBy: { builtAt: "asc" }, select: { id: true, packageNumber: true, name: true, kind: true, status: true, episode: true, createdAt: true, builtAt: true, uploadedAt: true, project: { select: { code: true } }, _count: { select: { items: true } }, }, }); const day = 24 * 3600_000; return rows.flatMap((row) => { const stage = handoffStage(row); if (!stage) return []; const builtAt = row.builtAt ?? row.createdAt; const age = Date.now() - builtAt.getTime(); return [ { id: row.id, packageNumber: row.packageNumber, name: row.name, projectCode: row.project.code, episode: row.episode, itemCount: row._count.items, stage, builtAt, severity: age >= 2 * day ? "critical" : age >= day ? "warning" : "normal", }, ]; }); } export interface CourtSide { count: number; items: { id: string; shotCode: string; label: string; href: string }[]; } export interface WhoseCourt { client: CourtSide; us: CourtSide; } /** * Which waits are ours and which are the client's. * * The distinction is the useful part: a version sitting unanswered for eleven * days is a phone call, while notes nobody has actioned are a scheduling * problem. Lumping both into "needs revisions" hid which was which. */ export async function getWhoseCourt(perSide = 4): Promise { const liveShot: Prisma.ShotWhereInput = { project: { status: { in: ["ACTIVE", "ON_HOLD"] } }, }; const [withClient, openNotes] = await Promise.all([ db.version.findMany({ where: { approvalStatus: "PENDING_REVIEW", isClientVisible: true, isLatest: true, shot: liveShot, }, // Longest wait first — that is the one worth chasing. orderBy: [{ sharedAt: { sort: "asc", nulls: "last" } }, { createdAt: "asc" }], take: perSide, select: { id: true, versionNumber: true, sharedAt: true, createdAt: true, shot: { select: { id: true, shotCode: true } }, }, }), // Unresolved notes on the version currently in play. Grouped in memory // because the interesting unit is the shot, not the comment. db.comment.findMany({ where: { isResolved: false, version: { isLatest: true, shot: liveShot }, }, select: { id: true, version: { select: { shot: { select: { id: true, shotCode: true } } } } }, }), ]); const byShot = new Map(); for (const note of openNotes) { const shot = note.version?.shot; if (!shot) continue; const entry = byShot.get(shot.id) ?? { shotCode: shot.shotCode, count: 0 }; entry.count++; byShot.set(shot.id, entry); } const ours = [...byShot.entries()].sort((a, b) => b[1].count - a[1].count); const days = (from: Date) => Math.max(0, Math.floor((Date.now() - from.getTime()) / DAY_MS)); return { client: { count: withClient.length, items: withClient.map((v) => ({ id: v.id, shotCode: v.shot?.shotCode ?? "—", label: `${days(v.sharedAt ?? v.createdAt)}d`, href: `/review/${v.id}`, })), }, us: { count: ours.length, items: ours.slice(0, perSide).map(([shotId, entry]) => ({ id: shotId, shotCode: entry.shotCode, label: `${entry.count} note${entry.count === 1 ? "" : "s"}`, href: `/shot-status`, })), }, }; } export interface FarmStatus { machines: Awaited>; online: number; total: number; queued: { render: number; preview: number; delivery: number; blender: number; total: number }; running: number; /** Age of the oldest job still waiting to be claimed, in ms. */ oldestQueuedMs: number | null; /** Highest worker version any machine reports, and who is behind it. */ latestWorkerVersion: string | null; behindMachines: { name: string; workerVersion: string | null }[]; } /** * Farm health in one strip: who is up, what is waiting, and — because a * worker's version is what decides whether it has a given fix — which * machines are running something older than the newest one out there. */ export async function getFarmStatus(): Promise { const [machines, queuedByType, running, oldest] = await Promise.all([ listMachines(), db.renderJob.groupBy({ by: ["type"], where: { status: "QUEUED" }, _count: { _all: true } }), db.renderJob.count({ where: { status: { in: ["CLAIMED", "RUNNING"] } } }), db.renderJob.findFirst({ where: { status: "QUEUED" }, orderBy: { createdAt: "asc" }, select: { createdAt: true }, }), ]); const countOf = (type: string) => queuedByType.find((q) => q.type === type)?._count._all ?? 0; const versions = machines .map((m) => m.workerVersion) .filter((v): v is string => Boolean(v)); const latest = versions.length ? versions.reduce((a, b) => (compareVersions(a, b) >= 0 ? a : b)) : null; return { machines, online: machines.filter((m) => m.status === "ONLINE").length, total: machines.length, queued: { render: countOf("AE_RENDER"), preview: countOf("PREVIEW_ONLY"), delivery: countOf("DELIVERY_BUILD"), blender: countOf("BLENDER_RENDER"), total: queuedByType.reduce((sum, q) => sum + q._count._all, 0), }, running, oldestQueuedMs: oldest ? Date.now() - oldest.createdAt.getTime() : null, latestWorkerVersion: latest, behindMachines: latest ? machines .filter((m) => m.workerVersion !== latest) .map((m) => ({ name: m.name, workerVersion: m.workerVersion })) : [], }; } /** * Compares "1.6.1+fe420b9" style versions numerically, ignoring the build * stamp publish.ps1 appends. A string compare would rank 1.10.0 below 1.9.0. */ export function compareVersions(a: string, b: string): number { const parts = (v: string) => v.split("+")[0].split(".").map((n) => Number.parseInt(n, 10) || 0); const [pa, pb] = [parts(a), parts(b)]; for (let i = 0; i < Math.max(pa.length, pb.length); i++) { const diff = (pa[i] ?? 0) - (pb[i] ?? 0); if (diff !== 0) return diff; } return 0; }