Files
twotalesanimation cc89415a29 feat(pipeline): render queue, worker service and automated preview generation
One "Queue Export" click now renders the EXR sequence, then rebuilds the shot
headlessly with the studio slate/overlay template to produce the delivery MOV
and review MP4. Implements RenderPipeline2 phases 1-2 plus the preview stage.

Server:
- New models Export, RenderJob, ExportEvent, Machine, WorkerHeartbeat, plus
  Project.deliveryConfig and per-submission slate fields (Export.vfxScope,
  Export.submissionNote, inherited from the shot's previous export).
  Both migrations are purely additive; no existing column is touched.
- lib/render-pipeline: server-enforced state machine, transactional version
  increment with supersede, atomic FOR UPDATE SKIP LOCKED claim gated by
  machine availability windows, and a lease reaper run from instrumentation.ts.
- /api/ext/* endpoints for the panel and workers; session-auth mirrors under
  /api/render and /api/machines for the web UI.
- Pipeline pages: render queue, export detail, machine monitoring, plus an
  Exports tab on shot detail.

RenderWorker (.NET 8 Windows service, new):
- Registration, heartbeat as cancel channel, claim loop, aerender runner with
  progress parsing and stall watchdog, crash recovery and disk-spooled
  reporting that survives server downtime.
- Preview stage: headless AE assembles the preview comp into a throwaway AEP
  with both output modules queued, then a single aerender pass renders them.
  Preview jobs are not claimed while an interactive AE session is open, so an
  artist's project is never taken over.

AE panel: Queue Export with live status polling, urgent flag, retry, and the
VFX Scope / Submission Note fields. Every existing panel action is unchanged.

Preview chaining ships disabled behind SystemConfig preview.enabled.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-06 15:46:53 +02:00

241 lines
7.9 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { Machine, Prisma } from "@prisma/client";
import { db } from "@/lib/db";
import { PipelineError } from "./errors";
import { getWorkerConfig } from "./config";
/**
* Machine availability (§7.10):
* ALWAYS — claims whenever idle (default when availability is null)
* SCHEDULE — claims only inside configured windows
* MANUAL — never claims unless explicitly triggered
* Overrides, in priority order: enabled=false beats everything; renderNowUntil
* allows normal claims until it expires; urgent jobs may claim anytime on
* machines with allowUrgentAnytime.
*/
export interface AvailabilityWindow {
days: string[]; // ["mon", ..., "sun"]
from: string; // "19:00"
to: string; // "08:00" — from > to spans midnight
}
export interface MachineAvailability {
mode?: "ALWAYS" | "SCHEDULE" | "MANUAL";
windows?: AvailabilityWindow[];
allowUrgentAnytime?: boolean;
}
const DAY_NAMES = ["sun", "mon", "tue", "wed", "thu", "fri", "sat"];
function parseHm(hm: string): number | null {
const m = /^(\d{1,2}):(\d{2})$/.exec(hm);
if (!m) return null;
const mins = parseInt(m[1], 10) * 60 + parseInt(m[2], 10);
return mins >= 0 && mins <= 24 * 60 ? mins : null;
}
export function isInWindow(windows: AvailabilityWindow[], now: Date): boolean {
const day = DAY_NAMES[now.getDay()];
const prevDay = DAY_NAMES[(now.getDay() + 6) % 7];
const t = now.getHours() * 60 + now.getMinutes();
for (const w of windows) {
const from = parseHm(w.from);
const to = parseHm(w.to);
if (from == null || to == null || !Array.isArray(w.days)) continue;
const days = w.days.map((d) => d.toLowerCase().slice(0, 3));
if (from < to) {
if (days.includes(day) && t >= from && t < to) return true;
} else if (from > to) {
// overnight window, e.g. 19:00 → 08:00
if (days.includes(day) && t >= from) return true;
if (days.includes(prevDay) && t < to) return true;
} else {
// from === to → full 24 h on listed days
if (days.includes(day)) return true;
}
}
return false;
}
export type ClaimGate = "normal" | "urgent-only" | "none";
export function machineCanClaim(machine: Machine, now: Date): ClaimGate {
if (!machine.enabled) return "none";
if (machine.renderNowUntil && machine.renderNowUntil > now) return "normal";
const availability = (machine.availability ?? null) as MachineAvailability | null;
const mode = availability?.mode ?? "ALWAYS";
const urgent = availability?.allowUrgentAnytime ? ("urgent-only" as const) : ("none" as const);
switch (mode) {
case "ALWAYS":
return "normal";
case "SCHEDULE":
return isInWindow(availability?.windows ?? [], now) ? "normal" : urgent;
case "MANUAL":
return urgent;
default:
return "normal";
}
}
/** E6 — idempotent register/upsert on machine name. */
export async function registerMachine(body: {
name?: string;
hostname?: string;
workerVersion?: string;
aeVersion?: string;
capabilities?: unknown;
}) {
const name = body.name?.trim();
if (!name) throw new PipelineError(422, "name is required");
const hostname = body.hostname?.trim() || name;
const machine = await db.machine.upsert({
where: { name },
create: {
name,
hostname,
status: "ONLINE",
lastSeenAt: new Date(),
workerVersion: body.workerVersion ?? null,
aeVersion: body.aeVersion ?? null,
capabilities: (body.capabilities ?? undefined) as Prisma.InputJsonValue | undefined,
},
update: {
hostname,
status: "ONLINE",
lastSeenAt: new Date(),
workerVersion: body.workerVersion ?? undefined,
aeVersion: body.aeVersion ?? undefined,
capabilities: (body.capabilities ?? undefined) as Prisma.InputJsonValue | undefined,
},
});
return {
machine: { id: machine.id, name: machine.name, enabled: machine.enabled },
config: await getWorkerConfig(),
};
}
/** E7 — heartbeat; also the server→worker command channel (cancel signals). */
export async function recordHeartbeat(
machineId: string,
body: { cpuPercent?: number; memPercent?: number; diskFreeGb?: number; currentJobId?: string | null }
) {
const machine = await db.machine.findUnique({ where: { id: machineId } });
if (!machine) throw new PipelineError(404, "Machine not found");
await db.$transaction([
db.machine.update({
where: { id: machineId },
data: { lastSeenAt: new Date(), status: machine.enabled ? "ONLINE" : "DISABLED" },
}),
db.workerHeartbeat.create({
data: {
machineId,
cpuPercent: body.cpuPercent ?? null,
memPercent: body.memPercent ?? null,
diskFreeGb: body.diskFreeGb ?? null,
currentJobId: body.currentJobId ?? null,
},
}),
]);
// Cancelled-but-unacknowledged jobs held by this machine → CANCEL_JOB commands
const cancelled = await db.renderJob.findMany({
where: { machineId, status: "CANCELLED", finishedAt: null },
select: { id: true },
});
return {
ok: true,
commands: cancelled.map((j) => ({ type: "CANCEL_JOB" as const, jobId: j.id })),
};
}
/** Machine list for the monitoring page, with latest heartbeat + current job. */
export async function listMachines() {
const heartbeatSeconds = 30;
const machines = await db.machine.findMany({
orderBy: { name: "asc" },
include: {
heartbeats: { orderBy: { createdAt: "desc" }, take: 1 },
renderJobs: {
where: { status: { in: ["CLAIMED", "RUNNING"] } },
take: 1,
include: { export: { select: { id: true, versionString: true, shot: { select: { shotCode: true } } } } },
},
},
});
const now = Date.now();
return machines.map((m) => ({
id: m.id,
name: m.name,
hostname: m.hostname,
enabled: m.enabled,
// ONLINE = heartbeat within 3× the heartbeat interval (§5.6)
status: !m.enabled
? "DISABLED"
: m.lastSeenAt && now - m.lastSeenAt.getTime() < heartbeatSeconds * 3 * 1000
? "ONLINE"
: "OFFLINE",
lastSeenAt: m.lastSeenAt,
workerVersion: m.workerVersion,
aeVersion: m.aeVersion,
capabilities: m.capabilities,
availability: m.availability,
renderNowUntil: m.renderNowUntil,
latestHeartbeat: m.heartbeats[0]
? {
createdAt: m.heartbeats[0].createdAt,
cpuPercent: m.heartbeats[0].cpuPercent,
memPercent: m.heartbeats[0].memPercent,
diskFreeGb: m.heartbeats[0].diskFreeGb,
}
: null,
currentJob: m.renderJobs[0]
? {
id: m.renderJobs[0].id,
exportId: m.renderJobs[0].exportId,
shotCode: m.renderJobs[0].export?.shot.shotCode ?? null,
versionString: m.renderJobs[0].export?.versionString ?? null,
progress: m.renderJobs[0].progress,
etaSeconds: m.renderJobs[0].etaSeconds,
}
: null,
}));
}
export async function updateMachine(
machineId: string,
patch: { enabled?: boolean; availability?: MachineAvailability | null; renderNowHours?: number | null }
) {
const machine = await db.machine.findUnique({ where: { id: machineId } });
if (!machine) throw new PipelineError(404, "Machine not found");
const data: Prisma.MachineUpdateInput = {};
if (patch.enabled !== undefined) {
data.enabled = patch.enabled;
data.status = patch.enabled ? machine.status === "DISABLED" ? "OFFLINE" : machine.status : "DISABLED";
}
if (patch.availability !== undefined) {
data.availability = patch.availability === null ? Prisma.DbNull : (patch.availability as unknown as Prisma.InputJsonValue);
}
if (patch.renderNowHours !== undefined) {
data.renderNowUntil =
patch.renderNowHours === null || patch.renderNowHours <= 0
? null
: new Date(Date.now() + patch.renderNowHours * 3600 * 1000);
}
const updated = await db.machine.update({ where: { id: machineId }, data });
return {
machine: {
id: updated.id,
name: updated.name,
enabled: updated.enabled,
availability: updated.availability,
renderNowUntil: updated.renderNowUntil,
},
};
}