feat(pipeline): render queue, worker service and automated preview generation
One "Queue Export" click now renders the EXR sequence, then rebuilds the shot headlessly with the studio slate/overlay template to produce the delivery MOV and review MP4. Implements RenderPipeline2 phases 1-2 plus the preview stage. Server: - New models Export, RenderJob, ExportEvent, Machine, WorkerHeartbeat, plus Project.deliveryConfig and per-submission slate fields (Export.vfxScope, Export.submissionNote, inherited from the shot's previous export). Both migrations are purely additive; no existing column is touched. - lib/render-pipeline: server-enforced state machine, transactional version increment with supersede, atomic FOR UPDATE SKIP LOCKED claim gated by machine availability windows, and a lease reaper run from instrumentation.ts. - /api/ext/* endpoints for the panel and workers; session-auth mirrors under /api/render and /api/machines for the web UI. - Pipeline pages: render queue, export detail, machine monitoring, plus an Exports tab on shot detail. RenderWorker (.NET 8 Windows service, new): - Registration, heartbeat as cancel channel, claim loop, aerender runner with progress parsing and stall watchdog, crash recovery and disk-spooled reporting that survives server downtime. - Preview stage: headless AE assembles the preview comp into a throwaway AEP with both output modules queued, then a single aerender pass renders them. Preview jobs are not claimed while an interactive AE session is open, so an artist's project is never taken over. AE panel: Queue Export with live status polling, urgent flag, retry, and the VFX Scope / Submission Note fields. Every existing panel action is unchanged. Preview chaining ships disabled behind SystemConfig preview.enabled. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
twotalesanimation
@@ -0,0 +1,317 @@
|
||||
import { Prisma, RenderJobType } from "@prisma/client";
|
||||
import { db } from "@/lib/db";
|
||||
import { PipelineError } from "./errors";
|
||||
import { applyTransition } from "./transitions";
|
||||
import { getConfigBoolean, getConfigNumber } from "./config";
|
||||
import { machineCanClaim } from "./machines";
|
||||
import { reapExpiredLeases } from "./reaper";
|
||||
import { buildPreviewManifest, createPreviewJob } from "./preview";
|
||||
|
||||
const JOB_TYPES: RenderJobType[] = ["AE_RENDER", "PREVIEW_ONLY", "DELIVERY_BUILD"];
|
||||
|
||||
/**
|
||||
* E8 — atomically claim the next queued job (§6.4). `FOR UPDATE SKIP LOCKED`
|
||||
* makes double-claims impossible under concurrent workers. All scheduling
|
||||
* policy (availability windows, Render Now, urgent priority) is enforced
|
||||
* here server-side — workers poll dumbly (§7.10).
|
||||
*/
|
||||
export async function claimNextJob(machineId: string, types?: string[]) {
|
||||
const machine = await db.machine.findUnique({ where: { id: machineId } });
|
||||
if (!machine) throw new PipelineError(404, "Machine not found — register first (POST /api/ext/workers/register)");
|
||||
|
||||
// Defensive sweep on each claim (studio decision 18.1-Q8)
|
||||
await reapExpiredLeases().catch((err) => console.error("[render-pipeline] claim-time reap failed", err));
|
||||
|
||||
const claimTypes = (types?.length ? types : ["AE_RENDER", "PREVIEW_ONLY", "DELIVERY_BUILD"]).filter(
|
||||
(t): t is RenderJobType => (JOB_TYPES as string[]).includes(t)
|
||||
);
|
||||
if (claimTypes.length === 0) throw new PipelineError(422, "No valid job types requested");
|
||||
|
||||
const gate = machineCanClaim(machine, new Date());
|
||||
if (gate === "none") return null;
|
||||
|
||||
const [leaseSeconds, urgentThreshold] = await Promise.all([
|
||||
getConfigNumber("render.leaseSeconds"),
|
||||
getConfigNumber("render.urgentPriorityThreshold"),
|
||||
]);
|
||||
|
||||
const urgentFilter =
|
||||
gate === "urgent-only" ? Prisma.sql`AND priority <= ${urgentThreshold}` : Prisma.empty;
|
||||
|
||||
const claimed = await db.$queryRaw<{ id: string }[]>(Prisma.sql`
|
||||
UPDATE "render_jobs"
|
||||
SET status = 'CLAIMED',
|
||||
"machineId" = ${machineId},
|
||||
"claimedAt" = now(),
|
||||
"leaseExpiresAt" = now() + (${leaseSeconds}::int * interval '1 second'),
|
||||
"updatedAt" = now()
|
||||
WHERE id = (
|
||||
SELECT id FROM "render_jobs"
|
||||
WHERE status = 'QUEUED'
|
||||
AND type::text = ANY(${claimTypes}::text[])
|
||||
${urgentFilter}
|
||||
ORDER BY priority ASC, "createdAt" ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING id
|
||||
`);
|
||||
if (claimed.length === 0) return null;
|
||||
|
||||
const job = await db.renderJob.findUniqueOrThrow({
|
||||
where: { id: claimed[0].id },
|
||||
include: { export: { select: { id: true, status: true, versionString: true } } },
|
||||
});
|
||||
|
||||
// T2: QUEUED → RENDERING
|
||||
if (job.export && job.export.status === "QUEUED") {
|
||||
await db.$transaction(async (tx) => {
|
||||
await applyTransition(tx, job.export!, "RENDERING", {
|
||||
type: "WORKER",
|
||||
id: machineId,
|
||||
note: `Claimed by ${machine.name} (attempt ${job.attempt})`,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
job: {
|
||||
id: job.id,
|
||||
type: job.type,
|
||||
attempt: job.attempt,
|
||||
maxAttempts: job.maxAttempts,
|
||||
priority: job.priority,
|
||||
exportId: job.exportId,
|
||||
deliveryId: job.deliveryId,
|
||||
leaseExpiresAt: job.leaseExpiresAt,
|
||||
manifest: job.manifest,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function getJobForReport(jobId: string, machineId: string) {
|
||||
const job = await db.renderJob.findUnique({
|
||||
where: { id: jobId },
|
||||
include: { export: { select: { id: true, status: true } } },
|
||||
});
|
||||
if (!job) throw new PipelineError(404, "Render job not found");
|
||||
// Zombie-worker guard (§7.4): reports from a machine that doesn't hold the lease are rejected
|
||||
if (job.machineId !== machineId) {
|
||||
throw new PipelineError(409, "Job is not held by this machine (lease reassigned?)");
|
||||
}
|
||||
return job;
|
||||
}
|
||||
|
||||
/** E9 — progress report; renews the lease. */
|
||||
export async function reportProgress(
|
||||
jobId: string,
|
||||
machineId: string,
|
||||
body: { progress?: number; currentFrame?: number; totalFrames?: number; etaSeconds?: number; logTail?: string }
|
||||
) {
|
||||
const job = await getJobForReport(jobId, machineId);
|
||||
|
||||
if (job.status === "CANCELLED") {
|
||||
return { ok: true, cancelRequested: true, leaseExpiresAt: job.leaseExpiresAt };
|
||||
}
|
||||
if (["COMPLETED", "FAILED", "EXPIRED"].includes(job.status)) {
|
||||
return { ok: true, alreadyApplied: true, cancelRequested: false, leaseExpiresAt: job.leaseExpiresAt };
|
||||
}
|
||||
|
||||
const leaseSeconds = await getConfigNumber("render.leaseSeconds");
|
||||
const leaseExpiresAt = new Date(Date.now() + leaseSeconds * 1000);
|
||||
await db.renderJob.update({
|
||||
where: { id: job.id },
|
||||
data: {
|
||||
status: "RUNNING",
|
||||
startedAt: job.startedAt ?? new Date(),
|
||||
leaseExpiresAt,
|
||||
...(body.progress != null ? { progress: Math.min(1, Math.max(0, body.progress)) } : {}),
|
||||
...(body.currentFrame != null ? { currentFrame: body.currentFrame } : {}),
|
||||
...(body.totalFrames != null ? { totalFrames: body.totalFrames } : {}),
|
||||
...(body.etaSeconds != null ? { etaSeconds: body.etaSeconds } : {}),
|
||||
...(body.logTail != null ? { logTail: body.logTail.slice(-20000) } : {}),
|
||||
},
|
||||
});
|
||||
return { ok: true, leaseExpiresAt, cancelRequested: false };
|
||||
}
|
||||
|
||||
/** E10 — failure report. Auto-requeues while attempts remain and the error is retryable. */
|
||||
export async function reportFail(
|
||||
jobId: string,
|
||||
machineId: string,
|
||||
body: {
|
||||
stage?: string;
|
||||
exitCode?: number;
|
||||
errorMessage?: string;
|
||||
logTail?: string;
|
||||
logFileKey?: string;
|
||||
retryable?: boolean;
|
||||
}
|
||||
) {
|
||||
return db.$transaction(async (tx) => {
|
||||
const job = await tx.renderJob.findUnique({
|
||||
where: { id: jobId },
|
||||
include: { export: { select: { id: true, status: true } } },
|
||||
});
|
||||
if (!job) throw new PipelineError(404, "Render job not found");
|
||||
if (job.machineId !== machineId) throw new PipelineError(409, "Job is not held by this machine (lease reassigned?)");
|
||||
|
||||
if (job.status === "FAILED") {
|
||||
return { job: { id: job.id, status: job.status }, export: job.export ? { id: job.export.id, status: job.export.status } : null, autoRequeued: false, alreadyApplied: true };
|
||||
}
|
||||
if (job.status === "CANCELLED") {
|
||||
// worker acknowledging a cancel — record the wind-down, stay CANCELLED
|
||||
await tx.renderJob.update({ where: { id: job.id }, data: { finishedAt: job.finishedAt ?? new Date(), logTail: body.logTail?.slice(-20000) ?? job.logTail } });
|
||||
return { job: { id: job.id, status: "CANCELLED" }, export: job.export ? { id: job.export.id, status: job.export.status } : null, autoRequeued: false, alreadyApplied: true };
|
||||
}
|
||||
if (job.status === "COMPLETED") {
|
||||
throw new PipelineError(409, "Job already reported complete — contradictory fail report rejected");
|
||||
}
|
||||
|
||||
await tx.renderJob.update({
|
||||
where: { id: job.id },
|
||||
data: {
|
||||
status: "FAILED",
|
||||
finishedAt: new Date(),
|
||||
exitCode: body.exitCode ?? null,
|
||||
errorMessage: body.errorMessage?.slice(0, 2000) ?? null,
|
||||
logTail: body.logTail?.slice(-20000) ?? job.logTail,
|
||||
logFileKey: body.logFileKey ?? job.logFileKey,
|
||||
},
|
||||
});
|
||||
|
||||
if (!job.export) {
|
||||
return { job: { id: job.id, status: "FAILED" }, export: null, autoRequeued: false };
|
||||
}
|
||||
|
||||
const retryable = body.retryable !== false;
|
||||
const autoRequeue = retryable && job.attempt < job.maxAttempts;
|
||||
const isPreview = job.type === "PREVIEW_ONLY";
|
||||
|
||||
if (autoRequeue) {
|
||||
const next = await tx.renderJob.create({
|
||||
data: {
|
||||
type: job.type,
|
||||
exportId: job.exportId,
|
||||
attempt: job.attempt + 1,
|
||||
maxAttempts: job.maxAttempts,
|
||||
priority: job.priority,
|
||||
manifest: job.manifest as Prisma.InputJsonValue,
|
||||
},
|
||||
});
|
||||
const note = `${body.stage ?? "RENDER"} failed (${body.errorMessage ?? "no message"}) — auto-requeued as attempt ${next.attempt}`;
|
||||
|
||||
if (isPreview) {
|
||||
// The export stays in GENERATING_PREVIEW across preview attempts —
|
||||
// the EXRs are untouched, only the preview build is repeated.
|
||||
await tx.exportEvent.create({
|
||||
data: {
|
||||
exportId: job.export.id,
|
||||
fromStatus: job.export.status,
|
||||
toStatus: job.export.status,
|
||||
actorType: "SYSTEM",
|
||||
actorId: machineId,
|
||||
note,
|
||||
},
|
||||
});
|
||||
return { job: { id: job.id, status: "FAILED" }, export: { id: job.export.id, status: job.export.status }, autoRequeued: true, nextJob: { id: next.id, attempt: next.attempt } };
|
||||
}
|
||||
|
||||
const exp = await applyTransition(tx, job.export, "QUEUED", { type: "SYSTEM", id: machineId, note });
|
||||
return { job: { id: job.id, status: "FAILED" }, export: { id: exp.id, status: exp.status }, autoRequeued: true, nextJob: { id: next.id, attempt: next.attempt } };
|
||||
}
|
||||
|
||||
const exp = await applyTransition(tx, job.export, isPreview ? "PREVIEW_FAILED" : "RENDER_FAILED", {
|
||||
type: "WORKER",
|
||||
id: machineId,
|
||||
note: `${body.stage ?? "RENDER"} failed: ${body.errorMessage ?? "no message"}${retryable ? " (max attempts reached)" : " (not retryable)"}`,
|
||||
});
|
||||
return { job: { id: job.id, status: "FAILED" }, export: { id: exp.id, status: exp.status }, autoRequeued: false };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* E11 — aerender finished OK.
|
||||
* Phase 2 interim (§17.3): transitions straight to a provisional READY_FOR_QC;
|
||||
* Phase 3 will insert VALIDATING between (see transitions.ts note).
|
||||
*/
|
||||
export async function completeRender(
|
||||
jobId: string,
|
||||
machineId: string,
|
||||
body: { renderSeconds?: number; logFileKey?: string; logTail?: string; exrFileCount?: number; exrTotalBytes?: number }
|
||||
) {
|
||||
return db.$transaction(async (tx) => {
|
||||
const job = await tx.renderJob.findUnique({
|
||||
where: { id: jobId },
|
||||
include: { export: { select: { id: true, status: true } } },
|
||||
});
|
||||
if (!job) throw new PipelineError(404, "Render job not found");
|
||||
if (job.machineId !== machineId) throw new PipelineError(409, "Job is not held by this machine (lease reassigned?)");
|
||||
|
||||
if (job.status === "COMPLETED") {
|
||||
return { job: { id: job.id, status: job.status }, export: job.export ? { id: job.export.id, status: job.export.status } : null, alreadyApplied: true };
|
||||
}
|
||||
if (["FAILED", "CANCELLED", "EXPIRED"].includes(job.status)) {
|
||||
throw new PipelineError(409, `Job is ${job.status} — contradictory complete report rejected`);
|
||||
}
|
||||
|
||||
await tx.renderJob.update({
|
||||
where: { id: job.id },
|
||||
data: {
|
||||
status: "COMPLETED",
|
||||
progress: 1,
|
||||
finishedAt: new Date(),
|
||||
exitCode: 0,
|
||||
logFileKey: body.logFileKey ?? job.logFileKey,
|
||||
logTail: body.logTail?.slice(-20000) ?? job.logTail,
|
||||
},
|
||||
});
|
||||
|
||||
let exportResult: { id: string; status: string } | null = null;
|
||||
let previewJob: { id: string; type: string } | null = null;
|
||||
|
||||
if (job.export) {
|
||||
const exportRow = await tx.export.findUniqueOrThrow({ where: { id: job.export.id } });
|
||||
const stats = {
|
||||
...(body.exrFileCount != null ? { exrFileCount: body.exrFileCount } : {}),
|
||||
...(body.exrTotalBytes != null ? { exrTotalBytes: BigInt(Math.round(body.exrTotalBytes)) } : {}),
|
||||
};
|
||||
|
||||
// The EXR render is only the first stage: hand off to the preview stage
|
||||
// (headless AE rebuild → delivery MOV + review MP4) when it is enabled.
|
||||
const previewEnabled = await getConfigBoolean("preview.enabled");
|
||||
if (previewEnabled) {
|
||||
const manifest = await buildPreviewManifest(exportRow);
|
||||
const created = await createPreviewJob(tx, exportRow, manifest, job.maxAttempts, job.priority);
|
||||
previewJob = { id: created.id, type: created.type };
|
||||
const exp = await applyTransition(tx, job.export, "GENERATING_PREVIEW", {
|
||||
type: "WORKER",
|
||||
id: machineId,
|
||||
note: `Render complete in ${body.renderSeconds ?? "?"}s — queued preview build`,
|
||||
}, stats);
|
||||
exportResult = { id: exp.id, status: exp.status };
|
||||
} else {
|
||||
const exp = await applyTransition(tx, job.export, "READY_FOR_QC", {
|
||||
type: "WORKER",
|
||||
id: machineId,
|
||||
note: `Render complete in ${body.renderSeconds ?? "?"}s (preview stage disabled)`,
|
||||
}, stats);
|
||||
exportResult = { id: exp.id, status: exp.status };
|
||||
}
|
||||
}
|
||||
return { job: { id: job.id, status: "COMPLETED" }, export: exportResult, previewJob, alreadyApplied: false };
|
||||
});
|
||||
}
|
||||
|
||||
/** E15 — cancel a single job from the web UI (delegates to export-level cancel when linked). */
|
||||
export async function getJobDetail(jobId: string) {
|
||||
const job = await db.renderJob.findUnique({
|
||||
where: { id: jobId },
|
||||
include: {
|
||||
machine: { select: { id: true, name: true } },
|
||||
export: { include: { shot: { select: { shotCode: true } } } },
|
||||
},
|
||||
});
|
||||
if (!job) throw new PipelineError(404, "Render job not found");
|
||||
return job;
|
||||
}
|
||||
Reference in New Issue
Block a user