feat(pipeline): render queue, worker service and automated preview generation
One "Queue Export" click now renders the EXR sequence, then rebuilds the shot headlessly with the studio slate/overlay template to produce the delivery MOV and review MP4. Implements RenderPipeline2 phases 1-2 plus the preview stage. Server: - New models Export, RenderJob, ExportEvent, Machine, WorkerHeartbeat, plus Project.deliveryConfig and per-submission slate fields (Export.vfxScope, Export.submissionNote, inherited from the shot's previous export). Both migrations are purely additive; no existing column is touched. - lib/render-pipeline: server-enforced state machine, transactional version increment with supersede, atomic FOR UPDATE SKIP LOCKED claim gated by machine availability windows, and a lease reaper run from instrumentation.ts. - /api/ext/* endpoints for the panel and workers; session-auth mirrors under /api/render and /api/machines for the web UI. - Pipeline pages: render queue, export detail, machine monitoring, plus an Exports tab on shot detail. RenderWorker (.NET 8 Windows service, new): - Registration, heartbeat as cancel channel, claim loop, aerender runner with progress parsing and stall watchdog, crash recovery and disk-spooled reporting that survives server downtime. - Preview stage: headless AE assembles the preview comp into a throwaway AEP with both output modules queued, then a single aerender pass renders them. Preview jobs are not claimed while an interactive AE session is open, so an artist's project is never taken over. AE panel: Queue Export with live status polling, urgent flag, retry, and the VFX Scope / Submission Note fields. Every existing panel action is unchanged. Preview chaining ships disabled behind SystemConfig preview.enabled. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
twotalesanimation
@@ -0,0 +1,83 @@
|
||||
import { db } from "@/lib/db";
|
||||
|
||||
/**
|
||||
* Pipeline-wide settings live in the existing SystemConfig key/value table
|
||||
* (RenderPipeline2 §14.1) so the fleet can be tuned without redeploys.
|
||||
*/
|
||||
export const PIPELINE_CONFIG_DEFAULTS = {
|
||||
"render.pollSeconds": 10,
|
||||
"render.heartbeatSeconds": 30,
|
||||
"render.leaseSeconds": 300,
|
||||
"render.maxAttempts": 3,
|
||||
"render.stallTimeoutSeconds": 600,
|
||||
"render.urgentPriorityThreshold": 20,
|
||||
"render.outputRoot": "//SAN/renders",
|
||||
"validation.sampleEvery": 25,
|
||||
"preview.maxWidth": 1920,
|
||||
// Preview stage (§9): headless AE rebuild of the render with slate/burn-ins
|
||||
"preview.enabled": "true",
|
||||
"preview.templateAep": "X:/shared_projects_2026/UNGO_VFX/production/working_files/UNG_VFXOVERLAY_SLATE_TEMPLATE.aep",
|
||||
"preview.templateComp": "UNG_EXPORT_TEMPLATE",
|
||||
"preview.overlayComp": "UNG_VFX_OVERLAY",
|
||||
"preview.lutComp": "_SHOW LUT",
|
||||
"preview.movTemplate": "4444 Tri",
|
||||
"preview.mp4Template": "REVIEW_PREVIEW",
|
||||
// Essential Property names on the NETFLIX_SLATE layer for the per-submission
|
||||
// fields. If a name is wrong the build logs the template's actual property
|
||||
// names as a warning, so it is a config fix rather than a code change.
|
||||
"preview.slateScopeProp": "Scope",
|
||||
"preview.slateSubmissionProp": "Notes",
|
||||
} as const;
|
||||
|
||||
export type PipelineConfigKey = keyof typeof PIPELINE_CONFIG_DEFAULTS;
|
||||
|
||||
export async function getConfigValue(key: PipelineConfigKey): Promise<string> {
|
||||
const row = await db.systemConfig.findUnique({ where: { key } });
|
||||
return row?.value ?? String(PIPELINE_CONFIG_DEFAULTS[key]);
|
||||
}
|
||||
|
||||
export async function getConfigNumber(key: PipelineConfigKey): Promise<number> {
|
||||
const raw = await getConfigValue(key);
|
||||
const n = Number(raw);
|
||||
return Number.isFinite(n) ? n : Number(PIPELINE_CONFIG_DEFAULTS[key]);
|
||||
}
|
||||
|
||||
export async function getConfigBoolean(key: PipelineConfigKey): Promise<boolean> {
|
||||
const raw = (await getConfigValue(key)).trim().toLowerCase();
|
||||
return raw === "true" || raw === "1" || raw === "yes";
|
||||
}
|
||||
|
||||
/** Preview-stage settings handed to the worker inside the PREVIEW_ONLY manifest. */
|
||||
export async function getPreviewConfig() {
|
||||
const [
|
||||
enabled, templateAep, templateComp, overlayComp, lutComp, movTemplate, mp4Template,
|
||||
slateScopeProp, slateSubmissionProp,
|
||||
] = await Promise.all([
|
||||
getConfigBoolean("preview.enabled"),
|
||||
getConfigValue("preview.templateAep"),
|
||||
getConfigValue("preview.templateComp"),
|
||||
getConfigValue("preview.overlayComp"),
|
||||
getConfigValue("preview.lutComp"),
|
||||
getConfigValue("preview.movTemplate"),
|
||||
getConfigValue("preview.mp4Template"),
|
||||
getConfigValue("preview.slateScopeProp"),
|
||||
getConfigValue("preview.slateSubmissionProp"),
|
||||
]);
|
||||
return {
|
||||
enabled, templateAep, templateComp, overlayComp, lutComp, movTemplate, mp4Template,
|
||||
slateScopeProp, slateSubmissionProp,
|
||||
};
|
||||
}
|
||||
|
||||
/** Worker-facing config bundle returned by E6 registration. */
|
||||
export async function getWorkerConfig() {
|
||||
const [pollSeconds, heartbeatSeconds, leaseSeconds, maxAttempts, stallTimeoutSeconds] =
|
||||
await Promise.all([
|
||||
getConfigNumber("render.pollSeconds"),
|
||||
getConfigNumber("render.heartbeatSeconds"),
|
||||
getConfigNumber("render.leaseSeconds"),
|
||||
getConfigNumber("render.maxAttempts"),
|
||||
getConfigNumber("render.stallTimeoutSeconds"),
|
||||
]);
|
||||
return { pollSeconds, heartbeatSeconds, leaseSeconds, maxAttempts, stallTimeoutSeconds };
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
/**
|
||||
* Typed service-layer error carrying an HTTP status so both auth fronts
|
||||
* (ext API-key routes and internal session routes) map errors identically.
|
||||
*/
|
||||
export class PipelineError extends Error {
|
||||
status: number;
|
||||
details?: unknown;
|
||||
|
||||
constructor(status: number, message: string, details?: unknown) {
|
||||
super(message);
|
||||
this.name = "PipelineError";
|
||||
this.status = status;
|
||||
this.details = details;
|
||||
}
|
||||
}
|
||||
|
||||
export function toErrorResponse(err: unknown): { body: { error: string; details?: unknown }; status: number } {
|
||||
if (err instanceof PipelineError) {
|
||||
return {
|
||||
body: { error: err.message, ...(err.details !== undefined ? { details: err.details } : {}) },
|
||||
status: err.status,
|
||||
};
|
||||
}
|
||||
console.error("[render-pipeline]", err);
|
||||
return { body: { error: "Internal server error" }, status: 500 };
|
||||
}
|
||||
@@ -0,0 +1,504 @@
|
||||
import { ExportStatus, Prisma } from "@prisma/client";
|
||||
import { db } from "@/lib/db";
|
||||
import { PipelineError } from "./errors";
|
||||
import { renderManifestSchema, type RenderManifest } from "./manifest";
|
||||
import { applyTransition, type TransitionActor } from "./transitions";
|
||||
import { getConfigNumber, getConfigValue } from "./config";
|
||||
|
||||
/** Statuses in which a shot has a live export in flight (§6.4 E1 409 rule). */
|
||||
const ACTIVE_STATUSES: ExportStatus[] = [
|
||||
"QUEUED",
|
||||
"RENDERING",
|
||||
"VALIDATING",
|
||||
"GENERATING_PREVIEW",
|
||||
];
|
||||
|
||||
/** Non-terminal statuses auto-superseded when a newer Export is queued (T1/T17). */
|
||||
const SUPERSEDABLE_STATUSES: ExportStatus[] = [
|
||||
"QUEUED",
|
||||
"RENDER_FAILED",
|
||||
"VALIDATION_FAILED",
|
||||
"PREVIEW_FAILED",
|
||||
"READY_FOR_QC",
|
||||
"QC_FAILED",
|
||||
"READY_FOR_DELIVERY",
|
||||
];
|
||||
|
||||
const OPEN_JOB_STATUSES = ["QUEUED", "CLAIMED", "RUNNING"] as const;
|
||||
|
||||
export function parseVersionString(v: string | null | undefined): number {
|
||||
const m = /^v(\d+)$/i.exec(v?.trim() ?? "");
|
||||
return m ? parseInt(m[1], 10) : 0;
|
||||
}
|
||||
|
||||
export function formatVersionString(n: number): string {
|
||||
return `v${String(n).padStart(3, "0")}`;
|
||||
}
|
||||
|
||||
/** `UNG_..._cmp_TT_v003` + 4 → `UNG_..._cmp_TT_v004`; falls back to house convention. */
|
||||
function nextExrOutputBase(current: string | null, shotCode: string, versionString: string): string {
|
||||
if (current && /_v\d+$/i.test(current)) return current.replace(/_v\d+$/i, `_${versionString}`);
|
||||
if (current) return `${current}_${versionString}`;
|
||||
return renderOutputBase(shotCode, versionString);
|
||||
}
|
||||
|
||||
/**
|
||||
* Rendered EXR naming for the pipeline — matches the panel's
|
||||
* "Queue EXR (Review)" convention (`{shotCode}_cmp_TT_{version}`), which is
|
||||
* shot-code based and deliberately excludes the source clip name that
|
||||
* `Shot.exrOutput` carries from the EDL import.
|
||||
*/
|
||||
function renderOutputBase(shotCode: string, versionString: string): string {
|
||||
return `${shotCode}_cmp_TT_${versionString}`;
|
||||
}
|
||||
|
||||
export interface CreateExportInput {
|
||||
manifest: unknown;
|
||||
submittedByEmail?: string | null;
|
||||
priority?: number;
|
||||
force?: boolean;
|
||||
/**
|
||||
* Per-submission slate fields. `undefined` inherits the previous export's
|
||||
* value for this shot; `null` or "" explicitly clears it.
|
||||
*/
|
||||
vfxScope?: string | null;
|
||||
submissionNote?: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* E1 — Queue Export (§6.4). Transactionally: decides the new version number
|
||||
* (server-side, race-free), updates Shot.shotVersion/exrOutput exactly as the
|
||||
* panel's legacy PATCH did, supersedes older non-terminal exports, creates
|
||||
* Export(QUEUED) + RenderJob attempt 1 with the manifest snapshot.
|
||||
*/
|
||||
export async function createExport(input: CreateExportInput) {
|
||||
const parsed = renderManifestSchema.safeParse(input.manifest);
|
||||
if (!parsed.success) {
|
||||
throw new PipelineError(422, "Manifest validation error", parsed.error.issues);
|
||||
}
|
||||
const manifest = parsed.data;
|
||||
|
||||
const shot = await db.shot.findUnique({
|
||||
where: { id: manifest.shotId },
|
||||
include: {
|
||||
project: { select: { id: true, code: true, showId: true } },
|
||||
tasks: { where: { type: "COMP" }, orderBy: { sortOrder: "asc" }, take: 1, select: { id: true } },
|
||||
},
|
||||
});
|
||||
if (!shot) throw new PipelineError(404, `Shot not found: ${manifest.shotId}`);
|
||||
|
||||
// Cross-checks (§6.2): manifest vs DB truth
|
||||
if (shot.shotCode !== manifest.shotCode) {
|
||||
throw new PipelineError(422, `Manifest shotCode "${manifest.shotCode}" does not match shot ${shot.id} ("${shot.shotCode}")`);
|
||||
}
|
||||
if (shot.project.code !== manifest.projectCode) {
|
||||
throw new PipelineError(422, `Manifest projectCode "${manifest.projectCode}" does not match shot's project ("${shot.project.code}")`);
|
||||
}
|
||||
// A 0/0 shot range means "never imported", not a real single-frame shot at 0
|
||||
const shotRangeKnown =
|
||||
shot.frameStart != null && shot.frameEnd != null && !(shot.frameStart === 0 && shot.frameEnd === 0);
|
||||
if (
|
||||
!input.force &&
|
||||
shotRangeKnown &&
|
||||
(shot.frameStart !== manifest.frameStart || shot.frameEnd !== manifest.frameEnd)
|
||||
) {
|
||||
throw new PipelineError(422, `Manifest frame range ${manifest.frameStart}-${manifest.frameEnd} does not match shot record ${shot.frameStart}-${shot.frameEnd} (pass force to override)`);
|
||||
}
|
||||
|
||||
const activeExport = await db.export.findFirst({
|
||||
where: { shotId: shot.id, status: { in: ACTIVE_STATUSES } },
|
||||
select: { id: true, status: true, versionString: true },
|
||||
});
|
||||
if (activeExport && !input.force) {
|
||||
throw new PipelineError(
|
||||
409,
|
||||
`An active export (${activeExport.versionString}, ${activeExport.status}) already exists for this shot — pass force to supersede it`,
|
||||
{ activeExportId: activeExport.id }
|
||||
);
|
||||
}
|
||||
|
||||
const submittedBy = input.submittedByEmail
|
||||
? await db.user.findUnique({ where: { email: input.submittedByEmail }, select: { id: true, name: true } })
|
||||
: null;
|
||||
|
||||
const [maxAttempts, outputRoot] = await Promise.all([
|
||||
getConfigNumber("render.maxAttempts"),
|
||||
getConfigValue("render.outputRoot"),
|
||||
]);
|
||||
|
||||
// Retry loop: @@unique([shotId, versionNumber]) guards the version-increment
|
||||
// race between two concurrent E1 calls; loser recomputes and retries.
|
||||
for (let tryNo = 0; tryNo < 3; tryNo++) {
|
||||
try {
|
||||
return await db.$transaction(async (tx) => {
|
||||
const latest = await tx.export.aggregate({
|
||||
where: { shotId: shot.id },
|
||||
_max: { versionNumber: true },
|
||||
});
|
||||
const freshShot = await tx.shot.findUniqueOrThrow({
|
||||
where: { id: shot.id },
|
||||
select: { shotVersion: true, exrOutput: true },
|
||||
});
|
||||
// Slate fields carry forward from the shot's previous submission
|
||||
// unless the caller supplies new ones.
|
||||
const previous = await tx.export.findFirst({
|
||||
where: { shotId: shot.id },
|
||||
orderBy: { versionNumber: "desc" },
|
||||
select: { vfxScope: true, submissionNote: true },
|
||||
});
|
||||
const vfxScope = input.vfxScope !== undefined ? input.vfxScope : (previous?.vfxScope ?? null);
|
||||
const submissionNote =
|
||||
input.submissionNote !== undefined ? input.submissionNote : (previous?.submissionNote ?? null);
|
||||
const versionNumber =
|
||||
Math.max(latest._max.versionNumber ?? 0, parseVersionString(freshShot.shotVersion)) + 1;
|
||||
const versionString = formatVersionString(versionNumber);
|
||||
const exrBase = nextExrOutputBase(freshShot.exrOutput, shot.shotCode, versionString);
|
||||
|
||||
const outputDir =
|
||||
manifest.outputDir === "auto"
|
||||
? [outputRoot, shot.project.code, ...(shot.episode ? [shot.episode] : []), shot.shotCode, versionString].join("/")
|
||||
: manifest.outputDir;
|
||||
const outputPattern =
|
||||
manifest.outputPattern === "auto"
|
||||
? `${renderOutputBase(shot.shotCode, versionString)}.[#####].exr`
|
||||
: manifest.outputPattern;
|
||||
|
||||
const resolvedManifest: RenderManifest = { ...manifest, outputDir, outputPattern };
|
||||
|
||||
// Supersede older non-terminal exports (T1/T17); cancel their open jobs.
|
||||
const toSupersede = await tx.export.findMany({
|
||||
where: {
|
||||
shotId: shot.id,
|
||||
status: { in: input.force ? [...SUPERSEDABLE_STATUSES, ...ACTIVE_STATUSES] : SUPERSEDABLE_STATUSES },
|
||||
},
|
||||
select: { id: true, status: true },
|
||||
});
|
||||
|
||||
const created = await tx.export.create({
|
||||
data: {
|
||||
shotId: shot.id,
|
||||
projectId: shot.project.id,
|
||||
taskId: shot.tasks[0]?.id ?? null,
|
||||
versionNumber,
|
||||
versionString,
|
||||
status: "QUEUED",
|
||||
aepPath: manifest.aepPath,
|
||||
compName: manifest.compName,
|
||||
rendererType: manifest.rendererType,
|
||||
outputDir,
|
||||
outputPattern,
|
||||
frameStart: manifest.frameStart,
|
||||
frameEnd: manifest.frameEnd,
|
||||
fps: manifest.fps,
|
||||
width: manifest.width,
|
||||
height: manifest.height,
|
||||
colorspace: manifest.expected?.colorspace ?? null,
|
||||
vfxScope,
|
||||
submissionNote,
|
||||
submittedById: submittedBy?.id ?? null,
|
||||
submittedByName: submittedBy?.name ?? input.submittedByEmail ?? null,
|
||||
},
|
||||
});
|
||||
|
||||
for (const old of toSupersede) {
|
||||
await tx.renderJob.updateMany({
|
||||
where: { exportId: old.id, status: { in: [...OPEN_JOB_STATUSES] } },
|
||||
data: { status: "CANCELLED", finishedAt: new Date(), errorMessage: `Superseded by export ${created.id} (${versionString})` },
|
||||
});
|
||||
await applyTransition(
|
||||
tx,
|
||||
old,
|
||||
"SUPERSEDED",
|
||||
{ type: "SYSTEM", note: `Superseded by ${versionString}` },
|
||||
{ supersededById: created.id }
|
||||
);
|
||||
}
|
||||
|
||||
const renderJob = await tx.renderJob.create({
|
||||
data: {
|
||||
type: "AE_RENDER",
|
||||
exportId: created.id,
|
||||
attempt: 1,
|
||||
maxAttempts,
|
||||
priority: input.priority ?? 50,
|
||||
manifest: resolvedManifest as unknown as Prisma.InputJsonValue,
|
||||
},
|
||||
});
|
||||
|
||||
await tx.exportEvent.create({
|
||||
data: {
|
||||
exportId: created.id,
|
||||
fromStatus: null,
|
||||
toStatus: "QUEUED",
|
||||
actorType: "USER",
|
||||
actorId: submittedBy?.id ?? null,
|
||||
note: `Queue Export ${versionString}${input.force ? " (force)" : ""}`,
|
||||
},
|
||||
});
|
||||
|
||||
// Mirror the legacy panel PATCH: Shot stays the canonical "current version"
|
||||
const updatedShot = await tx.shot.update({
|
||||
where: { id: shot.id },
|
||||
data: { shotVersion: versionString, exrOutput: exrBase },
|
||||
select: { id: true, shotVersion: true, exrOutput: true },
|
||||
});
|
||||
|
||||
return {
|
||||
export: {
|
||||
id: created.id,
|
||||
shotCode: shot.shotCode,
|
||||
versionNumber,
|
||||
versionString,
|
||||
status: created.status,
|
||||
outputDir,
|
||||
outputPattern,
|
||||
vfxScope,
|
||||
submissionNote,
|
||||
},
|
||||
renderJob: { id: renderJob.id, attempt: renderJob.attempt, priority: renderJob.priority },
|
||||
shot: updatedShot,
|
||||
superseded: toSupersede.map((s) => s.id),
|
||||
};
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof Prisma.PrismaClientKnownRequestError && err.code === "P2002" && tryNo < 2) {
|
||||
continue; // concurrent E1 won the version — recompute
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
throw new PipelineError(409, "Could not allocate a version number after retries");
|
||||
}
|
||||
|
||||
function serializeExport<T extends { exrTotalBytes: bigint | null }>(e: T) {
|
||||
return { ...e, exrTotalBytes: e.exrTotalBytes == null ? null : Number(e.exrTotalBytes) };
|
||||
}
|
||||
|
||||
/** E2 — latest export + status for the AE panel header (§6.4). */
|
||||
export async function getLatestExport(shotCode: string, projectCode?: string | null) {
|
||||
const shot = await db.shot.findFirst({
|
||||
where: { shotCode, ...(projectCode ? { project: { code: projectCode } } : {}) },
|
||||
select: { id: true },
|
||||
});
|
||||
if (!shot) throw new PipelineError(404, "Shot not found");
|
||||
|
||||
const exp = await db.export.findFirst({
|
||||
where: { shotId: shot.id },
|
||||
orderBy: { versionNumber: "desc" },
|
||||
include: {
|
||||
renderJobs: {
|
||||
orderBy: { attempt: "desc" },
|
||||
take: 1,
|
||||
include: { machine: { select: { name: true } } },
|
||||
},
|
||||
},
|
||||
});
|
||||
if (!exp) return { export: null };
|
||||
|
||||
const job = exp.renderJobs[0] ?? null;
|
||||
return {
|
||||
export: {
|
||||
id: exp.id,
|
||||
versionNumber: exp.versionNumber,
|
||||
versionString: exp.versionString,
|
||||
status: exp.status,
|
||||
statusChangedAt: exp.statusChangedAt,
|
||||
outputDir: exp.outputDir,
|
||||
outputPattern: exp.outputPattern,
|
||||
// Panel prefills its slate fields from these
|
||||
vfxScope: exp.vfxScope,
|
||||
submissionNote: exp.submissionNote,
|
||||
renderJob: job
|
||||
? {
|
||||
id: job.id,
|
||||
attempt: job.attempt,
|
||||
status: job.status,
|
||||
progress: job.progress,
|
||||
currentFrame: job.currentFrame,
|
||||
totalFrames: job.totalFrames,
|
||||
etaSeconds: job.etaSeconds,
|
||||
errorMessage: job.errorMessage,
|
||||
machineName: job.machine?.name ?? null,
|
||||
}
|
||||
: null,
|
||||
previewUrl: exp.previewMovKey ? `/api/files/${exp.previewMovKey}` : null,
|
||||
thumbnailUrl: exp.thumbnailKey ? `/api/files/${exp.thumbnailKey}` : null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** E3 — export detail incl. attempts and audit events. */
|
||||
export async function getExportDetail(exportId: string) {
|
||||
const exp = await db.export.findUnique({
|
||||
where: { id: exportId },
|
||||
include: {
|
||||
shot: { select: { id: true, shotCode: true, episode: true, project: { select: { id: true, name: true, code: true } } } },
|
||||
renderJobs: {
|
||||
orderBy: { attempt: "asc" },
|
||||
include: { machine: { select: { id: true, name: true } } },
|
||||
},
|
||||
events: { orderBy: { createdAt: "asc" } },
|
||||
},
|
||||
});
|
||||
if (!exp) throw new PipelineError(404, "Export not found");
|
||||
return { export: serializeExport(exp) };
|
||||
}
|
||||
|
||||
/**
|
||||
* E14 / E15 — retry a failed export: new RenderJob attempt, back to QUEUED.
|
||||
* PREVIEW_FAILED is routed to the preview-only retry (T11) so validated EXRs
|
||||
* are never re-rendered just to rebuild a slate.
|
||||
*/
|
||||
export async function retryExport(exportId: string, actor: TransitionActor) {
|
||||
const current = await db.export.findUnique({ where: { id: exportId }, select: { status: true } });
|
||||
if (!current) throw new PipelineError(404, "Export not found");
|
||||
if (current.status === "PREVIEW_FAILED") {
|
||||
const { retryPreview } = await import("./preview");
|
||||
return retryPreview(exportId, actor.id ?? undefined);
|
||||
}
|
||||
|
||||
return db.$transaction(async (tx) => {
|
||||
const exp = await tx.export.findUnique({
|
||||
where: { id: exportId },
|
||||
include: { renderJobs: { orderBy: { attempt: "desc" }, take: 1 } },
|
||||
});
|
||||
if (!exp) throw new PipelineError(404, "Export not found");
|
||||
if (!["RENDER_FAILED", "VALIDATION_FAILED"].includes(exp.status)) {
|
||||
throw new PipelineError(409, `Export is ${exp.status} — only RENDER_FAILED, VALIDATION_FAILED or PREVIEW_FAILED exports can be retried`);
|
||||
}
|
||||
const last = exp.renderJobs[0];
|
||||
if (!last) throw new PipelineError(500, "Export has no render jobs");
|
||||
|
||||
const job = await tx.renderJob.create({
|
||||
data: {
|
||||
type: last.type,
|
||||
exportId: exp.id,
|
||||
attempt: last.attempt + 1,
|
||||
maxAttempts: last.maxAttempts,
|
||||
priority: last.priority,
|
||||
manifest: last.manifest as Prisma.InputJsonValue,
|
||||
},
|
||||
});
|
||||
const updated = await applyTransition(tx, exp, "QUEUED", {
|
||||
...actor,
|
||||
note: actor.note ?? `Retry (attempt ${job.attempt})`,
|
||||
});
|
||||
return { export: { id: updated.id, status: updated.status }, renderJob: { id: job.id, attempt: job.attempt } };
|
||||
});
|
||||
}
|
||||
|
||||
/** E15 / T18 — cancel a queued or rendering export. Worker learns via heartbeat/progress. */
|
||||
export async function cancelExport(exportId: string, actor: TransitionActor) {
|
||||
return db.$transaction(async (tx) => {
|
||||
const exp = await tx.export.findUnique({ where: { id: exportId }, select: { id: true, status: true } });
|
||||
if (!exp) throw new PipelineError(404, "Export not found");
|
||||
if (exp.status === "CANCELLED") return { export: exp, alreadyApplied: true };
|
||||
if (!["QUEUED", "RENDERING"].includes(exp.status)) {
|
||||
throw new PipelineError(409, `Export is ${exp.status} — only QUEUED or RENDERING exports can be cancelled`);
|
||||
}
|
||||
await tx.renderJob.updateMany({
|
||||
where: { exportId: exp.id, status: "QUEUED" },
|
||||
data: { status: "CANCELLED", finishedAt: new Date(), errorMessage: "Cancelled by user" },
|
||||
});
|
||||
// Claimed/running jobs stay marked CANCELLED but keep finishedAt null until
|
||||
// the worker acknowledges (heartbeat CANCEL_JOB command / progress cancelRequested)
|
||||
await tx.renderJob.updateMany({
|
||||
where: { exportId: exp.id, status: { in: ["CLAIMED", "RUNNING"] } },
|
||||
data: { status: "CANCELLED", errorMessage: "Cancelled by user" },
|
||||
});
|
||||
const updated = await applyTransition(tx, exp, "CANCELLED", actor);
|
||||
return { export: { id: updated.id, status: updated.status }, alreadyApplied: false };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Phase 1 stopgap (§17.1): admin "mark as done manually" so the studio can
|
||||
* keep using the old render path while the queue is validated. Removed once
|
||||
* workers render for real.
|
||||
*/
|
||||
export async function markExportDoneManually(exportId: string, userId: string, note?: string) {
|
||||
return db.$transaction(async (tx) => {
|
||||
const exp = await tx.export.findUnique({ where: { id: exportId }, select: { id: true, status: true } });
|
||||
if (!exp) throw new PipelineError(404, "Export not found");
|
||||
if (!["QUEUED", "RENDERING"].includes(exp.status)) {
|
||||
throw new PipelineError(409, `Export is ${exp.status} — only QUEUED or RENDERING exports can be marked done`);
|
||||
}
|
||||
await tx.renderJob.updateMany({
|
||||
where: { exportId: exp.id, status: { in: [...OPEN_JOB_STATUSES] } },
|
||||
data: { status: "CANCELLED", finishedAt: new Date(), errorMessage: "Manually marked done (legacy render path)" },
|
||||
});
|
||||
const updated = await applyTransition(tx, exp, "READY_FOR_QC", {
|
||||
type: "USER",
|
||||
id: userId,
|
||||
note: note ?? "Manually marked done (rendered outside the pipeline)",
|
||||
});
|
||||
return { export: { id: updated.id, status: updated.status } };
|
||||
});
|
||||
}
|
||||
|
||||
export interface ListExportsFilter {
|
||||
projectId?: string;
|
||||
episode?: string;
|
||||
status?: ExportStatus;
|
||||
shotId?: string;
|
||||
page?: number;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
/** E21 — paginated export list for the web queue/monitoring pages. */
|
||||
export async function listExports(filter: ListExportsFilter) {
|
||||
const page = Math.max(1, filter.page ?? 1);
|
||||
const limit = Math.min(200, Math.max(1, filter.limit ?? 50));
|
||||
const where: Prisma.ExportWhereInput = {
|
||||
...(filter.projectId ? { projectId: filter.projectId } : {}),
|
||||
...(filter.status ? { status: filter.status } : {}),
|
||||
...(filter.shotId ? { shotId: filter.shotId } : {}),
|
||||
...(filter.episode ? { shot: { episode: filter.episode } } : {}),
|
||||
};
|
||||
const [total, rows] = await Promise.all([
|
||||
db.export.count({ where }),
|
||||
db.export.findMany({
|
||||
where,
|
||||
orderBy: { createdAt: "desc" },
|
||||
skip: (page - 1) * limit,
|
||||
take: limit,
|
||||
include: {
|
||||
shot: { select: { shotCode: true, episode: true, project: { select: { name: true, code: true } } } },
|
||||
renderJobs: {
|
||||
orderBy: { attempt: "desc" },
|
||||
take: 1,
|
||||
include: { machine: { select: { name: true } } },
|
||||
},
|
||||
},
|
||||
}),
|
||||
]);
|
||||
return {
|
||||
pagination: { page, limit, total, pages: Math.ceil(total / limit) },
|
||||
exports: rows.map((e) => ({
|
||||
id: e.id,
|
||||
shotCode: e.shot.shotCode,
|
||||
episode: e.shot.episode,
|
||||
projectName: e.shot.project.name,
|
||||
projectCode: e.shot.project.code,
|
||||
versionString: e.versionString,
|
||||
status: e.status,
|
||||
statusChangedAt: e.statusChangedAt,
|
||||
createdAt: e.createdAt,
|
||||
outputDir: e.outputDir,
|
||||
job: e.renderJobs[0]
|
||||
? {
|
||||
id: e.renderJobs[0].id,
|
||||
attempt: e.renderJobs[0].attempt,
|
||||
status: e.renderJobs[0].status,
|
||||
progress: e.renderJobs[0].progress,
|
||||
currentFrame: e.renderJobs[0].currentFrame,
|
||||
totalFrames: e.renderJobs[0].totalFrames,
|
||||
etaSeconds: e.renderJobs[0].etaSeconds,
|
||||
priority: e.renderJobs[0].priority,
|
||||
machineName: e.renderJobs[0].machine?.name ?? null,
|
||||
errorMessage: e.renderJobs[0].errorMessage,
|
||||
}
|
||||
: null,
|
||||
})),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import { NextRequest } from "next/server";
|
||||
|
||||
/**
|
||||
* Shared API-key auth for /api/ext/* pipeline routes — same contract as the
|
||||
* existing ext endpoints (Authorization: Bearer <key> or X-Api-Key header).
|
||||
* One shared API_SECRET_KEY for now (studio decision 18.1-Q7).
|
||||
*/
|
||||
export function isExtAuthorized(req: NextRequest): boolean {
|
||||
const apiKey = process.env.API_SECRET_KEY;
|
||||
if (!apiKey) return false;
|
||||
const authHeader = req.headers.get("authorization") ?? "";
|
||||
if (authHeader.startsWith("Bearer ")) return authHeader.slice(7) === apiKey;
|
||||
return (req.headers.get("x-api-key") ?? "") === apiKey;
|
||||
}
|
||||
@@ -0,0 +1,317 @@
|
||||
import { Prisma, RenderJobType } from "@prisma/client";
|
||||
import { db } from "@/lib/db";
|
||||
import { PipelineError } from "./errors";
|
||||
import { applyTransition } from "./transitions";
|
||||
import { getConfigBoolean, getConfigNumber } from "./config";
|
||||
import { machineCanClaim } from "./machines";
|
||||
import { reapExpiredLeases } from "./reaper";
|
||||
import { buildPreviewManifest, createPreviewJob } from "./preview";
|
||||
|
||||
const JOB_TYPES: RenderJobType[] = ["AE_RENDER", "PREVIEW_ONLY", "DELIVERY_BUILD"];
|
||||
|
||||
/**
|
||||
* E8 — atomically claim the next queued job (§6.4). `FOR UPDATE SKIP LOCKED`
|
||||
* makes double-claims impossible under concurrent workers. All scheduling
|
||||
* policy (availability windows, Render Now, urgent priority) is enforced
|
||||
* here server-side — workers poll dumbly (§7.10).
|
||||
*/
|
||||
export async function claimNextJob(machineId: string, types?: string[]) {
|
||||
const machine = await db.machine.findUnique({ where: { id: machineId } });
|
||||
if (!machine) throw new PipelineError(404, "Machine not found — register first (POST /api/ext/workers/register)");
|
||||
|
||||
// Defensive sweep on each claim (studio decision 18.1-Q8)
|
||||
await reapExpiredLeases().catch((err) => console.error("[render-pipeline] claim-time reap failed", err));
|
||||
|
||||
const claimTypes = (types?.length ? types : ["AE_RENDER", "PREVIEW_ONLY", "DELIVERY_BUILD"]).filter(
|
||||
(t): t is RenderJobType => (JOB_TYPES as string[]).includes(t)
|
||||
);
|
||||
if (claimTypes.length === 0) throw new PipelineError(422, "No valid job types requested");
|
||||
|
||||
const gate = machineCanClaim(machine, new Date());
|
||||
if (gate === "none") return null;
|
||||
|
||||
const [leaseSeconds, urgentThreshold] = await Promise.all([
|
||||
getConfigNumber("render.leaseSeconds"),
|
||||
getConfigNumber("render.urgentPriorityThreshold"),
|
||||
]);
|
||||
|
||||
const urgentFilter =
|
||||
gate === "urgent-only" ? Prisma.sql`AND priority <= ${urgentThreshold}` : Prisma.empty;
|
||||
|
||||
const claimed = await db.$queryRaw<{ id: string }[]>(Prisma.sql`
|
||||
UPDATE "render_jobs"
|
||||
SET status = 'CLAIMED',
|
||||
"machineId" = ${machineId},
|
||||
"claimedAt" = now(),
|
||||
"leaseExpiresAt" = now() + (${leaseSeconds}::int * interval '1 second'),
|
||||
"updatedAt" = now()
|
||||
WHERE id = (
|
||||
SELECT id FROM "render_jobs"
|
||||
WHERE status = 'QUEUED'
|
||||
AND type::text = ANY(${claimTypes}::text[])
|
||||
${urgentFilter}
|
||||
ORDER BY priority ASC, "createdAt" ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING id
|
||||
`);
|
||||
if (claimed.length === 0) return null;
|
||||
|
||||
const job = await db.renderJob.findUniqueOrThrow({
|
||||
where: { id: claimed[0].id },
|
||||
include: { export: { select: { id: true, status: true, versionString: true } } },
|
||||
});
|
||||
|
||||
// T2: QUEUED → RENDERING
|
||||
if (job.export && job.export.status === "QUEUED") {
|
||||
await db.$transaction(async (tx) => {
|
||||
await applyTransition(tx, job.export!, "RENDERING", {
|
||||
type: "WORKER",
|
||||
id: machineId,
|
||||
note: `Claimed by ${machine.name} (attempt ${job.attempt})`,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
job: {
|
||||
id: job.id,
|
||||
type: job.type,
|
||||
attempt: job.attempt,
|
||||
maxAttempts: job.maxAttempts,
|
||||
priority: job.priority,
|
||||
exportId: job.exportId,
|
||||
deliveryId: job.deliveryId,
|
||||
leaseExpiresAt: job.leaseExpiresAt,
|
||||
manifest: job.manifest,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function getJobForReport(jobId: string, machineId: string) {
|
||||
const job = await db.renderJob.findUnique({
|
||||
where: { id: jobId },
|
||||
include: { export: { select: { id: true, status: true } } },
|
||||
});
|
||||
if (!job) throw new PipelineError(404, "Render job not found");
|
||||
// Zombie-worker guard (§7.4): reports from a machine that doesn't hold the lease are rejected
|
||||
if (job.machineId !== machineId) {
|
||||
throw new PipelineError(409, "Job is not held by this machine (lease reassigned?)");
|
||||
}
|
||||
return job;
|
||||
}
|
||||
|
||||
/** E9 — progress report; renews the lease. */
|
||||
export async function reportProgress(
|
||||
jobId: string,
|
||||
machineId: string,
|
||||
body: { progress?: number; currentFrame?: number; totalFrames?: number; etaSeconds?: number; logTail?: string }
|
||||
) {
|
||||
const job = await getJobForReport(jobId, machineId);
|
||||
|
||||
if (job.status === "CANCELLED") {
|
||||
return { ok: true, cancelRequested: true, leaseExpiresAt: job.leaseExpiresAt };
|
||||
}
|
||||
if (["COMPLETED", "FAILED", "EXPIRED"].includes(job.status)) {
|
||||
return { ok: true, alreadyApplied: true, cancelRequested: false, leaseExpiresAt: job.leaseExpiresAt };
|
||||
}
|
||||
|
||||
const leaseSeconds = await getConfigNumber("render.leaseSeconds");
|
||||
const leaseExpiresAt = new Date(Date.now() + leaseSeconds * 1000);
|
||||
await db.renderJob.update({
|
||||
where: { id: job.id },
|
||||
data: {
|
||||
status: "RUNNING",
|
||||
startedAt: job.startedAt ?? new Date(),
|
||||
leaseExpiresAt,
|
||||
...(body.progress != null ? { progress: Math.min(1, Math.max(0, body.progress)) } : {}),
|
||||
...(body.currentFrame != null ? { currentFrame: body.currentFrame } : {}),
|
||||
...(body.totalFrames != null ? { totalFrames: body.totalFrames } : {}),
|
||||
...(body.etaSeconds != null ? { etaSeconds: body.etaSeconds } : {}),
|
||||
...(body.logTail != null ? { logTail: body.logTail.slice(-20000) } : {}),
|
||||
},
|
||||
});
|
||||
return { ok: true, leaseExpiresAt, cancelRequested: false };
|
||||
}
|
||||
|
||||
/** E10 — failure report. Auto-requeues while attempts remain and the error is retryable. */
|
||||
export async function reportFail(
|
||||
jobId: string,
|
||||
machineId: string,
|
||||
body: {
|
||||
stage?: string;
|
||||
exitCode?: number;
|
||||
errorMessage?: string;
|
||||
logTail?: string;
|
||||
logFileKey?: string;
|
||||
retryable?: boolean;
|
||||
}
|
||||
) {
|
||||
return db.$transaction(async (tx) => {
|
||||
const job = await tx.renderJob.findUnique({
|
||||
where: { id: jobId },
|
||||
include: { export: { select: { id: true, status: true } } },
|
||||
});
|
||||
if (!job) throw new PipelineError(404, "Render job not found");
|
||||
if (job.machineId !== machineId) throw new PipelineError(409, "Job is not held by this machine (lease reassigned?)");
|
||||
|
||||
if (job.status === "FAILED") {
|
||||
return { job: { id: job.id, status: job.status }, export: job.export ? { id: job.export.id, status: job.export.status } : null, autoRequeued: false, alreadyApplied: true };
|
||||
}
|
||||
if (job.status === "CANCELLED") {
|
||||
// worker acknowledging a cancel — record the wind-down, stay CANCELLED
|
||||
await tx.renderJob.update({ where: { id: job.id }, data: { finishedAt: job.finishedAt ?? new Date(), logTail: body.logTail?.slice(-20000) ?? job.logTail } });
|
||||
return { job: { id: job.id, status: "CANCELLED" }, export: job.export ? { id: job.export.id, status: job.export.status } : null, autoRequeued: false, alreadyApplied: true };
|
||||
}
|
||||
if (job.status === "COMPLETED") {
|
||||
throw new PipelineError(409, "Job already reported complete — contradictory fail report rejected");
|
||||
}
|
||||
|
||||
await tx.renderJob.update({
|
||||
where: { id: job.id },
|
||||
data: {
|
||||
status: "FAILED",
|
||||
finishedAt: new Date(),
|
||||
exitCode: body.exitCode ?? null,
|
||||
errorMessage: body.errorMessage?.slice(0, 2000) ?? null,
|
||||
logTail: body.logTail?.slice(-20000) ?? job.logTail,
|
||||
logFileKey: body.logFileKey ?? job.logFileKey,
|
||||
},
|
||||
});
|
||||
|
||||
if (!job.export) {
|
||||
return { job: { id: job.id, status: "FAILED" }, export: null, autoRequeued: false };
|
||||
}
|
||||
|
||||
const retryable = body.retryable !== false;
|
||||
const autoRequeue = retryable && job.attempt < job.maxAttempts;
|
||||
const isPreview = job.type === "PREVIEW_ONLY";
|
||||
|
||||
if (autoRequeue) {
|
||||
const next = await tx.renderJob.create({
|
||||
data: {
|
||||
type: job.type,
|
||||
exportId: job.exportId,
|
||||
attempt: job.attempt + 1,
|
||||
maxAttempts: job.maxAttempts,
|
||||
priority: job.priority,
|
||||
manifest: job.manifest as Prisma.InputJsonValue,
|
||||
},
|
||||
});
|
||||
const note = `${body.stage ?? "RENDER"} failed (${body.errorMessage ?? "no message"}) — auto-requeued as attempt ${next.attempt}`;
|
||||
|
||||
if (isPreview) {
|
||||
// The export stays in GENERATING_PREVIEW across preview attempts —
|
||||
// the EXRs are untouched, only the preview build is repeated.
|
||||
await tx.exportEvent.create({
|
||||
data: {
|
||||
exportId: job.export.id,
|
||||
fromStatus: job.export.status,
|
||||
toStatus: job.export.status,
|
||||
actorType: "SYSTEM",
|
||||
actorId: machineId,
|
||||
note,
|
||||
},
|
||||
});
|
||||
return { job: { id: job.id, status: "FAILED" }, export: { id: job.export.id, status: job.export.status }, autoRequeued: true, nextJob: { id: next.id, attempt: next.attempt } };
|
||||
}
|
||||
|
||||
const exp = await applyTransition(tx, job.export, "QUEUED", { type: "SYSTEM", id: machineId, note });
|
||||
return { job: { id: job.id, status: "FAILED" }, export: { id: exp.id, status: exp.status }, autoRequeued: true, nextJob: { id: next.id, attempt: next.attempt } };
|
||||
}
|
||||
|
||||
const exp = await applyTransition(tx, job.export, isPreview ? "PREVIEW_FAILED" : "RENDER_FAILED", {
|
||||
type: "WORKER",
|
||||
id: machineId,
|
||||
note: `${body.stage ?? "RENDER"} failed: ${body.errorMessage ?? "no message"}${retryable ? " (max attempts reached)" : " (not retryable)"}`,
|
||||
});
|
||||
return { job: { id: job.id, status: "FAILED" }, export: { id: exp.id, status: exp.status }, autoRequeued: false };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* E11 — aerender finished OK.
|
||||
* Phase 2 interim (§17.3): transitions straight to a provisional READY_FOR_QC;
|
||||
* Phase 3 will insert VALIDATING between (see transitions.ts note).
|
||||
*/
|
||||
export async function completeRender(
|
||||
jobId: string,
|
||||
machineId: string,
|
||||
body: { renderSeconds?: number; logFileKey?: string; logTail?: string; exrFileCount?: number; exrTotalBytes?: number }
|
||||
) {
|
||||
return db.$transaction(async (tx) => {
|
||||
const job = await tx.renderJob.findUnique({
|
||||
where: { id: jobId },
|
||||
include: { export: { select: { id: true, status: true } } },
|
||||
});
|
||||
if (!job) throw new PipelineError(404, "Render job not found");
|
||||
if (job.machineId !== machineId) throw new PipelineError(409, "Job is not held by this machine (lease reassigned?)");
|
||||
|
||||
if (job.status === "COMPLETED") {
|
||||
return { job: { id: job.id, status: job.status }, export: job.export ? { id: job.export.id, status: job.export.status } : null, alreadyApplied: true };
|
||||
}
|
||||
if (["FAILED", "CANCELLED", "EXPIRED"].includes(job.status)) {
|
||||
throw new PipelineError(409, `Job is ${job.status} — contradictory complete report rejected`);
|
||||
}
|
||||
|
||||
await tx.renderJob.update({
|
||||
where: { id: job.id },
|
||||
data: {
|
||||
status: "COMPLETED",
|
||||
progress: 1,
|
||||
finishedAt: new Date(),
|
||||
exitCode: 0,
|
||||
logFileKey: body.logFileKey ?? job.logFileKey,
|
||||
logTail: body.logTail?.slice(-20000) ?? job.logTail,
|
||||
},
|
||||
});
|
||||
|
||||
let exportResult: { id: string; status: string } | null = null;
|
||||
let previewJob: { id: string; type: string } | null = null;
|
||||
|
||||
if (job.export) {
|
||||
const exportRow = await tx.export.findUniqueOrThrow({ where: { id: job.export.id } });
|
||||
const stats = {
|
||||
...(body.exrFileCount != null ? { exrFileCount: body.exrFileCount } : {}),
|
||||
...(body.exrTotalBytes != null ? { exrTotalBytes: BigInt(Math.round(body.exrTotalBytes)) } : {}),
|
||||
};
|
||||
|
||||
// The EXR render is only the first stage: hand off to the preview stage
|
||||
// (headless AE rebuild → delivery MOV + review MP4) when it is enabled.
|
||||
const previewEnabled = await getConfigBoolean("preview.enabled");
|
||||
if (previewEnabled) {
|
||||
const manifest = await buildPreviewManifest(exportRow);
|
||||
const created = await createPreviewJob(tx, exportRow, manifest, job.maxAttempts, job.priority);
|
||||
previewJob = { id: created.id, type: created.type };
|
||||
const exp = await applyTransition(tx, job.export, "GENERATING_PREVIEW", {
|
||||
type: "WORKER",
|
||||
id: machineId,
|
||||
note: `Render complete in ${body.renderSeconds ?? "?"}s — queued preview build`,
|
||||
}, stats);
|
||||
exportResult = { id: exp.id, status: exp.status };
|
||||
} else {
|
||||
const exp = await applyTransition(tx, job.export, "READY_FOR_QC", {
|
||||
type: "WORKER",
|
||||
id: machineId,
|
||||
note: `Render complete in ${body.renderSeconds ?? "?"}s (preview stage disabled)`,
|
||||
}, stats);
|
||||
exportResult = { id: exp.id, status: exp.status };
|
||||
}
|
||||
}
|
||||
return { job: { id: job.id, status: "COMPLETED" }, export: exportResult, previewJob, alreadyApplied: false };
|
||||
});
|
||||
}
|
||||
|
||||
/** E15 — cancel a single job from the web UI (delegates to export-level cancel when linked). */
|
||||
export async function getJobDetail(jobId: string) {
|
||||
const job = await db.renderJob.findUnique({
|
||||
where: { id: jobId },
|
||||
include: {
|
||||
machine: { select: { id: true, name: true } },
|
||||
export: { include: { shot: { select: { shotCode: true } } } },
|
||||
},
|
||||
});
|
||||
if (!job) throw new PipelineError(404, "Render job not found");
|
||||
return job;
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
import { Machine, Prisma } from "@prisma/client";
|
||||
import { db } from "@/lib/db";
|
||||
import { PipelineError } from "./errors";
|
||||
import { getWorkerConfig } from "./config";
|
||||
|
||||
/**
|
||||
* Machine availability (§7.10):
|
||||
* ALWAYS — claims whenever idle (default when availability is null)
|
||||
* SCHEDULE — claims only inside configured windows
|
||||
* MANUAL — never claims unless explicitly triggered
|
||||
* Overrides, in priority order: enabled=false beats everything; renderNowUntil
|
||||
* allows normal claims until it expires; urgent jobs may claim anytime on
|
||||
* machines with allowUrgentAnytime.
|
||||
*/
|
||||
export interface AvailabilityWindow {
|
||||
days: string[]; // ["mon", ..., "sun"]
|
||||
from: string; // "19:00"
|
||||
to: string; // "08:00" — from > to spans midnight
|
||||
}
|
||||
|
||||
export interface MachineAvailability {
|
||||
mode?: "ALWAYS" | "SCHEDULE" | "MANUAL";
|
||||
windows?: AvailabilityWindow[];
|
||||
allowUrgentAnytime?: boolean;
|
||||
}
|
||||
|
||||
const DAY_NAMES = ["sun", "mon", "tue", "wed", "thu", "fri", "sat"];
|
||||
|
||||
function parseHm(hm: string): number | null {
|
||||
const m = /^(\d{1,2}):(\d{2})$/.exec(hm);
|
||||
if (!m) return null;
|
||||
const mins = parseInt(m[1], 10) * 60 + parseInt(m[2], 10);
|
||||
return mins >= 0 && mins <= 24 * 60 ? mins : null;
|
||||
}
|
||||
|
||||
export function isInWindow(windows: AvailabilityWindow[], now: Date): boolean {
|
||||
const day = DAY_NAMES[now.getDay()];
|
||||
const prevDay = DAY_NAMES[(now.getDay() + 6) % 7];
|
||||
const t = now.getHours() * 60 + now.getMinutes();
|
||||
|
||||
for (const w of windows) {
|
||||
const from = parseHm(w.from);
|
||||
const to = parseHm(w.to);
|
||||
if (from == null || to == null || !Array.isArray(w.days)) continue;
|
||||
const days = w.days.map((d) => d.toLowerCase().slice(0, 3));
|
||||
if (from < to) {
|
||||
if (days.includes(day) && t >= from && t < to) return true;
|
||||
} else if (from > to) {
|
||||
// overnight window, e.g. 19:00 → 08:00
|
||||
if (days.includes(day) && t >= from) return true;
|
||||
if (days.includes(prevDay) && t < to) return true;
|
||||
} else {
|
||||
// from === to → full 24 h on listed days
|
||||
if (days.includes(day)) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export type ClaimGate = "normal" | "urgent-only" | "none";
|
||||
|
||||
export function machineCanClaim(machine: Machine, now: Date): ClaimGate {
|
||||
if (!machine.enabled) return "none";
|
||||
if (machine.renderNowUntil && machine.renderNowUntil > now) return "normal";
|
||||
|
||||
const availability = (machine.availability ?? null) as MachineAvailability | null;
|
||||
const mode = availability?.mode ?? "ALWAYS";
|
||||
const urgent = availability?.allowUrgentAnytime ? ("urgent-only" as const) : ("none" as const);
|
||||
|
||||
switch (mode) {
|
||||
case "ALWAYS":
|
||||
return "normal";
|
||||
case "SCHEDULE":
|
||||
return isInWindow(availability?.windows ?? [], now) ? "normal" : urgent;
|
||||
case "MANUAL":
|
||||
return urgent;
|
||||
default:
|
||||
return "normal";
|
||||
}
|
||||
}
|
||||
|
||||
/** E6 — idempotent register/upsert on machine name. */
|
||||
export async function registerMachine(body: {
|
||||
name?: string;
|
||||
hostname?: string;
|
||||
workerVersion?: string;
|
||||
aeVersion?: string;
|
||||
capabilities?: unknown;
|
||||
}) {
|
||||
const name = body.name?.trim();
|
||||
if (!name) throw new PipelineError(422, "name is required");
|
||||
const hostname = body.hostname?.trim() || name;
|
||||
|
||||
const machine = await db.machine.upsert({
|
||||
where: { name },
|
||||
create: {
|
||||
name,
|
||||
hostname,
|
||||
status: "ONLINE",
|
||||
lastSeenAt: new Date(),
|
||||
workerVersion: body.workerVersion ?? null,
|
||||
aeVersion: body.aeVersion ?? null,
|
||||
capabilities: (body.capabilities ?? undefined) as Prisma.InputJsonValue | undefined,
|
||||
},
|
||||
update: {
|
||||
hostname,
|
||||
status: "ONLINE",
|
||||
lastSeenAt: new Date(),
|
||||
workerVersion: body.workerVersion ?? undefined,
|
||||
aeVersion: body.aeVersion ?? undefined,
|
||||
capabilities: (body.capabilities ?? undefined) as Prisma.InputJsonValue | undefined,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
machine: { id: machine.id, name: machine.name, enabled: machine.enabled },
|
||||
config: await getWorkerConfig(),
|
||||
};
|
||||
}
|
||||
|
||||
/** E7 — heartbeat; also the server→worker command channel (cancel signals). */
|
||||
export async function recordHeartbeat(
|
||||
machineId: string,
|
||||
body: { cpuPercent?: number; memPercent?: number; diskFreeGb?: number; currentJobId?: string | null }
|
||||
) {
|
||||
const machine = await db.machine.findUnique({ where: { id: machineId } });
|
||||
if (!machine) throw new PipelineError(404, "Machine not found");
|
||||
|
||||
await db.$transaction([
|
||||
db.machine.update({
|
||||
where: { id: machineId },
|
||||
data: { lastSeenAt: new Date(), status: machine.enabled ? "ONLINE" : "DISABLED" },
|
||||
}),
|
||||
db.workerHeartbeat.create({
|
||||
data: {
|
||||
machineId,
|
||||
cpuPercent: body.cpuPercent ?? null,
|
||||
memPercent: body.memPercent ?? null,
|
||||
diskFreeGb: body.diskFreeGb ?? null,
|
||||
currentJobId: body.currentJobId ?? null,
|
||||
},
|
||||
}),
|
||||
]);
|
||||
|
||||
// Cancelled-but-unacknowledged jobs held by this machine → CANCEL_JOB commands
|
||||
const cancelled = await db.renderJob.findMany({
|
||||
where: { machineId, status: "CANCELLED", finishedAt: null },
|
||||
select: { id: true },
|
||||
});
|
||||
return {
|
||||
ok: true,
|
||||
commands: cancelled.map((j) => ({ type: "CANCEL_JOB" as const, jobId: j.id })),
|
||||
};
|
||||
}
|
||||
|
||||
/** Machine list for the monitoring page, with latest heartbeat + current job. */
|
||||
export async function listMachines() {
|
||||
const heartbeatSeconds = 30;
|
||||
const machines = await db.machine.findMany({
|
||||
orderBy: { name: "asc" },
|
||||
include: {
|
||||
heartbeats: { orderBy: { createdAt: "desc" }, take: 1 },
|
||||
renderJobs: {
|
||||
where: { status: { in: ["CLAIMED", "RUNNING"] } },
|
||||
take: 1,
|
||||
include: { export: { select: { id: true, versionString: true, shot: { select: { shotCode: true } } } } },
|
||||
},
|
||||
},
|
||||
});
|
||||
const now = Date.now();
|
||||
return machines.map((m) => ({
|
||||
id: m.id,
|
||||
name: m.name,
|
||||
hostname: m.hostname,
|
||||
enabled: m.enabled,
|
||||
// ONLINE = heartbeat within 3× the heartbeat interval (§5.6)
|
||||
status: !m.enabled
|
||||
? "DISABLED"
|
||||
: m.lastSeenAt && now - m.lastSeenAt.getTime() < heartbeatSeconds * 3 * 1000
|
||||
? "ONLINE"
|
||||
: "OFFLINE",
|
||||
lastSeenAt: m.lastSeenAt,
|
||||
workerVersion: m.workerVersion,
|
||||
aeVersion: m.aeVersion,
|
||||
capabilities: m.capabilities,
|
||||
availability: m.availability,
|
||||
renderNowUntil: m.renderNowUntil,
|
||||
latestHeartbeat: m.heartbeats[0]
|
||||
? {
|
||||
createdAt: m.heartbeats[0].createdAt,
|
||||
cpuPercent: m.heartbeats[0].cpuPercent,
|
||||
memPercent: m.heartbeats[0].memPercent,
|
||||
diskFreeGb: m.heartbeats[0].diskFreeGb,
|
||||
}
|
||||
: null,
|
||||
currentJob: m.renderJobs[0]
|
||||
? {
|
||||
id: m.renderJobs[0].id,
|
||||
exportId: m.renderJobs[0].exportId,
|
||||
shotCode: m.renderJobs[0].export?.shot.shotCode ?? null,
|
||||
versionString: m.renderJobs[0].export?.versionString ?? null,
|
||||
progress: m.renderJobs[0].progress,
|
||||
etaSeconds: m.renderJobs[0].etaSeconds,
|
||||
}
|
||||
: null,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function updateMachine(
|
||||
machineId: string,
|
||||
patch: { enabled?: boolean; availability?: MachineAvailability | null; renderNowHours?: number | null }
|
||||
) {
|
||||
const machine = await db.machine.findUnique({ where: { id: machineId } });
|
||||
if (!machine) throw new PipelineError(404, "Machine not found");
|
||||
|
||||
const data: Prisma.MachineUpdateInput = {};
|
||||
if (patch.enabled !== undefined) {
|
||||
data.enabled = patch.enabled;
|
||||
data.status = patch.enabled ? machine.status === "DISABLED" ? "OFFLINE" : machine.status : "DISABLED";
|
||||
}
|
||||
if (patch.availability !== undefined) {
|
||||
data.availability = patch.availability === null ? Prisma.DbNull : (patch.availability as unknown as Prisma.InputJsonValue);
|
||||
}
|
||||
if (patch.renderNowHours !== undefined) {
|
||||
data.renderNowUntil =
|
||||
patch.renderNowHours === null || patch.renderNowHours <= 0
|
||||
? null
|
||||
: new Date(Date.now() + patch.renderNowHours * 3600 * 1000);
|
||||
}
|
||||
const updated = await db.machine.update({ where: { id: machineId }, data });
|
||||
return {
|
||||
machine: {
|
||||
id: updated.id,
|
||||
name: updated.name,
|
||||
enabled: updated.enabled,
|
||||
availability: updated.availability,
|
||||
renderNowUntil: updated.renderNowUntil,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
import { z } from "zod";
|
||||
|
||||
/**
|
||||
* The Render Manifest (RenderPipeline2 §6.2).
|
||||
* Uploaded by the AE panel on Queue Export, snapshotted verbatim into
|
||||
* RenderJob.manifest, handed to the worker on claim.
|
||||
*
|
||||
* `outputDir` / `outputPattern` may be the literal string "auto" — the server
|
||||
* generates them from the shot's exrOutput convention and the NEW version
|
||||
* number it decides (§6.4 E1).
|
||||
*/
|
||||
export const renderManifestSchema = z
|
||||
.object({
|
||||
manifestVersion: z.number().int().min(1).default(1),
|
||||
projectCode: z.string().min(1),
|
||||
shotCode: z.string().regex(/^[A-Za-z0-9_-]+$/, "shotCode contains invalid characters"),
|
||||
shotId: z.string().min(1),
|
||||
aepPath: z.string().min(1),
|
||||
compName: z.string().min(1),
|
||||
rendererType: z.string().default("aerender"),
|
||||
aeVersionHint: z.string().optional(),
|
||||
outputDir: z.string().min(1),
|
||||
outputPattern: z.string().min(1),
|
||||
outputModuleTemplate: z.string().optional(),
|
||||
renderSettingsTemplate: z.string().optional(),
|
||||
frameStart: z.number().int(),
|
||||
frameEnd: z.number().int(),
|
||||
fps: z.number().positive(),
|
||||
width: z.number().int().positive(),
|
||||
height: z.number().int().positive(),
|
||||
expected: z
|
||||
.object({
|
||||
colorspace: z.string().optional(),
|
||||
bitDepth: z.string().optional(),
|
||||
exrCompression: z.string().optional(),
|
||||
alpha: z.boolean().optional(),
|
||||
timecodeStart: z.string().optional(),
|
||||
})
|
||||
.optional(),
|
||||
preview: z
|
||||
.object({
|
||||
template: z.string().optional(),
|
||||
burnins: z.boolean().optional(),
|
||||
})
|
||||
.optional(),
|
||||
})
|
||||
.refine((m) => m.frameEnd >= m.frameStart, {
|
||||
message: "frameEnd must be >= frameStart",
|
||||
path: ["frameEnd"],
|
||||
});
|
||||
|
||||
export type RenderManifest = z.infer<typeof renderManifestSchema>;
|
||||
@@ -0,0 +1,295 @@
|
||||
import { Prisma } from "@prisma/client";
|
||||
import { db } from "@/lib/db";
|
||||
import { PipelineError } from "./errors";
|
||||
import { applyTransition } from "./transitions";
|
||||
import { getConfigNumber, getPreviewConfig } from "./config";
|
||||
|
||||
/** Slate date format used by the panel's getDateString (YYYY/MM/DD). */
|
||||
function slateDate(d = new Date()): string {
|
||||
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
||||
const dd = String(d.getDate()).padStart(2, "0");
|
||||
return `${d.getFullYear()}/${mm}/${dd}`;
|
||||
}
|
||||
|
||||
export interface PreviewManifest {
|
||||
stage: "PREVIEW";
|
||||
exportId: string;
|
||||
shotCode: string;
|
||||
versionString: string;
|
||||
outputDir: string;
|
||||
outputPattern: string;
|
||||
frameStart: number;
|
||||
frameEnd: number;
|
||||
fps: number;
|
||||
templateAep: string;
|
||||
templateComp: string;
|
||||
overlayComp: string;
|
||||
lutComp: string;
|
||||
movTemplate: string;
|
||||
mp4Template: string;
|
||||
movOutput: string;
|
||||
mp4Output: string;
|
||||
slateScopeProp: string;
|
||||
slateSubmissionProp: string;
|
||||
slate: {
|
||||
versionName: string;
|
||||
date: string;
|
||||
description: string | null;
|
||||
notes: string | null;
|
||||
shotCode: string;
|
||||
episode: string | null;
|
||||
scene: string | null;
|
||||
vfxScope: string | null;
|
||||
submissionNote: string | null;
|
||||
};
|
||||
}
|
||||
|
||||
type ExportForPreview = {
|
||||
id: string;
|
||||
shotId: string;
|
||||
versionString: string;
|
||||
outputDir: string;
|
||||
outputPattern: string;
|
||||
frameStart: number;
|
||||
frameEnd: number;
|
||||
fps: number;
|
||||
vfxScope: string | null;
|
||||
submissionNote: string | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Builds the PREVIEW_ONLY job manifest (RenderPipeline2 §9): everything the
|
||||
* headless AE build script needs to rebuild the shot around the rendered EXRs
|
||||
* and queue the delivery MOV + review MP4.
|
||||
*
|
||||
* The MOV and MP4 are written beside the EXR sequence, so every artifact for a
|
||||
* version lives in one folder and Phase 5 can package them together.
|
||||
*/
|
||||
export async function buildPreviewManifest(exportRow: ExportForPreview): Promise<PreviewManifest> {
|
||||
const shot = await db.shot.findUnique({
|
||||
where: { id: exportRow.shotId },
|
||||
select: { shotCode: true, episode: true, scene: true, description: true, notes: true },
|
||||
});
|
||||
if (!shot) throw new PipelineError(404, "Shot not found for export");
|
||||
|
||||
const cfg = await getPreviewConfig();
|
||||
const base = `${shot.shotCode}_cmp_TT_${exportRow.versionString}`;
|
||||
|
||||
return {
|
||||
stage: "PREVIEW",
|
||||
exportId: exportRow.id,
|
||||
shotCode: shot.shotCode,
|
||||
versionString: exportRow.versionString,
|
||||
outputDir: exportRow.outputDir,
|
||||
outputPattern: exportRow.outputPattern,
|
||||
frameStart: exportRow.frameStart,
|
||||
frameEnd: exportRow.frameEnd,
|
||||
fps: exportRow.fps,
|
||||
templateAep: cfg.templateAep,
|
||||
templateComp: cfg.templateComp,
|
||||
overlayComp: cfg.overlayComp,
|
||||
lutComp: cfg.lutComp,
|
||||
movTemplate: cfg.movTemplate,
|
||||
mp4Template: cfg.mp4Template,
|
||||
movOutput: `${exportRow.outputDir}/${base}.mov`,
|
||||
mp4Output: `${exportRow.outputDir}/${base}.mp4`,
|
||||
slateScopeProp: cfg.slateScopeProp,
|
||||
slateSubmissionProp: cfg.slateSubmissionProp,
|
||||
slate: {
|
||||
versionName: base,
|
||||
date: slateDate(),
|
||||
description: shot.description,
|
||||
notes: shot.notes,
|
||||
shotCode: shot.shotCode,
|
||||
episode: shot.episode,
|
||||
scene: shot.scene,
|
||||
vfxScope: exportRow.vfxScope,
|
||||
submissionNote: exportRow.submissionNote,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Creates the PREVIEW_ONLY job that follows a successful EXR render. */
|
||||
export async function createPreviewJob(
|
||||
tx: Prisma.TransactionClient,
|
||||
exportRow: ExportForPreview,
|
||||
manifest: PreviewManifest,
|
||||
maxAttempts: number,
|
||||
priority: number
|
||||
) {
|
||||
return tx.renderJob.create({
|
||||
data: {
|
||||
type: "PREVIEW_ONLY",
|
||||
exportId: exportRow.id,
|
||||
attempt: 1,
|
||||
maxAttempts,
|
||||
priority,
|
||||
manifest: manifest as unknown as Prisma.InputJsonValue,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export interface FinalizeInput {
|
||||
artifacts?: {
|
||||
deliveryMovPath?: string;
|
||||
previewMovKey?: string;
|
||||
thumbnailKey?: string;
|
||||
metadataKey?: string;
|
||||
logFileKey?: string;
|
||||
};
|
||||
renderStats?: { renderSeconds?: number; previewSeconds?: number };
|
||||
media?: { width?: number; height?: number; frameCount?: number; fps?: number; fileName?: string };
|
||||
}
|
||||
|
||||
/**
|
||||
* E13 — preview artifacts are done: register the review MP4 as an ordinary
|
||||
* `Version` and move the Export to READY_FOR_QC.
|
||||
*
|
||||
* Pipeline mode (§6.4 / T12): the Version is **never client-visible** and
|
||||
* **no task or shot status is touched**. Delivery QC runs on shots the client
|
||||
* has usually already approved, so a post-approval technical render must not
|
||||
* resurface in the review flow or the client portal.
|
||||
*/
|
||||
export async function finalizePreview(jobId: string, machineId: string, body: FinalizeInput) {
|
||||
return db.$transaction(async (tx) => {
|
||||
const job = await tx.renderJob.findUnique({
|
||||
where: { id: jobId },
|
||||
include: { export: true },
|
||||
});
|
||||
if (!job) throw new PipelineError(404, "Render job not found");
|
||||
if (job.machineId !== machineId) {
|
||||
throw new PipelineError(409, "Job is not held by this machine (lease reassigned?)");
|
||||
}
|
||||
const exportRow = job.export;
|
||||
if (!exportRow) throw new PipelineError(422, "Job has no export to finalize");
|
||||
|
||||
if (exportRow.status === "READY_FOR_QC" && exportRow.versionId) {
|
||||
return {
|
||||
export: { id: exportRow.id, status: exportRow.status },
|
||||
version: { id: exportRow.versionId },
|
||||
alreadyApplied: true,
|
||||
};
|
||||
}
|
||||
|
||||
const artifacts = body.artifacts ?? {};
|
||||
const media = body.media ?? {};
|
||||
|
||||
await tx.renderJob.update({
|
||||
where: { id: job.id },
|
||||
data: {
|
||||
status: "COMPLETED",
|
||||
progress: 1,
|
||||
finishedAt: new Date(),
|
||||
exitCode: 0,
|
||||
logFileKey: artifacts.logFileKey ?? job.logFileKey,
|
||||
},
|
||||
});
|
||||
|
||||
let versionId: string | null = null;
|
||||
if (artifacts.previewMovKey) {
|
||||
// Version numbering stays monotonic for the task (existing review flows
|
||||
// rely on the newest version having the highest number) while matching
|
||||
// the export version whenever it is ahead.
|
||||
const taskMax = exportRow.taskId
|
||||
? await tx.version.aggregate({
|
||||
where: { taskId: exportRow.taskId },
|
||||
_max: { versionNumber: true },
|
||||
})
|
||||
: { _max: { versionNumber: null as number | null } };
|
||||
const versionNumber = Math.max((taskMax._max.versionNumber ?? 0) + 1, exportRow.versionNumber);
|
||||
|
||||
if (exportRow.taskId) {
|
||||
await tx.version.updateMany({
|
||||
where: { taskId: exportRow.taskId },
|
||||
data: { isLatest: false },
|
||||
});
|
||||
}
|
||||
|
||||
const version = await tx.version.create({
|
||||
data: {
|
||||
versionNumber,
|
||||
shotId: exportRow.shotId,
|
||||
taskId: exportRow.taskId,
|
||||
artistId: exportRow.submittedById,
|
||||
fileUrl: `/api/files/${artifacts.previewMovKey}`,
|
||||
fileName: media.fileName ?? `${exportRow.versionString}.mp4`,
|
||||
mimeType: "video/mp4",
|
||||
thumbnailUrl: artifacts.thumbnailKey ? `/api/files/${artifacts.thumbnailKey}` : undefined,
|
||||
fps: media.fps ?? exportRow.fps,
|
||||
frameCount: media.frameCount ?? exportRow.frameEnd - exportRow.frameStart + 1,
|
||||
width: media.width ?? exportRow.width,
|
||||
height: media.height ?? exportRow.height,
|
||||
notes: `Pipeline render ${exportRow.versionString} (internal only)`,
|
||||
isLatest: true,
|
||||
isClientVisible: false, // never shared to the client portal (§10.0)
|
||||
},
|
||||
});
|
||||
versionId = version.id;
|
||||
}
|
||||
|
||||
const updated = await applyTransition(
|
||||
tx,
|
||||
exportRow,
|
||||
"READY_FOR_QC",
|
||||
{
|
||||
type: "WORKER",
|
||||
id: machineId,
|
||||
note: `Preview complete in ${body.renderStats?.previewSeconds ?? "?"}s${versionId ? " — review version created" : " — no preview media reported"}`,
|
||||
},
|
||||
{
|
||||
...(versionId ? { versionId } : {}),
|
||||
...(artifacts.deliveryMovPath ? { deliveryMovPath: artifacts.deliveryMovPath } : {}),
|
||||
...(artifacts.previewMovKey ? { previewMovKey: artifacts.previewMovKey } : {}),
|
||||
...(artifacts.thumbnailKey ? { thumbnailKey: artifacts.thumbnailKey } : {}),
|
||||
...(artifacts.metadataKey ? { metadataKey: artifacts.metadataKey } : {}),
|
||||
}
|
||||
);
|
||||
|
||||
return {
|
||||
export: { id: updated.id, status: updated.status },
|
||||
version: versionId ? { id: versionId } : null,
|
||||
alreadyApplied: false,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** T11 — retry the preview stage only; validated EXRs are left alone. */
|
||||
export async function retryPreview(exportId: string, actorId?: string) {
|
||||
const maxAttempts = await getConfigNumber("render.maxAttempts");
|
||||
|
||||
return db.$transaction(async (tx) => {
|
||||
const exportRow = await tx.export.findUnique({ where: { id: exportId } });
|
||||
if (!exportRow) throw new PipelineError(404, "Export not found");
|
||||
if (exportRow.status !== "PREVIEW_FAILED") {
|
||||
throw new PipelineError(409, `Export is ${exportRow.status} — only PREVIEW_FAILED exports can retry the preview`);
|
||||
}
|
||||
|
||||
const manifest = await buildPreviewManifest(exportRow);
|
||||
const last = await tx.renderJob.findFirst({
|
||||
where: { exportId, type: "PREVIEW_ONLY" },
|
||||
orderBy: { attempt: "desc" },
|
||||
});
|
||||
|
||||
const job = await tx.renderJob.create({
|
||||
data: {
|
||||
type: "PREVIEW_ONLY",
|
||||
exportId,
|
||||
attempt: (last?.attempt ?? 0) + 1,
|
||||
maxAttempts,
|
||||
priority: last?.priority ?? 50,
|
||||
manifest: manifest as unknown as Prisma.InputJsonValue,
|
||||
},
|
||||
});
|
||||
|
||||
const updated = await applyTransition(tx, exportRow, "GENERATING_PREVIEW", {
|
||||
type: "USER",
|
||||
id: actorId,
|
||||
note: `Retry preview (attempt ${job.attempt})`,
|
||||
});
|
||||
|
||||
return {
|
||||
export: { id: updated.id, status: updated.status },
|
||||
renderJob: { id: job.id, attempt: job.attempt },
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
import { Prisma } from "@prisma/client";
|
||||
import { db } from "@/lib/db";
|
||||
import { applyTransition } from "./transitions";
|
||||
import { getConfigNumber } from "./config";
|
||||
|
||||
/**
|
||||
* Server-side reaper (RenderPipeline2 §7.7) — the only background job the web
|
||||
* app gains. Covers worker power loss, BSOD and network partition with no
|
||||
* worker cooperation:
|
||||
* 1. expired leases → job EXPIRED; requeue (T5) or Export → RENDER_FAILED (T4)
|
||||
* 2. silent machines → OFFLINE
|
||||
* 3. prune heartbeats older than 7 days
|
||||
*/
|
||||
export async function reapExpiredLeases() {
|
||||
const expired = await db.renderJob.findMany({
|
||||
where: { status: { in: ["CLAIMED", "RUNNING"] }, leaseExpiresAt: { lt: new Date() } },
|
||||
include: { export: { select: { id: true, status: true } } },
|
||||
});
|
||||
|
||||
for (const job of expired) {
|
||||
try {
|
||||
await db.$transaction(async (tx) => {
|
||||
// Re-check inside the transaction — the worker may have reported in between
|
||||
const fresh = await tx.renderJob.findUnique({ where: { id: job.id }, select: { status: true, leaseExpiresAt: true } });
|
||||
if (!fresh || !["CLAIMED", "RUNNING"].includes(fresh.status) || !fresh.leaseExpiresAt || fresh.leaseExpiresAt >= new Date()) {
|
||||
return;
|
||||
}
|
||||
await tx.renderJob.update({
|
||||
where: { id: job.id },
|
||||
data: { status: "EXPIRED", finishedAt: new Date(), errorMessage: "Lease expired (worker unreachable)" },
|
||||
});
|
||||
if (!job.export) return;
|
||||
|
||||
if (job.attempt < job.maxAttempts) {
|
||||
const next = await tx.renderJob.create({
|
||||
data: {
|
||||
type: job.type,
|
||||
exportId: job.exportId,
|
||||
attempt: job.attempt + 1,
|
||||
maxAttempts: job.maxAttempts,
|
||||
priority: job.priority,
|
||||
manifest: job.manifest as Prisma.InputJsonValue,
|
||||
},
|
||||
});
|
||||
if (job.export.status === "RENDERING") {
|
||||
await applyTransition(tx, job.export, "QUEUED", {
|
||||
type: "SYSTEM",
|
||||
note: `Lease expired on attempt ${job.attempt} — requeued as attempt ${next.attempt}`,
|
||||
});
|
||||
}
|
||||
} else if (job.export.status === "RENDERING") {
|
||||
await applyTransition(tx, job.export, "RENDER_FAILED", {
|
||||
type: "SYSTEM",
|
||||
note: `Lease expired on attempt ${job.attempt}/${job.maxAttempts} — no retries remain`,
|
||||
});
|
||||
}
|
||||
});
|
||||
} catch (err) {
|
||||
console.error(`[render-pipeline] reaper failed for job ${job.id}`, err);
|
||||
}
|
||||
}
|
||||
return expired.length;
|
||||
}
|
||||
|
||||
export async function reaperSweep() {
|
||||
const heartbeatSeconds = await getConfigNumber("render.heartbeatSeconds");
|
||||
|
||||
await reapExpiredLeases();
|
||||
|
||||
await db.machine.updateMany({
|
||||
where: {
|
||||
status: "ONLINE",
|
||||
OR: [{ lastSeenAt: null }, { lastSeenAt: { lt: new Date(Date.now() - heartbeatSeconds * 3 * 1000) } }],
|
||||
},
|
||||
data: { status: "OFFLINE" },
|
||||
});
|
||||
|
||||
await db.workerHeartbeat.deleteMany({
|
||||
where: { createdAt: { lt: new Date(Date.now() - 7 * 24 * 3600 * 1000) } },
|
||||
});
|
||||
}
|
||||
|
||||
const globalForReaper = globalThis as unknown as { renderPipelineReaper?: ReturnType<typeof setInterval> };
|
||||
|
||||
/** Started once from instrumentation.ts (Next.js server startup hook). */
|
||||
export function startReaper(intervalMs = 60_000) {
|
||||
if (globalForReaper.renderPipelineReaper) return;
|
||||
globalForReaper.renderPipelineReaper = setInterval(() => {
|
||||
reaperSweep().catch((err) => console.error("[render-pipeline] reaper sweep failed", err));
|
||||
}, intervalMs);
|
||||
// Don't hold the process open on shutdown
|
||||
globalForReaper.renderPipelineReaper.unref?.();
|
||||
console.log("[render-pipeline] reaper started (interval", intervalMs, "ms)");
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { auth } from "@/auth";
|
||||
import { PipelineError } from "./errors";
|
||||
|
||||
export type SessionUser = { id: string; role: string; name?: string | null; email?: string | null };
|
||||
|
||||
/** Any signed-in studio user (clients never see pipeline pages). */
|
||||
export async function requirePipelineUser(): Promise<SessionUser> {
|
||||
const session = await auth();
|
||||
const user = session?.user as (SessionUser & { role?: string }) | undefined;
|
||||
if (!user?.id) throw new PipelineError(401, "Unauthorized");
|
||||
if (user.role === "CLIENT") throw new PipelineError(403, "Forbidden");
|
||||
return user as SessionUser;
|
||||
}
|
||||
|
||||
/** Admin-level pipeline actions (machine kill-switch, manual mark-done). */
|
||||
export async function requirePipelineAdmin(): Promise<SessionUser> {
|
||||
const user = await requirePipelineUser();
|
||||
if (!["ADMIN", "PRODUCER", "SUPERVISOR"].includes(user.role)) {
|
||||
throw new PipelineError(403, "Forbidden — requires admin/producer/supervisor role");
|
||||
}
|
||||
return user;
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import { ExportStatus, Prisma } from "@prisma/client";
|
||||
import { PipelineError } from "./errors";
|
||||
|
||||
/**
|
||||
* Export lifecycle state machine (RenderPipeline2 §4).
|
||||
* The server is the sole authority on state; workers *request* transitions.
|
||||
*
|
||||
* NOTE (Phase 2 interim): RENDERING → READY_FOR_QC is temporarily legal.
|
||||
* Phase 3 inserts VALIDATING between them (§17.3) — remove READY_FOR_QC from
|
||||
* RENDERING's list when the validation engine ships.
|
||||
*/
|
||||
export const EXPORT_TRANSITIONS: Record<ExportStatus, ExportStatus[]> = {
|
||||
QUEUED: ["RENDERING", "CANCELLED", "SUPERSEDED", "READY_FOR_QC"], // READY_FOR_QC: manual mark-done only
|
||||
RENDERING: [
|
||||
"VALIDATING",
|
||||
"GENERATING_PREVIEW", // interim: preview stage runs before validation exists
|
||||
"RENDER_FAILED",
|
||||
"QUEUED", // lease expiry / retryable fail with attempts remaining (T5)
|
||||
"CANCELLED",
|
||||
"READY_FOR_QC", // preview disabled / manual mark-done
|
||||
],
|
||||
RENDER_FAILED: ["QUEUED", "SUPERSEDED"],
|
||||
VALIDATING: ["VALIDATION_FAILED", "GENERATING_PREVIEW"],
|
||||
VALIDATION_FAILED: ["QUEUED", "SUPERSEDED"],
|
||||
GENERATING_PREVIEW: ["PREVIEW_FAILED", "READY_FOR_QC"],
|
||||
PREVIEW_FAILED: ["GENERATING_PREVIEW", "SUPERSEDED"],
|
||||
READY_FOR_QC: ["QC_FAILED", "READY_FOR_DELIVERY", "SUPERSEDED"],
|
||||
QC_FAILED: ["SUPERSEDED"],
|
||||
READY_FOR_DELIVERY: ["PACKAGED", "SUPERSEDED"],
|
||||
PACKAGED: ["DELIVERED"],
|
||||
DELIVERED: ["ARCHIVED"],
|
||||
SUPERSEDED: [],
|
||||
ARCHIVED: [],
|
||||
CANCELLED: [],
|
||||
};
|
||||
|
||||
export type TransitionActor = {
|
||||
type: "WORKER" | "USER" | "SYSTEM";
|
||||
id?: string | null;
|
||||
note?: string | null;
|
||||
};
|
||||
|
||||
export function isTransitionAllowed(from: ExportStatus, to: ExportStatus): boolean {
|
||||
return EXPORT_TRANSITIONS[from]?.includes(to) ?? false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a state transition inside an existing transaction: validates
|
||||
* legality, stamps statusChangedAt, appends the ExportEvent audit row.
|
||||
* Throws PipelineError(409) on an illegal transition (§4.2 rules).
|
||||
*/
|
||||
export async function applyTransition(
|
||||
tx: Prisma.TransactionClient,
|
||||
exportRow: { id: string; status: ExportStatus },
|
||||
to: ExportStatus,
|
||||
actor: TransitionActor,
|
||||
extraData: Prisma.ExportUpdateInput = {}
|
||||
) {
|
||||
if (!isTransitionAllowed(exportRow.status, to)) {
|
||||
throw new PipelineError(409, `Invalid transition ${exportRow.status} → ${to}`);
|
||||
}
|
||||
const updated = await tx.export.update({
|
||||
where: { id: exportRow.id },
|
||||
data: { status: to, statusChangedAt: new Date(), ...extraData },
|
||||
});
|
||||
await tx.exportEvent.create({
|
||||
data: {
|
||||
exportId: exportRow.id,
|
||||
fromStatus: exportRow.status,
|
||||
toStatus: to,
|
||||
actorType: actor.type,
|
||||
actorId: actor.id ?? null,
|
||||
note: actor.note ?? null,
|
||||
},
|
||||
});
|
||||
return updated;
|
||||
}
|
||||
Reference in New Issue
Block a user