Files
vfxreview/lib/render-pipeline/deliveries.ts
T
twotalesanimation 82dfaee56f
Deploy / deploy (push) Successful in 6m37s
fix(delivery): stop the 422 retry loop on failed delivery items
A FAILED delivery item never sets fileCount/totalBytes, and the worker's
Dictionary-based JSON payload serializes those as explicit nulls (the
WhenWritingNull option only suppresses nulls on POCO properties, not
dictionary values). The server's schema allowed the fields to be omitted
but not null, so the report was permanently rejected with 422 - and the
worker only ever treated 409/404 as terminal, so it retried the same
malformed payload forever across restarts.
2026-08-21 12:07:30 +02:00

635 lines
22 KiB
TypeScript

import { DeliveryStatus, Prisma } from "@prisma/client";
import { z } from "zod";
import { db } from "@/lib/db";
import { PipelineError } from "./errors";
import { applyTransition } from "./transitions";
import { getConfigNumber, getConfigValue } from "./config";
/**
* Delivery builder (RenderPipeline2 §11). The web app decides *what* goes in a
* package and records it; a worker does the file work through the same queue
* and claim path as a render.
*
* Version locking: a package references Export rows and denormalises
* shotCode/versionString at creation, so a later re-render can never mutate
* delivery history.
*/
// ── Config (§11.2) ───────────────────────────────────────────────────────────
export interface DeliveryConfig {
packageNameTemplate: string;
layout: { packageDir: string; shotDir: string; reportFile: string };
deliveryRoot: string;
includeMovs: boolean;
checksumAlgo: string;
reportFormats: string[];
copyMode: "auto" | "always-copy";
/**
* "auto" verifies copies by hash and hard links by identity — a link is the
* same data as the source, so re-reading it cannot disagree with what
* validation already hashed, and at SAN speeds costs ~30 s per shot to prove
* nothing. "always-hash" reads every delivered file back regardless.
*/
verifyMode: "auto" | "always-hash";
}
export const DELIVERY_CONFIG_DEFAULT: DeliveryConfig = {
packageNameTemplate: "{yymmdd}_Delivery",
layout: {
packageDir: "{deliveryRoot}/{episode}/{packageName}",
shotDir: "{packageDir}/{shotCode}",
reportFile: "{packageDir}/{packageName}_report.csv",
},
deliveryRoot: "//SAN/DELIVERY",
includeMovs: true,
checksumAlgo: "xxh64",
reportFormats: ["csv", "json"],
copyMode: "auto",
verifyMode: "auto",
};
/**
* Project config → SystemConfig default → built-in. Merged shallowly per top
* level key so a project can override just `deliveryRoot` without restating the
* whole layout.
*/
export async function resolveDeliveryConfig(projectDeliveryConfig: unknown): Promise<DeliveryConfig> {
let systemDefault: Partial<DeliveryConfig> = {};
const raw = await getConfigValue("delivery.defaultConfig");
if (raw?.trim() && raw.trim() !== "{}") {
try {
systemDefault = JSON.parse(raw) as Partial<DeliveryConfig>;
} catch {
console.error("[render-pipeline] delivery.defaultConfig is not valid JSON — ignoring");
}
}
const project = (projectDeliveryConfig ?? {}) as Partial<DeliveryConfig>;
return {
...DELIVERY_CONFIG_DEFAULT,
...systemDefault,
...project,
layout: {
...DELIVERY_CONFIG_DEFAULT.layout,
...(systemDefault.layout ?? {}),
...(project.layout ?? {}),
},
};
}
export interface TokenValues {
showId: string;
episode: string;
yymmdd: string;
yyyymmdd: string;
packageName: string;
packageDir: string;
deliveryRoot: string;
shotCode: string;
versionString: string;
packageNumber: string;
}
export function applyTokens(template: string, tokens: Partial<TokenValues>): string {
return template.replace(/\{(\w+)\}/g, (whole, key: string) => {
const value = (tokens as Record<string, string | undefined>)[key];
return value === undefined ? whole : value;
});
}
export function dateTokens(date: Date) {
const yyyy = String(date.getFullYear());
const mm = String(date.getMonth() + 1).padStart(2, "0");
const dd = String(date.getDate()).padStart(2, "0");
return { yymmdd: `${yyyy.slice(2)}${mm}${dd}`, yyyymmdd: `${yyyy}${mm}${dd}` };
}
// ── E18: what is ready ───────────────────────────────────────────────────────
/**
* READY_FOR_DELIVERY exports grouped by episode. Only the newest ready export
* per shot is offered: an older ready version of the same shot is history, not
* a second thing to ship.
*/
export async function getReadyForDelivery(projectId: string, episode?: string | null) {
const rows = await db.export.findMany({
where: {
projectId,
status: "READY_FOR_DELIVERY",
...(episode ? { shot: { episode } } : {}),
},
orderBy: [{ shotId: "asc" }, { versionNumber: "desc" }],
include: {
shot: { select: { id: true, shotCode: true, episode: true } },
qcReviews: { orderBy: { createdAt: "desc" }, take: 1, select: { result: true, reviewerName: true, createdAt: true } },
},
});
const newestPerShot = new Map<string, (typeof rows)[number]>();
for (const row of rows) {
if (!newestPerShot.has(row.shotId)) newestPerShot.set(row.shotId, row);
}
const byEpisode = new Map<string, ReturnType<typeof serialiseReady>[]>();
for (const row of newestPerShot.values()) {
const key = row.shot.episode ?? "";
if (!byEpisode.has(key)) byEpisode.set(key, []);
byEpisode.get(key)!.push(serialiseReady(row));
}
return {
episodes: [...byEpisode.entries()]
.sort((a, b) => a[0].localeCompare(b[0]))
.map(([ep, exports]) => ({
episode: ep,
count: exports.length,
exports: exports.sort((a, b) => a.shotCode.localeCompare(b.shotCode)),
})),
};
}
function serialiseReady(row: {
id: string;
versionString: string;
outputDir: string;
exrFileCount: number | null;
exrTotalBytes: bigint | null;
deliveryMovPath: string | null;
checksum: string | null;
frameStart: number;
frameEnd: number;
shot: { id: string; shotCode: string; episode: string | null };
qcReviews: { result: string; reviewerName: string | null; createdAt: Date }[];
}) {
return {
exportId: row.id,
shotId: row.shot.id,
shotCode: row.shot.shotCode,
episode: row.shot.episode ?? "",
versionString: row.versionString,
outputDir: row.outputDir,
frameStart: row.frameStart,
frameEnd: row.frameEnd,
fileCount: row.exrFileCount,
totalBytes: row.exrTotalBytes == null ? null : Number(row.exrTotalBytes),
deliveryMovPath: row.deliveryMovPath,
hasChecksum: row.checksum != null,
qc: row.qcReviews[0]
? { result: row.qcReviews[0].result, reviewer: row.qcReviews[0].reviewerName, at: row.qcReviews[0].createdAt }
: null,
};
}
// ── E16: create a package ────────────────────────────────────────────────────
export const createDeliverySchema = z.object({
projectId: z.string().min(1),
episode: z.string().max(50).optional().nullable(),
/** Omitted → every ready export in the episode. */
exportIds: z.array(z.string().min(1)).max(500).optional(),
options: z
.object({
date: z.string().regex(/^\d{4}-\d{2}-\d{2}$/).optional(),
name: z.string().max(120).optional(),
includeMovs: z.boolean().optional(),
copyMode: z.enum(["auto", "always-copy"]).optional(),
})
.optional(),
});
export type CreateDeliveryInput = z.infer<typeof createDeliverySchema>;
export async function createDeliveryPackage(input: CreateDeliveryInput, userId: string) {
const project = await db.project.findUnique({
where: { id: input.projectId },
select: { id: true, code: true, showId: true, deliveryConfig: true },
});
if (!project) throw new PipelineError(404, "Project not found");
const config = await resolveDeliveryConfig(project.deliveryConfig);
if (input.options?.includeMovs !== undefined) config.includeMovs = input.options.includeMovs;
if (input.options?.copyMode) config.copyMode = input.options.copyMode;
const buildDate = input.options?.date ? new Date(`${input.options.date}T00:00:00`) : new Date();
if (Number.isNaN(buildDate.getTime())) throw new PipelineError(422, "options.date is not a valid date");
const maxAttempts = await getConfigNumber("render.maxAttempts");
return db.$transaction(async (tx) => {
const exports = await tx.export.findMany({
where: {
projectId: project.id,
status: "READY_FOR_DELIVERY",
...(input.exportIds?.length ? { id: { in: input.exportIds } } : {}),
...(input.episode ? { shot: { episode: input.episode } } : {}),
},
include: { shot: { select: { id: true, shotCode: true, episode: true } } },
orderBy: { shotId: "asc" },
});
if (exports.length === 0) {
throw new PipelineError(422, "No READY_FOR_DELIVERY exports match — nothing to package");
}
if (input.exportIds?.length && exports.length !== input.exportIds.length) {
const found = new Set(exports.map((e) => e.id));
throw new PipelineError(
409,
"Some exports are no longer ready for delivery (re-rendered or already packaged?)",
{ missing: input.exportIds.filter((id) => !found.has(id)) }
);
}
// One export per shot: two versions of the same shot in one package is
// always a mistake, and it is cheaper to refuse than to ship it.
const shotIds = new Set<string>();
for (const e of exports) {
if (shotIds.has(e.shotId)) {
throw new PipelineError(422, `Two versions of ${e.shot.shotCode} selected — pick one`);
}
shotIds.add(e.shotId);
}
const episode = input.episode ?? exports[0].shot.episode ?? "";
const dates = dateTokens(buildDate);
const showId = project.showId || project.code;
const lastNumber = await tx.deliveryPackage.aggregate({
where: { projectId: project.id },
_max: { packageNumber: true },
});
const packageNumber = (lastNumber._max.packageNumber ?? 0) + 1;
const tokens: TokenValues = {
showId,
episode,
...dates,
packageName: "",
packageDir: "",
deliveryRoot: config.deliveryRoot,
shotCode: "",
versionString: "",
packageNumber: String(packageNumber),
};
tokens.packageName = input.options?.name ?? applyTokens(config.packageNameTemplate, tokens);
tokens.packageDir = applyTokens(config.layout.packageDir, tokens);
const pkg = await tx.deliveryPackage.create({
data: {
projectId: project.id,
episode,
name: tokens.packageName,
packageNumber,
status: "QUEUED",
rootPath: tokens.packageDir,
// Snapshot: historic packages always show the rules they were built with.
configSnapshot: { ...config, tokens } as unknown as Prisma.InputJsonValue,
createdById: userId,
},
});
await tx.deliveryItem.createMany({
data: exports.map((e) => ({
packageId: pkg.id,
exportId: e.id,
shotId: e.shotId,
shotCode: e.shot.shotCode,
versionString: e.versionString,
})),
});
const manifest = await buildDeliveryJobManifest(tx, pkg.id);
const job = await tx.renderJob.create({
data: {
type: "DELIVERY_BUILD",
deliveryId: pkg.id,
attempt: 1,
maxAttempts,
// Deliveries are interactive: someone is waiting on the folder, and the
// work is file copying rather than hours of CPU, so it jumps renders.
priority: 10,
manifest: manifest as unknown as Prisma.InputJsonValue,
},
});
return {
package: {
id: pkg.id,
name: pkg.name,
packageNumber,
episode,
status: pkg.status,
rootPath: pkg.rootPath,
itemCount: exports.length,
},
renderJob: { id: job.id },
};
});
}
// ── The DELIVERY_BUILD manifest ──────────────────────────────────────────────
export interface DeliveryManifestItem {
itemId: string;
exportId: string;
shotCode: string;
versionString: string;
sourceDir: string;
outputPattern: string;
frameStart: number;
frameEnd: number;
fileCount: number | null;
checksum: string | null;
deliveryMovPath: string | null;
shotDir: string;
/** When validation last ran for this export; null when it never did. */
validatedAt: string | null;
}
export interface DeliveryManifest {
stage: "DELIVERY";
packageId: string;
packageName: string;
packageNumber: number;
episode: string;
packageDir: string;
reportFile: string;
includeMovs: boolean;
copyMode: string;
verifyMode: string;
checksumAlgo: string;
reportFormats: string[];
items: DeliveryManifestItem[];
}
export async function buildDeliveryJobManifest(
tx: Prisma.TransactionClient,
packageId: string
): Promise<DeliveryManifest> {
const pkg = await tx.deliveryPackage.findUniqueOrThrow({
where: { id: packageId },
include: {
items: {
include: {
export: {
include: {
// Newest validation run: a linked package uses it to detect a
// source file changed after it passed QC.
validations: { orderBy: { createdAt: "desc" }, take: 1, select: { createdAt: true } },
},
},
},
},
project: { select: { code: true, showId: true } },
},
});
const snapshot = pkg.configSnapshot as unknown as DeliveryConfig & { tokens: TokenValues };
const baseTokens: TokenValues = {
...snapshot.tokens,
packageName: pkg.name,
packageDir: pkg.rootPath ?? snapshot.tokens.packageDir,
episode: pkg.episode,
};
return {
stage: "DELIVERY",
packageId: pkg.id,
packageName: pkg.name,
packageNumber: pkg.packageNumber,
episode: pkg.episode,
packageDir: baseTokens.packageDir,
reportFile: applyTokens(snapshot.layout.reportFile, baseTokens),
includeMovs: snapshot.includeMovs,
copyMode: snapshot.copyMode ?? "auto",
verifyMode: snapshot.verifyMode ?? "auto",
checksumAlgo: snapshot.checksumAlgo,
reportFormats: snapshot.reportFormats,
items: pkg.items.map((item) => ({
itemId: item.id,
exportId: item.exportId,
shotCode: item.shotCode,
versionString: item.versionString,
sourceDir: item.export.outputDir,
outputPattern: item.export.outputPattern,
frameStart: item.export.frameStart,
frameEnd: item.export.frameEnd,
fileCount: item.export.exrFileCount,
checksum: item.export.checksum,
deliveryMovPath: snapshot.includeMovs ? item.export.deliveryMovPath : null,
validatedAt: item.export.validations[0]?.createdAt.toISOString() ?? null,
shotDir: applyTokens(snapshot.layout.shotDir, {
...baseTokens,
shotCode: item.shotCode,
versionString: item.versionString,
}),
})),
};
}
// ── Worker reporting ─────────────────────────────────────────────────────────
export const deliveryReportSchema = z.object({
machineId: z.string().min(1),
items: z
.array(
z.object({
itemId: z.string().min(1),
status: z.enum(["COPIED", "VERIFIED", "FAILED"]),
exrPath: z.string().max(1000).optional().nullable(),
movPath: z.string().max(1000).optional().nullable(),
fileCount: z.number().int().nonnegative().optional().nullable(),
totalBytes: z.number().nonnegative().optional().nullable(),
checksum: z.string().max(200).optional().nullable(),
linkMode: z.enum(["hardlink", "copy"]).optional().nullable(),
errorMessage: z.string().max(2000).optional().nullable(),
})
)
.max(500),
manifest: z.unknown().optional(),
reportKey: z.string().max(500).optional().nullable(),
rootPath: z.string().max(1000).optional().nullable(),
errorMessage: z.string().max(2000).optional().nullable(),
/** false → the package failed as a whole. */
ok: z.boolean(),
});
export type DeliveryReport = z.infer<typeof deliveryReportSchema>;
/**
* The worker finished (or gave up on) a DELIVERY_BUILD job. Exports move to
* PACKAGED only for items that actually verified — a package with a failed item
* is FAILED, and nothing about it is silently treated as delivered.
*/
export async function reportDeliveryBuild(jobId: string, body: DeliveryReport) {
return db.$transaction(async (tx) => {
const job = await tx.renderJob.findUnique({ where: { id: jobId } });
if (!job) throw new PipelineError(404, "Render job not found");
if (job.machineId !== body.machineId) {
throw new PipelineError(409, "Job is not held by this machine (lease reassigned?)");
}
if (!job.deliveryId) throw new PipelineError(422, "Job has no delivery package");
const pkg = await tx.deliveryPackage.findUnique({
where: { id: job.deliveryId },
include: { items: true },
});
if (!pkg) throw new PipelineError(404, "Delivery package not found");
if (pkg.status === "READY" || pkg.status === "DELIVERED") {
return { package: { id: pkg.id, status: pkg.status }, alreadyApplied: true };
}
await tx.renderJob.update({
where: { id: job.id },
data: {
status: body.ok ? "COMPLETED" : "FAILED",
progress: 1,
finishedAt: new Date(),
errorMessage: body.errorMessage ?? null,
},
});
for (const reported of body.items) {
await tx.deliveryItem.updateMany({
where: { id: reported.itemId, packageId: pkg.id },
data: {
status: reported.status,
exrPath: reported.exrPath ?? null,
movPath: reported.movPath ?? null,
fileCount: reported.fileCount ?? null,
totalBytes: reported.totalBytes == null ? null : BigInt(Math.round(reported.totalBytes)),
checksum: reported.checksum ?? null,
linkMode: reported.linkMode ?? null,
errorMessage: reported.errorMessage ?? null,
},
});
}
const verified = body.items.filter((i) => i.status === "VERIFIED");
const failed = body.items.filter((i) => i.status === "FAILED");
const success = body.ok && failed.length === 0 && verified.length === pkg.items.length;
if (success) {
for (const item of pkg.items) {
const exportRow = await tx.export.findUnique({ where: { id: item.exportId }, select: { id: true, status: true } });
if (exportRow?.status === "READY_FOR_DELIVERY") {
await applyTransition(tx, exportRow, "PACKAGED", {
type: "WORKER",
id: body.machineId,
note: `Packaged into #${pkg.packageNumber} ${pkg.name}`,
});
}
}
}
const updated = await tx.deliveryPackage.update({
where: { id: pkg.id },
data: {
status: success ? "READY" : "FAILED",
builtAt: new Date(),
rootPath: body.rootPath ?? pkg.rootPath,
manifest: (body.manifest ?? undefined) as Prisma.InputJsonValue | undefined,
reportKey: body.reportKey ?? pkg.reportKey,
errorMessage: success
? null
: (body.errorMessage ??
(failed.length ? `${failed.length} item(s) failed: ${failed.map((f) => f.itemId).join(", ")}` : "Build did not complete")),
},
});
return {
package: { id: updated.id, status: updated.status },
verified: verified.length,
failed: failed.length,
alreadyApplied: false,
};
});
}
// ── E17 / E19 / cancel ───────────────────────────────────────────────────────
export async function listDeliveryPackages(filter: { projectId?: string; episode?: string; status?: DeliveryStatus }) {
const packages = await db.deliveryPackage.findMany({
where: {
...(filter.projectId ? { projectId: filter.projectId } : {}),
...(filter.episode ? { episode: filter.episode } : {}),
...(filter.status ? { status: filter.status } : {}),
},
orderBy: { createdAt: "desc" },
include: {
items: { select: { id: true, shotCode: true, versionString: true, status: true, fileCount: true, linkMode: true, errorMessage: true } },
project: { select: { id: true, name: true, code: true } },
},
});
return {
packages: packages.map((p) => ({
id: p.id,
packageNumber: p.packageNumber,
name: p.name,
episode: p.episode,
status: p.status,
rootPath: p.rootPath,
errorMessage: p.errorMessage,
projectName: p.project.name,
projectCode: p.project.code,
createdAt: p.createdAt,
builtAt: p.builtAt,
deliveredAt: p.deliveredAt,
itemCount: p.items.length,
verifiedCount: p.items.filter((i) => i.status === "VERIFIED").length,
items: p.items,
})),
};
}
/** E19 — the human uploaded the folder to the client and is confirming it. */
export async function markDelivered(packageId: string, userId: string) {
return db.$transaction(async (tx) => {
const pkg = await tx.deliveryPackage.findUnique({ where: { id: packageId }, include: { items: true } });
if (!pkg) throw new PipelineError(404, "Delivery package not found");
if (pkg.status === "DELIVERED") {
return { package: { id: pkg.id, status: pkg.status }, alreadyApplied: true };
}
if (pkg.status !== "READY") {
throw new PipelineError(409, `Package is ${pkg.status} — only a READY package can be marked delivered`);
}
for (const item of pkg.items) {
const exportRow = await tx.export.findUnique({ where: { id: item.exportId }, select: { id: true, status: true } });
if (exportRow?.status === "PACKAGED") {
await applyTransition(tx, exportRow, "DELIVERED", {
type: "USER",
id: userId,
note: `Delivered in #${pkg.packageNumber} ${pkg.name}`,
});
}
}
const updated = await tx.deliveryPackage.update({
where: { id: pkg.id },
data: { status: "DELIVERED", deliveredAt: new Date(), deliveredById: userId },
});
return { package: { id: updated.id, status: updated.status }, alreadyApplied: false };
});
}
/** A mistaken package is cancelled, never deleted (§11.5). */
export async function cancelDeliveryPackage(packageId: string, userId: string) {
return db.$transaction(async (tx) => {
const pkg = await tx.deliveryPackage.findUnique({ where: { id: packageId } });
if (!pkg) throw new PipelineError(404, "Delivery package not found");
if (!["DRAFT", "QUEUED", "FAILED"].includes(pkg.status)) {
throw new PipelineError(409, `Package is ${pkg.status} — only DRAFT, QUEUED or FAILED packages can be cancelled`);
}
await tx.renderJob.updateMany({
where: { deliveryId: pkg.id, status: { in: ["QUEUED", "CLAIMED", "RUNNING"] } },
data: { status: "CANCELLED", finishedAt: new Date(), errorMessage: "Delivery package cancelled" },
});
const updated = await tx.deliveryPackage.update({
where: { id: pkg.id },
data: { status: "CANCELLED", errorMessage: `Cancelled by user ${userId}` },
});
return { package: { id: updated.id, status: updated.status } };
});
}