feat(pipeline): standalone Blender renders on the same queue
Deploy / deploy (push) Successful in 7m2s

Blender work is not a shot submission: no Export, no version, no
validation, QC or delivery. It is "Ctrl+F12, but on the farm", so it is
modelled as what it is — a queue entry and nothing more.

That turned out to need very little. Claim, progress, fail and
complete-render already tolerated a job with no Export behind them, so a
Blender job reuses the whole lifecycle; RenderJob gains projectId, label
and submittedBy so a job with no shot code still has something to show.

Capability gating is new and was already missing: the claim now filters
job types by the machine's reported tools. Without it a workstation
without Blender would claim Blender work and fail it repeatedly, burning
the attempts while a machine that could have rendered it sat idle. The
worker asks for Blender jobs unconditionally and the server refuses them
— trusting the worker to self-select would have left the same hole.

The runner treats a clean exit with no saved frames as a failure:
Blender exits 0 when a frame range is empty or an output path is
unwritable, and a job that silently rendered nothing is the worst
outcome available.

Standalone jobs auto-requeue on a transient failure like any other work;
delivery builds are deliberately excluded, since a half-written package
wants a human first.

Adds the Blender add-on (BlenderPanel/) and its docs. The add-on refuses
to submit an unsaved file — the farm opens the .blend from disk, so
unsaved changes would render the previous state without warning. Its
payload is pinned by a contract test, since add-on and server only ever
meet over JSON.

Worker 1.5.0. Adds migration 20260815000000_blender_render_jobs.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
twotalesanimation
2026-08-15 18:53:19 +02:00
parent bada122f42
commit 70cf45ead4
24 changed files with 1347 additions and 9 deletions
+50
View File
@@ -0,0 +1,50 @@
import { describe, it, expect } from "vitest";
import { blenderManifestSchema, createBlenderJobSchema } from "@/lib/render-pipeline/blender";
/**
* The add-on and the server are separate codebases that only meet over JSON,
* so the exact payload the add-on builds is pinned here. A field renamed on
* either side otherwise surfaces as a 422 in front of an artist.
*/
describe("Blender submission contract", () => {
// Verbatim shape of scene_summary() in BlenderPanel/vfxreview_submit.py
const addonManifest = {
blendFile: "X:/shows/UNG/cg/UNG_110_030_010_smoke.blend",
scene: "Scene",
frameStart: 1001,
frameEnd: 1110,
outputPath: "X:/shows/UNG/cg/renders/smoke/frame_",
engine: "CYCLES",
blenderVersion: "4.2.1",
};
it("accepts what the add-on sends", () => {
const parsed = blenderManifestSchema.safeParse(addonManifest);
expect(parsed.success).toBe(true);
});
it("accepts the full submission body", () => {
const parsed = createBlenderJobSchema.safeParse({
projectCode: "UNG_S1",
label: "UNG_110_030_010_smoke",
manifest: addonManifest,
submittedByEmail: "chris@twotalesanimation.com",
urgent: false,
});
expect(parsed.success).toBe(true);
});
it("defaults the stage so the add-on need not send it", () => {
const parsed = blenderManifestSchema.parse(addonManifest);
expect(parsed.stage).toBe("BLENDER");
});
it("rejects a backwards frame range rather than queueing a no-op render", () => {
const parsed = blenderManifestSchema.safeParse({ ...addonManifest, frameEnd: 900 });
expect(parsed.success).toBe(false);
});
it("accepts EEVEE, which is what the add-on reports on a non-Cycles scene", () => {
expect(blenderManifestSchema.safeParse({ ...addonManifest, engine: "BLENDER_EEVEE_NEXT" }).success).toBe(true);
});
});
+189
View File
@@ -0,0 +1,189 @@
import { Prisma, RenderJobStatus } from "@prisma/client";
import { z } from "zod";
import { db } from "@/lib/db";
import { PipelineError } from "./errors";
import { getConfigNumber } from "./config";
/**
* Standalone Blender renders.
*
* Deliberately the simplest thing in the pipeline: a queue entry and nothing
* more. No Export, no shot version, no validation, QC or delivery — the .blend
* is already set up to render, and this is "Ctrl+F12, but on the farm". Every
* lifecycle call it needs (claim, progress, fail, complete) already tolerates a
* job with no Export behind it.
*/
export const blenderManifestSchema = z.object({
stage: z.literal("BLENDER").default("BLENDER"),
blendFile: z.string().min(1),
/** Blank renders the .blend's active scene, which is usually what is wanted. */
scene: z.string().max(200).optional().nullable(),
frameStart: z.number().int(),
frameEnd: z.number().int(),
/**
* Read from the .blend at submission and passed through only so the queue can
* show where frames will land. Left unset, the file's own output path is used
* — the file is expected to be ready to render.
*/
outputPath: z.string().max(1000).optional().nullable(),
/** Overrides, all optional: an unset value leaves the .blend's own setting. */
engine: z.enum(["CYCLES", "BLENDER_EEVEE_NEXT", "BLENDER_EEVEE", "BLENDER_WORKBENCH"]).optional().nullable(),
device: z.enum(["CPU", "GPU"]).optional().nullable(),
samples: z.number().int().positive().max(100000).optional().nullable(),
resolutionPercent: z.number().int().min(1).max(100).optional().nullable(),
/** Blender version this was authored in, for diagnosing a farm-vs-desk difference. */
blenderVersion: z.string().max(50).optional().nullable(),
}).refine((m) => m.frameEnd >= m.frameStart, {
message: "frameEnd must be >= frameStart",
path: ["frameEnd"],
});
export type BlenderManifest = z.infer<typeof blenderManifestSchema>;
export const createBlenderJobSchema = z.object({
projectCode: z.string().min(1),
label: z.string().min(1).max(200),
manifest: blenderManifestSchema,
submittedByEmail: z.string().max(200).optional().nullable(),
urgent: z.boolean().optional(),
});
export type CreateBlenderJobInput = z.infer<typeof createBlenderJobSchema>;
export async function createBlenderJob(input: CreateBlenderJobInput) {
const project = await db.project.findUnique({
where: { code: input.projectCode },
select: { id: true, code: true, name: true },
});
if (!project) throw new PipelineError(404, `Project not found: ${input.projectCode}`);
const [maxAttempts, urgentThreshold] = await Promise.all([
getConfigNumber("render.maxAttempts"),
getConfigNumber("render.urgentPriorityThreshold"),
]);
const job = await db.renderJob.create({
data: {
type: "BLENDER_RENDER",
projectId: project.id,
label: input.label.trim(),
submittedBy: input.submittedByEmail?.trim() || null,
attempt: 1,
maxAttempts,
priority: input.urgent ? urgentThreshold : 50,
manifest: input.manifest as unknown as Prisma.InputJsonValue,
},
});
return {
job: {
id: job.id,
label: job.label,
status: job.status,
priority: job.priority,
frames: input.manifest.frameEnd - input.manifest.frameStart + 1,
},
project: { code: project.code, name: project.name },
};
}
export interface BlenderJobFilter {
projectId?: string;
status?: RenderJobStatus;
limit?: number;
}
export async function listBlenderJobs(filter: BlenderJobFilter = {}) {
const jobs = await db.renderJob.findMany({
where: {
type: "BLENDER_RENDER",
...(filter.projectId ? { projectId: filter.projectId } : {}),
...(filter.status ? { status: filter.status } : {}),
},
orderBy: [{ createdAt: "desc" }],
take: Math.min(200, Math.max(1, filter.limit ?? 50)),
include: { machine: { select: { name: true } } },
});
const projectIds = [...new Set(jobs.map((j) => j.projectId).filter((id): id is string => !!id))];
const projects = projectIds.length
? await db.project.findMany({ where: { id: { in: projectIds } }, select: { id: true, code: true, name: true } })
: [];
const byId = new Map(projects.map((p) => [p.id, p]));
return {
jobs: jobs.map((j) => {
const m = j.manifest as unknown as Partial<BlenderManifest>;
return {
id: j.id,
label: j.label,
status: j.status,
attempt: j.attempt,
maxAttempts: j.maxAttempts,
priority: j.priority,
progress: j.progress,
currentFrame: j.currentFrame,
totalFrames: j.totalFrames,
etaSeconds: j.etaSeconds,
errorMessage: j.errorMessage,
machineName: j.machine?.name ?? null,
submittedBy: j.submittedBy,
createdAt: j.createdAt,
startedAt: j.startedAt,
finishedAt: j.finishedAt,
projectCode: j.projectId ? byId.get(j.projectId)?.code ?? null : null,
blendFile: m.blendFile ?? null,
outputPath: m.outputPath ?? null,
frameStart: m.frameStart ?? null,
frameEnd: m.frameEnd ?? null,
};
}),
};
}
/** Cancel a queued or running Blender job. The worker learns via its progress report. */
export async function cancelBlenderJob(jobId: string, actorId?: string) {
const job = await db.renderJob.findUnique({ where: { id: jobId }, select: { id: true, type: true, status: true } });
if (!job) throw new PipelineError(404, "Render job not found");
if (job.type !== "BLENDER_RENDER") throw new PipelineError(422, "Not a Blender job");
if (!["QUEUED", "CLAIMED", "RUNNING"].includes(job.status)) {
throw new PipelineError(409, `Job is ${job.status} — only a queued or running job can be cancelled`);
}
const updated = await db.renderJob.update({
where: { id: job.id },
data: {
status: "CANCELLED",
errorMessage: `Cancelled by user${actorId ? ` ${actorId}` : ""}`,
// A claimed job keeps finishedAt null until the worker acknowledges, so a
// render still winding down is not reported as already stopped.
...(job.status === "QUEUED" ? { finishedAt: new Date() } : {}),
},
});
return { job: { id: updated.id, status: updated.status } };
}
/** Requeue a failed Blender job as a fresh attempt, reusing its manifest. */
export async function retryBlenderJob(jobId: string) {
const job = await db.renderJob.findUnique({ where: { id: jobId } });
if (!job) throw new PipelineError(404, "Render job not found");
if (job.type !== "BLENDER_RENDER") throw new PipelineError(422, "Not a Blender job");
if (!["FAILED", "CANCELLED", "EXPIRED"].includes(job.status)) {
throw new PipelineError(409, `Job is ${job.status} — only a finished-unsuccessfully job can be retried`);
}
const next = await db.renderJob.create({
data: {
type: "BLENDER_RENDER",
projectId: job.projectId,
label: job.label,
submittedBy: job.submittedBy,
attempt: job.attempt + 1,
maxAttempts: job.maxAttempts,
priority: job.priority,
manifest: job.manifest as Prisma.InputJsonValue,
},
});
return { job: { id: next.id, attempt: next.attempt, status: next.status } };
}
+36 -4
View File
@@ -3,11 +3,11 @@ import { db } from "@/lib/db";
import { PipelineError } from "./errors";
import { applyTransition } from "./transitions";
import { getConfigBoolean, getConfigNumber } from "./config";
import { machineCanClaim } from "./machines";
import { machineCanClaim, machineSupportedTypes } from "./machines";
import { reapExpiredLeases } from "./reaper";
import { buildPreviewManifest, createPreviewJob, inheritPreviewMedia } from "./preview";
const JOB_TYPES: RenderJobType[] = ["AE_RENDER", "PREVIEW_ONLY", "DELIVERY_BUILD"];
const JOB_TYPES: RenderJobType[] = ["AE_RENDER", "PREVIEW_ONLY", "DELIVERY_BUILD", "BLENDER_RENDER"];
/**
* E8 — atomically claim the next queued job (§6.4). `FOR UPDATE SKIP LOCKED`
@@ -22,10 +22,14 @@ export async function claimNextJob(machineId: string, types?: string[]) {
// Defensive sweep on each claim (studio decision 18.1-Q8)
await reapExpiredLeases().catch((err) => console.error("[render-pipeline] claim-time reap failed", err));
const claimTypes = (types?.length ? types : ["AE_RENDER", "PREVIEW_ONLY", "DELIVERY_BUILD"]).filter(
const requested = (types?.length ? types : ["AE_RENDER", "PREVIEW_ONLY", "DELIVERY_BUILD"]).filter(
(t): t is RenderJobType => (JOB_TYPES as string[]).includes(t)
);
if (claimTypes.length === 0) throw new PipelineError(422, "No valid job types requested");
// Enforced here rather than trusted to the worker: a machine without the tool
// installed must not be able to claim work it can only fail.
const claimTypes = machineSupportedTypes(machine, requested) as RenderJobType[];
if (requested.length === 0) throw new PipelineError(422, "No valid job types requested");
if (claimTypes.length === 0) return null;
const gate = machineCanClaim(machine, new Date());
if (gate === "none") return null;
@@ -91,6 +95,7 @@ export async function claimNextJob(machineId: string, types?: string[]) {
priority: job.priority,
exportId: job.exportId,
deliveryId: job.deliveryId,
label: job.label,
leaseExpiresAt: job.leaseExpiresAt,
manifest: job.manifest,
},
@@ -189,6 +194,33 @@ export async function reportFail(
});
if (!job.export) {
// A standalone job (a Blender render) has no Export to move, but a
// transient failure should still be retried like any other work.
// Delivery builds are deliberately excluded: a half-written package
// folder wants a human to look at it before it is rebuilt.
const retryableStandalone =
!job.deliveryId && body.retryable !== false && job.attempt < job.maxAttempts;
if (retryableStandalone) {
const next = await tx.renderJob.create({
data: {
type: job.type,
projectId: job.projectId,
label: job.label,
submittedBy: job.submittedBy,
deliveryId: job.deliveryId,
attempt: job.attempt + 1,
maxAttempts: job.maxAttempts,
priority: job.priority,
manifest: job.manifest as Prisma.InputJsonValue,
},
});
return {
job: { id: job.id, status: "FAILED" },
export: null,
autoRequeued: true,
nextJob: { id: next.id, attempt: next.attempt },
};
}
return { job: { id: job.id, status: "FAILED" }, export: null, autoRequeued: false };
}
+13
View File
@@ -59,6 +59,19 @@ export function isInWindow(windows: AvailabilityWindow[], now: Date): boolean {
export type ClaimGate = "normal" | "urgent-only" | "none";
/**
* Job types a machine is equipped for. Capabilities are reported at
* registration; a machine without Blender installed must never be handed a
* Blender job, or it claims work it can only fail — repeatedly, until the
* attempts run out — while a machine that could have rendered it sits idle.
*/
export function machineSupportedTypes(machine: { capabilities: unknown }, requested: string[]): string[] {
const tools = ((machine.capabilities as { tools?: Record<string, unknown> } | null)?.tools ?? {}) as Record<string, unknown>;
const hasBlender = Boolean(tools.blender);
return requested.filter((type) => (type === "BLENDER_RENDER" ? hasBlender : true));
}
export function machineCanClaim(machine: Machine, now: Date): ClaimGate {
if (!machine.enabled) return "none";
if (machine.renderNowUntil && machine.renderNowUntil > now) return "normal";