Files
vfxreview/lib/render-pipeline/machines.ts
T
twotalesanimation 70cf45ead4
Deploy / deploy (push) Successful in 7m2s
feat(pipeline): standalone Blender renders on the same queue
Blender work is not a shot submission: no Export, no version, no
validation, QC or delivery. It is "Ctrl+F12, but on the farm", so it is
modelled as what it is — a queue entry and nothing more.

That turned out to need very little. Claim, progress, fail and
complete-render already tolerated a job with no Export behind them, so a
Blender job reuses the whole lifecycle; RenderJob gains projectId, label
and submittedBy so a job with no shot code still has something to show.

Capability gating is new and was already missing: the claim now filters
job types by the machine's reported tools. Without it a workstation
without Blender would claim Blender work and fail it repeatedly, burning
the attempts while a machine that could have rendered it sat idle. The
worker asks for Blender jobs unconditionally and the server refuses them
— trusting the worker to self-select would have left the same hole.

The runner treats a clean exit with no saved frames as a failure:
Blender exits 0 when a frame range is empty or an output path is
unwritable, and a job that silently rendered nothing is the worst
outcome available.

Standalone jobs auto-requeue on a transient failure like any other work;
delivery builds are deliberately excluded, since a half-written package
wants a human first.

Adds the Blender add-on (BlenderPanel/) and its docs. The add-on refuses
to submit an unsaved file — the farm opens the .blend from disk, so
unsaved changes would render the previous state without warning. Its
payload is pinned by a contract test, since add-on and server only ever
meet over JSON.

Worker 1.5.0. Adds migration 20260815000000_blender_render_jobs.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-15 18:53:19 +02:00

254 lines
8.6 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { Machine, Prisma } from "@prisma/client";
import { db } from "@/lib/db";
import { PipelineError } from "./errors";
import { getWorkerConfig } from "./config";
/**
* Machine availability (§7.10):
* ALWAYS — claims whenever idle (default when availability is null)
* SCHEDULE — claims only inside configured windows
* MANUAL — never claims unless explicitly triggered
* Overrides, in priority order: enabled=false beats everything; renderNowUntil
* allows normal claims until it expires; urgent jobs may claim anytime on
* machines with allowUrgentAnytime.
*/
export interface AvailabilityWindow {
days: string[]; // ["mon", ..., "sun"]
from: string; // "19:00"
to: string; // "08:00" — from > to spans midnight
}
export interface MachineAvailability {
mode?: "ALWAYS" | "SCHEDULE" | "MANUAL";
windows?: AvailabilityWindow[];
allowUrgentAnytime?: boolean;
}
const DAY_NAMES = ["sun", "mon", "tue", "wed", "thu", "fri", "sat"];
function parseHm(hm: string): number | null {
const m = /^(\d{1,2}):(\d{2})$/.exec(hm);
if (!m) return null;
const mins = parseInt(m[1], 10) * 60 + parseInt(m[2], 10);
return mins >= 0 && mins <= 24 * 60 ? mins : null;
}
export function isInWindow(windows: AvailabilityWindow[], now: Date): boolean {
const day = DAY_NAMES[now.getDay()];
const prevDay = DAY_NAMES[(now.getDay() + 6) % 7];
const t = now.getHours() * 60 + now.getMinutes();
for (const w of windows) {
const from = parseHm(w.from);
const to = parseHm(w.to);
if (from == null || to == null || !Array.isArray(w.days)) continue;
const days = w.days.map((d) => d.toLowerCase().slice(0, 3));
if (from < to) {
if (days.includes(day) && t >= from && t < to) return true;
} else if (from > to) {
// overnight window, e.g. 19:00 → 08:00
if (days.includes(day) && t >= from) return true;
if (days.includes(prevDay) && t < to) return true;
} else {
// from === to → full 24 h on listed days
if (days.includes(day)) return true;
}
}
return false;
}
export type ClaimGate = "normal" | "urgent-only" | "none";
/**
* Job types a machine is equipped for. Capabilities are reported at
* registration; a machine without Blender installed must never be handed a
* Blender job, or it claims work it can only fail — repeatedly, until the
* attempts run out — while a machine that could have rendered it sits idle.
*/
export function machineSupportedTypes(machine: { capabilities: unknown }, requested: string[]): string[] {
const tools = ((machine.capabilities as { tools?: Record<string, unknown> } | null)?.tools ?? {}) as Record<string, unknown>;
const hasBlender = Boolean(tools.blender);
return requested.filter((type) => (type === "BLENDER_RENDER" ? hasBlender : true));
}
export function machineCanClaim(machine: Machine, now: Date): ClaimGate {
if (!machine.enabled) return "none";
if (machine.renderNowUntil && machine.renderNowUntil > now) return "normal";
const availability = (machine.availability ?? null) as MachineAvailability | null;
const mode = availability?.mode ?? "ALWAYS";
const urgent = availability?.allowUrgentAnytime ? ("urgent-only" as const) : ("none" as const);
switch (mode) {
case "ALWAYS":
return "normal";
case "SCHEDULE":
return isInWindow(availability?.windows ?? [], now) ? "normal" : urgent;
case "MANUAL":
return urgent;
default:
return "normal";
}
}
/** E6 — idempotent register/upsert on machine name. */
export async function registerMachine(body: {
name?: string;
hostname?: string;
workerVersion?: string;
aeVersion?: string;
capabilities?: unknown;
}) {
const name = body.name?.trim();
if (!name) throw new PipelineError(422, "name is required");
const hostname = body.hostname?.trim() || name;
const machine = await db.machine.upsert({
where: { name },
create: {
name,
hostname,
status: "ONLINE",
lastSeenAt: new Date(),
workerVersion: body.workerVersion ?? null,
aeVersion: body.aeVersion ?? null,
capabilities: (body.capabilities ?? undefined) as Prisma.InputJsonValue | undefined,
},
update: {
hostname,
status: "ONLINE",
lastSeenAt: new Date(),
workerVersion: body.workerVersion ?? undefined,
aeVersion: body.aeVersion ?? undefined,
capabilities: (body.capabilities ?? undefined) as Prisma.InputJsonValue | undefined,
},
});
return {
machine: { id: machine.id, name: machine.name, enabled: machine.enabled },
config: await getWorkerConfig(),
};
}
/** E7 — heartbeat; also the server→worker command channel (cancel signals). */
export async function recordHeartbeat(
machineId: string,
body: { cpuPercent?: number; memPercent?: number; diskFreeGb?: number; currentJobId?: string | null }
) {
const machine = await db.machine.findUnique({ where: { id: machineId } });
if (!machine) throw new PipelineError(404, "Machine not found");
await db.$transaction([
db.machine.update({
where: { id: machineId },
data: { lastSeenAt: new Date(), status: machine.enabled ? "ONLINE" : "DISABLED" },
}),
db.workerHeartbeat.create({
data: {
machineId,
cpuPercent: body.cpuPercent ?? null,
memPercent: body.memPercent ?? null,
diskFreeGb: body.diskFreeGb ?? null,
currentJobId: body.currentJobId ?? null,
},
}),
]);
// Cancelled-but-unacknowledged jobs held by this machine → CANCEL_JOB commands
const cancelled = await db.renderJob.findMany({
where: { machineId, status: "CANCELLED", finishedAt: null },
select: { id: true },
});
return {
ok: true,
commands: cancelled.map((j) => ({ type: "CANCEL_JOB" as const, jobId: j.id })),
};
}
/** Machine list for the monitoring page, with latest heartbeat + current job. */
export async function listMachines() {
const heartbeatSeconds = 30;
const machines = await db.machine.findMany({
orderBy: { name: "asc" },
include: {
heartbeats: { orderBy: { createdAt: "desc" }, take: 1 },
renderJobs: {
where: { status: { in: ["CLAIMED", "RUNNING"] } },
take: 1,
include: { export: { select: { id: true, versionString: true, shot: { select: { shotCode: true } } } } },
},
},
});
const now = Date.now();
return machines.map((m) => ({
id: m.id,
name: m.name,
hostname: m.hostname,
enabled: m.enabled,
// ONLINE = heartbeat within 3× the heartbeat interval (§5.6)
status: !m.enabled
? "DISABLED"
: m.lastSeenAt && now - m.lastSeenAt.getTime() < heartbeatSeconds * 3 * 1000
? "ONLINE"
: "OFFLINE",
lastSeenAt: m.lastSeenAt,
workerVersion: m.workerVersion,
aeVersion: m.aeVersion,
capabilities: m.capabilities,
availability: m.availability,
renderNowUntil: m.renderNowUntil,
latestHeartbeat: m.heartbeats[0]
? {
createdAt: m.heartbeats[0].createdAt,
cpuPercent: m.heartbeats[0].cpuPercent,
memPercent: m.heartbeats[0].memPercent,
diskFreeGb: m.heartbeats[0].diskFreeGb,
}
: null,
currentJob: m.renderJobs[0]
? {
id: m.renderJobs[0].id,
exportId: m.renderJobs[0].exportId,
shotCode: m.renderJobs[0].export?.shot.shotCode ?? null,
versionString: m.renderJobs[0].export?.versionString ?? null,
progress: m.renderJobs[0].progress,
etaSeconds: m.renderJobs[0].etaSeconds,
}
: null,
}));
}
export async function updateMachine(
machineId: string,
patch: { enabled?: boolean; availability?: MachineAvailability | null; renderNowHours?: number | null }
) {
const machine = await db.machine.findUnique({ where: { id: machineId } });
if (!machine) throw new PipelineError(404, "Machine not found");
const data: Prisma.MachineUpdateInput = {};
if (patch.enabled !== undefined) {
data.enabled = patch.enabled;
data.status = patch.enabled ? machine.status === "DISABLED" ? "OFFLINE" : machine.status : "DISABLED";
}
if (patch.availability !== undefined) {
data.availability = patch.availability === null ? Prisma.DbNull : (patch.availability as unknown as Prisma.InputJsonValue);
}
if (patch.renderNowHours !== undefined) {
data.renderNowUntil =
patch.renderNowHours === null || patch.renderNowHours <= 0
? null
: new Date(Date.now() + patch.renderNowHours * 3600 * 1000);
}
const updated = await db.machine.update({ where: { id: machineId }, data });
return {
machine: {
id: updated.id,
name: updated.name,
enabled: updated.enabled,
availability: updated.availability,
renderNowUntil: updated.renderNowUntil,
},
};
}