feat(pipeline): render queue, worker service and automated preview generation

One "Queue Export" click now renders the EXR sequence, then rebuilds the shot
headlessly with the studio slate/overlay template to produce the delivery MOV
and review MP4. Implements RenderPipeline2 phases 1-2 plus the preview stage.

Server:
- New models Export, RenderJob, ExportEvent, Machine, WorkerHeartbeat, plus
  Project.deliveryConfig and per-submission slate fields (Export.vfxScope,
  Export.submissionNote, inherited from the shot's previous export).
  Both migrations are purely additive; no existing column is touched.
- lib/render-pipeline: server-enforced state machine, transactional version
  increment with supersede, atomic FOR UPDATE SKIP LOCKED claim gated by
  machine availability windows, and a lease reaper run from instrumentation.ts.
- /api/ext/* endpoints for the panel and workers; session-auth mirrors under
  /api/render and /api/machines for the web UI.
- Pipeline pages: render queue, export detail, machine monitoring, plus an
  Exports tab on shot detail.

RenderWorker (.NET 8 Windows service, new):
- Registration, heartbeat as cancel channel, claim loop, aerender runner with
  progress parsing and stall watchdog, crash recovery and disk-spooled
  reporting that survives server downtime.
- Preview stage: headless AE assembles the preview comp into a throwaway AEP
  with both output modules queued, then a single aerender pass renders them.
  Preview jobs are not claimed while an interactive AE session is open, so an
  artist's project is never taken over.

AE panel: Queue Export with live status polling, urgent flag, retry, and the
VFX Scope / Submission Note fields. Every existing panel action is unchanged.

Preview chaining ships disabled behind SystemConfig preview.enabled.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
twotalesanimation
2026-08-02 14:34:34 +02:00
parent 6b15bae62a
commit cc89415a29
71 changed files with 11767 additions and 1 deletions
+138
View File
@@ -0,0 +1,138 @@
<#
.SYNOPSIS
Headless AE spike / debug tool for the preview build stage.
.DESCRIPTION
Runs vfxr_build_preview.jsx exactly the way the RenderWorker does, but
standalone, so the risky part (AfterFX.com -noui -r under a service
account) can be proven before wiring the whole pipeline together.
It does NOT render - it builds the preview comp, queues the MOV + MP4
output modules and saves a temp AEP, then prints the build result.
IMPORTANT: close After Effects first. `AfterFX.com -r` hands the script to
an already-running instance, which would open the template in your session
and then quit it.
.EXAMPLE
.\test-preview-build.ps1 -ExrDir "V:\_EXPORTS\UNG\renders\UNG_S1\111\UNG_111_001_030\v002" `
-ShotCode "UNG_111_001_030" -Version "v002"
.EXAMPLE
# then render what it built, exactly as the worker would:
& "C:\Program Files\Adobe\Adobe After Effects 2026\Support Files\aerender.exe" -project "$env:ProgramData\VFXReviewWorker\scratch\spike_preview.aep"
#>
[CmdletBinding()]
param(
[Parameter(Mandatory = $true)][string]$ExrDir,
[Parameter(Mandatory = $true)][string]$ShotCode,
[string]$Version = "v001",
[string]$TemplateAep = "X:/shared_projects_2026/UNGO_VFX/production/working_files/UNG_VFXOVERLAY_SLATE_TEMPLATE.aep",
[string]$TemplateComp = "UNG_EXPORT_TEMPLATE",
[string]$OverlayComp = "UNG_VFX_OVERLAY",
[string]$LutComp = "_SHOW LUT",
[string]$MovTemplate = "4444 Tri",
[string]$Mp4Template = "REVIEW_PREVIEW",
[string]$AfterFx = "C:\Program Files\Adobe\Adobe After Effects 2026\Support Files\AfterFX.com",
[int]$FrameStart = 1001,
[double]$Fps = 24,
[int]$TimeoutSeconds = 300
)
$ErrorActionPreference = "Stop"
$running = Get-Process AfterFX -ErrorAction SilentlyContinue
if ($running) {
Write-Warning "After Effects is running (PID $($running.Id -join ', '))."
Write-Warning "AfterFX.com -r would target that instance and quit it. Close AE first."
return
}
$scratch = Join-Path $env:ProgramData "VFXReviewWorker\scratch"
if (-not (Test-Path $scratch)) { New-Item -ItemType Directory -Force -Path $scratch | Out-Null }
$scriptPath = Join-Path $PSScriptRoot "vfxr_build_preview.jsx"
if (-not (Test-Path $scriptPath)) { throw "Build script not found: $scriptPath" }
if (-not (Test-Path $AfterFx)) { throw "AfterFX.com not found: $AfterFx" }
if (-not (Test-Path $ExrDir)) { throw "EXR folder not found: $ExrDir" }
$contextPath = Join-Path $scratch "spike_context.json"
$resultPath = Join-Path $scratch "spike_result.json"
$wrapperPath = Join-Path $scratch "spike_run.jsx"
$tempAep = Join-Path $scratch "spike_preview.aep"
$base = "${ShotCode}_cmp_TT_${Version}"
Remove-Item $resultPath -ErrorAction SilentlyContinue
$context = [ordered]@{
jobId = "spike"
exportId = "spike"
shotCode = $ShotCode
versionString = $Version
outputDirLocal = $ExrDir
templateAep = $TemplateAep
templateComp = $TemplateComp
overlayComp = $OverlayComp
lutComp = $LutComp
movTemplate = $MovTemplate
mp4Template = $Mp4Template
movOutput = (Join-Path $ExrDir "$base.mov")
mp4Output = (Join-Path $ExrDir "$base.mp4")
tempAep = $tempAep
resultPath = $resultPath
frameStart = $FrameStart
fps = $Fps
slate = [ordered]@{
versionName = $base
date = (Get-Date -Format "yyyy/MM/dd")
description = "Spike test"
notes = ""
shotCode = $ShotCode
episode = ""
scene = ""
}
}
$context | ConvertTo-Json -Depth 6 | Out-File -FilePath $contextPath -Encoding utf8
$ctxJs = ($contextPath -replace '\\', '/') | ConvertTo-Json
$scriptJs = ($scriptPath -replace '\\', '/') | ConvertTo-Json
$wrapper = @(
"// generated by test-preview-build.ps1",
("var VFXR_CONTEXT_PATH = {0};" -f $ctxJs),
('$.evalFile({0});' -f $scriptJs)
) -join "`r`n"
$wrapper | Out-File -FilePath $wrapperPath -Encoding utf8
Write-Host "Launching headless AE build..." -ForegroundColor Cyan
$proc = Start-Process -FilePath $AfterFx -ArgumentList @("-noui", "-r", $wrapperPath) -PassThru -NoNewWindow
if (-not $proc.WaitForExit($TimeoutSeconds * 1000)) {
Write-Warning "Timed out after $TimeoutSeconds s - killing AE."
try { $proc.Kill($true) } catch {}
}
if (-not (Test-Path $resultPath)) {
Write-Host "FAILED: no result file produced." -ForegroundColor Red
Write-Host "AE could not run the script headlessly under this account. See RenderWorker/README.md fallbacks." -ForegroundColor Red
return
}
$result = Get-Content $resultPath -Raw | ConvertFrom-Json
if ($result.ok) {
Write-Host "BUILD OK" -ForegroundColor Green
Write-Host " temp AEP : $($result.tempAep)"
Write-Host " preview comp: $($result.previewComp)"
Write-Host (" frames : {0} ({1} x {2})" -f $result.frameCount, $result.width, $result.height)
Write-Host (" queued : {0} output(s)" -f @($result.queued).Count)
foreach ($q in @($result.queued)) {
Write-Host (" - {0}: {1} -> {2}" -f $q.label, $q.template, $q.output)
}
Write-Host ""
Write-Host "Render it with:" -ForegroundColor Cyan
Write-Host (' "{0}\aerender.exe" -project "{1}"' -f (Split-Path $AfterFx), $result.tempAep)
} else {
Write-Host "BUILD FAILED: $($result.error)" -ForegroundColor Red
}
if ($result.warnings -and $result.warnings.Count -gt 0) {
Write-Host "Warnings:" -ForegroundColor Yellow
$result.warnings | ForEach-Object { Write-Host " - $_" -ForegroundColor Yellow }
}
+527
View File
@@ -0,0 +1,527 @@
/*
vfxr_build_preview.jsx — VFXReview render pipeline, preview build stage.
Runs headlessly: AfterFX.com -noui -r <wrapper>.jsx
where the wrapper sets var VFXR_CONTEXT_PATH = "...json"; then
$.evalFile()s this script.
It does NOT render. It opens the studio slate/overlay template, rebuilds
the shot around the *rendered* EXR sequence (OCIO → show LUT → overlay),
duplicates the export template into a preview comp with the slate filled
in, queues the MOV and MP4 output modules, and saves a throwaway AEP.
The worker then runs `aerender -project <that aep>` with no -comp, which
renders the whole queue in one launch.
Build logic mirrors the VFXReviewConnector panel functions
buildShotFromCode / updateOverlayForComp / buildPreviewForShot so the
farm and the artist produce identical slates and burn-ins.
Everything is reported back through the result JSON — never a dialog.
*/
(function vfxrBuildPreview() {
var ctx = null;
var result = {
ok: false,
error: null,
warnings: [],
tempAep: null,
previewComp: null,
queued: [],
frameCount: null,
width: null,
height: null
};
var resultPath = null;
// ── tiny helpers (ES3 / ExtendScript safe) ───────────────────────────────
function readTextFile(path) {
var f = new File(path);
var text;
if (!f.exists) {
return null;
}
f.encoding = "UTF-8";
if (!f.open("r")) {
return null;
}
text = f.read();
f.close();
return text;
}
function writeTextFile(path, text) {
var f = new File(path);
try {
f.encoding = "UTF-8";
if (!f.open("w")) {
return false;
}
f.write(text);
f.close();
return true;
} catch (writeError) {
return false;
}
}
function jsonEscape(value) {
var text = String(value);
var out = "";
var i;
var ch;
var code;
for (i = 0; i < text.length; i += 1) {
ch = text.charAt(i);
code = text.charCodeAt(i);
if (ch === "\"") {
out += "\\\"";
} else if (ch === "\\") {
out += "\\\\";
} else if (ch === "\n") {
out += "\\n";
} else if (ch === "\r") {
out += "\\r";
} else if (ch === "\t") {
out += "\\t";
} else if (code < 32) {
out += "\\u" + ("000" + code.toString(16)).slice(-4);
} else {
out += ch;
}
}
return out;
}
function toJSON(value) {
var parts = [];
var i;
var key;
if (value === null || value === undefined) {
return "null";
}
if (typeof value === "number") {
return isFinite(value) ? String(value) : "null";
}
if (typeof value === "boolean") {
return String(value);
}
if (typeof value === "string") {
return "\"" + jsonEscape(value) + "\"";
}
if (value instanceof Array) {
for (i = 0; i < value.length; i += 1) {
parts.push(toJSON(value[i]));
}
return "[" + parts.join(",") + "]";
}
for (key in value) {
if (value.hasOwnProperty(key) && value[key] !== undefined) {
parts.push("\"" + jsonEscape(key) + "\":" + toJSON(value[key]));
}
}
return "{" + parts.join(",") + "}";
}
function warn(message) {
result.warnings.push(String(message));
}
function findComp(name) {
var i;
var item;
for (i = 1; i <= app.project.numItems; i += 1) {
item = app.project.item(i);
if (item instanceof CompItem && item.name === name) {
return item;
}
}
return null;
}
function findFolder(name) {
var i;
var item;
for (i = 1; i <= app.project.numItems; i += 1) {
item = app.project.item(i);
if (item instanceof FolderItem && item.name === name) {
return item;
}
}
return null;
}
// Mirrors the panel's addOCIOToLayer: effect added but disabled, so the
// artist/farm chain matches exactly.
function addOCIOToLayer(layer) {
var ocio;
var outputProp;
try {
ocio = layer.property("Effects").addProperty("OCIO Color Space Transform");
ocio.enabled = false;
} catch (ocioError) {
warn("OCIO effect unavailable: " + ocioError.toString());
return;
}
try {
outputProp = ocio.property("Output Color Space");
} catch (byNameError) {
outputProp = null;
}
if (!outputProp) {
try {
outputProp = ocio.property(2);
} catch (byIndexError) {
outputProp = null;
}
}
if (outputProp) {
try {
outputProp.setValue(93);
} catch (setError) {
warn("Could not set OCIO output colour space: " + setError.toString());
}
}
}
function setEssentialProperty(layer, propName, value) {
if (value === null || value === undefined || value === "") {
return false;
}
try {
layer.property("Essential Properties").property(propName).setValue(value);
return true;
} catch (setError) {
return false;
}
}
function setEssentialPropertyByIndex(layer, index, value) {
try {
layer.property("Essential Properties").property(index).setValue(value);
return true;
} catch (setError) {
return false;
}
}
function listEssentialPropertyNames(layer) {
var names = [];
var group;
var i;
try {
group = layer.property("Essential Properties");
for (i = 1; i <= group.numProperties; i += 1) {
try {
names.push(i + ":" + group.property(i).name);
} catch (oneError) {
}
}
} catch (groupError) {
}
return names;
}
// Per-submission fields (VFX Scope / Submission Note). The property names
// come from config; if one does not exist, report the template's actual
// property names so it can be corrected without another build.
function setSlateField(layer, propName, value, label) {
if (!propName) {
return;
}
if (value === null || value === undefined || value === "") {
return; // nothing to write; leave the template's own default
}
if (!setEssentialProperty(layer, propName, value)) {
warn(label + ": Essential Property \"" + propName + "\" not found or not settable. " +
"Available: " + listEssentialPropertyNames(layer).join(", "));
}
}
function setCompStartFrame(comp, frameNumber) {
try {
comp.displayStartFrame = frameNumber;
return;
} catch (frameError) {
}
try {
comp.displayStartTime = frameNumber / comp.frameRate;
} catch (timeError) {
}
}
function applyOutputModuleTemplate(outputModule, templateName) {
var templates;
var i;
try {
templates = outputModule.templates;
for (i = 0; i < templates.length; i += 1) {
if (templates[i] === templateName) {
outputModule.applyTemplate(templateName);
return true;
}
}
} catch (templateError) {
}
return false;
}
function importExrSequence(dirPath) {
var folder = new Folder(dirPath);
var files;
var importOptions;
var footage;
if (!folder.exists) {
throw new Error("Render output folder not found: " + dirPath);
}
files = folder.getFiles("*.exr");
if (!files || files.length < 1) {
throw new Error("No EXR files found in " + dirPath);
}
files.sort();
importOptions = new ImportOptions(files[0]);
importOptions.sequence = true;
footage = app.project.importFile(importOptions);
try {
footage.mainSource.conformFrameRate = ctx.fps || 24;
} catch (conformError) {
warn("Could not conform frame rate: " + conformError.toString());
}
return footage;
}
function queueOutput(comp, templateName, outputPath, label) {
var item;
var outputModule;
if (!templateName || !outputPath) {
warn("Skipping " + label + ": no template or output path configured");
return false;
}
try {
item = app.project.renderQueue.items.add(comp);
outputModule = item.outputModule(1);
if (!applyOutputModuleTemplate(outputModule, templateName)) {
item.remove();
throw new Error("Output module template not found: \"" + templateName + "\"");
}
outputModule.file = new File(outputPath);
result.queued.push({ label: label, template: templateName, output: outputPath });
return true;
} catch (queueError) {
try {
if (item) {
item.remove();
}
} catch (removeError) {
}
throw queueError;
}
}
// ── main ─────────────────────────────────────────────────────────────────
try {
if (typeof VFXR_CONTEXT_PATH === "undefined" || !VFXR_CONTEXT_PATH) {
throw new Error("VFXR_CONTEXT_PATH was not set by the wrapper script");
}
var contextText = readTextFile(VFXR_CONTEXT_PATH);
if (!contextText) {
throw new Error("Could not read job context: " + VFXR_CONTEXT_PATH);
}
ctx = eval("(" + contextText + ")");
resultPath = ctx.resultPath;
try {
app.beginSuppressDialogs();
} catch (suppressError) {
}
// 1. Open the studio template (slate, overlay, show LUT, export template)
var templateFile = new File(ctx.templateAep);
if (!templateFile.exists) {
throw new Error("Template project not found: " + ctx.templateAep);
}
app.open(templateFile);
var templateComp = findComp(ctx.templateComp);
if (!templateComp) {
throw new Error("Export template comp \"" + ctx.templateComp + "\" not found in template project");
}
// 2. Import the rendered EXR sequence
var footage = importExrSequence(ctx.outputDirLocal);
var footageFolder = findFolder("_FOOTAGE_4K");
if (footageFolder) {
footage.parentFolder = footageFolder;
}
footage.name = ctx.shotCode + "_" + ctx.versionString;
var width = footage.width;
var height = footage.height;
var duration = footage.duration;
var fps = ctx.fps || 24;
result.width = width;
result.height = height;
result.frameCount = Math.round(duration * fps);
// 3. Footage precomp (mirrors buildShotFromCode)
var precompFolder = findFolder("_PRECOMPS");
var footageComp = app.project.items.addComp(
ctx.shotCode + "_FOOTAGE", width, height, 1, duration, fps);
if (precompFolder) {
footageComp.parentFolder = precompFolder;
}
if (ctx.frameStart !== null && ctx.frameStart !== undefined) {
setCompStartFrame(footageComp, ctx.frameStart);
}
addOCIOToLayer(footageComp.layers.add(footage));
// 4. Shot comp: footage + show LUT + overlay
var shotComp = app.project.items.addComp(ctx.shotCode, width, height, 1, duration, fps);
if (ctx.frameStart !== null && ctx.frameStart !== undefined) {
setCompStartFrame(shotComp, ctx.frameStart);
}
shotComp.layers.add(footageComp);
if (ctx.lutComp) {
var showLut = findComp(ctx.lutComp);
if (showLut) {
var lutLayer = shotComp.layers.add(showLut);
lutLayer.collapseTransformation = true;
} else {
warn("Show LUT comp \"" + ctx.lutComp + "\" not found — rendering without it");
}
}
if (ctx.overlayComp) {
var overlayComp = findComp(ctx.overlayComp);
if (overlayComp) {
var overlayLayer = shotComp.layers.add(overlayComp);
overlayLayer.moveToBeginning();
overlayLayer.enabled = true;
setEssentialProperty(overlayLayer, "DATE", ctx.slate.date);
setEssentialProperty(overlayLayer, "SHOT NAME", ctx.slate.versionName);
} else {
warn("Overlay comp \"" + ctx.overlayComp + "\" not found — rendering without burn-ins");
}
}
// 5. Preview comp from the export template (mirrors buildPreviewForShot)
var previewsFolder = findFolder("_PREVIEWS");
var previewComp = templateComp.duplicate();
previewComp.name = ctx.shotCode + "_PREVIEW";
if (previewsFolder) {
previewComp.parentFolder = previewsFolder;
}
previewComp.layer("SHOT").replaceSource(shotComp, false);
try {
previewComp.layer("THUMBNAIL").replaceSource(shotComp, false);
} catch (thumbError) {
warn("THUMBNAIL layer not updated: " + thumbError.toString());
}
var slateLayer = previewComp.layer("NETFLIX_SLATE");
if (slateLayer) {
// Index-addressed properties match the panel exactly (3 = version
// name, 10 = shot code); the rest are addressed by name.
setEssentialPropertyByIndex(slateLayer, 3, ctx.slate.versionName);
setEssentialProperty(slateLayer, "Date", ctx.slate.date);
setEssentialProperty(slateLayer, "Desc", ctx.slate.description);
setEssentialPropertyByIndex(slateLayer, 10, ctx.slate.shotCode);
setEssentialProperty(slateLayer, "Episode", ctx.slate.episode);
setEssentialProperty(slateLayer, "Scene", ctx.slate.scene);
setEssentialProperty(slateLayer, "Frames", result.frameCount);
setSlateField(slateLayer, ctx.slateScopeProp, ctx.slate.vfxScope, "VFX Scope");
// The slate's Notes field carries this submission's note; when the
// artist left it blank, fall back to the shot's own notes so the
// slate looks the same as an artist-built preview.
setSlateField(slateLayer, ctx.slateSubmissionProp,
(ctx.slate.submissionNote !== null &&
ctx.slate.submissionNote !== undefined &&
ctx.slate.submissionNote !== "")
? ctx.slate.submissionNote
: ctx.slate.notes,
"Submission Note");
} else {
warn("NETFLIX_SLATE layer not found in template — slate fields not filled");
}
var shotLayer = previewComp.layer("SHOT");
var targetOutPoint = shotLayer.inPoint + shotComp.duration;
if (targetOutPoint > previewComp.duration) {
previewComp.duration = targetOutPoint;
}
shotLayer.outPoint = targetOutPoint;
previewComp.duration = targetOutPoint;
result.previewComp = previewComp.name;
// 6. Queue both outputs against the same preview comp. aerender will
// render the whole queue in one launch.
app.project.renderQueue.showWindow(false);
while (app.project.renderQueue.numItems > 0) {
app.project.renderQueue.item(1).remove();
}
queueOutput(previewComp, ctx.movTemplate, ctx.movOutput, "mov");
queueOutput(previewComp, ctx.mp4Template, ctx.mp4Output, "mp4");
if (result.queued.length < 1) {
throw new Error("Nothing was queued — check output module templates");
}
// 7. Save the throwaway project the worker will hand to aerender
var tempFile = new File(ctx.tempAep);
var tempParent = tempFile.parent;
if (tempParent && !tempParent.exists) {
tempParent.create();
}
app.project.save(tempFile);
result.tempAep = ctx.tempAep;
result.ok = true;
} catch (buildError) {
result.ok = false;
result.error = buildError && buildError.toString ? buildError.toString() : String(buildError);
try {
if (buildError && buildError.line) {
result.error += " (line " + buildError.line + ")";
}
} catch (lineError) {
}
}
try {
app.endSuppressDialogs(false);
} catch (endSuppressError) {
}
if (!resultPath && ctx && ctx.resultPath) {
resultPath = ctx.resultPath;
}
if (resultPath) {
writeTextFile(resultPath, toJSON(result));
}
// -noui leaves the app running otherwise; the worker waits on exit.
try {
app.quit();
} catch (quitError) {
}
}());