feat(render-pipeline): validation, QC and delivery builder

Completes RenderPipeline2 phases 3-5. Additive throughout: with
validation.enabled=false the pipeline behaves exactly as before.

Validation (§8) runs on the worker straight after a render, while it
still holds the job and the files are local. Everything is read from the
EXR header — one 64 KB read per sampled frame, no pixel decode — so the
engine needs no oiiotool. The render job deliberately stays open through
the stage, so a worker that dies there is reaped and retried instead of
stranding the export in VALIDATING.

Timecode is the primary check and is anchored on the plate, never the
API: measured on the live show, Shot.timecodeStart is the editorial
in-point and sits one handle-length (8f) ahead of the plate, so checking
against it would pass the defective exports and fail the correct ones.
Reference precedence is manifest timing plate -> plate found on disk ->
API value, and the API rank is WARN-only. The panel now resolves plate
provenance by walking the comp's own footage layers, so the worker never
guesses which element carries the comp's timing.

Verified against real studio renders via the new --validate mode: it
catches both defect classes seen in shipped work — the +8 handle offset
and frame-derived timecode (00:00:41:17 = frame 1001 at 24fps).

QC (§10) is one service behind two front doors, API-key for the panel
and session for the web. PASS means ready to upload and nothing else;
FAIL sets the comp task to CHANGES. The panel gains a QC section that
imports the real EXR sequence at full res rather than the H.264.

Delivery (§11) rides the existing queue as a DELIVERY_BUILD job: hard
links with a copy fallback, verification by re-hashing delivered bytes
against the validation-time digest, manifest.json and the CSV report.
fsutil hardlink list is unavailable over SMB, so link identity is never
queried — reading the bytes back covers links and copies alike.

Also fixes ApiClient.ProgressAsync throwing on transport failure, which
would have aborted work in flight during a brief server outage; progress
reporting is best-effort by contract.

Removes the stale RenderWorker/ rule from the root .gitignore. The
worker's source is tracked like the rest of the repo and its own
.gitignore already covers bin/, obj/ and publish/; the rule silently
excluded new files while the existing 20 stayed tracked.

Not yet applied: migration 20260813000000 was hand-written because the
dev database was unavailable — run prisma migrate deploy and verify
before relying on it. None of this has run against a live server.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
twotalesanimation
2026-08-13 13:49:51 +02:00
parent 7f2f5e1005
commit feb1c7b3e9
48 changed files with 5786 additions and 238 deletions
+155 -1
View File
@@ -14,15 +14,20 @@ public sealed class JobExecutor
private readonly ApiClient _api;
private readonly AerenderRunner _runner;
private readonly PreviewStage _preview;
private readonly ValidationStage _validation;
private readonly DeliveryStage _delivery;
private readonly ReportSpooler _spooler;
private readonly ILogger<JobExecutor> _log;
public JobExecutor(WorkerOptions options, ApiClient api, AerenderRunner runner, PreviewStage preview, ReportSpooler spooler, ILogger<JobExecutor> log)
public JobExecutor(WorkerOptions options, ApiClient api, AerenderRunner runner, PreviewStage preview,
ValidationStage validation, DeliveryStage delivery, ReportSpooler spooler, ILogger<JobExecutor> log)
{
_options = options;
_api = api;
_runner = runner;
_preview = preview;
_validation = validation;
_delivery = delivery;
_spooler = spooler;
_log = log;
}
@@ -35,6 +40,12 @@ public sealed class JobExecutor
return;
}
if (job.Type == "DELIVERY_BUILD")
{
await RunDeliveryAsync(job, machineId, cancelJob, shutdown);
return;
}
var manifest = job.AsRenderManifest();
var mapper = new PathMapper(_options.PathMappings);
var outputDirLocal = mapper.Map(manifest.OutputDir);
@@ -101,6 +112,15 @@ public sealed class JobExecutor
["exrFileCount"] = fileCount,
["exrTotalBytes"] = totalBytes,
});
// Validation runs here, while we still hold the job and the
// files are local. It is queued behind complete-render, and the
// spooler's in-order replay is what guarantees the server has
// moved the Export to VALIDATING before this lands (§8).
if (config.ValidationEnabled)
{
await RunValidationAsync(job, manifest, machineId, outputDirLocal, config, shutdown);
}
}
else if (result.ErrorMessage == "Cancelled")
{
@@ -149,6 +169,140 @@ public sealed class JobExecutor
}
}
/// <summary>
/// DELIVERY_BUILD job: build the dated package folder from already-QC'd
/// exports. No AE involved — folders, links and verification only.
/// </summary>
private async Task RunDeliveryAsync(ClaimedJob job, string machineId, CancellationToken cancelJob, CancellationToken shutdown)
{
var manifest = job.AsDeliveryManifest();
WriteState(new CurrentJobState
{
JobId = job.Id,
ExportId = null,
MachineId = machineId,
OutputDirLocal = new PathMapper(_options.PathMappings).Map(manifest.PackageDir),
ClaimedAt = DateTimeOffset.UtcNow,
});
try
{
var result = await _delivery.RunAsync(job, manifest, machineId, cancelJob, shutdown);
_log.LogInformation("Delivery {Name}: {Ok} ({Verified}/{Total} shots verified)",
manifest.PackageName, result.Ok ? "READY" : "FAILED",
result.Items.Count(i => i.Status == "VERIFIED"), manifest.Items.Count);
string? reportKey = null;
if (result.ReportPath is not null)
{
reportKey = await _api.UploadArtifactAsync(job.Id, machineId, "report", result.ReportPath, "text/csv", shutdown);
}
_spooler.Enqueue("delivery", job.Id, new Dictionary<string, object?>
{
["machineId"] = machineId,
["ok"] = result.Ok,
["rootPath"] = result.PackageDir,
["reportKey"] = reportKey,
["items"] = result.Items.Select(i => new Dictionary<string, object?>
{
["itemId"] = i.ItemId,
["status"] = i.Status,
["exrPath"] = i.ExrPath,
["movPath"] = i.MovPath,
["fileCount"] = i.FileCount,
["totalBytes"] = i.TotalBytes,
["checksum"] = i.Checksum,
["linkMode"] = i.LinkMode,
["errorMessage"] = i.ErrorMessage,
}).ToList(),
["manifest"] = new Dictionary<string, object?>
{
["packageDir"] = result.PackageDir,
["reportPath"] = result.ReportPath,
["files"] = result.Files,
},
["errorMessage"] = result.Ok
? null
: string.Join("; ", result.Items.Where(i => i.Status == "FAILED").Select(i => $"{i.ShotCode}: {i.ErrorMessage}").Take(5)),
});
}
catch (Exception ex)
{
_log.LogError(ex, "Unexpected delivery error for job {JobId}", job.Id);
_spooler.Enqueue("delivery", job.Id, new Dictionary<string, object?>
{
["machineId"] = machineId,
["ok"] = false,
["items"] = new List<object>(),
["errorMessage"] = $"Worker exception: {ex.Message}",
});
}
finally
{
ClearState();
}
}
/// <summary>
/// Runs the validation engine and spools the E12 report. Deliberately
/// non-fatal on its own errors: a crash in the checks must not lose a
/// finished render, so it is reported as a failed validation run rather
/// than being allowed to escape.
/// </summary>
private async Task RunValidationAsync(
ClaimedJob job, RenderManifest manifest, string machineId, string outputDirLocal,
ServerConfig config, CancellationToken shutdown)
{
var started = DateTimeOffset.UtcNow;
try
{
// Keep the lease alive: the job stays open through validation.
await _api.ProgressAsync(job.Id, machineId, 1.0, manifest.FrameEnd, manifest.TotalFrames, 0,
"Validating rendered sequence…", shutdown);
var outcome = await Task.Run(() => _validation.Run(manifest, outputDirLocal, config), shutdown);
var seconds = (DateTimeOffset.UtcNow - started).TotalSeconds;
var failed = outcome.Checks.Where(c => c.Status == "FAIL").Select(c => c.CheckName).ToList();
if (failed.Count > 0)
{
_log.LogWarning("Job {JobId} validation FAILED in {Sec:F1}s: {Checks}", job.Id, seconds, string.Join(", ", failed));
}
else
{
_log.LogInformation("Job {JobId} validation passed in {Sec:F1}s ({Count} checks, {Files} files)",
job.Id, seconds, outcome.Checks.Count, outcome.FileCount);
}
_spooler.Enqueue("validation", job.Id, new Dictionary<string, object?>
{
["machineId"] = machineId,
["checks"] = outcome.Checks,
["checksum"] = outcome.Checksum,
["checksumAlgo"] = "xxh64",
["exrFileCount"] = outcome.FileCount,
["exrTotalBytes"] = outcome.TotalBytes,
["validationSeconds"] = (int)seconds,
});
}
catch (Exception ex)
{
_log.LogError(ex, "Validation engine error for job {JobId}", job.Id);
_spooler.Enqueue("validation", job.Id, new Dictionary<string, object?>
{
["machineId"] = machineId,
["checks"] = new[]
{
ValidationCheck.Fail("engine_error", "validation completes", "worker exception",
$"The validation engine threw: {ex.Message}"),
},
});
}
}
/// <summary>
/// PREVIEW_ONLY job: rebuild the shot around the rendered EXRs with slate
/// and burn-ins, render the delivery MOV + review MP4, upload the review