feat(render-pipeline): validation, QC and delivery builder

Completes RenderPipeline2 phases 3-5. Additive throughout: with
validation.enabled=false the pipeline behaves exactly as before.

Validation (§8) runs on the worker straight after a render, while it
still holds the job and the files are local. Everything is read from the
EXR header — one 64 KB read per sampled frame, no pixel decode — so the
engine needs no oiiotool. The render job deliberately stays open through
the stage, so a worker that dies there is reaped and retried instead of
stranding the export in VALIDATING.

Timecode is the primary check and is anchored on the plate, never the
API: measured on the live show, Shot.timecodeStart is the editorial
in-point and sits one handle-length (8f) ahead of the plate, so checking
against it would pass the defective exports and fail the correct ones.
Reference precedence is manifest timing plate -> plate found on disk ->
API value, and the API rank is WARN-only. The panel now resolves plate
provenance by walking the comp's own footage layers, so the worker never
guesses which element carries the comp's timing.

Verified against real studio renders via the new --validate mode: it
catches both defect classes seen in shipped work — the +8 handle offset
and frame-derived timecode (00:00:41:17 = frame 1001 at 24fps).

QC (§10) is one service behind two front doors, API-key for the panel
and session for the web. PASS means ready to upload and nothing else;
FAIL sets the comp task to CHANGES. The panel gains a QC section that
imports the real EXR sequence at full res rather than the H.264.

Delivery (§11) rides the existing queue as a DELIVERY_BUILD job: hard
links with a copy fallback, verification by re-hashing delivered bytes
against the validation-time digest, manifest.json and the CSV report.
fsutil hardlink list is unavailable over SMB, so link identity is never
queried — reading the bytes back covers links and copies alike.

Also fixes ApiClient.ProgressAsync throwing on transport failure, which
would have aborted work in flight during a brief server outage; progress
reporting is best-effort by contract.

Removes the stale RenderWorker/ rule from the root .gitignore. The
worker's source is tracked like the rest of the repo and its own
.gitignore already covers bin/, obj/ and publish/; the rule silently
excluded new files while the existing 20 stayed tracked.

Not yet applied: migration 20260813000000 was hand-written because the
dev database was unavailable — run prisma migrate deploy and verify
before relying on it. None of this has run against a live server.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
twotalesanimation
2026-08-13 13:49:51 +02:00
parent 7f2f5e1005
commit feb1c7b3e9
48 changed files with 5786 additions and 238 deletions
@@ -0,0 +1,177 @@
using Microsoft.Extensions.Logging.Abstractions;
using VFXReviewWorker;
using Xunit;
namespace VFXReviewWorker.Tests;
/// <summary>
/// Packaging is the last step before files leave the studio, so the parts that
/// could silently ship the wrong thing — checksum verification, frame-count
/// drift, hard-link vs copy — are pinned here.
/// </summary>
public class DeliveryStageTests : IDisposable
{
private readonly string _root = Path.Combine(Path.GetTempPath(), $"vfxr_delivery_{Guid.NewGuid():N}");
private readonly string _sourceDir;
private readonly string _packageDir;
public DeliveryStageTests()
{
_sourceDir = Path.Combine(_root, "renders", "SHOT_001", "v001");
_packageDir = Path.Combine(_root, "DELIVERY", "101", "260813_Delivery");
Directory.CreateDirectory(_sourceDir);
}
public void Dispose()
{
try { Directory.Delete(_root, recursive: true); } catch { }
}
private void WriteFrames(int count, int startFrame = 1001)
{
for (var i = 0; i < count; i++)
{
var name = $"SHOT_001_cmp_TT_v001.{startFrame + i:D5}.exr";
File.WriteAllText(Path.Combine(_sourceDir, name), $"frame {startFrame + i} pixels");
}
}
private DeliveryManifest Manifest(string? checksum = null, int fileCount = 3, string copyMode = "auto") => new()
{
PackageId = "pkg1",
PackageName = "260813_Delivery",
PackageNumber = 1,
Episode = "101",
PackageDir = _packageDir,
ReportFile = Path.Combine(_packageDir, "260813_Delivery_report.csv"),
CopyMode = copyMode,
Items = new List<DeliveryManifestItem>
{
new()
{
ItemId = "item1",
ExportId = "exp1",
ShotCode = "SHOT_001",
VersionString = "v001",
SourceDir = _sourceDir,
OutputPattern = "SHOT_001_cmp_TT_v001.[#####].exr",
FrameStart = 1001,
FrameEnd = 1000 + fileCount,
FileCount = fileCount,
Checksum = checksum,
ShotDir = Path.Combine(_packageDir, "SHOT_001"),
},
},
};
private DeliveryStage NewStage() => new(
new WorkerOptions { ServerUrl = "n/a", ApiKey = "n/a" },
new ApiClient(new WorkerOptions { ServerUrl = "http://localhost:1", ApiKey = "x" }, NullLogger<ApiClient>.Instance),
NullLogger<DeliveryStage>.Instance);
[Fact]
public async Task PackagesFramesAndWritesReport()
{
WriteFrames(3);
var result = await NewStage().RunAsync(new ClaimedJob { Id = "job1" }, Manifest(), "machine1",
CancellationToken.None, CancellationToken.None);
Assert.True(result.Ok);
var item = Assert.Single(result.Items);
Assert.Equal("VERIFIED", item.Status);
Assert.Equal(3, item.FileCount);
var delivered = Directory.GetFiles(Path.Combine(_packageDir, "SHOT_001"), "*.exr");
Assert.Equal(3, delivered.Length);
Assert.True(File.Exists(Path.Combine(_packageDir, "manifest.json")));
var report = File.ReadAllText(Path.Combine(_packageDir, "260813_Delivery_report.csv"));
Assert.Contains("Shot,Version,FrameStart", report);
Assert.Contains("SHOT_001,v001,1001,1003,3,3", report);
}
[Fact]
public async Task HardLinksOnTheSameVolumeSoNoExtraBytesAreWritten()
{
WriteFrames(2);
var result = await NewStage().RunAsync(new ClaimedJob { Id = "job1" }, Manifest(fileCount: 2), "machine1",
CancellationToken.None, CancellationToken.None);
Assert.Equal("hardlink", result.Items[0].LinkMode);
// A hard link is the same file: writing through one name is visible via
// the other. That is what makes packaging near-instant and free.
var source = Path.Combine(_sourceDir, "SHOT_001_cmp_TT_v001.01001.exr");
var linked = Path.Combine(_packageDir, "SHOT_001", "SHOT_001_cmp_TT_v001.01001.exr");
File.WriteAllText(source, "rewritten");
Assert.Equal("rewritten", File.ReadAllText(linked));
}
[Fact]
public async Task AlwaysCopyModeProducesIndependentFiles()
{
WriteFrames(2);
var result = await NewStage().RunAsync(new ClaimedJob { Id = "job1" }, Manifest(fileCount: 2, copyMode: "always-copy"),
"machine1", CancellationToken.None, CancellationToken.None);
Assert.Equal("copy", result.Items[0].LinkMode);
var source = Path.Combine(_sourceDir, "SHOT_001_cmp_TT_v001.01001.exr");
var copied = Path.Combine(_packageDir, "SHOT_001", "SHOT_001_cmp_TT_v001.01001.exr");
File.WriteAllText(source, "rewritten");
Assert.NotEqual("rewritten", File.ReadAllText(copied));
}
[Fact]
public async Task ChecksumMismatchFailsTheItem()
{
WriteFrames(3);
var result = await NewStage().RunAsync(new ClaimedJob { Id = "job1" },
Manifest(checksum: "deadbeefdeadbeef"), "machine1", CancellationToken.None, CancellationToken.None);
Assert.False(result.Ok);
Assert.Equal("FAILED", result.Items[0].Status);
Assert.Contains("Checksum mismatch", result.Items[0].ErrorMessage!);
}
[Fact]
public async Task FrameCountDriftSinceQcFailsTheItem()
{
// The export recorded 3 frames at validation; the folder now has 2.
WriteFrames(2);
var result = await NewStage().RunAsync(new ClaimedJob { Id = "job1" }, Manifest(fileCount: 3), "machine1",
CancellationToken.None, CancellationToken.None);
Assert.False(result.Ok);
Assert.Contains("the render changed since QC", result.Items[0].ErrorMessage!);
}
[Fact]
public async Task MissingSourceFolderFailsWithThePath()
{
var manifest = Manifest();
manifest.Items[0].SourceDir = Path.Combine(_root, "does-not-exist");
var result = await NewStage().RunAsync(new ClaimedJob { Id = "job1" }, manifest, "machine1",
CancellationToken.None, CancellationToken.None);
Assert.False(result.Ok);
Assert.Contains("Source folder not found", result.Items[0].ErrorMessage!);
}
[Fact]
public async Task ChecksumMatchesTheValidationTimeDigestForIdenticalContent()
{
WriteFrames(3);
var first = await NewStage().RunAsync(new ClaimedJob { Id = "job1" }, Manifest(), "machine1",
CancellationToken.None, CancellationToken.None);
// Feeding the recorded digest back in must verify, which is exactly what
// the delivery builder does against Export.checksum.
Directory.Delete(_packageDir, recursive: true);
var second = await NewStage().RunAsync(new ClaimedJob { Id = "job1" },
Manifest(checksum: first.Items[0].Checksum), "machine1", CancellationToken.None, CancellationToken.None);
Assert.True(second.Ok);
}
}
@@ -0,0 +1,193 @@
using VFXReviewWorker;
using Xunit;
namespace VFXReviewWorker.Tests;
/// <summary>
/// The timecode check is the reason the validation engine exists (§8.1a), so
/// the header parse and the timecode arithmetic are pinned here against a
/// synthetic EXR built byte by byte.
/// </summary>
public class ExrHeaderTests
{
[Fact]
public void ReadsTimecodeResolutionChannelsAndFps()
{
var path = ExrFixture.Write(timecode: "18:58:57:12", width: 1920, height: 1080,
channels: new[] { ("A", 1), ("B", 1), ("G", 1), ("R", 1) }, fpsNum: 24, compression: 4);
try
{
var header = ExrHeader.Read(path);
Assert.Equal(new Timecode(18, 58, 57, 12, false), header.Timecode);
Assert.Equal(1920, header.DataWidth);
Assert.Equal(1080, header.DataHeight);
Assert.Equal(24, header.FramesPerSecond);
Assert.Equal("half", header.BitDepth);
Assert.True(header.HasAlpha);
Assert.Equal("PIZ", header.CompressionName);
}
finally { File.Delete(path); }
}
[Fact]
public void MissingTimecodeIsReportedAsAbsentNotAsZero()
{
var path = ExrFixture.Write(timecode: null, width: 1920, height: 1080,
channels: new[] { ("B", 1), ("G", 1), ("R", 1) }, fpsNum: 24, compression: 3);
try
{
var header = ExrHeader.Read(path);
Assert.Null(header.Timecode);
Assert.False(header.HasAlpha);
}
finally { File.Delete(path); }
}
[Fact]
public void NonExrFileThrows()
{
var path = Path.GetTempFileName();
try
{
File.WriteAllText(path, "this is not an exr at all, not even close");
Assert.Throws<InvalidDataException>(() => ExrHeader.Read(path));
Assert.Null(ExrHeader.TryRead(path));
}
finally { File.Delete(path); }
}
[Theory]
// The BCD packing means the raw value reads back as its own hex digits.
[InlineData(0x18585712u, 18, 58, 57, 12)]
[InlineData(0x00004117u, 0, 0, 41, 17)]
[InlineData(0x00000000u, 0, 0, 0, 0)]
public void DecodesPackedSmpteTimecode(uint packed, int hh, int mm, int ss, int ff)
{
Assert.Equal(new Timecode(hh, mm, ss, ff, false), Timecode.FromPacked(packed));
}
[Fact]
public void FrameArithmeticRoundTrips()
{
var tc = new Timecode(18, 58, 57, 12, false);
Assert.Equal(tc, Timecode.FromFrames(tc.ToFrames(24), 24));
// 133 frames on from 18:58:57:12 at 24 fps is 18:59:03:01 — the real
// first/last frame pair measured on UNG_102_019_010 v005.
Assert.Equal(new Timecode(18, 59, 3, 1, false), Timecode.FromFrames(tc.ToFrames(24) + 133, 24));
}
[Fact]
public void ApiTimecodeIsOneHandleAheadOfThePlate()
{
// The defect class from §8.1a: the API stores the editorial in-point and
// the plate starts 8 frames earlier, so a comp anchored to the API value
// renders 8 frames late.
var plate = Timecode.Parse("13:44:06:02")!.Value;
var api = Timecode.Parse("13:44:06:10")!.Value;
Assert.Equal(8, api.ToFrames(24) - plate.ToFrames(24));
}
[Fact]
public void FrameDerivedTimecodeIsFrameNumberOverFps()
{
// 00:00:41:17 is frame 1001 at 24 fps — the signature of a comp built
// with no source timecode at all.
Assert.Equal(1001, Timecode.Parse("00:00:41:17")!.Value.ToFrames(24));
}
[Theory]
[InlineData("UNG_102_019_010_cmp_TT_v005.01001.exr", 1001)]
[InlineData("UNG_108_030_010_cmp_TT_v003.01204.exr", 1204)]
[InlineData("no_frame_number.exr", null)]
public void ParsesFrameNumbersFromFilenames(string name, int? expected)
{
Assert.Equal(expected, SequencePattern.FrameFromName(name));
}
[Fact]
public void ParsesOutputPattern()
{
var p = SequencePattern.FromOutputPattern("UNG_102_019_010_cmp_TT_v005.[#####].exr");
Assert.Equal("UNG_102_019_010_cmp_TT_v005", p.Prefix);
Assert.Equal(".exr", p.Extension);
Assert.Equal(5, p.Padding);
}
}
/// <summary>Builds a minimal but structurally real EXR header on disk.</summary>
internal static class ExrFixture
{
public static string Write(string? timecode, int width, int height,
(string Name, int PixelType)[] channels, int fpsNum, byte compression)
{
var path = Path.Combine(Path.GetTempPath(), $"vfxr_test_{Guid.NewGuid():N}.exr");
using var fs = new FileStream(path, FileMode.Create, FileAccess.Write);
using var w = new BinaryWriter(fs);
w.Write(ExrHeader.Magic);
w.Write(2); // version 2, no flags
// channels (chlist)
using (var chData = new MemoryStream())
using (var cw = new BinaryWriter(chData))
{
foreach (var (name, pixelType) in channels)
{
WriteCString(cw, name);
cw.Write(pixelType);
cw.Write((byte)0); // pLinear
cw.Write(new byte[3]); // reserved
cw.Write(1); // xSampling
cw.Write(1); // ySampling
}
cw.Write((byte)0); // list terminator
WriteAttr(w, "channels", "chlist", chData.ToArray());
}
WriteAttr(w, "compression", "compression", new[] { compression });
var window = new byte[16];
BitConverter.GetBytes(0).CopyTo(window, 0);
BitConverter.GetBytes(0).CopyTo(window, 4);
BitConverter.GetBytes(width - 1).CopyTo(window, 8);
BitConverter.GetBytes(height - 1).CopyTo(window, 12);
WriteAttr(w, "dataWindow", "box2i", window);
WriteAttr(w, "displayWindow", "box2i", window);
var rational = new byte[8];
BitConverter.GetBytes(fpsNum).CopyTo(rational, 0);
BitConverter.GetBytes(1u).CopyTo(rational, 4);
WriteAttr(w, "framesPerSecond", "rational", rational);
if (timecode is not null)
{
var parts = timecode.Split(':');
uint Bcd(int v) => (uint)((v / 10) * 16 + v % 10);
var packed = (Bcd(int.Parse(parts[0])) << 24) | (Bcd(int.Parse(parts[1])) << 16)
| (Bcd(int.Parse(parts[2])) << 8) | Bcd(int.Parse(parts[3]));
var tcData = new byte[8];
BitConverter.GetBytes(packed).CopyTo(tcData, 0);
WriteAttr(w, "timeCode", "timecode", tcData);
WriteAttr(w, "timeCodeString", "string", System.Text.Encoding.ASCII.GetBytes(timecode));
}
w.Write((byte)0); // end of header
return path;
}
private static void WriteAttr(BinaryWriter w, string name, string type, byte[] data)
{
WriteCString(w, name);
WriteCString(w, type);
w.Write(data.Length);
w.Write(data);
}
private static void WriteCString(BinaryWriter w, string s)
{
w.Write(System.Text.Encoding.ASCII.GetBytes(s));
w.Write((byte)0);
}
}
@@ -0,0 +1,61 @@
using System.Text;
using VFXReviewWorker;
using Xunit;
namespace VFXReviewWorker.Tests;
/// <summary>
/// Reference vectors from the xxHash specification. The delivery builder
/// verifies packaged files against digests recorded here at validation time, so
/// a wrong implementation would not be caught by round-tripping our own code —
/// only by pinning the published values.
/// </summary>
public class XxHash64Tests
{
[Theory]
[InlineData("", 0xEF46DB3751D8E999UL)]
[InlineData("a", 0xD24EC4F1A98C6E5BUL)]
[InlineData("abc", 0x44BC2CF5AD770999UL)]
public void MatchesReferenceVectors(string input, ulong expected)
{
Assert.Equal(expected, XxHash64.Hash(Encoding.ASCII.GetBytes(input)));
}
[Fact]
public void StreamingInChunksMatchesSingleShot()
{
var data = new byte[1000];
new Random(42).NextBytes(data);
var oneShot = XxHash64.Hash(data);
// Chunk sizes deliberately straddle the 32-byte stripe boundary.
foreach (var chunk in new[] { 1, 7, 31, 32, 33, 64, 999 })
{
var state = new XxHash64.State(0);
for (var i = 0; i < data.Length; i += chunk)
{
state.Update(new ReadOnlySpan<byte>(data, i, Math.Min(chunk, data.Length - i)));
}
Assert.Equal(oneShot, state.Digest());
}
}
[Fact]
public void HashFileIsStableAndContentSensitive()
{
var path = Path.GetTempFileName();
try
{
File.WriteAllText(path, "frame data");
var first = XxHash64.HashFile(path);
Assert.Equal(first, XxHash64.HashFile(path));
File.WriteAllText(path, "frame datb");
Assert.NotEqual(first, XxHash64.HashFile(path));
}
finally
{
File.Delete(path);
}
}
}