3 Commits

Author SHA1 Message Date
twotalesanimation 708e8a0d8f fix(worker): parse Blender 5.x output so rendered frames are counted
Deploy / deploy (push) Successful in 3m28s
Blender 5 logs through CLOG, prefixing every line with elapsed time and a
category ("00:02.954  render | Saved: '...'"), and reports errors as
"ERROR ..." instead of "Error: ...". The parser anchored on "Saved:" at the
start of the line, so it counted no frames from a render that wrote all of
them and the job failed as "saved no frames". Progress (Fra:) and error
detection were dead on 5.x for the same reason.

The prefix is now stripped before matching; 4.x output parses as before.
Warnings no longer count as errors: add-ons print them on every launch and
one would otherwise be reported as the reason a render failed. A run that
saves nothing now reports Blender's own error when there is one.

Verified on the real DH2 test.blend under Blender 5.2: the 1.6.2 parser
counts 0 of 3 saved frames, this one counts 3.

Bump worker to 1.6.3.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-18 04:42:33 +02:00
twotalesanimation 97246bcbc3 fix(worker): Blender jobs always failed with "Collection was modified"
LogTail was read by the progress loop while the output pumps appended to
it. Blender rewrites its status line many times a second, so the first
progress report of every job threw; aerender hit the same race only rarely.
LogTail now locks internally.

The exception also escaped BlenderRunner without ending the process, leaving
blender.exe rendering unsupervised on the node on every failed attempt. Any
unexpected exception in the monitor loop now kills the process tree.

Bump worker to 1.6.2.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-18 04:21:54 +02:00
twotalesanimation 400cf3f6ea feat(validation): try the comp's other delivered plates before failing timecode
When the timing plate disagrees with the render, the worker now checks the
manifest's other plates[] that sit under a plate search root (or beside the
timing plate's footage folder). A match is a PASS naming that element; no
match still FAILs against the timing plate, listing the alternates tried.
Pre-renders are excluded because they agree with the render by construction.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-18 04:21:53 +02:00
9 changed files with 346 additions and 85 deletions
+2
View File
@@ -1073,6 +1073,8 @@ Timecode is the check this engine exists for. It is validated **against the sour
| 2 | Plate re-derived on the worker from the footage root + shot code (manifestVersion 1, or a timing plate that no longer resolves). If elements disagree, the one matching the render's frame count wins; still ambiguous → WARN, never PASS | PASS / FAIL, `reference: "derived"` |
| 3 | API `Shot.timecodeStart` **minus `handleFrames`** | **WARN only, never PASS** — `reference: "api"`, message states the plate could not be read |
**Several delivered elements.** When the rank-1 timing plate does not match, the worker tries the manifest's other `plates[]` before failing — the panel's longest-duration pick is not always the element the comp was anchored to. A match against one of them is a PASS naming that element; no match still FAILs against the timing plate, listing the others tried. Only delivered footage qualifies: plates under a configured plate search root, or (with none configured) alongside the timing plate in its footage folder. `plates[]` also holds pre-renders, which agree with the render by construction and must never rescue it.
Rank 3 is a diagnostic, not a verification: it depends on the show's handle count being correct for that specific plate, which is exactly the assumption the defects above violate.
**Comparison, given the plate reference:**
@@ -73,6 +73,46 @@ public class BlenderProgressParserTests
Assert.Equal(1002, frames[^1]);
}
/// <summary>
/// Blender 5.x prefixes every line with elapsed time and a log category.
/// Anchored on "Saved:", the parser counted no frames from a render that
/// wrote every one, and the job failed as "saved no frames".
/// These lines are verbatim Blender 5.2 output.
/// </summary>
[Fact]
public void ParsesBlender5PrefixedOutput()
{
Assert.Equal(@"X:\renders\frame_0001.png",
BlenderProgressParser.ParseSaved(@"00:02.954 render | Saved: 'X:\renders\frame_0001.png'"));
Assert.Equal(1, BlenderProgressParser.ParseFrame("00:02.469 render | Fra: 1 | Mem: 0M | Synchronizing object | Cube"));
Assert.Null(BlenderProgressParser.ParseSaved("00:02.954 render | Time: 00:00.87 (Saving: 00:00.14)"));
Assert.Null(BlenderProgressParser.ParseFrame("00:02.079 render | Rendering frame 1"));
}
[Fact]
public void RecognisesBlender5Errors()
{
const string unwritable = @"00:03.063 reports | ERROR Render error (No such file or directory) cannot save: 'Q:\out\f_0001.png'";
Assert.True(BlenderProgressParser.IsErrorLine(unwritable));
Assert.True(BlenderProgressParser.IsNonRetryableError(unwritable));
const string missing = @"00:13.250 reports | ERROR Cannot read file ""X:\gone.blend"": No such file or directory";
Assert.True(BlenderProgressParser.IsErrorLine(missing));
Assert.True(BlenderProgressParser.IsNonRetryableError(missing));
}
/// <summary>
/// Add-ons print warnings on every launch; one must never become the
/// reported reason a render failed.
/// </summary>
[Theory]
[InlineData("Warning: add-on missing 'bl_info', this can cause poor performance!: 'G:\\addons\\x\\__init__.py'")]
[InlineData("00:01.000 addon | WARNING something harmless")]
public void WarningsAreNotErrors(string line)
{
Assert.False(BlenderProgressParser.IsErrorLine(line));
}
/// <summary>Mirrors BlenderRunner.PumpAsync: CR and LF both end a line.</summary>
private static List<string> SplitLikeThePump(string text)
{
@@ -117,4 +117,21 @@ public class LogTailTests
for (var i = 1; i <= 5; i++) tail.Add($"line {i}");
Assert.Equal("line 3\nline 4\nline 5", tail.ToString());
}
[Fact]
public async Task ReadingWhileAnotherThreadWritesDoesNotThrow()
{
// The Blender runner's output pumps add lines while its progress loop
// reads the tail; unsynchronised this threw "Collection was modified".
var tail = new LogTail(200);
using var stop = new CancellationTokenSource(TimeSpan.FromSeconds(1));
var writer = Task.Run(() =>
{
var i = 0;
while (!stop.IsCancellationRequested) tail.Add($"Fra:1001 Sample {i++}/128");
});
while (!stop.IsCancellationRequested) _ = tail.ToString();
await writer;
}
}
@@ -109,6 +109,74 @@ public class ValidationStageTests : IDisposable
Assert.Contains("8-frame handle", tc.Message);
}
/// <summary>Writes a one-frame plate into a footage-style element folder.</summary>
private string WritePlate(string element, string timecode, int fps = 24)
{
var plate = Path.Combine(_dir, "footage", $"UNG_107_051_010_{element}_TT_v001", $"UNG_107_051_010_{element}_TT_v001.01001.exr");
Directory.CreateDirectory(Path.GetDirectoryName(plate)!);
File.Move(ExrFixture.Write(timecode, 3840, 2160, new[] { ("B", 1), ("G", 1), ("R", 1) }, fps, 4), plate);
return plate;
}
private static ManifestSource Source(string timingPlate, params string[] plates) => new()
{
TimingPlate = new PlateRef { Path = timingPlate, FrameStart = 1001 },
Plates = plates.Select(p => new PlateRef { Path = p, FrameStart = 1001 }).ToList(),
HandleFrames = 8,
};
[Fact]
public void AnotherDeliveredPlatePassesWhenTheTimingPlateDoesNot()
{
// The comp was anchored to FG01, but the panel named BG01 (longest) as
// the timing plate — the two carry unrelated camera timecodes.
WriteSequence(105, "09:12:00:00");
var bg = WritePlate("BG01", "14:26:31:08");
var fg = WritePlate("FG01", "09:12:00:00");
var manifest = Manifest(_dir, null, 1001);
manifest.Source = Source(bg, bg, fg);
var tc = Check(NewStage().Run(manifest, _dir, new ServerConfig()), "timecode_plate_match");
Assert.Equal("PASS", tc.Status);
Assert.Contains("UNG_107_051_010_FG01_TT_v001.01001.exr", tc.Message);
Assert.Contains("BG01", tc.Message);
}
[Fact]
public void NoMatchingPlateStillFailsAgainstTheTimingPlate()
{
WriteSequence(105, "14:26:31:16");
var bg = WritePlate("BG01", "14:26:31:08");
var fg = WritePlate("FG01", "09:12:00:00");
var manifest = Manifest(_dir, null, 1001);
manifest.Source = Source(bg, bg, fg);
var tc = Check(NewStage().Run(manifest, _dir, new ServerConfig()), "timecode_plate_match");
Assert.Equal("FAIL", tc.Status);
Assert.Equal("14:26:31:08", tc.Expected);
Assert.Contains("8-frame handle", tc.Message);
Assert.Contains("1 other delivered plate", tc.Message);
}
[Fact]
public void APreRenderIsNeverUsedAsTheFallbackPlate()
{
// plates[] is every file in the comp. A pre-render was rendered from
// this comp, so it agrees with the render whether or not either is right.
WriteSequence(105, "14:26:31:16");
var bg = WritePlate("BG01", "14:26:31:08");
var preRender = Path.Combine(_dir, "renders", "prerender", "UNG_107_051_010_PRERENDER_01001.exr");
Directory.CreateDirectory(Path.GetDirectoryName(preRender)!);
File.Move(ExrFixture.Write("14:26:31:16", 3840, 2160, new[] { ("B", 1), ("G", 1), ("R", 1) }, 24, 4), preRender);
var manifest = Manifest(_dir, null, 1001);
manifest.Source = Source(bg, bg, preRender);
Assert.Equal("FAIL", Check(NewStage().Run(manifest, _dir, new ServerConfig()), "timecode_plate_match").Status);
}
[Fact]
public void APlateAtADifferentRateFailsPlateFps()
{
+16 -2
View File
@@ -280,11 +280,19 @@ public sealed class AerenderRunner
}
}
/// <summary>Ring buffer of the last N output lines (§5.3 logTail).</summary>
/// <summary>
/// Ring buffer of the last N output lines (§5.3 logTail).
///
/// Thread-safe: output readers add lines while the progress loop reads the
/// tail for its report. Unsynchronised, ToString enumerates the queue while a
/// reader enqueues and throws "Collection was modified" — Blender, which
/// writes a status line many times a second, hit this on every job.
/// </summary>
public sealed class LogTail
{
private readonly Queue<string> _lines;
private readonly int _capacity;
private readonly object _gate = new();
public LogTail(int capacity)
{
@@ -293,10 +301,16 @@ public sealed class LogTail
}
public void Add(string line)
{
lock (_gate)
{
_lines.Enqueue(line);
while (_lines.Count > _capacity) _lines.Dequeue();
}
}
public override string ToString() => string.Join('\n', _lines);
public override string ToString()
{
lock (_gate) return string.Join('\n', _lines);
}
}
@@ -8,18 +8,36 @@ namespace VFXReviewWorker;
/// with "Saved: '...'". The saved lines are the reliable signal — a frame can
/// be reported for a long time while it renders — so progress counts those and
/// falls back to Fra: for a first sign of life.
///
/// Blender 5.x logs through CLOG, which prefixes every line with an elapsed
/// time and a category ("00:02.954 render | Saved: '...'") and
/// reports errors as "ERROR ..." rather than "Error: ...". The prefix is
/// stripped before matching, so both formats parse the same way.
/// </summary>
public static partial class BlenderProgressParser
{
[GeneratedRegex(@"^Fra:(\d+)", RegexOptions.Compiled)]
[GeneratedRegex(@"^(?:\d+:)?\d{2}:\d{2}\.\d{3}\s+\S+\s*\| ?", RegexOptions.Compiled)]
private static partial Regex LogPrefix();
[GeneratedRegex(@"^Fra:\s*(\d+)", RegexOptions.Compiled)]
private static partial Regex FrameLine();
[GeneratedRegex(@"^Saved:\s*'(.+)'", RegexOptions.Compiled)]
private static partial Regex SavedLine();
[GeneratedRegex(@"^(Error|Warning): |^Error: ", RegexOptions.Compiled | RegexOptions.IgnoreCase)]
// Errors only. Warnings are not failures, and add-ons print them on every
// launch ("Warning: add-on missing 'bl_info'"), so counting them made an
// add-on's complaint the reported reason for a failed render.
[GeneratedRegex(@"(?i:^Error: )|^(ERROR|FATAL) ", RegexOptions.Compiled)]
private static partial Regex ErrorLine();
/// <summary>The line without Blender 5.x's "elapsed category | " prefix.</summary>
private static string Message(string line)
{
var m = LogPrefix().Match(line);
return m.Success ? line[m.Length..] : line;
}
/// <summary>
/// Failures that will repeat identically on another machine — a missing
/// file, an unknown scene, an unwritable output. Retrying these burns the
@@ -35,7 +53,9 @@ public static partial class BlenderProgressParser
"unknown scene",
"scene not found",
"unable to create directory",
"couldn't create directory",
"cannot write file",
"cannot save:",
"no camera",
"python script fail",
"unknown argument",
@@ -44,18 +64,18 @@ public static partial class BlenderProgressParser
/// <summary>The frame Blender is currently working on, or null.</summary>
public static int? ParseFrame(string line)
{
var m = FrameLine().Match(line);
var m = FrameLine().Match(Message(line));
return m.Success && int.TryParse(m.Groups[1].Value, out var f) ? f : null;
}
/// <summary>The path of a frame Blender has just written, or null.</summary>
public static string? ParseSaved(string line)
{
var m = SavedLine().Match(line);
var m = SavedLine().Match(Message(line));
return m.Success ? m.Groups[1].Value : null;
}
public static bool IsErrorLine(string line) => ErrorLine().IsMatch(line);
public static bool IsErrorLine(string line) => ErrorLine().IsMatch(Message(line));
public static bool IsNonRetryableError(string line)
{
+19 -1
View File
@@ -190,6 +190,21 @@ public sealed class BlenderRunner
PumpAsync(process.StandardError, line => HandleLine(line, isStderr: true)),
};
try
{
return await MonitorAsync();
}
catch
{
// Disposing a Process does not end it: without this, any exception
// in the monitor loop leaves Blender rendering on the node,
// unsupervised, while the job is retried alongside it.
KillTree(process);
throw;
}
async Task<RenderResult> MonitorAsync()
{
while (!process.HasExited)
{
await Task.Delay(TimeSpan.FromSeconds(2), CancellationToken.None);
@@ -253,13 +268,16 @@ public sealed class BlenderRunner
if (framesSaved == 0)
{
return new RenderResult(false, 0, Retryable: false,
"Blender exited cleanly but saved no frames — check the output path and frame range in the .blend",
firstError is not null
? $"Blender exited cleanly but saved no frames: {firstError}"
: "Blender exited cleanly but saved no frames — check the output path and frame range in the .blend",
tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
}
_log.LogInformation("Job {JobId}: {Saved}/{Total} frames in {Sec:F0}s", jobId, framesSaved, totalFrames, sw.Elapsed.TotalSeconds);
return new RenderResult(true, 0, true, null, tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
}
}
/// <summary>
/// Reads a stream treating both CR and LF as line terminators, so
@@ -15,7 +15,7 @@
every change that gets published. publish.ps1 appends the git SHA via
SourceRevisionId, giving "1.1.0+feb1c7b".
-->
<Version>1.6.1</Version>
<Version>1.6.3</Version>
</PropertyGroup>
<ItemGroup>
+99 -17
View File
@@ -350,6 +350,31 @@ public sealed class ValidationStage
return;
}
var deltaOf = (TimecodeReference r) =>
renderTc.ToFrames(fps) - (r.PlateTimecode.ToFrames(fps) + (renderFirstFrame - r.PlateFirstFrame));
// A shot with several delivered elements: the panel names the longest
// as the timing plate, but the comp may have been anchored to another
// (BG01 and FG01 carry unrelated camera timecodes). When the timing
// plate disagrees, the manifest's other delivered plates are tried
// before failing. The comparison is unchanged — it is still a plate, so
// a match is a real PASS and no match still FAILs against the timing plate.
var timingPlate = reference;
var alternatesTried = new List<object>();
if (reference.Kind == "manifest" && deltaOf(reference) != 0)
{
foreach (var alternate in ResolveAlternatePlates(m, renderFirstFrame, reference))
{
var altDelta = deltaOf(alternate);
alternatesTried.Add(new { platePath = alternate.PlatePath, plateTimecode = alternate.PlateTimecode.ToString(), deltaFrames = altDelta });
if (altDelta == 0)
{
reference = alternate;
break;
}
}
}
// plate_fps — the render's rate against the plate's own.
//
// timecode_fps only compares the render to the manifest, and both come
@@ -377,8 +402,8 @@ public sealed class ValidationStage
var expectedFrames = reference.PlateTimecode.ToFrames(fps) + (renderFirstFrame - reference.PlateFirstFrame);
var expected = Timecode.FromFrames(expectedFrames, fps);
var actualFrames = renderTc.ToFrames(fps);
var deltaFrames = actualFrames - expectedFrames;
var deltaFrames = deltaOf(reference);
var matchedAlternate = !ReferenceEquals(reference, timingPlate);
var details = new
{
reference = reference.Kind,
@@ -387,10 +412,14 @@ public sealed class ValidationStage
plateFirstFrame = reference.PlateFirstFrame,
renderFirstFrame,
deltaFrames,
timingPlatePath = timingPlate.PlatePath,
matchedAlternate,
alternatesTried,
};
if (deltaFrames == 0)
{
var timingDelta = deltaOf(timingPlate);
// The API fallback proves nothing on its own (§8.1a): it is the
// editorial in-point minus a handle count that may not be this
// plate's, so a match is reported as a WARN, never a PASS.
@@ -398,7 +427,11 @@ public sealed class ValidationStage
? ValidationCheck.Warn("timecode_plate_match", expected.ToString(), renderTc.ToString(),
"Matches the API timecode less handles, but the plate itself could not be read — unverified.", details)
: ValidationCheck.Pass("timecode_plate_match", expected.ToString(), renderTc.ToString(),
$"Matches {reference.Kind} plate {Path.GetFileName(reference.PlatePath ?? "")}", details));
$"Matches {reference.Kind} plate {Path.GetFileName(reference.PlatePath ?? "")}" +
(matchedAlternate
? $" (the timing plate {Path.GetFileName(timingPlate.PlatePath ?? "")} is {(timingDelta > 0 ? "+" : "")}{timingDelta} frames off — the comp was anchored to this element instead)"
: ""),
details));
return;
}
@@ -409,6 +442,9 @@ public sealed class ValidationStage
? $"That is exactly the {reference.HandleFrames}-frame handle: the comp was anchored to the editorial in-point instead of the plate's first frame."
: renderTc.Hours == 0 && renderTc.Minutes == 0
? "A near-zero timecode means the comp had no source timecode at all and AE derived it from the frame number."
: "") +
(alternatesTried.Count > 0
? $" Also checked {alternatesTried.Count} other delivered plate{(alternatesTried.Count == 1 ? "" : "s")} in the manifest: none match."
: "");
checks.Add(reference.Kind == "api"
@@ -430,20 +466,9 @@ public sealed class ValidationStage
var timing = m.Source?.TimingPlate;
if (timing is not null && !string.IsNullOrWhiteSpace(timing.Path))
{
var local = mapper.Map(timing.Path);
var header = File.Exists(local) ? ExrHeader.TryRead(local) : null;
if (header?.Timecode is not null)
{
// The plate's own file numbering, never the comp's frame range:
// the render's first frame is in output numbering, so anchoring
// on a comp frame skews the comparison by the gap between them.
var frame = SequencePattern.FrameFromName(Path.GetFileName(local))
?? timing.FrameStart
?? renderFirstFrame;
return new TimecodeReference("manifest", local, header.Timecode.Value, frame, handleFrames,
header.FramesPerSecond);
}
_log.LogWarning("Timing plate from manifest is unreadable: {Path}", local);
var manifestPlate = ReadManifestPlate(timing, mapper, handleFrames, renderFirstFrame);
if (manifestPlate is not null) return manifestPlate;
_log.LogWarning("Timing plate from manifest is unreadable: {Path}", mapper.Map(timing.Path));
}
// Rank 2 — derive the plate from a configured search root.
@@ -469,6 +494,63 @@ public sealed class ValidationStage
return null;
}
private static TimecodeReference? ReadManifestPlate(PlateRef plate, PathMapper mapper, int handleFrames, int renderFirstFrame)
{
var local = mapper.Map(plate.Path);
var header = File.Exists(local) ? ExrHeader.TryRead(local) : null;
if (header?.Timecode is null) return null;
// The plate's own file numbering, never the comp's frame range: the
// render's first frame is in output numbering, so anchoring on a comp
// frame skews the comparison by the gap between them.
var frame = SequencePattern.FrameFromName(Path.GetFileName(local))
?? plate.FrameStart
?? renderFirstFrame;
return new TimecodeReference("manifest", local, header.Timecode.Value, frame, handleFrames,
header.FramesPerSecond);
}
/// <summary>
/// The manifest's other delivered plates, tried when the timing plate does
/// not match. Only real footage qualifies — under a configured plate search
/// root or, with none configured, alongside the timing plate in its footage
/// folder — because plates[] is every file in the comp, and a pre-render
/// agrees with the render whether or not either is right.
/// </summary>
private IEnumerable<TimecodeReference> ResolveAlternatePlates(RenderManifest m, int renderFirstFrame, TimecodeReference timingPlate)
{
var plates = m.Source?.Plates;
if (plates is null || plates.Count == 0 || timingPlate.PlatePath is null) yield break;
var mapper = new PathMapper(_options.PathMappings);
static string AsFolder(string dir) => Path.TrimEndingDirectorySeparator(Path.GetFullPath(dir)) + Path.DirectorySeparatorChar;
var deliveredRoots = _options.PlateSearchRoots.Count > 0
? _options.PlateSearchRoots.Select(r => AsFolder(mapper.Map(r))).ToList()
// {footage}/{shot}_{element}/file.exr or {footage}/{shot}/{element}/file.exr:
// two levels up from the timing plate is the folder its siblings share.
: Path.GetDirectoryName(Path.GetDirectoryName(Path.GetFullPath(timingPlate.PlatePath))) is { } footage
? new List<string> { AsFolder(footage) }
: new List<string>();
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { Path.GetFullPath(timingPlate.PlatePath) };
foreach (var plate in plates)
{
if (string.IsNullOrWhiteSpace(plate.Path)) continue;
var local = Path.GetFullPath(mapper.Map(plate.Path));
if (!seen.Add(local)) continue;
if (!deliveredRoots.Any(root => local.StartsWith(root, StringComparison.OrdinalIgnoreCase))) continue;
var reference = ReadManifestPlate(plate, mapper, timingPlate.HandleFrames, renderFirstFrame);
if (reference is null)
{
_log.LogWarning("Alternate plate from manifest is unreadable: {Path}", local);
continue;
}
yield return reference;
}
}
/// <summary>
/// Rank 2: look for the shot's plate folders under the configured roots,
/// mirroring the panel's two on-disk layouts ({shotCode}/{element}/ and