fix(worker): Blender jobs always failed with "Collection was modified"

LogTail was read by the progress loop while the output pumps appended to
it. Blender rewrites its status line many times a second, so the first
progress report of every job threw; aerender hit the same race only rarely.
LogTail now locks internally.

The exception also escaped BlenderRunner without ending the process, leaving
blender.exe rendering unsupervised on the node on every failed attempt. Any
unexpected exception in the monitor loop now kills the process tree.

Bump worker to 1.6.2.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
twotalesanimation
2026-09-18 04:21:54 +02:00
parent 400cf3f6ea
commit 97246bcbc3
4 changed files with 109 additions and 62 deletions
@@ -117,4 +117,21 @@ public class LogTailTests
for (var i = 1; i <= 5; i++) tail.Add($"line {i}");
Assert.Equal("line 3\nline 4\nline 5", tail.ToString());
}
[Fact]
public async Task ReadingWhileAnotherThreadWritesDoesNotThrow()
{
// The Blender runner's output pumps add lines while its progress loop
// reads the tail; unsynchronised this threw "Collection was modified".
var tail = new LogTail(200);
using var stop = new CancellationTokenSource(TimeSpan.FromSeconds(1));
var writer = Task.Run(() =>
{
var i = 0;
while (!stop.IsCancellationRequested) tail.Add($"Fra:1001 Sample {i++}/128");
});
while (!stop.IsCancellationRequested) _ = tail.ToString();
await writer;
}
}
+17 -3
View File
@@ -280,11 +280,19 @@ public sealed class AerenderRunner
}
}
/// <summary>Ring buffer of the last N output lines (§5.3 logTail).</summary>
/// <summary>
/// Ring buffer of the last N output lines (§5.3 logTail).
///
/// Thread-safe: output readers add lines while the progress loop reads the
/// tail for its report. Unsynchronised, ToString enumerates the queue while a
/// reader enqueues and throws "Collection was modified" — Blender, which
/// writes a status line many times a second, hit this on every job.
/// </summary>
public sealed class LogTail
{
private readonly Queue<string> _lines;
private readonly int _capacity;
private readonly object _gate = new();
public LogTail(int capacity)
{
@@ -293,10 +301,16 @@ public sealed class LogTail
}
public void Add(string line)
{
lock (_gate)
{
_lines.Enqueue(line);
while (_lines.Count > _capacity) _lines.Dequeue();
}
public override string ToString() => string.Join('\n', _lines);
}
public override string ToString()
{
lock (_gate) return string.Join('\n', _lines);
}
}
@@ -190,6 +190,21 @@ public sealed class BlenderRunner
PumpAsync(process.StandardError, line => HandleLine(line, isStderr: true)),
};
try
{
return await MonitorAsync();
}
catch
{
// Disposing a Process does not end it: without this, any exception
// in the monitor loop leaves Blender rendering on the node,
// unsupervised, while the job is retried alongside it.
KillTree(process);
throw;
}
async Task<RenderResult> MonitorAsync()
{
while (!process.HasExited)
{
await Task.Delay(TimeSpan.FromSeconds(2), CancellationToken.None);
@@ -260,6 +275,7 @@ public sealed class BlenderRunner
_log.LogInformation("Job {JobId}: {Saved}/{Total} frames in {Sec:F0}s", jobId, framesSaved, totalFrames, sw.Elapsed.TotalSeconds);
return new RenderResult(true, 0, true, null, tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
}
}
/// <summary>
/// Reads a stream treating both CR and LF as line terminators, so
@@ -15,7 +15,7 @@
every change that gets published. publish.ps1 appends the git SHA via
SourceRevisionId, giving "1.1.0+feb1c7b".
-->
<Version>1.6.1</Version>
<Version>1.6.2</Version>
</PropertyGroup>
<ItemGroup>