fix(worker): Blender jobs always failed with "Collection was modified"
LogTail was read by the progress loop while the output pumps appended to it. Blender rewrites its status line many times a second, so the first progress report of every job threw; aerender hit the same race only rarely. LogTail now locks internally. The exception also escaped BlenderRunner without ending the process, leaving blender.exe rendering unsupervised on the node on every failed attempt. Any unexpected exception in the monitor loop now kills the process tree. Bump worker to 1.6.2. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -117,4 +117,21 @@ public class LogTailTests
|
||||
for (var i = 1; i <= 5; i++) tail.Add($"line {i}");
|
||||
Assert.Equal("line 3\nline 4\nline 5", tail.ToString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReadingWhileAnotherThreadWritesDoesNotThrow()
|
||||
{
|
||||
// The Blender runner's output pumps add lines while its progress loop
|
||||
// reads the tail; unsynchronised this threw "Collection was modified".
|
||||
var tail = new LogTail(200);
|
||||
using var stop = new CancellationTokenSource(TimeSpan.FromSeconds(1));
|
||||
var writer = Task.Run(() =>
|
||||
{
|
||||
var i = 0;
|
||||
while (!stop.IsCancellationRequested) tail.Add($"Fra:1001 Sample {i++}/128");
|
||||
});
|
||||
|
||||
while (!stop.IsCancellationRequested) _ = tail.ToString();
|
||||
await writer;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -280,11 +280,19 @@ public sealed class AerenderRunner
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Ring buffer of the last N output lines (§5.3 logTail).</summary>
|
||||
/// <summary>
|
||||
/// Ring buffer of the last N output lines (§5.3 logTail).
|
||||
///
|
||||
/// Thread-safe: output readers add lines while the progress loop reads the
|
||||
/// tail for its report. Unsynchronised, ToString enumerates the queue while a
|
||||
/// reader enqueues and throws "Collection was modified" — Blender, which
|
||||
/// writes a status line many times a second, hit this on every job.
|
||||
/// </summary>
|
||||
public sealed class LogTail
|
||||
{
|
||||
private readonly Queue<string> _lines;
|
||||
private readonly int _capacity;
|
||||
private readonly object _gate = new();
|
||||
|
||||
public LogTail(int capacity)
|
||||
{
|
||||
@@ -293,10 +301,16 @@ public sealed class LogTail
|
||||
}
|
||||
|
||||
public void Add(string line)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_lines.Enqueue(line);
|
||||
while (_lines.Count > _capacity) _lines.Dequeue();
|
||||
}
|
||||
}
|
||||
|
||||
public override string ToString() => string.Join('\n', _lines);
|
||||
public override string ToString()
|
||||
{
|
||||
lock (_gate) return string.Join('\n', _lines);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -190,6 +190,21 @@ public sealed class BlenderRunner
|
||||
PumpAsync(process.StandardError, line => HandleLine(line, isStderr: true)),
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
return await MonitorAsync();
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Disposing a Process does not end it: without this, any exception
|
||||
// in the monitor loop leaves Blender rendering on the node,
|
||||
// unsupervised, while the job is retried alongside it.
|
||||
KillTree(process);
|
||||
throw;
|
||||
}
|
||||
|
||||
async Task<RenderResult> MonitorAsync()
|
||||
{
|
||||
while (!process.HasExited)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(2), CancellationToken.None);
|
||||
@@ -260,6 +275,7 @@ public sealed class BlenderRunner
|
||||
_log.LogInformation("Job {JobId}: {Saved}/{Total} frames in {Sec:F0}s", jobId, framesSaved, totalFrames, sw.Elapsed.TotalSeconds);
|
||||
return new RenderResult(true, 0, true, null, tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a stream treating both CR and LF as line terminators, so
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
every change that gets published. publish.ps1 appends the git SHA via
|
||||
SourceRevisionId, giving "1.1.0+feb1c7b".
|
||||
-->
|
||||
<Version>1.6.1</Version>
|
||||
<Version>1.6.2</Version>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
Reference in New Issue
Block a user