Compare commits
3 Commits
2dee1bf6fc
...
708e8a0d8f
| Author | SHA1 | Date | |
|---|---|---|---|
| 708e8a0d8f | |||
| 97246bcbc3 | |||
| 400cf3f6ea |
@@ -1073,6 +1073,8 @@ Timecode is the check this engine exists for. It is validated **against the sour
|
|||||||
| 2 | Plate re-derived on the worker from the footage root + shot code (manifestVersion 1, or a timing plate that no longer resolves). If elements disagree, the one matching the render's frame count wins; still ambiguous → WARN, never PASS | PASS / FAIL, `reference: "derived"` |
|
| 2 | Plate re-derived on the worker from the footage root + shot code (manifestVersion 1, or a timing plate that no longer resolves). If elements disagree, the one matching the render's frame count wins; still ambiguous → WARN, never PASS | PASS / FAIL, `reference: "derived"` |
|
||||||
| 3 | API `Shot.timecodeStart` **minus `handleFrames`** | **WARN only, never PASS** — `reference: "api"`, message states the plate could not be read |
|
| 3 | API `Shot.timecodeStart` **minus `handleFrames`** | **WARN only, never PASS** — `reference: "api"`, message states the plate could not be read |
|
||||||
|
|
||||||
|
**Several delivered elements.** When the rank-1 timing plate does not match, the worker tries the manifest's other `plates[]` before failing — the panel's longest-duration pick is not always the element the comp was anchored to. A match against one of them is a PASS naming that element; no match still FAILs against the timing plate, listing the others tried. Only delivered footage qualifies: plates under a configured plate search root, or (with none configured) alongside the timing plate in its footage folder. `plates[]` also holds pre-renders, which agree with the render by construction and must never rescue it.
|
||||||
|
|
||||||
Rank 3 is a diagnostic, not a verification: it depends on the show's handle count being correct for that specific plate, which is exactly the assumption the defects above violate.
|
Rank 3 is a diagnostic, not a verification: it depends on the show's handle count being correct for that specific plate, which is exactly the assumption the defects above violate.
|
||||||
|
|
||||||
**Comparison, given the plate reference:**
|
**Comparison, given the plate reference:**
|
||||||
|
|||||||
@@ -73,6 +73,46 @@ public class BlenderProgressParserTests
|
|||||||
Assert.Equal(1002, frames[^1]);
|
Assert.Equal(1002, frames[^1]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Blender 5.x prefixes every line with elapsed time and a log category.
|
||||||
|
/// Anchored on "Saved:", the parser counted no frames from a render that
|
||||||
|
/// wrote every one, and the job failed as "saved no frames".
|
||||||
|
/// These lines are verbatim Blender 5.2 output.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void ParsesBlender5PrefixedOutput()
|
||||||
|
{
|
||||||
|
Assert.Equal(@"X:\renders\frame_0001.png",
|
||||||
|
BlenderProgressParser.ParseSaved(@"00:02.954 render | Saved: 'X:\renders\frame_0001.png'"));
|
||||||
|
Assert.Equal(1, BlenderProgressParser.ParseFrame("00:02.469 render | Fra: 1 | Mem: 0M | Synchronizing object | Cube"));
|
||||||
|
Assert.Null(BlenderProgressParser.ParseSaved("00:02.954 render | Time: 00:00.87 (Saving: 00:00.14)"));
|
||||||
|
Assert.Null(BlenderProgressParser.ParseFrame("00:02.079 render | Rendering frame 1"));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void RecognisesBlender5Errors()
|
||||||
|
{
|
||||||
|
const string unwritable = @"00:03.063 reports | ERROR Render error (No such file or directory) cannot save: 'Q:\out\f_0001.png'";
|
||||||
|
Assert.True(BlenderProgressParser.IsErrorLine(unwritable));
|
||||||
|
Assert.True(BlenderProgressParser.IsNonRetryableError(unwritable));
|
||||||
|
|
||||||
|
const string missing = @"00:13.250 reports | ERROR Cannot read file ""X:\gone.blend"": No such file or directory";
|
||||||
|
Assert.True(BlenderProgressParser.IsErrorLine(missing));
|
||||||
|
Assert.True(BlenderProgressParser.IsNonRetryableError(missing));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Add-ons print warnings on every launch; one must never become the
|
||||||
|
/// reported reason a render failed.
|
||||||
|
/// </summary>
|
||||||
|
[Theory]
|
||||||
|
[InlineData("Warning: add-on missing 'bl_info', this can cause poor performance!: 'G:\\addons\\x\\__init__.py'")]
|
||||||
|
[InlineData("00:01.000 addon | WARNING something harmless")]
|
||||||
|
public void WarningsAreNotErrors(string line)
|
||||||
|
{
|
||||||
|
Assert.False(BlenderProgressParser.IsErrorLine(line));
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>Mirrors BlenderRunner.PumpAsync: CR and LF both end a line.</summary>
|
/// <summary>Mirrors BlenderRunner.PumpAsync: CR and LF both end a line.</summary>
|
||||||
private static List<string> SplitLikeThePump(string text)
|
private static List<string> SplitLikeThePump(string text)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -117,4 +117,21 @@ public class LogTailTests
|
|||||||
for (var i = 1; i <= 5; i++) tail.Add($"line {i}");
|
for (var i = 1; i <= 5; i++) tail.Add($"line {i}");
|
||||||
Assert.Equal("line 3\nline 4\nline 5", tail.ToString());
|
Assert.Equal("line 3\nline 4\nline 5", tail.ToString());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ReadingWhileAnotherThreadWritesDoesNotThrow()
|
||||||
|
{
|
||||||
|
// The Blender runner's output pumps add lines while its progress loop
|
||||||
|
// reads the tail; unsynchronised this threw "Collection was modified".
|
||||||
|
var tail = new LogTail(200);
|
||||||
|
using var stop = new CancellationTokenSource(TimeSpan.FromSeconds(1));
|
||||||
|
var writer = Task.Run(() =>
|
||||||
|
{
|
||||||
|
var i = 0;
|
||||||
|
while (!stop.IsCancellationRequested) tail.Add($"Fra:1001 Sample {i++}/128");
|
||||||
|
});
|
||||||
|
|
||||||
|
while (!stop.IsCancellationRequested) _ = tail.ToString();
|
||||||
|
await writer;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -109,6 +109,74 @@ public class ValidationStageTests : IDisposable
|
|||||||
Assert.Contains("8-frame handle", tc.Message);
|
Assert.Contains("8-frame handle", tc.Message);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>Writes a one-frame plate into a footage-style element folder.</summary>
|
||||||
|
private string WritePlate(string element, string timecode, int fps = 24)
|
||||||
|
{
|
||||||
|
var plate = Path.Combine(_dir, "footage", $"UNG_107_051_010_{element}_TT_v001", $"UNG_107_051_010_{element}_TT_v001.01001.exr");
|
||||||
|
Directory.CreateDirectory(Path.GetDirectoryName(plate)!);
|
||||||
|
File.Move(ExrFixture.Write(timecode, 3840, 2160, new[] { ("B", 1), ("G", 1), ("R", 1) }, fps, 4), plate);
|
||||||
|
return plate;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ManifestSource Source(string timingPlate, params string[] plates) => new()
|
||||||
|
{
|
||||||
|
TimingPlate = new PlateRef { Path = timingPlate, FrameStart = 1001 },
|
||||||
|
Plates = plates.Select(p => new PlateRef { Path = p, FrameStart = 1001 }).ToList(),
|
||||||
|
HandleFrames = 8,
|
||||||
|
};
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void AnotherDeliveredPlatePassesWhenTheTimingPlateDoesNot()
|
||||||
|
{
|
||||||
|
// The comp was anchored to FG01, but the panel named BG01 (longest) as
|
||||||
|
// the timing plate — the two carry unrelated camera timecodes.
|
||||||
|
WriteSequence(105, "09:12:00:00");
|
||||||
|
var bg = WritePlate("BG01", "14:26:31:08");
|
||||||
|
var fg = WritePlate("FG01", "09:12:00:00");
|
||||||
|
|
||||||
|
var manifest = Manifest(_dir, null, 1001);
|
||||||
|
manifest.Source = Source(bg, bg, fg);
|
||||||
|
|
||||||
|
var tc = Check(NewStage().Run(manifest, _dir, new ServerConfig()), "timecode_plate_match");
|
||||||
|
Assert.Equal("PASS", tc.Status);
|
||||||
|
Assert.Contains("UNG_107_051_010_FG01_TT_v001.01001.exr", tc.Message);
|
||||||
|
Assert.Contains("BG01", tc.Message);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void NoMatchingPlateStillFailsAgainstTheTimingPlate()
|
||||||
|
{
|
||||||
|
WriteSequence(105, "14:26:31:16");
|
||||||
|
var bg = WritePlate("BG01", "14:26:31:08");
|
||||||
|
var fg = WritePlate("FG01", "09:12:00:00");
|
||||||
|
|
||||||
|
var manifest = Manifest(_dir, null, 1001);
|
||||||
|
manifest.Source = Source(bg, bg, fg);
|
||||||
|
|
||||||
|
var tc = Check(NewStage().Run(manifest, _dir, new ServerConfig()), "timecode_plate_match");
|
||||||
|
Assert.Equal("FAIL", tc.Status);
|
||||||
|
Assert.Equal("14:26:31:08", tc.Expected);
|
||||||
|
Assert.Contains("8-frame handle", tc.Message);
|
||||||
|
Assert.Contains("1 other delivered plate", tc.Message);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void APreRenderIsNeverUsedAsTheFallbackPlate()
|
||||||
|
{
|
||||||
|
// plates[] is every file in the comp. A pre-render was rendered from
|
||||||
|
// this comp, so it agrees with the render whether or not either is right.
|
||||||
|
WriteSequence(105, "14:26:31:16");
|
||||||
|
var bg = WritePlate("BG01", "14:26:31:08");
|
||||||
|
var preRender = Path.Combine(_dir, "renders", "prerender", "UNG_107_051_010_PRERENDER_01001.exr");
|
||||||
|
Directory.CreateDirectory(Path.GetDirectoryName(preRender)!);
|
||||||
|
File.Move(ExrFixture.Write("14:26:31:16", 3840, 2160, new[] { ("B", 1), ("G", 1), ("R", 1) }, 24, 4), preRender);
|
||||||
|
|
||||||
|
var manifest = Manifest(_dir, null, 1001);
|
||||||
|
manifest.Source = Source(bg, bg, preRender);
|
||||||
|
|
||||||
|
Assert.Equal("FAIL", Check(NewStage().Run(manifest, _dir, new ServerConfig()), "timecode_plate_match").Status);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void APlateAtADifferentRateFailsPlateFps()
|
public void APlateAtADifferentRateFailsPlateFps()
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -280,11 +280,19 @@ public sealed class AerenderRunner
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>Ring buffer of the last N output lines (§5.3 logTail).</summary>
|
/// <summary>
|
||||||
|
/// Ring buffer of the last N output lines (§5.3 logTail).
|
||||||
|
///
|
||||||
|
/// Thread-safe: output readers add lines while the progress loop reads the
|
||||||
|
/// tail for its report. Unsynchronised, ToString enumerates the queue while a
|
||||||
|
/// reader enqueues and throws "Collection was modified" — Blender, which
|
||||||
|
/// writes a status line many times a second, hit this on every job.
|
||||||
|
/// </summary>
|
||||||
public sealed class LogTail
|
public sealed class LogTail
|
||||||
{
|
{
|
||||||
private readonly Queue<string> _lines;
|
private readonly Queue<string> _lines;
|
||||||
private readonly int _capacity;
|
private readonly int _capacity;
|
||||||
|
private readonly object _gate = new();
|
||||||
|
|
||||||
public LogTail(int capacity)
|
public LogTail(int capacity)
|
||||||
{
|
{
|
||||||
@@ -294,9 +302,15 @@ public sealed class LogTail
|
|||||||
|
|
||||||
public void Add(string line)
|
public void Add(string line)
|
||||||
{
|
{
|
||||||
_lines.Enqueue(line);
|
lock (_gate)
|
||||||
while (_lines.Count > _capacity) _lines.Dequeue();
|
{
|
||||||
|
_lines.Enqueue(line);
|
||||||
|
while (_lines.Count > _capacity) _lines.Dequeue();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public override string ToString() => string.Join('\n', _lines);
|
public override string ToString()
|
||||||
|
{
|
||||||
|
lock (_gate) return string.Join('\n', _lines);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,18 +8,36 @@ namespace VFXReviewWorker;
|
|||||||
/// with "Saved: '...'". The saved lines are the reliable signal — a frame can
|
/// with "Saved: '...'". The saved lines are the reliable signal — a frame can
|
||||||
/// be reported for a long time while it renders — so progress counts those and
|
/// be reported for a long time while it renders — so progress counts those and
|
||||||
/// falls back to Fra: for a first sign of life.
|
/// falls back to Fra: for a first sign of life.
|
||||||
|
///
|
||||||
|
/// Blender 5.x logs through CLOG, which prefixes every line with an elapsed
|
||||||
|
/// time and a category ("00:02.954 render | Saved: '...'") and
|
||||||
|
/// reports errors as "ERROR ..." rather than "Error: ...". The prefix is
|
||||||
|
/// stripped before matching, so both formats parse the same way.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public static partial class BlenderProgressParser
|
public static partial class BlenderProgressParser
|
||||||
{
|
{
|
||||||
[GeneratedRegex(@"^Fra:(\d+)", RegexOptions.Compiled)]
|
[GeneratedRegex(@"^(?:\d+:)?\d{2}:\d{2}\.\d{3}\s+\S+\s*\| ?", RegexOptions.Compiled)]
|
||||||
|
private static partial Regex LogPrefix();
|
||||||
|
|
||||||
|
[GeneratedRegex(@"^Fra:\s*(\d+)", RegexOptions.Compiled)]
|
||||||
private static partial Regex FrameLine();
|
private static partial Regex FrameLine();
|
||||||
|
|
||||||
[GeneratedRegex(@"^Saved:\s*'(.+)'", RegexOptions.Compiled)]
|
[GeneratedRegex(@"^Saved:\s*'(.+)'", RegexOptions.Compiled)]
|
||||||
private static partial Regex SavedLine();
|
private static partial Regex SavedLine();
|
||||||
|
|
||||||
[GeneratedRegex(@"^(Error|Warning): |^Error: ", RegexOptions.Compiled | RegexOptions.IgnoreCase)]
|
// Errors only. Warnings are not failures, and add-ons print them on every
|
||||||
|
// launch ("Warning: add-on missing 'bl_info'"), so counting them made an
|
||||||
|
// add-on's complaint the reported reason for a failed render.
|
||||||
|
[GeneratedRegex(@"(?i:^Error: )|^(ERROR|FATAL) ", RegexOptions.Compiled)]
|
||||||
private static partial Regex ErrorLine();
|
private static partial Regex ErrorLine();
|
||||||
|
|
||||||
|
/// <summary>The line without Blender 5.x's "elapsed category | " prefix.</summary>
|
||||||
|
private static string Message(string line)
|
||||||
|
{
|
||||||
|
var m = LogPrefix().Match(line);
|
||||||
|
return m.Success ? line[m.Length..] : line;
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Failures that will repeat identically on another machine — a missing
|
/// Failures that will repeat identically on another machine — a missing
|
||||||
/// file, an unknown scene, an unwritable output. Retrying these burns the
|
/// file, an unknown scene, an unwritable output. Retrying these burns the
|
||||||
@@ -35,7 +53,9 @@ public static partial class BlenderProgressParser
|
|||||||
"unknown scene",
|
"unknown scene",
|
||||||
"scene not found",
|
"scene not found",
|
||||||
"unable to create directory",
|
"unable to create directory",
|
||||||
|
"couldn't create directory",
|
||||||
"cannot write file",
|
"cannot write file",
|
||||||
|
"cannot save:",
|
||||||
"no camera",
|
"no camera",
|
||||||
"python script fail",
|
"python script fail",
|
||||||
"unknown argument",
|
"unknown argument",
|
||||||
@@ -44,18 +64,18 @@ public static partial class BlenderProgressParser
|
|||||||
/// <summary>The frame Blender is currently working on, or null.</summary>
|
/// <summary>The frame Blender is currently working on, or null.</summary>
|
||||||
public static int? ParseFrame(string line)
|
public static int? ParseFrame(string line)
|
||||||
{
|
{
|
||||||
var m = FrameLine().Match(line);
|
var m = FrameLine().Match(Message(line));
|
||||||
return m.Success && int.TryParse(m.Groups[1].Value, out var f) ? f : null;
|
return m.Success && int.TryParse(m.Groups[1].Value, out var f) ? f : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>The path of a frame Blender has just written, or null.</summary>
|
/// <summary>The path of a frame Blender has just written, or null.</summary>
|
||||||
public static string? ParseSaved(string line)
|
public static string? ParseSaved(string line)
|
||||||
{
|
{
|
||||||
var m = SavedLine().Match(line);
|
var m = SavedLine().Match(Message(line));
|
||||||
return m.Success ? m.Groups[1].Value : null;
|
return m.Success ? m.Groups[1].Value : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static bool IsErrorLine(string line) => ErrorLine().IsMatch(line);
|
public static bool IsErrorLine(string line) => ErrorLine().IsMatch(Message(line));
|
||||||
|
|
||||||
public static bool IsNonRetryableError(string line)
|
public static bool IsNonRetryableError(string line)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -190,75 +190,93 @@ public sealed class BlenderRunner
|
|||||||
PumpAsync(process.StandardError, line => HandleLine(line, isStderr: true)),
|
PumpAsync(process.StandardError, line => HandleLine(line, isStderr: true)),
|
||||||
};
|
};
|
||||||
|
|
||||||
while (!process.HasExited)
|
try
|
||||||
{
|
{
|
||||||
await Task.Delay(TimeSpan.FromSeconds(2), CancellationToken.None);
|
return await MonitorAsync();
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// Disposing a Process does not end it: without this, any exception
|
||||||
|
// in the monitor loop leaves Blender rendering on the node,
|
||||||
|
// unsupervised, while the job is retried alongside it.
|
||||||
|
KillTree(process);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
|
||||||
if (cancelJob.IsCancellationRequested || shutdown.IsCancellationRequested)
|
async Task<RenderResult> MonitorAsync()
|
||||||
|
{
|
||||||
|
while (!process.HasExited)
|
||||||
{
|
{
|
||||||
_log.LogWarning("Cancellation requested — killing Blender tree for job {JobId}", jobId);
|
await Task.Delay(TimeSpan.FromSeconds(2), CancellationToken.None);
|
||||||
KillTree(process);
|
|
||||||
cancelledByServer = cancelJob.IsCancellationRequested;
|
if (cancelJob.IsCancellationRequested || shutdown.IsCancellationRequested)
|
||||||
break;
|
{
|
||||||
|
_log.LogWarning("Cancellation requested — killing Blender tree for job {JobId}", jobId);
|
||||||
|
KillTree(process);
|
||||||
|
cancelledByServer = cancelJob.IsCancellationRequested;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ((DateTimeOffset.UtcNow - lastProgressAt).TotalSeconds > config.StallTimeoutSeconds)
|
||||||
|
{
|
||||||
|
_log.LogError("Blender stalled ({Sec}s without output) — killing job {JobId}", config.StallTimeoutSeconds, jobId);
|
||||||
|
KillTree(process);
|
||||||
|
await WaitForExitSafe(process);
|
||||||
|
return new RenderResult(false, -2, Retryable: true,
|
||||||
|
$"Blender produced no output for {config.StallTimeoutSeconds}s (stall timeout)",
|
||||||
|
tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
|
||||||
|
}
|
||||||
|
|
||||||
|
if ((DateTimeOffset.UtcNow - lastReportAt).TotalSeconds >= 10)
|
||||||
|
{
|
||||||
|
lastReportAt = DateTimeOffset.UtcNow;
|
||||||
|
var progress = totalFrames > 0 ? Math.Min(1.0, (double)framesSaved / totalFrames) : 0;
|
||||||
|
var resp = await _api.ProgressAsync(jobId, machineId, progress, currentFrame, totalFrames,
|
||||||
|
eta.EstimateSeconds(totalFrames - framesSaved), tail.ToString(), shutdown);
|
||||||
|
if (resp?.CancelRequested == true)
|
||||||
|
{
|
||||||
|
_log.LogWarning("Server requested cancel for job {JobId}", jobId);
|
||||||
|
KillTree(process);
|
||||||
|
cancelledByServer = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if ((DateTimeOffset.UtcNow - lastProgressAt).TotalSeconds > config.StallTimeoutSeconds)
|
await WaitForExitSafe(process);
|
||||||
|
// Let the readers drain what is still buffered before judging the run —
|
||||||
|
// the last frame's "Saved:" often arrives after the process exits.
|
||||||
|
try { await Task.WhenAll(pumps).WaitAsync(TimeSpan.FromSeconds(10)); } catch { }
|
||||||
|
sw.Stop();
|
||||||
|
lock (logLock) { fullLog.Flush(); }
|
||||||
|
|
||||||
|
if (cancelledByServer)
|
||||||
{
|
{
|
||||||
_log.LogError("Blender stalled ({Sec}s without output) — killing job {JobId}", config.StallTimeoutSeconds, jobId);
|
return new RenderResult(false, -3, Retryable: false, "Cancelled", tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
|
||||||
KillTree(process);
|
}
|
||||||
await WaitForExitSafe(process);
|
|
||||||
return new RenderResult(false, -2, Retryable: true,
|
var exitCode = process.ExitCode;
|
||||||
$"Blender produced no output for {config.StallTimeoutSeconds}s (stall timeout)",
|
if (exitCode != 0)
|
||||||
|
{
|
||||||
|
return new RenderResult(false, exitCode, Retryable: !nonRetryable,
|
||||||
|
firstError ?? $"Blender exited with code {exitCode}",
|
||||||
tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
|
tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
|
||||||
}
|
}
|
||||||
|
|
||||||
if ((DateTimeOffset.UtcNow - lastReportAt).TotalSeconds >= 10)
|
// Blender exits 0 having rendered nothing if the range is empty or the
|
||||||
|
// output path is unwritable, so a silent success is treated as failure.
|
||||||
|
if (framesSaved == 0)
|
||||||
{
|
{
|
||||||
lastReportAt = DateTimeOffset.UtcNow;
|
return new RenderResult(false, 0, Retryable: false,
|
||||||
var progress = totalFrames > 0 ? Math.Min(1.0, (double)framesSaved / totalFrames) : 0;
|
firstError is not null
|
||||||
var resp = await _api.ProgressAsync(jobId, machineId, progress, currentFrame, totalFrames,
|
? $"Blender exited cleanly but saved no frames: {firstError}"
|
||||||
eta.EstimateSeconds(totalFrames - framesSaved), tail.ToString(), shutdown);
|
: "Blender exited cleanly but saved no frames — check the output path and frame range in the .blend",
|
||||||
if (resp?.CancelRequested == true)
|
tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
|
||||||
{
|
|
||||||
_log.LogWarning("Server requested cancel for job {JobId}", jobId);
|
|
||||||
KillTree(process);
|
|
||||||
cancelledByServer = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_log.LogInformation("Job {JobId}: {Saved}/{Total} frames in {Sec:F0}s", jobId, framesSaved, totalFrames, sw.Elapsed.TotalSeconds);
|
||||||
|
return new RenderResult(true, 0, true, null, tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
|
||||||
}
|
}
|
||||||
|
|
||||||
await WaitForExitSafe(process);
|
|
||||||
// Let the readers drain what is still buffered before judging the run —
|
|
||||||
// the last frame's "Saved:" often arrives after the process exits.
|
|
||||||
try { await Task.WhenAll(pumps).WaitAsync(TimeSpan.FromSeconds(10)); } catch { }
|
|
||||||
sw.Stop();
|
|
||||||
lock (logLock) { fullLog.Flush(); }
|
|
||||||
|
|
||||||
if (cancelledByServer)
|
|
||||||
{
|
|
||||||
return new RenderResult(false, -3, Retryable: false, "Cancelled", tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
|
|
||||||
}
|
|
||||||
|
|
||||||
var exitCode = process.ExitCode;
|
|
||||||
if (exitCode != 0)
|
|
||||||
{
|
|
||||||
return new RenderResult(false, exitCode, Retryable: !nonRetryable,
|
|
||||||
firstError ?? $"Blender exited with code {exitCode}",
|
|
||||||
tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Blender exits 0 having rendered nothing if the range is empty or the
|
|
||||||
// output path is unwritable, so a silent success is treated as failure.
|
|
||||||
if (framesSaved == 0)
|
|
||||||
{
|
|
||||||
return new RenderResult(false, 0, Retryable: false,
|
|
||||||
"Blender exited cleanly but saved no frames — check the output path and frame range in the .blend",
|
|
||||||
tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
|
|
||||||
}
|
|
||||||
|
|
||||||
_log.LogInformation("Job {JobId}: {Saved}/{Total} frames in {Sec:F0}s", jobId, framesSaved, totalFrames, sw.Elapsed.TotalSeconds);
|
|
||||||
return new RenderResult(true, 0, true, null, tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|||||||
@@ -15,7 +15,7 @@
|
|||||||
every change that gets published. publish.ps1 appends the git SHA via
|
every change that gets published. publish.ps1 appends the git SHA via
|
||||||
SourceRevisionId, giving "1.1.0+feb1c7b".
|
SourceRevisionId, giving "1.1.0+feb1c7b".
|
||||||
-->
|
-->
|
||||||
<Version>1.6.1</Version>
|
<Version>1.6.3</Version>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|||||||
@@ -350,6 +350,31 @@ public sealed class ValidationStage
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var deltaOf = (TimecodeReference r) =>
|
||||||
|
renderTc.ToFrames(fps) - (r.PlateTimecode.ToFrames(fps) + (renderFirstFrame - r.PlateFirstFrame));
|
||||||
|
|
||||||
|
// A shot with several delivered elements: the panel names the longest
|
||||||
|
// as the timing plate, but the comp may have been anchored to another
|
||||||
|
// (BG01 and FG01 carry unrelated camera timecodes). When the timing
|
||||||
|
// plate disagrees, the manifest's other delivered plates are tried
|
||||||
|
// before failing. The comparison is unchanged — it is still a plate, so
|
||||||
|
// a match is a real PASS and no match still FAILs against the timing plate.
|
||||||
|
var timingPlate = reference;
|
||||||
|
var alternatesTried = new List<object>();
|
||||||
|
if (reference.Kind == "manifest" && deltaOf(reference) != 0)
|
||||||
|
{
|
||||||
|
foreach (var alternate in ResolveAlternatePlates(m, renderFirstFrame, reference))
|
||||||
|
{
|
||||||
|
var altDelta = deltaOf(alternate);
|
||||||
|
alternatesTried.Add(new { platePath = alternate.PlatePath, plateTimecode = alternate.PlateTimecode.ToString(), deltaFrames = altDelta });
|
||||||
|
if (altDelta == 0)
|
||||||
|
{
|
||||||
|
reference = alternate;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// plate_fps — the render's rate against the plate's own.
|
// plate_fps — the render's rate against the plate's own.
|
||||||
//
|
//
|
||||||
// timecode_fps only compares the render to the manifest, and both come
|
// timecode_fps only compares the render to the manifest, and both come
|
||||||
@@ -377,8 +402,8 @@ public sealed class ValidationStage
|
|||||||
|
|
||||||
var expectedFrames = reference.PlateTimecode.ToFrames(fps) + (renderFirstFrame - reference.PlateFirstFrame);
|
var expectedFrames = reference.PlateTimecode.ToFrames(fps) + (renderFirstFrame - reference.PlateFirstFrame);
|
||||||
var expected = Timecode.FromFrames(expectedFrames, fps);
|
var expected = Timecode.FromFrames(expectedFrames, fps);
|
||||||
var actualFrames = renderTc.ToFrames(fps);
|
var deltaFrames = deltaOf(reference);
|
||||||
var deltaFrames = actualFrames - expectedFrames;
|
var matchedAlternate = !ReferenceEquals(reference, timingPlate);
|
||||||
var details = new
|
var details = new
|
||||||
{
|
{
|
||||||
reference = reference.Kind,
|
reference = reference.Kind,
|
||||||
@@ -387,10 +412,14 @@ public sealed class ValidationStage
|
|||||||
plateFirstFrame = reference.PlateFirstFrame,
|
plateFirstFrame = reference.PlateFirstFrame,
|
||||||
renderFirstFrame,
|
renderFirstFrame,
|
||||||
deltaFrames,
|
deltaFrames,
|
||||||
|
timingPlatePath = timingPlate.PlatePath,
|
||||||
|
matchedAlternate,
|
||||||
|
alternatesTried,
|
||||||
};
|
};
|
||||||
|
|
||||||
if (deltaFrames == 0)
|
if (deltaFrames == 0)
|
||||||
{
|
{
|
||||||
|
var timingDelta = deltaOf(timingPlate);
|
||||||
// The API fallback proves nothing on its own (§8.1a): it is the
|
// The API fallback proves nothing on its own (§8.1a): it is the
|
||||||
// editorial in-point minus a handle count that may not be this
|
// editorial in-point minus a handle count that may not be this
|
||||||
// plate's, so a match is reported as a WARN, never a PASS.
|
// plate's, so a match is reported as a WARN, never a PASS.
|
||||||
@@ -398,7 +427,11 @@ public sealed class ValidationStage
|
|||||||
? ValidationCheck.Warn("timecode_plate_match", expected.ToString(), renderTc.ToString(),
|
? ValidationCheck.Warn("timecode_plate_match", expected.ToString(), renderTc.ToString(),
|
||||||
"Matches the API timecode less handles, but the plate itself could not be read — unverified.", details)
|
"Matches the API timecode less handles, but the plate itself could not be read — unverified.", details)
|
||||||
: ValidationCheck.Pass("timecode_plate_match", expected.ToString(), renderTc.ToString(),
|
: ValidationCheck.Pass("timecode_plate_match", expected.ToString(), renderTc.ToString(),
|
||||||
$"Matches {reference.Kind} plate {Path.GetFileName(reference.PlatePath ?? "")}", details));
|
$"Matches {reference.Kind} plate {Path.GetFileName(reference.PlatePath ?? "")}" +
|
||||||
|
(matchedAlternate
|
||||||
|
? $" (the timing plate {Path.GetFileName(timingPlate.PlatePath ?? "")} is {(timingDelta > 0 ? "+" : "")}{timingDelta} frames off — the comp was anchored to this element instead)"
|
||||||
|
: ""),
|
||||||
|
details));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -409,7 +442,10 @@ public sealed class ValidationStage
|
|||||||
? $"That is exactly the {reference.HandleFrames}-frame handle: the comp was anchored to the editorial in-point instead of the plate's first frame."
|
? $"That is exactly the {reference.HandleFrames}-frame handle: the comp was anchored to the editorial in-point instead of the plate's first frame."
|
||||||
: renderTc.Hours == 0 && renderTc.Minutes == 0
|
: renderTc.Hours == 0 && renderTc.Minutes == 0
|
||||||
? "A near-zero timecode means the comp had no source timecode at all and AE derived it from the frame number."
|
? "A near-zero timecode means the comp had no source timecode at all and AE derived it from the frame number."
|
||||||
: "");
|
: "") +
|
||||||
|
(alternatesTried.Count > 0
|
||||||
|
? $" Also checked {alternatesTried.Count} other delivered plate{(alternatesTried.Count == 1 ? "" : "s")} in the manifest: none match."
|
||||||
|
: "");
|
||||||
|
|
||||||
checks.Add(reference.Kind == "api"
|
checks.Add(reference.Kind == "api"
|
||||||
? ValidationCheck.Warn("timecode_plate_match", expected.ToString(), renderTc.ToString(),
|
? ValidationCheck.Warn("timecode_plate_match", expected.ToString(), renderTc.ToString(),
|
||||||
@@ -430,20 +466,9 @@ public sealed class ValidationStage
|
|||||||
var timing = m.Source?.TimingPlate;
|
var timing = m.Source?.TimingPlate;
|
||||||
if (timing is not null && !string.IsNullOrWhiteSpace(timing.Path))
|
if (timing is not null && !string.IsNullOrWhiteSpace(timing.Path))
|
||||||
{
|
{
|
||||||
var local = mapper.Map(timing.Path);
|
var manifestPlate = ReadManifestPlate(timing, mapper, handleFrames, renderFirstFrame);
|
||||||
var header = File.Exists(local) ? ExrHeader.TryRead(local) : null;
|
if (manifestPlate is not null) return manifestPlate;
|
||||||
if (header?.Timecode is not null)
|
_log.LogWarning("Timing plate from manifest is unreadable: {Path}", mapper.Map(timing.Path));
|
||||||
{
|
|
||||||
// The plate's own file numbering, never the comp's frame range:
|
|
||||||
// the render's first frame is in output numbering, so anchoring
|
|
||||||
// on a comp frame skews the comparison by the gap between them.
|
|
||||||
var frame = SequencePattern.FrameFromName(Path.GetFileName(local))
|
|
||||||
?? timing.FrameStart
|
|
||||||
?? renderFirstFrame;
|
|
||||||
return new TimecodeReference("manifest", local, header.Timecode.Value, frame, handleFrames,
|
|
||||||
header.FramesPerSecond);
|
|
||||||
}
|
|
||||||
_log.LogWarning("Timing plate from manifest is unreadable: {Path}", local);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Rank 2 — derive the plate from a configured search root.
|
// Rank 2 — derive the plate from a configured search root.
|
||||||
@@ -469,6 +494,63 @@ public sealed class ValidationStage
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static TimecodeReference? ReadManifestPlate(PlateRef plate, PathMapper mapper, int handleFrames, int renderFirstFrame)
|
||||||
|
{
|
||||||
|
var local = mapper.Map(plate.Path);
|
||||||
|
var header = File.Exists(local) ? ExrHeader.TryRead(local) : null;
|
||||||
|
if (header?.Timecode is null) return null;
|
||||||
|
|
||||||
|
// The plate's own file numbering, never the comp's frame range: the
|
||||||
|
// render's first frame is in output numbering, so anchoring on a comp
|
||||||
|
// frame skews the comparison by the gap between them.
|
||||||
|
var frame = SequencePattern.FrameFromName(Path.GetFileName(local))
|
||||||
|
?? plate.FrameStart
|
||||||
|
?? renderFirstFrame;
|
||||||
|
return new TimecodeReference("manifest", local, header.Timecode.Value, frame, handleFrames,
|
||||||
|
header.FramesPerSecond);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The manifest's other delivered plates, tried when the timing plate does
|
||||||
|
/// not match. Only real footage qualifies — under a configured plate search
|
||||||
|
/// root or, with none configured, alongside the timing plate in its footage
|
||||||
|
/// folder — because plates[] is every file in the comp, and a pre-render
|
||||||
|
/// agrees with the render whether or not either is right.
|
||||||
|
/// </summary>
|
||||||
|
private IEnumerable<TimecodeReference> ResolveAlternatePlates(RenderManifest m, int renderFirstFrame, TimecodeReference timingPlate)
|
||||||
|
{
|
||||||
|
var plates = m.Source?.Plates;
|
||||||
|
if (plates is null || plates.Count == 0 || timingPlate.PlatePath is null) yield break;
|
||||||
|
|
||||||
|
var mapper = new PathMapper(_options.PathMappings);
|
||||||
|
static string AsFolder(string dir) => Path.TrimEndingDirectorySeparator(Path.GetFullPath(dir)) + Path.DirectorySeparatorChar;
|
||||||
|
|
||||||
|
var deliveredRoots = _options.PlateSearchRoots.Count > 0
|
||||||
|
? _options.PlateSearchRoots.Select(r => AsFolder(mapper.Map(r))).ToList()
|
||||||
|
// {footage}/{shot}_{element}/file.exr or {footage}/{shot}/{element}/file.exr:
|
||||||
|
// two levels up from the timing plate is the folder its siblings share.
|
||||||
|
: Path.GetDirectoryName(Path.GetDirectoryName(Path.GetFullPath(timingPlate.PlatePath))) is { } footage
|
||||||
|
? new List<string> { AsFolder(footage) }
|
||||||
|
: new List<string>();
|
||||||
|
|
||||||
|
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { Path.GetFullPath(timingPlate.PlatePath) };
|
||||||
|
foreach (var plate in plates)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(plate.Path)) continue;
|
||||||
|
var local = Path.GetFullPath(mapper.Map(plate.Path));
|
||||||
|
if (!seen.Add(local)) continue;
|
||||||
|
if (!deliveredRoots.Any(root => local.StartsWith(root, StringComparison.OrdinalIgnoreCase))) continue;
|
||||||
|
|
||||||
|
var reference = ReadManifestPlate(plate, mapper, timingPlate.HandleFrames, renderFirstFrame);
|
||||||
|
if (reference is null)
|
||||||
|
{
|
||||||
|
_log.LogWarning("Alternate plate from manifest is unreadable: {Path}", local);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
yield return reference;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Rank 2: look for the shot's plate folders under the configured roots,
|
/// Rank 2: look for the shot's plate folders under the configured roots,
|
||||||
/// mirroring the panel's two on-disk layouts ({shotCode}/{element}/ and
|
/// mirroring the panel's two on-disk layouts ({shotCode}/{element}/ and
|
||||||
|
|||||||
Reference in New Issue
Block a user