feat(pipeline): standalone Blender renders on the same queue
Deploy / deploy (push) Successful in 7m2s

Blender work is not a shot submission: no Export, no version, no
validation, QC or delivery. It is "Ctrl+F12, but on the farm", so it is
modelled as what it is — a queue entry and nothing more.

That turned out to need very little. Claim, progress, fail and
complete-render already tolerated a job with no Export behind them, so a
Blender job reuses the whole lifecycle; RenderJob gains projectId, label
and submittedBy so a job with no shot code still has something to show.

Capability gating is new and was already missing: the claim now filters
job types by the machine's reported tools. Without it a workstation
without Blender would claim Blender work and fail it repeatedly, burning
the attempts while a machine that could have rendered it sat idle. The
worker asks for Blender jobs unconditionally and the server refuses them
— trusting the worker to self-select would have left the same hole.

The runner treats a clean exit with no saved frames as a failure:
Blender exits 0 when a frame range is empty or an output path is
unwritable, and a job that silently rendered nothing is the worst
outcome available.

Standalone jobs auto-requeue on a transient failure like any other work;
delivery builds are deliberately excluded, since a half-written package
wants a human first.

Adds the Blender add-on (BlenderPanel/) and its docs. The add-on refuses
to submit an unsaved file — the farm opens the .blend from disk, so
unsaved changes would render the previous state without warning. Its
payload is pinned by a contract test, since add-on and server only ever
meet over JSON.

Worker 1.5.0. Adds migration 20260815000000_blender_render_jobs.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
twotalesanimation
2026-08-15 18:53:19 +02:00
parent bada122f42
commit 70cf45ead4
24 changed files with 1347 additions and 9 deletions
+14 -1
View File
@@ -37,7 +37,20 @@ public sealed class ApiClient
hostname = Dns.GetHostName(),
workerVersion = o.WorkerVersion,
aeVersion = o.AeVersion,
capabilities = new { maxConcurrentJobs = 1, tools = new { ffmpeg = o.FfmpegPath != null, oiiotool = o.OiiotoolPath != null } },
// The server hands out BLENDER_RENDER work on the strength of the
// blender entry, so it reports the resolved path rather than a bare
// flag: a machine claiming Blender jobs it cannot run is worse than
// one that never claims them.
capabilities = new
{
maxConcurrentJobs = 1,
tools = new
{
ffmpeg = o.FfmpegPath != null,
oiiotool = o.OiiotoolPath != null,
blender = !string.IsNullOrWhiteSpace(o.BlenderPath) && File.Exists(o.BlenderPath) ? o.BlenderPath : null,
},
},
}, Json, ct);
res.EnsureSuccessStatusCode();
return (await res.Content.ReadFromJsonAsync<RegisterResponse>(Json, ct))!;
@@ -0,0 +1,68 @@
using System.Text.RegularExpressions;
namespace VFXReviewWorker;
/// <summary>
/// Parses `blender -b` stdout. Blender reports the frame it is working on in
/// its status lines ("Fra:1042 Mem:512.00M ...") and confirms each written file
/// with "Saved: '...'". The saved lines are the reliable signal — a frame can
/// be reported for a long time while it renders — so progress counts those and
/// falls back to Fra: for a first sign of life.
/// </summary>
public static partial class BlenderProgressParser
{
[GeneratedRegex(@"^Fra:(\d+)", RegexOptions.Compiled)]
private static partial Regex FrameLine();
[GeneratedRegex(@"^Saved:\s*'(.+)'", RegexOptions.Compiled)]
private static partial Regex SavedLine();
[GeneratedRegex(@"^(Error|Warning): |^Error: ", RegexOptions.Compiled | RegexOptions.IgnoreCase)]
private static partial Regex ErrorLine();
/// <summary>
/// Failures that will repeat identically on another machine — a missing
/// file, an unknown scene, an unwritable output. Retrying these burns the
/// queue for nothing; a human has to fix the .blend or the path.
/// </summary>
private static readonly string[] NonRetryablePatterns =
{
"cannot read file",
"file not found",
"unable to open",
"is not a blend file",
"failed to read blend file",
"unknown scene",
"scene not found",
"unable to create directory",
"cannot write file",
"no camera",
"python script fail",
"unknown argument",
};
/// <summary>The frame Blender is currently working on, or null.</summary>
public static int? ParseFrame(string line)
{
var m = FrameLine().Match(line);
return m.Success && int.TryParse(m.Groups[1].Value, out var f) ? f : null;
}
/// <summary>The path of a frame Blender has just written, or null.</summary>
public static string? ParseSaved(string line)
{
var m = SavedLine().Match(line);
return m.Success ? m.Groups[1].Value : null;
}
public static bool IsErrorLine(string line) => ErrorLine().IsMatch(line);
public static bool IsNonRetryableError(string line)
{
foreach (var p in NonRetryablePatterns)
{
if (line.Contains(p, StringComparison.OrdinalIgnoreCase)) return true;
}
return false;
}
}
@@ -0,0 +1,260 @@
using System.Diagnostics;
using System.Text;
using Microsoft.Extensions.Logging;
namespace VFXReviewWorker;
/// <summary>
/// Standalone Blender renders: `blender -b file.blend -s N -e M -a`.
///
/// The .blend is expected to be ready to render — output path, format and
/// frame range already set by whoever built it. Overrides are only applied
/// when the submission asked for them, so this stays "Ctrl+F12 on the farm"
/// rather than a second place where render settings can drift.
/// </summary>
public sealed class BlenderRunner
{
private readonly WorkerOptions _options;
private readonly ApiClient _api;
private readonly ILogger<BlenderRunner> _log;
public BlenderRunner(WorkerOptions options, ApiClient api, ILogger<BlenderRunner> log)
{
_options = options;
_api = api;
_log = log;
}
public async Task<RenderResult> RunAsync(
ClaimedJob job,
BlenderManifest m,
string machineId,
ServerConfig config,
CancellationToken cancelJob,
CancellationToken shutdown)
{
var mapper = new PathMapper(_options.PathMappings);
var blendLocal = mapper.Map(m.BlendFile);
var totalFrames = m.TotalFrames;
if (!File.Exists(blendLocal))
{
return new RenderResult(false, -1, Retryable: false,
$"Blend file not found on this machine: {blendLocal}", "", "", 0);
}
// Argument order matters to Blender: everything before -a configures the
// render, and -a must come last or it renders with the old settings.
var args = new List<string> { "-b", blendLocal };
if (!string.IsNullOrWhiteSpace(m.Scene)) { args.Add("-S"); args.Add(m.Scene!); }
if (!string.IsNullOrWhiteSpace(m.OutputPath))
{
args.Add("-o");
args.Add(mapper.Map(m.OutputPath!));
}
if (!string.IsNullOrWhiteSpace(m.Engine)) { args.Add("-E"); args.Add(m.Engine!); }
var pythonOverrides = BuildOverrideExpression(m);
if (pythonOverrides is not null) { args.Add("--python-expr"); args.Add(pythonOverrides); }
args.Add("-s"); args.Add(m.FrameStart.ToString());
args.Add("-e"); args.Add(m.FrameEnd.ToString());
args.Add("-a");
_log.LogInformation("Rendering {Label}: {Blend} frames {Start}-{End}",
job.Label ?? job.Id, blendLocal, m.FrameStart, m.FrameEnd);
return await ExecuteAsync(job.Id, machineId, args, totalFrames, m.FrameStart, config,
$"blender_{job.Id}.log", cancelJob, shutdown);
}
/// <summary>
/// Settings Blender has no command-line flag for. Returns null when the
/// submission asked for no overrides, so the file renders exactly as saved.
/// </summary>
private static string? BuildOverrideExpression(BlenderManifest m)
{
var parts = new List<string>();
if (m.Samples is int samples)
{
parts.Add($"getattr(S.cycles, 'samples', None) and setattr(S.cycles, 'samples', {samples})");
}
if (m.ResolutionPercent is int pct)
{
parts.Add($"setattr(S.render, 'resolution_percentage', {pct})");
}
if (!string.IsNullOrWhiteSpace(m.Device))
{
var deviceValue = m.Device!.Equals("GPU", StringComparison.OrdinalIgnoreCase) ? "GPU" : "CPU";
parts.Add($"hasattr(S, 'cycles') and setattr(S.cycles, 'device', '{deviceValue}')");
}
if (parts.Count == 0) return null;
return "import bpy; S = bpy.context.scene; " + string.Join("; ", parts);
}
private async Task<RenderResult> ExecuteAsync(
string jobId,
string machineId,
List<string> args,
int totalFrames,
int frameStart,
ServerConfig config,
string logFileName,
CancellationToken cancelJob,
CancellationToken shutdown)
{
var fullLogPath = Path.Combine(WorkerOptions.LogsDir, logFileName);
Directory.CreateDirectory(WorkerOptions.LogsDir);
var psi = new ProcessStartInfo
{
FileName = _options.BlenderPath,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true,
StandardOutputEncoding = Encoding.UTF8,
StandardErrorEncoding = Encoding.UTF8,
};
foreach (var a in args) psi.ArgumentList.Add(a);
var tail = new LogTail(200);
var eta = new EtaCalculator();
var sw = Stopwatch.StartNew();
var lastProgressAt = DateTimeOffset.UtcNow;
var lastReportAt = DateTimeOffset.MinValue;
var framesSaved = 0;
var currentFrame = frameStart;
string? firstError = null;
var nonRetryable = false;
var cancelledByServer = false;
using var process = new Process { StartInfo = psi };
await using var fullLog = new StreamWriter(fullLogPath, append: false, Encoding.UTF8);
var logLock = new object();
void HandleLine(string? line, bool isStderr)
{
if (line is null) return;
lock (logLock)
{
fullLog.WriteLine(line);
tail.Add(line);
}
var frame = BlenderProgressParser.ParseFrame(line);
if (frame is not null)
{
// A frame line only proves Blender is alive; a long frame emits
// many of them, so this feeds the stall watchdog, not progress.
currentFrame = frame.Value;
lastProgressAt = DateTimeOffset.UtcNow;
}
if (BlenderProgressParser.ParseSaved(line) is not null)
{
framesSaved++;
lastProgressAt = DateTimeOffset.UtcNow;
eta.RecordFrame(lastProgressAt);
}
if ((isStderr || BlenderProgressParser.IsErrorLine(line)) && !string.IsNullOrWhiteSpace(line))
{
if (BlenderProgressParser.IsErrorLine(line)) firstError ??= line.Trim();
if (BlenderProgressParser.IsNonRetryableError(line)) nonRetryable = true;
}
}
process.OutputDataReceived += (_, e) => HandleLine(e.Data, isStderr: false);
process.ErrorDataReceived += (_, e) => HandleLine(e.Data, isStderr: true);
try
{
process.Start();
}
catch (Exception ex)
{
return new RenderResult(false, -1, Retryable: false,
$"Failed to launch Blender at {_options.BlenderPath}: {ex.Message}",
tail.ToString(), fullLogPath, 0);
}
process.BeginOutputReadLine();
process.BeginErrorReadLine();
while (!process.HasExited)
{
await Task.Delay(TimeSpan.FromSeconds(2), CancellationToken.None);
if (cancelJob.IsCancellationRequested || shutdown.IsCancellationRequested)
{
_log.LogWarning("Cancellation requested — killing Blender tree for job {JobId}", jobId);
KillTree(process);
cancelledByServer = cancelJob.IsCancellationRequested;
break;
}
if ((DateTimeOffset.UtcNow - lastProgressAt).TotalSeconds > config.StallTimeoutSeconds)
{
_log.LogError("Blender stalled ({Sec}s without output) — killing job {JobId}", config.StallTimeoutSeconds, jobId);
KillTree(process);
await WaitForExitSafe(process);
return new RenderResult(false, -2, Retryable: true,
$"Blender produced no output for {config.StallTimeoutSeconds}s (stall timeout)",
tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
}
if ((DateTimeOffset.UtcNow - lastReportAt).TotalSeconds >= 10)
{
lastReportAt = DateTimeOffset.UtcNow;
var progress = totalFrames > 0 ? Math.Min(1.0, (double)framesSaved / totalFrames) : 0;
var resp = await _api.ProgressAsync(jobId, machineId, progress, currentFrame, totalFrames,
eta.EstimateSeconds(totalFrames - framesSaved), tail.ToString(), shutdown);
if (resp?.CancelRequested == true)
{
_log.LogWarning("Server requested cancel for job {JobId}", jobId);
KillTree(process);
cancelledByServer = true;
break;
}
}
}
await WaitForExitSafe(process);
sw.Stop();
lock (logLock) { fullLog.Flush(); }
if (cancelledByServer)
{
return new RenderResult(false, -3, Retryable: false, "Cancelled", tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
}
var exitCode = process.ExitCode;
if (exitCode != 0)
{
return new RenderResult(false, exitCode, Retryable: !nonRetryable,
firstError ?? $"Blender exited with code {exitCode}",
tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
}
// Blender exits 0 having rendered nothing if the range is empty or the
// output path is unwritable, so a silent success is treated as failure.
if (framesSaved == 0)
{
return new RenderResult(false, 0, Retryable: false,
"Blender exited cleanly but saved no frames — check the output path and frame range in the .blend",
tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
}
_log.LogInformation("Job {JobId}: {Saved}/{Total} frames in {Sec:F0}s", jobId, framesSaved, totalFrames, sw.Elapsed.TotalSeconds);
return new RenderResult(true, 0, true, null, tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
}
private static void KillTree(Process process)
{
try { if (!process.HasExited) process.Kill(entireProcessTree: true); } catch { }
}
private static async Task WaitForExitSafe(Process process)
{
try { await process.WaitForExitAsync(); } catch { }
}
}
+29
View File
@@ -49,6 +49,9 @@ public sealed class ClaimedJob
public int Attempt { get; set; }
public int MaxAttempts { get; set; }
public string? ExportId { get; set; }
public string? DeliveryId { get; set; }
/// <summary>Set for jobs with no shot behind them; shown instead of a shot code.</summary>
public string? Label { get; set; }
public DateTimeOffset? LeaseExpiresAt { get; set; }
/// <summary>Manifest shape depends on the job type — deserialize per stage.</summary>
@@ -65,6 +68,32 @@ public sealed class ClaimedJob
public DeliveryManifest AsDeliveryManifest() =>
Manifest.Deserialize<DeliveryManifest>(WorkerOptions.JsonOpts)
?? throw new InvalidOperationException("Job manifest is not a delivery manifest");
public BlenderManifest AsBlenderManifest() =>
Manifest.Deserialize<BlenderManifest>(WorkerOptions.JsonOpts)
?? throw new InvalidOperationException("Job manifest is not a Blender manifest");
}
/// <summary>
/// A standalone Blender render. Overrides are all optional: an unset value
/// leaves whatever the .blend was saved with, which is the point — the file is
/// expected to be ready to render.
/// </summary>
public sealed class BlenderManifest
{
public string Stage { get; set; } = "BLENDER";
public string BlendFile { get; set; } = "";
public string? Scene { get; set; }
public int FrameStart { get; set; }
public int FrameEnd { get; set; }
public string? OutputPath { get; set; }
public string? Engine { get; set; }
public string? Device { get; set; }
public int? Samples { get; set; }
public int? ResolutionPercent { get; set; }
public string? BlenderVersion { get; set; }
public int TotalFrames => FrameEnd - FrameStart + 1;
}
/// <summary>DELIVERY_BUILD manifest (server-built, RenderPipeline2 §11).</summary>
+77 -1
View File
@@ -16,11 +16,12 @@ public sealed class JobExecutor
private readonly PreviewStage _preview;
private readonly ValidationStage _validation;
private readonly DeliveryStage _delivery;
private readonly BlenderRunner _blender;
private readonly ReportSpooler _spooler;
private readonly ILogger<JobExecutor> _log;
public JobExecutor(WorkerOptions options, ApiClient api, AerenderRunner runner, PreviewStage preview,
ValidationStage validation, DeliveryStage delivery, ReportSpooler spooler, ILogger<JobExecutor> log)
ValidationStage validation, DeliveryStage delivery, BlenderRunner blender, ReportSpooler spooler, ILogger<JobExecutor> log)
{
_options = options;
_api = api;
@@ -28,6 +29,7 @@ public sealed class JobExecutor
_preview = preview;
_validation = validation;
_delivery = delivery;
_blender = blender;
_spooler = spooler;
_log = log;
}
@@ -46,6 +48,12 @@ public sealed class JobExecutor
return;
}
if (job.Type == "BLENDER_RENDER")
{
await RunBlenderAsync(job, machineId, config, cancelJob, shutdown);
return;
}
var manifest = job.AsRenderManifest();
var mapper = new PathMapper(_options.PathMappings);
var outputDirLocal = mapper.Map(manifest.OutputDir);
@@ -169,6 +177,74 @@ public sealed class JobExecutor
}
}
/// <summary>
/// BLENDER_RENDER job: a standalone render with no Export behind it, so
/// there is nothing to validate, preview or QC — it finishes at
/// complete-render and that is the whole lifecycle.
/// </summary>
private async Task RunBlenderAsync(ClaimedJob job, string machineId, ServerConfig config,
CancellationToken cancelJob, CancellationToken shutdown)
{
var manifest = job.AsBlenderManifest();
WriteState(new CurrentJobState
{
JobId = job.Id,
ExportId = null,
MachineId = machineId,
OutputDirLocal = manifest.OutputPath ?? "",
ClaimedAt = DateTimeOffset.UtcNow,
});
try
{
var result = await _blender.RunAsync(job, manifest, machineId, config, cancelJob, shutdown);
var logKey = string.IsNullOrEmpty(result.FullLogPath)
? null
: await _api.UploadLogAsync(job.Id, machineId, result.FullLogPath, shutdown);
if (result.Success)
{
_log.LogInformation("Blender job {JobId} complete: {Frames} frames in {Sec:F0}s",
job.Id, manifest.TotalFrames, result.RenderSeconds);
_spooler.Enqueue("complete-render", job.Id, new Dictionary<string, object?>
{
["machineId"] = machineId,
["renderSeconds"] = (int)result.RenderSeconds,
["logFileKey"] = logKey,
["logTail"] = result.LogTail,
});
return;
}
_spooler.Enqueue("fail", job.Id, new Dictionary<string, object?>
{
["machineId"] = machineId,
["stage"] = "BLENDER",
["exitCode"] = result.ExitCode,
["errorMessage"] = result.ErrorMessage,
["logTail"] = result.LogTail,
["logFileKey"] = logKey,
["retryable"] = result.ErrorMessage == "Cancelled" ? false : result.Retryable,
});
}
catch (Exception ex)
{
_log.LogError(ex, "Unexpected Blender error for job {JobId}", job.Id);
_spooler.Enqueue("fail", job.Id, new Dictionary<string, object?>
{
["machineId"] = machineId,
["stage"] = "BLENDER",
["errorMessage"] = $"Worker exception: {ex.Message}",
["retryable"] = true,
});
}
finally
{
ClearState();
}
}
/// <summary>
/// DELIVERY_BUILD job: build the dated package folder from already-QC'd
/// exports. No AE involved — folders, links and verification only.
@@ -38,6 +38,11 @@ public static class Preflight
Console.WriteLine();
ok &= Report("aerender.exe", options.AerenderPath, File.Exists(options.AerenderPath));
// Not required: a machine without Blender simply never claims Blender
// jobs, so this reports rather than fails.
Report("blender.exe", options.BlenderPath ?? "(not configured)",
!string.IsNullOrWhiteSpace(options.BlenderPath) && File.Exists(options.BlenderPath),
"needed only for Blender render jobs");
var afterFx = options.ResolveAfterFxPath();
ok &= Report("AfterFX.com", afterFx, File.Exists(afterFx),
"needed for preview builds only");
+1
View File
@@ -41,6 +41,7 @@ builder.Services.AddSingleton<AerenderRunner>();
builder.Services.AddSingleton<PreviewStage>();
builder.Services.AddSingleton<ValidationStage>();
builder.Services.AddSingleton<DeliveryStage>();
builder.Services.AddSingleton<BlenderRunner>();
builder.Services.AddSingleton<JobExecutor>();
builder.Services.AddHostedService<WorkerService>();
@@ -15,7 +15,7 @@
every change that gets published. publish.ps1 appends the git SHA via
SourceRevisionId, giving "1.1.0+feb1c7b".
-->
<Version>1.4.0</Version>
<Version>1.5.0</Version>
</PropertyGroup>
<ItemGroup>
@@ -20,6 +20,12 @@ public sealed class WorkerOptions
/// <summary>vfxr_build_preview.jsx — defaults to scripts\ beside the worker exe.</summary>
public string? PreviewScriptPath { get; set; }
public string? FfmpegPath { get; set; }
/// <summary>
/// blender.exe. Its presence is what makes this machine eligible for
/// Blender jobs: the version is reported at registration and the server
/// only hands BLENDER_RENDER work to machines that have it.
/// </summary>
public string? BlenderPath { get; set; }
public string? OiiotoolPath { get; set; }
public List<PathMapping> PathMappings { get; set; } = new();
/// <summary>
@@ -64,9 +64,11 @@ public sealed class WorkerService : BackgroundService
}
// Delivery builds are file work with no AE involvement, so they
// are always claimable — someone is usually waiting on the folder.
// Blender is asked for unconditionally; the server refuses it to
// machines whose capabilities do not include Blender.
var types = aeOpen
? new[] { "AE_RENDER", "DELIVERY_BUILD" }
: new[] { "AE_RENDER", "PREVIEW_ONLY", "DELIVERY_BUILD" };
? new[] { "AE_RENDER", "DELIVERY_BUILD", "BLENDER_RENDER" }
: new[] { "AE_RENDER", "PREVIEW_ONLY", "DELIVERY_BUILD", "BLENDER_RENDER" };
var job = await _api.ClaimAsync(_machineId!, types, stoppingToken);
if (job is not null)