feat(pipeline): render queue, worker service and automated preview generation
One "Queue Export" click now renders the EXR sequence, then rebuilds the shot headlessly with the studio slate/overlay template to produce the delivery MOV and review MP4. Implements RenderPipeline2 phases 1-2 plus the preview stage. Server: - New models Export, RenderJob, ExportEvent, Machine, WorkerHeartbeat, plus Project.deliveryConfig and per-submission slate fields (Export.vfxScope, Export.submissionNote, inherited from the shot's previous export). Both migrations are purely additive; no existing column is touched. - lib/render-pipeline: server-enforced state machine, transactional version increment with supersede, atomic FOR UPDATE SKIP LOCKED claim gated by machine availability windows, and a lease reaper run from instrumentation.ts. - /api/ext/* endpoints for the panel and workers; session-auth mirrors under /api/render and /api/machines for the web UI. - Pipeline pages: render queue, export detail, machine monitoring, plus an Exports tab on shot detail. RenderWorker (.NET 8 Windows service, new): - Registration, heartbeat as cancel channel, claim loop, aerender runner with progress parsing and stall watchdog, crash recovery and disk-spooled reporting that survives server downtime. - Preview stage: headless AE assembles the preview comp into a throwaway AEP with both output modules queued, then a single aerender pass renders them. Preview jobs are not claimed while an interactive AE session is open, so an artist's project is never taken over. AE panel: Queue Export with live status polling, urgent flag, retry, and the VFX Scope / Submission Note fields. Every existing panel action is unchanged. Preview chaining ships disabled behind SystemConfig preview.enabled. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
twotalesanimation
@@ -0,0 +1,33 @@
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace VFXReviewWorker;
|
||||
|
||||
/// <summary>
|
||||
/// Detects an *interactive* After Effects session on this machine.
|
||||
///
|
||||
/// This matters because `AfterFX.com -noui -r script.jsx` is handed to an
|
||||
/// already-running After Effects instance rather than starting an isolated
|
||||
/// one — so running the preview build while an artist has AE open would take
|
||||
/// over their session and quit it, losing unsaved work.
|
||||
///
|
||||
/// Deliberately conservative: this matches any After Effects process, the
|
||||
/// interactive `AfterFX.exe` as well as the `AfterFX.com` render engine that
|
||||
/// aerender drives. Waiting for either to finish also avoids two AE instances
|
||||
/// competing for the same box, and the worst case is only that a preview build
|
||||
/// starts a little later.
|
||||
/// </summary>
|
||||
public static class AeSession
|
||||
{
|
||||
public static bool InteractiveRunning()
|
||||
{
|
||||
try
|
||||
{
|
||||
return Process.GetProcessesByName("AfterFX").Length > 0;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// If we cannot tell, assume an artist is working — never risk their session.
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace VFXReviewWorker;
|
||||
|
||||
public sealed record RenderResult(
|
||||
bool Success,
|
||||
int ExitCode,
|
||||
bool Retryable,
|
||||
string? ErrorMessage,
|
||||
string LogTail,
|
||||
string FullLogPath,
|
||||
double RenderSeconds);
|
||||
|
||||
/// <summary>
|
||||
/// Stage 1: aerender execution (§7.3). Launches aerender.exe, parses stdout
|
||||
/// for PROGRESS/ERROR lines, reports progress (renewing the lease), enforces
|
||||
/// the stall timeout, and honours cancellation by killing the process tree.
|
||||
/// </summary>
|
||||
public sealed class AerenderRunner
|
||||
{
|
||||
private readonly WorkerOptions _options;
|
||||
private readonly ApiClient _api;
|
||||
private readonly ILogger<AerenderRunner> _log;
|
||||
|
||||
public AerenderRunner(WorkerOptions options, ApiClient api, ILogger<AerenderRunner> log)
|
||||
{
|
||||
_options = options;
|
||||
_api = api;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Renders every queued item in an already-prepared project (no -comp), used
|
||||
/// by the preview stage: the build script saved an AEP with the MOV and MP4
|
||||
/// output modules already queued, so one launch produces both.
|
||||
/// </summary>
|
||||
public Task<RenderResult> RunProjectAsync(
|
||||
string jobId,
|
||||
string machineId,
|
||||
string projectPathLocal,
|
||||
int totalFrames,
|
||||
ServerConfig config,
|
||||
CancellationToken cancelJob,
|
||||
CancellationToken shutdown)
|
||||
{
|
||||
var args = new List<string> { "-project", projectPathLocal, "-mp" };
|
||||
_log.LogInformation("Rendering prepared project for job {JobId}: {Project}", jobId, projectPathLocal);
|
||||
return ExecuteAsync(jobId, machineId, args, totalFrames, 0, config, $"aerender_{jobId}_preview.log", cancelJob, shutdown);
|
||||
}
|
||||
|
||||
public async Task<RenderResult> RunAsync(
|
||||
ClaimedJob job,
|
||||
RenderManifest m,
|
||||
string machineId,
|
||||
string outputDirLocal,
|
||||
ServerConfig config,
|
||||
CancellationToken cancelJob,
|
||||
CancellationToken shutdown)
|
||||
{
|
||||
var mapper = new PathMapper(_options.PathMappings);
|
||||
var aepLocal = mapper.Map(m.AepPath);
|
||||
var outputArg = Path.Combine(outputDirLocal, m.OutputPattern);
|
||||
|
||||
PrepareOutputDir(outputDirLocal, m.OutputPattern);
|
||||
|
||||
var args = new List<string>
|
||||
{
|
||||
"-project", aepLocal,
|
||||
"-comp", m.CompName,
|
||||
"-s", m.FrameStart.ToString(),
|
||||
"-e", m.FrameEnd.ToString(),
|
||||
};
|
||||
if (!string.IsNullOrEmpty(m.RenderSettingsTemplate)) { args.Add("-RStemplate"); args.Add(m.RenderSettingsTemplate); }
|
||||
if (!string.IsNullOrEmpty(m.OutputModuleTemplate)) { args.Add("-OMtemplate"); args.Add(m.OutputModuleTemplate); }
|
||||
args.Add("-output"); args.Add(outputArg);
|
||||
// NB: no "-continueOnMissingFootage" — the real flag takes no value
|
||||
// (passing "false" is an aerender SYNTAX ERROR), and its *presence*
|
||||
// enables skipping missing footage. aerender's default is to stop on
|
||||
// missing footage, which is exactly what the pipeline wants.
|
||||
|
||||
_log.LogInformation("Launching aerender for job {JobId}: {Aep} comp \"{Comp}\" frames {S}-{E} → {Out}",
|
||||
job.Id, aepLocal, m.CompName, m.FrameStart, m.FrameEnd, outputArg);
|
||||
|
||||
return await ExecuteAsync(job.Id, machineId, args, m.TotalFrames, m.FrameStart, config,
|
||||
$"aerender_{job.Id}.log", cancelJob, shutdown);
|
||||
}
|
||||
|
||||
private async Task<RenderResult> ExecuteAsync(
|
||||
string jobId,
|
||||
string machineId,
|
||||
List<string> args,
|
||||
int totalFrames,
|
||||
int frameStart,
|
||||
ServerConfig config,
|
||||
string logFileName,
|
||||
CancellationToken cancelJob,
|
||||
CancellationToken shutdown)
|
||||
{
|
||||
var fullLogPath = Path.Combine(WorkerOptions.LogsDir, logFileName);
|
||||
Directory.CreateDirectory(WorkerOptions.LogsDir);
|
||||
|
||||
var psi = new ProcessStartInfo
|
||||
{
|
||||
FileName = _options.AerenderPath,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false,
|
||||
CreateNoWindow = true,
|
||||
StandardOutputEncoding = Encoding.UTF8,
|
||||
StandardErrorEncoding = Encoding.UTF8,
|
||||
};
|
||||
foreach (var a in args) psi.ArgumentList.Add(a);
|
||||
|
||||
var tail = new LogTail(200);
|
||||
var eta = new EtaCalculator();
|
||||
var sw = Stopwatch.StartNew();
|
||||
var lastProgressAt = DateTimeOffset.UtcNow;
|
||||
var lastReportAt = DateTimeOffset.MinValue;
|
||||
int framesDone = 0;
|
||||
string? firstError = null;
|
||||
bool nonRetryable = false;
|
||||
bool cancelledByServer = false;
|
||||
|
||||
using var process = new Process { StartInfo = psi };
|
||||
await using var fullLog = new StreamWriter(fullLogPath, append: false, Encoding.UTF8);
|
||||
var logLock = new object();
|
||||
|
||||
void HandleLine(string? line, bool isStderr)
|
||||
{
|
||||
if (line is null) return;
|
||||
lock (logLock)
|
||||
{
|
||||
fullLog.WriteLine(line);
|
||||
tail.Add(line);
|
||||
}
|
||||
var frame = ProgressParser.ParseFrame(line);
|
||||
if (frame is not null)
|
||||
{
|
||||
framesDone = Math.Max(framesDone, frame.Value);
|
||||
lastProgressAt = DateTimeOffset.UtcNow;
|
||||
eta.RecordFrame(lastProgressAt);
|
||||
}
|
||||
if ((isStderr || ProgressParser.IsErrorLine(line)) && !string.IsNullOrWhiteSpace(line))
|
||||
{
|
||||
if (ProgressParser.IsErrorLine(line)) firstError ??= line.Trim();
|
||||
if (ProgressParser.IsNonRetryableError(line)) nonRetryable = true;
|
||||
}
|
||||
}
|
||||
|
||||
process.OutputDataReceived += (_, e) => HandleLine(e.Data, isStderr: false);
|
||||
process.ErrorDataReceived += (_, e) => HandleLine(e.Data, isStderr: true);
|
||||
|
||||
try
|
||||
{
|
||||
process.Start();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return new RenderResult(false, -1, Retryable: false,
|
||||
$"Failed to launch aerender at {_options.AerenderPath}: {ex.Message}",
|
||||
tail.ToString(), fullLogPath, 0);
|
||||
}
|
||||
process.BeginOutputReadLine();
|
||||
process.BeginErrorReadLine();
|
||||
|
||||
var total = totalFrames;
|
||||
while (!process.HasExited)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(2), CancellationToken.None);
|
||||
|
||||
if (cancelJob.IsCancellationRequested || shutdown.IsCancellationRequested)
|
||||
{
|
||||
_log.LogWarning("Cancellation requested — killing aerender tree for job {JobId}", jobId);
|
||||
KillTree(process);
|
||||
cancelledByServer = cancelJob.IsCancellationRequested;
|
||||
break;
|
||||
}
|
||||
|
||||
// Stall watchdog (§7.3): no progress line for stallTimeoutSeconds → kill
|
||||
if ((DateTimeOffset.UtcNow - lastProgressAt).TotalSeconds > config.StallTimeoutSeconds)
|
||||
{
|
||||
_log.LogError("aerender stalled ({Sec}s without progress) — killing job {JobId}", config.StallTimeoutSeconds, jobId);
|
||||
KillTree(process);
|
||||
await WaitForExitSafe(process);
|
||||
return new RenderResult(false, -2, Retryable: true,
|
||||
$"aerender produced no progress for {config.StallTimeoutSeconds}s (stall timeout)",
|
||||
tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
|
||||
}
|
||||
|
||||
// Progress report every ~10 s (renews lease); best-effort
|
||||
if ((DateTimeOffset.UtcNow - lastReportAt).TotalSeconds >= 10)
|
||||
{
|
||||
lastReportAt = DateTimeOffset.UtcNow;
|
||||
var progress = total > 0 ? Math.Min(1.0, (double)framesDone / total) : 0;
|
||||
try
|
||||
{
|
||||
var resp = await _api.ProgressAsync(jobId, machineId, progress,
|
||||
framesDone > 0 ? frameStart + framesDone - 1 : null,
|
||||
total, eta.EstimateSeconds(total - framesDone), tail.ToString(), shutdown);
|
||||
if (resp?.CancelRequested == true)
|
||||
{
|
||||
_log.LogWarning("Server requested cancel for job {JobId}", jobId);
|
||||
KillTree(process);
|
||||
cancelledByServer = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.LogDebug(ex, "Progress report failed (non-fatal; lease covered by reaper)");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await WaitForExitSafe(process);
|
||||
sw.Stop();
|
||||
lock (logLock) { fullLog.Flush(); }
|
||||
|
||||
if (cancelledByServer)
|
||||
{
|
||||
return new RenderResult(false, -3, Retryable: false, "Cancelled", tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
|
||||
}
|
||||
|
||||
var exitCode = process.ExitCode;
|
||||
if (exitCode == 0 && firstError is null)
|
||||
{
|
||||
return new RenderResult(true, 0, true, null, tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
|
||||
}
|
||||
return new RenderResult(false, exitCode, Retryable: !nonRetryable,
|
||||
firstError ?? $"aerender exited with code {exitCode}",
|
||||
tail.ToString(), fullLogPath, sw.Elapsed.TotalSeconds);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// §7.3: outputDir is created before launch; a pre-existing non-empty dir
|
||||
/// can only hold output from a previous crashed attempt of this same
|
||||
/// immutable version. Safety guard: only files matching the job's own
|
||||
/// output pattern prefix are ever deleted.
|
||||
/// </summary>
|
||||
/// <summary>Filename prefix of an output pattern ("X.[####].exr" → "X").</summary>
|
||||
internal static string OutputPatternPrefix(string outputPattern)
|
||||
{
|
||||
var bracket = outputPattern.IndexOf('[');
|
||||
return bracket > 0 ? outputPattern[..bracket].TrimEnd('.') : Path.GetFileNameWithoutExtension(outputPattern);
|
||||
}
|
||||
|
||||
internal static void PrepareOutputDir(string outputDirLocal, string outputPattern)
|
||||
{
|
||||
Directory.CreateDirectory(outputDirLocal);
|
||||
var prefix = OutputPatternPrefix(outputPattern);
|
||||
if (string.IsNullOrEmpty(prefix)) return;
|
||||
foreach (var file in Directory.EnumerateFiles(outputDirLocal))
|
||||
{
|
||||
if (Path.GetFileName(file).StartsWith(prefix, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
try { File.Delete(file); } catch { /* locked leftovers surface as an aerender error */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void KillTree(Process process)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!process.HasExited) process.Kill(entireProcessTree: true);
|
||||
}
|
||||
catch { /* already gone */ }
|
||||
}
|
||||
|
||||
private static async Task WaitForExitSafe(Process process)
|
||||
{
|
||||
try { await process.WaitForExitAsync(new CancellationTokenSource(TimeSpan.FromSeconds(30)).Token); }
|
||||
catch { try { process.Kill(entireProcessTree: true); } catch { } }
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Ring buffer of the last N output lines (§5.3 logTail).</summary>
|
||||
public sealed class LogTail
|
||||
{
|
||||
private readonly Queue<string> _lines;
|
||||
private readonly int _capacity;
|
||||
|
||||
public LogTail(int capacity)
|
||||
{
|
||||
_capacity = capacity;
|
||||
_lines = new Queue<string>(capacity);
|
||||
}
|
||||
|
||||
public void Add(string line)
|
||||
{
|
||||
_lines.Enqueue(line);
|
||||
while (_lines.Count > _capacity) _lines.Dequeue();
|
||||
}
|
||||
|
||||
public override string ToString() => string.Join('\n', _lines);
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Net.Http.Json;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace VFXReviewWorker;
|
||||
|
||||
/// <summary>
|
||||
/// HTTP client for the VFXReview /api/ext pipeline endpoints. Workers talk
|
||||
/// only HTTP — never a database (§2.1 principle 4).
|
||||
/// </summary>
|
||||
public sealed class ApiClient
|
||||
{
|
||||
private readonly HttpClient _http;
|
||||
private readonly ILogger<ApiClient> _log;
|
||||
|
||||
public ApiClient(WorkerOptions options, ILogger<ApiClient> log)
|
||||
{
|
||||
_log = log;
|
||||
_http = new HttpClient
|
||||
{
|
||||
BaseAddress = new Uri(options.ServerUrl.TrimEnd('/') + "/"),
|
||||
Timeout = TimeSpan.FromSeconds(30),
|
||||
};
|
||||
_http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", options.ApiKey);
|
||||
}
|
||||
|
||||
private static readonly JsonSerializerOptions Json = WorkerOptions.JsonOpts;
|
||||
|
||||
// E6
|
||||
public async Task<RegisterResponse> RegisterAsync(WorkerOptions o, CancellationToken ct)
|
||||
{
|
||||
var res = await _http.PostAsJsonAsync("api/ext/workers/register", new
|
||||
{
|
||||
name = o.MachineName,
|
||||
hostname = Dns.GetHostName(),
|
||||
workerVersion = o.WorkerVersion,
|
||||
aeVersion = o.AeVersion,
|
||||
capabilities = new { maxConcurrentJobs = 1, tools = new { ffmpeg = o.FfmpegPath != null, oiiotool = o.OiiotoolPath != null } },
|
||||
}, Json, ct);
|
||||
res.EnsureSuccessStatusCode();
|
||||
return (await res.Content.ReadFromJsonAsync<RegisterResponse>(Json, ct))!;
|
||||
}
|
||||
|
||||
// E7
|
||||
public async Task<HeartbeatResponse> HeartbeatAsync(string machineId, string? currentJobId, CancellationToken ct)
|
||||
{
|
||||
var res = await _http.PostAsJsonAsync($"api/ext/workers/{machineId}/heartbeat", new
|
||||
{
|
||||
cpuPercent = (double?)null,
|
||||
memPercent = (double?)null,
|
||||
diskFreeGb = GetDiskFreeGb(),
|
||||
currentJobId,
|
||||
}, Json, ct);
|
||||
res.EnsureSuccessStatusCode();
|
||||
return (await res.Content.ReadFromJsonAsync<HeartbeatResponse>(Json, ct))!;
|
||||
}
|
||||
|
||||
// E8 — null when the queue is empty / machine outside its render window (204)
|
||||
public async Task<ClaimedJob?> ClaimAsync(string machineId, string[] types, CancellationToken ct)
|
||||
{
|
||||
var res = await _http.PostAsJsonAsync("api/ext/render/jobs/claim", new
|
||||
{
|
||||
machineId,
|
||||
types,
|
||||
}, Json, ct);
|
||||
if (res.StatusCode == HttpStatusCode.NoContent) return null;
|
||||
res.EnsureSuccessStatusCode();
|
||||
var body = await res.Content.ReadFromJsonAsync<ClaimResponse>(Json, ct);
|
||||
return body?.Job;
|
||||
}
|
||||
|
||||
// E9 — best-effort; failures are swallowed by the caller (lease/reaper covers us)
|
||||
public async Task<ProgressResponse?> ProgressAsync(string jobId, string machineId, double progress,
|
||||
int? currentFrame, int? totalFrames, int? etaSeconds, string? logTail, CancellationToken ct)
|
||||
{
|
||||
using var req = new HttpRequestMessage(HttpMethod.Patch, $"api/ext/render/jobs/{jobId}/progress")
|
||||
{
|
||||
Content = JsonContent.Create(new { machineId, progress, currentFrame, totalFrames, etaSeconds, logTail }, options: Json),
|
||||
};
|
||||
var res = await _http.SendAsync(req, ct);
|
||||
if (!res.IsSuccessStatusCode) return null;
|
||||
return await res.Content.ReadFromJsonAsync<ProgressResponse>(Json, ct);
|
||||
}
|
||||
|
||||
// E10 / E11 — lifecycle reports, sent via the spooler for durability.
|
||||
// Returns true when the server accepted (2xx) OR permanently rejected (409
|
||||
// contradictory / 404 gone) — both mean "stop retrying".
|
||||
public async Task<bool> SendLifecycleAsync(string kind, string jobId, Dictionary<string, object?> payload, CancellationToken ct)
|
||||
{
|
||||
var path = kind switch
|
||||
{
|
||||
"fail" => $"api/ext/render/jobs/{jobId}/fail",
|
||||
"complete-render" => $"api/ext/render/jobs/{jobId}/complete-render",
|
||||
"finalize" => $"api/ext/render/jobs/{jobId}/finalize",
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(kind), kind, "unknown lifecycle report kind"),
|
||||
};
|
||||
var res = await _http.PostAsJsonAsync(path, payload, Json, ct);
|
||||
if (res.IsSuccessStatusCode) return true;
|
||||
if (res.StatusCode is HttpStatusCode.Conflict or HttpStatusCode.NotFound)
|
||||
{
|
||||
_log.LogWarning("Server permanently rejected {Kind} for job {JobId}: {Status} {Body}",
|
||||
kind, jobId, (int)res.StatusCode, await res.Content.ReadAsStringAsync(ct));
|
||||
return true; // don't retry contradictory/vanished reports
|
||||
}
|
||||
_log.LogWarning("Lifecycle report {Kind} for {JobId} failed with {Status}; will retry", kind, jobId, (int)res.StatusCode);
|
||||
return false;
|
||||
}
|
||||
|
||||
// E20 + PUT upload. Returns the storage key, or null when the upload failed.
|
||||
public async Task<string?> UploadArtifactAsync(
|
||||
string jobId, string machineId, string kind, string filePath, string contentType, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!File.Exists(filePath))
|
||||
{
|
||||
_log.LogWarning("Artifact {Kind} not found at {Path}", kind, filePath);
|
||||
return null;
|
||||
}
|
||||
|
||||
var presignRes = await _http.PostAsJsonAsync($"api/ext/render/jobs/{jobId}/artifact-presign", new
|
||||
{
|
||||
machineId,
|
||||
kind,
|
||||
fileName = Path.GetFileName(filePath),
|
||||
contentType,
|
||||
}, Json, ct);
|
||||
if (!presignRes.IsSuccessStatusCode)
|
||||
{
|
||||
_log.LogWarning("Presign for {Kind} failed: {Status}", kind, (int)presignRes.StatusCode);
|
||||
return null;
|
||||
}
|
||||
var presign = await presignRes.Content.ReadFromJsonAsync<PresignResponse>(Json, ct);
|
||||
if (presign is null) return null;
|
||||
|
||||
// Media files can be large — generous timeout, separate client so the
|
||||
// presigned URL is not sent with our Authorization header.
|
||||
using var upload = new HttpClient { Timeout = TimeSpan.FromMinutes(30) };
|
||||
await using var stream = File.OpenRead(filePath);
|
||||
var content = new StreamContent(stream);
|
||||
content.Headers.ContentType = new MediaTypeHeaderValue(contentType);
|
||||
var putRes = await upload.PutAsync(presign.PresignedUrl, content, ct);
|
||||
if (!putRes.IsSuccessStatusCode)
|
||||
{
|
||||
_log.LogWarning("Upload of {Kind} failed: {Status}", kind, (int)putRes.StatusCode);
|
||||
return null;
|
||||
}
|
||||
return presign.Key;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.LogWarning(ex, "Artifact upload ({Kind}) failed for job {JobId}", kind, jobId);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public Task<string?> UploadLogAsync(string jobId, string machineId, string logFilePath, CancellationToken ct) =>
|
||||
UploadArtifactAsync(jobId, machineId, "log", logFilePath, "text/plain", ct);
|
||||
|
||||
// E3 — used by crash-recovery reconcile (§7.5)
|
||||
public async Task<ExportDetail?> GetExportAsync(string exportId, CancellationToken ct)
|
||||
{
|
||||
var res = await _http.GetAsync($"api/ext/exports/{exportId}", ct);
|
||||
if (!res.IsSuccessStatusCode) return null;
|
||||
var body = await res.Content.ReadFromJsonAsync<ExportDetailResponse>(Json, ct);
|
||||
return body?.Export;
|
||||
}
|
||||
|
||||
private static double? GetDiskFreeGb()
|
||||
{
|
||||
try
|
||||
{
|
||||
var root = Path.GetPathRoot(Environment.SystemDirectory);
|
||||
if (root is null) return null;
|
||||
return Math.Round(new DriveInfo(root).AvailableFreeSpace / 1_073_741_824.0, 1);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace VFXReviewWorker;
|
||||
|
||||
// Wire types for the /api/ext pipeline endpoints (RenderPipeline2 §6).
|
||||
|
||||
public sealed class RegisterResponse
|
||||
{
|
||||
public MachineInfo Machine { get; set; } = new();
|
||||
public ServerConfig Config { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class MachineInfo
|
||||
{
|
||||
public string Id { get; set; } = "";
|
||||
public string Name { get; set; } = "";
|
||||
public bool Enabled { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Server-supplied tuning (E6) — fleet-tunable without touching installs.</summary>
|
||||
public sealed class ServerConfig
|
||||
{
|
||||
public int PollSeconds { get; set; } = 10;
|
||||
public int HeartbeatSeconds { get; set; } = 30;
|
||||
public int LeaseSeconds { get; set; } = 300;
|
||||
public int MaxAttempts { get; set; } = 3;
|
||||
public int StallTimeoutSeconds { get; set; } = 600;
|
||||
}
|
||||
|
||||
public sealed class ClaimResponse
|
||||
{
|
||||
public ClaimedJob Job { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class ClaimedJob
|
||||
{
|
||||
public string Id { get; set; } = "";
|
||||
public string Type { get; set; } = "AE_RENDER";
|
||||
public int Attempt { get; set; }
|
||||
public int MaxAttempts { get; set; }
|
||||
public string? ExportId { get; set; }
|
||||
public DateTimeOffset? LeaseExpiresAt { get; set; }
|
||||
|
||||
/// <summary>Manifest shape depends on the job type — deserialize per stage.</summary>
|
||||
public JsonElement Manifest { get; set; }
|
||||
|
||||
public RenderManifest AsRenderManifest() =>
|
||||
Manifest.Deserialize<RenderManifest>(WorkerOptions.JsonOpts)
|
||||
?? throw new InvalidOperationException("Job manifest is not a render manifest");
|
||||
|
||||
public PreviewManifest AsPreviewManifest() =>
|
||||
Manifest.Deserialize<PreviewManifest>(WorkerOptions.JsonOpts)
|
||||
?? throw new InvalidOperationException("Job manifest is not a preview manifest");
|
||||
}
|
||||
|
||||
/// <summary>PREVIEW_ONLY manifest (server-built, RenderPipeline2 §9).</summary>
|
||||
public sealed class PreviewManifest
|
||||
{
|
||||
public string Stage { get; set; } = "PREVIEW";
|
||||
public string ExportId { get; set; } = "";
|
||||
public string ShotCode { get; set; } = "";
|
||||
public string VersionString { get; set; } = "";
|
||||
public string OutputDir { get; set; } = "";
|
||||
public string OutputPattern { get; set; } = "";
|
||||
public int FrameStart { get; set; }
|
||||
public int FrameEnd { get; set; }
|
||||
public double Fps { get; set; } = 24;
|
||||
public string TemplateAep { get; set; } = "";
|
||||
public string TemplateComp { get; set; } = "";
|
||||
public string? OverlayComp { get; set; }
|
||||
public string? LutComp { get; set; }
|
||||
public string MovTemplate { get; set; } = "";
|
||||
public string Mp4Template { get; set; } = "";
|
||||
public string MovOutput { get; set; } = "";
|
||||
public string Mp4Output { get; set; } = "";
|
||||
public string? SlateScopeProp { get; set; }
|
||||
public string? SlateSubmissionProp { get; set; }
|
||||
public SlateInfo Slate { get; set; } = new();
|
||||
|
||||
public int TotalFrames => FrameEnd - FrameStart + 1;
|
||||
}
|
||||
|
||||
public sealed class SlateInfo
|
||||
{
|
||||
public string VersionName { get; set; } = "";
|
||||
public string Date { get; set; } = "";
|
||||
public string? Description { get; set; }
|
||||
public string? Notes { get; set; }
|
||||
public string ShotCode { get; set; } = "";
|
||||
public string? Episode { get; set; }
|
||||
public string? Scene { get; set; }
|
||||
public string? VfxScope { get; set; }
|
||||
public string? SubmissionNote { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Result JSON written by vfxr_build_preview.jsx.</summary>
|
||||
public sealed class PreviewBuildResult
|
||||
{
|
||||
public bool Ok { get; set; }
|
||||
public string? Error { get; set; }
|
||||
public List<string> Warnings { get; set; } = new();
|
||||
public string? TempAep { get; set; }
|
||||
public string? PreviewComp { get; set; }
|
||||
public int? FrameCount { get; set; }
|
||||
public int? Width { get; set; }
|
||||
public int? Height { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>The Render Manifest (§6.2) — only the fields the Phase 2 render stage needs; the rest is preserved raw.</summary>
|
||||
public sealed class RenderManifest
|
||||
{
|
||||
public string AepPath { get; set; } = "";
|
||||
public string CompName { get; set; } = "";
|
||||
public string OutputDir { get; set; } = "";
|
||||
public string OutputPattern { get; set; } = "";
|
||||
public string? OutputModuleTemplate { get; set; }
|
||||
public string? RenderSettingsTemplate { get; set; }
|
||||
public int FrameStart { get; set; }
|
||||
public int FrameEnd { get; set; }
|
||||
|
||||
[JsonExtensionData]
|
||||
public Dictionary<string, JsonElement>? Extra { get; set; }
|
||||
|
||||
public int TotalFrames => FrameEnd - FrameStart + 1;
|
||||
}
|
||||
|
||||
public sealed class ProgressResponse
|
||||
{
|
||||
public bool Ok { get; set; }
|
||||
public bool CancelRequested { get; set; }
|
||||
public DateTimeOffset? LeaseExpiresAt { get; set; }
|
||||
}
|
||||
|
||||
public sealed class HeartbeatResponse
|
||||
{
|
||||
public bool Ok { get; set; }
|
||||
public List<WorkerCommand> Commands { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class WorkerCommand
|
||||
{
|
||||
public string Type { get; set; } = "";
|
||||
public string? JobId { get; set; }
|
||||
}
|
||||
|
||||
public sealed class PresignResponse
|
||||
{
|
||||
public string PresignedUrl { get; set; } = "";
|
||||
public string Key { get; set; } = "";
|
||||
public string Url { get; set; } = "";
|
||||
}
|
||||
|
||||
public sealed class ExportDetailResponse
|
||||
{
|
||||
public ExportDetail Export { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class ExportDetail
|
||||
{
|
||||
public string Id { get; set; } = "";
|
||||
public string Status { get; set; } = "";
|
||||
public List<RenderJobDetail> RenderJobs { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class RenderJobDetail
|
||||
{
|
||||
public string Id { get; set; } = "";
|
||||
public string Status { get; set; } = "";
|
||||
}
|
||||
|
||||
/// <summary>Durable lifecycle report persisted to the spool directory (§7.6).</summary>
|
||||
public sealed class SpooledReport
|
||||
{
|
||||
public string Kind { get; set; } = ""; // "fail" | "complete-render"
|
||||
public string JobId { get; set; } = "";
|
||||
public Dictionary<string, object?> Payload { get; set; } = new();
|
||||
public DateTimeOffset CreatedAt { get; set; } = DateTimeOffset.UtcNow;
|
||||
}
|
||||
|
||||
/// <summary>current-job.json — crash recovery state (§7.5).</summary>
|
||||
public sealed class CurrentJobState
|
||||
{
|
||||
public string JobId { get; set; } = "";
|
||||
public string? ExportId { get; set; } = "";
|
||||
public string MachineId { get; set; } = "";
|
||||
public string OutputDirLocal { get; set; } = "";
|
||||
public DateTimeOffset ClaimedAt { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace VFXReviewWorker;
|
||||
|
||||
/// <summary>
|
||||
/// Minimal rolling file logger (§7.9): one file per day under
|
||||
/// %ProgramData%\VFXReviewWorker\logs, files older than 14 days pruned on
|
||||
/// startup and at midnight rollover. No external logging dependencies.
|
||||
/// </summary>
|
||||
public sealed class FileLoggerProvider : ILoggerProvider
|
||||
{
|
||||
private readonly object _lock = new();
|
||||
private StreamWriter? _writer;
|
||||
private DateOnly _currentDay;
|
||||
|
||||
public FileLoggerProvider()
|
||||
{
|
||||
Directory.CreateDirectory(WorkerOptions.LogsDir);
|
||||
Prune();
|
||||
}
|
||||
|
||||
public ILogger CreateLogger(string categoryName) => new FileLogger(this, categoryName);
|
||||
|
||||
internal void Write(string category, LogLevel level, string message, Exception? ex)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
var today = DateOnly.FromDateTime(DateTime.Now);
|
||||
if (_writer is null || today != _currentDay)
|
||||
{
|
||||
_writer?.Dispose();
|
||||
_currentDay = today;
|
||||
_writer = new StreamWriter(
|
||||
Path.Combine(WorkerOptions.LogsDir, $"worker_{today:yyyyMMdd}.log"),
|
||||
append: true) { AutoFlush = true };
|
||||
Prune();
|
||||
}
|
||||
_writer.WriteLine($"{DateTime.Now:HH:mm:ss.fff} [{level,-5}] {Shorten(category)}: {message}{(ex is null ? "" : $"\n{ex}")}");
|
||||
}
|
||||
}
|
||||
|
||||
private static string Shorten(string category) => category[(category.LastIndexOf('.') + 1)..];
|
||||
|
||||
private static void Prune()
|
||||
{
|
||||
try
|
||||
{
|
||||
var cutoff = DateTime.Now.AddDays(-14);
|
||||
foreach (var f in Directory.EnumerateFiles(WorkerOptions.LogsDir, "*.log"))
|
||||
{
|
||||
if (File.GetLastWriteTime(f) < cutoff) File.Delete(f);
|
||||
}
|
||||
}
|
||||
catch { /* best effort */ }
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
lock (_lock) { _writer?.Dispose(); _writer = null; }
|
||||
}
|
||||
|
||||
private sealed class FileLogger : ILogger
|
||||
{
|
||||
private readonly FileLoggerProvider _provider;
|
||||
private readonly string _category;
|
||||
|
||||
public FileLogger(FileLoggerProvider provider, string category)
|
||||
{
|
||||
_provider = provider;
|
||||
_category = category;
|
||||
}
|
||||
|
||||
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
||||
public bool IsEnabled(LogLevel logLevel) => logLevel >= LogLevel.Information;
|
||||
|
||||
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
|
||||
{
|
||||
if (!IsEnabled(logLevel)) return;
|
||||
_provider.Write(_category, logLevel, formatter(state, exception), exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace VFXReviewWorker;
|
||||
|
||||
/// <summary>
|
||||
/// Executes one claimed job end-to-end: writes crash-recovery state, runs
|
||||
/// aerender, uploads the full log, and enqueues the durable complete/fail
|
||||
/// report. One job at a time per machine (§7.2).
|
||||
/// </summary>
|
||||
public sealed class JobExecutor
|
||||
{
|
||||
private readonly WorkerOptions _options;
|
||||
private readonly ApiClient _api;
|
||||
private readonly AerenderRunner _runner;
|
||||
private readonly PreviewStage _preview;
|
||||
private readonly ReportSpooler _spooler;
|
||||
private readonly ILogger<JobExecutor> _log;
|
||||
|
||||
public JobExecutor(WorkerOptions options, ApiClient api, AerenderRunner runner, PreviewStage preview, ReportSpooler spooler, ILogger<JobExecutor> log)
|
||||
{
|
||||
_options = options;
|
||||
_api = api;
|
||||
_runner = runner;
|
||||
_preview = preview;
|
||||
_spooler = spooler;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
public async Task RunAsync(ClaimedJob job, string machineId, ServerConfig config, CancellationToken cancelJob, CancellationToken shutdown)
|
||||
{
|
||||
if (job.Type == "PREVIEW_ONLY")
|
||||
{
|
||||
await RunPreviewAsync(job, machineId, config, cancelJob, shutdown);
|
||||
return;
|
||||
}
|
||||
|
||||
var manifest = job.AsRenderManifest();
|
||||
var mapper = new PathMapper(_options.PathMappings);
|
||||
var outputDirLocal = mapper.Map(manifest.OutputDir);
|
||||
|
||||
WriteState(new CurrentJobState
|
||||
{
|
||||
JobId = job.Id,
|
||||
ExportId = job.ExportId,
|
||||
MachineId = machineId,
|
||||
OutputDirLocal = outputDirLocal,
|
||||
ClaimedAt = DateTimeOffset.UtcNow,
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
var result = await _runner.RunAsync(job, manifest, machineId, outputDirLocal, config, cancelJob, shutdown);
|
||||
var logKey = await _api.UploadLogAsync(job.Id, machineId, result.FullLogPath, shutdown);
|
||||
|
||||
if (result.Success)
|
||||
{
|
||||
long totalBytes = 0;
|
||||
int fileCount = 0;
|
||||
var prefix = AerenderRunner.OutputPatternPrefix(manifest.OutputPattern);
|
||||
try
|
||||
{
|
||||
foreach (var f in Directory.EnumerateFiles(outputDirLocal))
|
||||
{
|
||||
if (!Path.GetFileName(f).StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) continue;
|
||||
fileCount++;
|
||||
totalBytes += new FileInfo(f).Length;
|
||||
}
|
||||
}
|
||||
catch { /* stats are best-effort in Phase 2; validation owns this in Phase 3 */ }
|
||||
|
||||
if (fileCount == 0)
|
||||
{
|
||||
// aerender can exit 0 having rendered nothing (bad frame
|
||||
// range, template mismatch, silently swallowed error).
|
||||
// Until Phase 3 validation exists, an empty output dir must
|
||||
// never become READY_FOR_QC.
|
||||
_log.LogError("Job {JobId}: aerender exited 0 but no \"{Prefix}*\" files exist in {Dir} — reporting failure",
|
||||
job.Id, prefix, outputDirLocal);
|
||||
_spooler.Enqueue("fail", job.Id, new Dictionary<string, object?>
|
||||
{
|
||||
["machineId"] = machineId,
|
||||
["stage"] = "RENDER",
|
||||
["exitCode"] = 0,
|
||||
["errorMessage"] = "aerender exited 0 but produced no output files in " + outputDirLocal,
|
||||
["logTail"] = result.LogTail,
|
||||
["logFileKey"] = logKey,
|
||||
["retryable"] = false,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
_log.LogInformation("Job {JobId} complete: {Frames} frames in {Sec:F0}s ({Files} files, {Bytes} bytes)",
|
||||
job.Id, manifest.TotalFrames, result.RenderSeconds, fileCount, totalBytes);
|
||||
_spooler.Enqueue("complete-render", job.Id, new Dictionary<string, object?>
|
||||
{
|
||||
["machineId"] = machineId,
|
||||
["renderSeconds"] = (int)result.RenderSeconds,
|
||||
["logFileKey"] = logKey,
|
||||
["logTail"] = result.LogTail,
|
||||
["exrFileCount"] = fileCount,
|
||||
["exrTotalBytes"] = totalBytes,
|
||||
});
|
||||
}
|
||||
else if (result.ErrorMessage == "Cancelled")
|
||||
{
|
||||
// Ack the cancel so the server clears the CANCEL_JOB command
|
||||
_log.LogInformation("Job {JobId} cancelled — acknowledging", job.Id);
|
||||
_spooler.Enqueue("fail", job.Id, new Dictionary<string, object?>
|
||||
{
|
||||
["machineId"] = machineId,
|
||||
["stage"] = "RENDER",
|
||||
["errorMessage"] = "Cancelled by server",
|
||||
["logTail"] = result.LogTail,
|
||||
["logFileKey"] = logKey,
|
||||
["retryable"] = false,
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
_log.LogError("Job {JobId} failed (exit {Exit}, retryable {Retryable}): {Error}",
|
||||
job.Id, result.ExitCode, result.Retryable, result.ErrorMessage);
|
||||
_spooler.Enqueue("fail", job.Id, new Dictionary<string, object?>
|
||||
{
|
||||
["machineId"] = machineId,
|
||||
["stage"] = "RENDER",
|
||||
["exitCode"] = result.ExitCode,
|
||||
["errorMessage"] = result.ErrorMessage,
|
||||
["logTail"] = result.LogTail,
|
||||
["logFileKey"] = logKey,
|
||||
["retryable"] = result.Retryable,
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.LogError(ex, "Unexpected executor error for job {JobId}", job.Id);
|
||||
_spooler.Enqueue("fail", job.Id, new Dictionary<string, object?>
|
||||
{
|
||||
["machineId"] = machineId,
|
||||
["stage"] = "RENDER",
|
||||
["errorMessage"] = $"Worker exception: {ex.Message}",
|
||||
["retryable"] = true,
|
||||
});
|
||||
}
|
||||
finally
|
||||
{
|
||||
ClearState();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// PREVIEW_ONLY job: rebuild the shot around the rendered EXRs with slate
|
||||
/// and burn-ins, render the delivery MOV + review MP4, upload the review
|
||||
/// media, and finalize (which registers the internal-only Version).
|
||||
/// </summary>
|
||||
private async Task RunPreviewAsync(ClaimedJob job, string machineId, ServerConfig config, CancellationToken cancelJob, CancellationToken shutdown)
|
||||
{
|
||||
var manifest = job.AsPreviewManifest();
|
||||
|
||||
WriteState(new CurrentJobState
|
||||
{
|
||||
JobId = job.Id,
|
||||
ExportId = job.ExportId,
|
||||
MachineId = machineId,
|
||||
OutputDirLocal = new PathMapper(_options.PathMappings).Map(manifest.OutputDir),
|
||||
ClaimedAt = DateTimeOffset.UtcNow,
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
var result = await _preview.RunAsync(job, manifest, machineId, config, cancelJob, shutdown);
|
||||
|
||||
if (!result.Success)
|
||||
{
|
||||
_log.LogError("Preview job {JobId} failed (retryable {Retryable}): {Error}",
|
||||
job.Id, result.Retryable, result.ErrorMessage);
|
||||
_spooler.Enqueue("fail", job.Id, new Dictionary<string, object?>
|
||||
{
|
||||
["machineId"] = machineId,
|
||||
["stage"] = "PREVIEW",
|
||||
["errorMessage"] = result.ErrorMessage,
|
||||
["retryable"] = result.Retryable,
|
||||
});
|
||||
return; // temp AEP is deliberately kept for debugging
|
||||
}
|
||||
|
||||
var previewKey = await _api.UploadArtifactAsync(
|
||||
job.Id, machineId, "preview", result.Mp4Path!, "video/mp4", shutdown);
|
||||
if (previewKey is null)
|
||||
{
|
||||
// The renders are on the SAN and fine — only the upload failed,
|
||||
// so this is worth retrying without rebuilding anything.
|
||||
_spooler.Enqueue("fail", job.Id, new Dictionary<string, object?>
|
||||
{
|
||||
["machineId"] = machineId,
|
||||
["stage"] = "PREVIEW",
|
||||
["errorMessage"] = "Preview MP4 upload failed",
|
||||
["retryable"] = true,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
string? thumbKey = null;
|
||||
if (result.ThumbnailPath is not null)
|
||||
{
|
||||
thumbKey = await _api.UploadArtifactAsync(
|
||||
job.Id, machineId, "thumbnail", result.ThumbnailPath, "image/jpeg", shutdown);
|
||||
}
|
||||
|
||||
_spooler.Enqueue("finalize", job.Id, new Dictionary<string, object?>
|
||||
{
|
||||
["machineId"] = machineId,
|
||||
["artifacts"] = new Dictionary<string, object?>
|
||||
{
|
||||
["deliveryMovPath"] = manifest.MovOutput,
|
||||
["previewMovKey"] = previewKey,
|
||||
["thumbnailKey"] = thumbKey,
|
||||
},
|
||||
["renderStats"] = new Dictionary<string, object?>
|
||||
{
|
||||
["previewSeconds"] = (int)result.Seconds,
|
||||
},
|
||||
["media"] = new Dictionary<string, object?>
|
||||
{
|
||||
["width"] = result.Build?.Width,
|
||||
["height"] = result.Build?.Height,
|
||||
["frameCount"] = result.Build?.FrameCount,
|
||||
["fps"] = manifest.Fps,
|
||||
["fileName"] = Path.GetFileName(result.Mp4Path!),
|
||||
},
|
||||
});
|
||||
|
||||
// Only clean up once the outputs are safely reported.
|
||||
CleanScratch(job.Id, result.TempAep);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.LogError(ex, "Unexpected preview error for job {JobId}", job.Id);
|
||||
_spooler.Enqueue("fail", job.Id, new Dictionary<string, object?>
|
||||
{
|
||||
["machineId"] = machineId,
|
||||
["stage"] = "PREVIEW",
|
||||
["errorMessage"] = $"Worker exception: {ex.Message}",
|
||||
["retryable"] = true,
|
||||
});
|
||||
}
|
||||
finally
|
||||
{
|
||||
ClearState();
|
||||
}
|
||||
}
|
||||
|
||||
private static void CleanScratch(string jobId, string? tempAep)
|
||||
{
|
||||
PreviewStage.TryDelete(Path.Combine(WorkerOptions.ScratchDir, $"job_{jobId}_context.json"));
|
||||
PreviewStage.TryDelete(Path.Combine(WorkerOptions.ScratchDir, $"job_{jobId}_result.json"));
|
||||
PreviewStage.TryDelete(Path.Combine(WorkerOptions.ScratchDir, $"job_{jobId}_run.jsx"));
|
||||
PreviewStage.TryDelete(Path.Combine(WorkerOptions.ScratchDir, $"job_{jobId}_thumb.jpg"));
|
||||
if (tempAep is not null) PreviewStage.TryDelete(tempAep);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Crash recovery (§7.5): on service start, if current-job.json names a
|
||||
/// job, ask the server what became of it. Still ours (CLAIMED/RUNNING) →
|
||||
/// we crashed mid-render; report a retryable fail so the server requeues
|
||||
/// (never resume a partial aerender). Reassigned/finished → discard state;
|
||||
/// the next attempt's PrepareOutputDir clears any orphaned frames.
|
||||
/// </summary>
|
||||
public async Task ReconcileAsync(string machineId, CancellationToken ct)
|
||||
{
|
||||
var state = ReadState();
|
||||
if (state is null) return;
|
||||
_log.LogWarning("Found crash-recovery state for job {JobId} — reconciling", state.JobId);
|
||||
try
|
||||
{
|
||||
var stillOurs = false;
|
||||
if (!string.IsNullOrEmpty(state.ExportId))
|
||||
{
|
||||
var export = await _api.GetExportAsync(state.ExportId!, ct);
|
||||
var job = export?.RenderJobs.FirstOrDefault(j => j.Id == state.JobId);
|
||||
stillOurs = job?.Status is "CLAIMED" or "RUNNING";
|
||||
}
|
||||
if (stillOurs)
|
||||
{
|
||||
_log.LogWarning("Server still shows job {JobId} as ours — reporting crash for requeue", state.JobId);
|
||||
_spooler.Enqueue("fail", state.JobId, new Dictionary<string, object?>
|
||||
{
|
||||
["machineId"] = machineId,
|
||||
["stage"] = "RENDER",
|
||||
["errorMessage"] = "Worker restarted mid-render (crash/power loss); never resuming a partial render",
|
||||
["retryable"] = true,
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.LogWarning(ex, "Reconcile failed for job {JobId}; discarding local state (reaper covers the lease)", state.JobId);
|
||||
}
|
||||
ClearState();
|
||||
}
|
||||
|
||||
private static void WriteState(CurrentJobState state)
|
||||
{
|
||||
Directory.CreateDirectory(WorkerOptions.DataDir);
|
||||
File.WriteAllText(WorkerOptions.StateFilePath, JsonSerializer.Serialize(state, WorkerOptions.JsonOpts));
|
||||
}
|
||||
|
||||
private static CurrentJobState? ReadState()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!File.Exists(WorkerOptions.StateFilePath)) return null;
|
||||
return JsonSerializer.Deserialize<CurrentJobState>(File.ReadAllText(WorkerOptions.StateFilePath), WorkerOptions.JsonOpts);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static void ClearState()
|
||||
{
|
||||
try { File.Delete(WorkerOptions.StateFilePath); } catch { }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
namespace VFXReviewWorker;
|
||||
|
||||
/// <summary>
|
||||
/// Translates canonical UNC paths from manifests to this machine's local
|
||||
/// drive mappings (§7.8 pathMappings). Comparison is case-insensitive and
|
||||
/// slash-direction tolerant; the first matching mapping wins.
|
||||
/// </summary>
|
||||
public sealed class PathMapper
|
||||
{
|
||||
private readonly List<(string From, string To)> _mappings;
|
||||
|
||||
public PathMapper(IEnumerable<PathMapping> mappings)
|
||||
{
|
||||
_mappings = mappings
|
||||
.Where(m => !string.IsNullOrWhiteSpace(m.From))
|
||||
.Select(m => (Normalize(m.From), m.To))
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static string Normalize(string p) => p.Replace('\\', '/');
|
||||
|
||||
public string Map(string path)
|
||||
{
|
||||
if (string.IsNullOrEmpty(path)) return path;
|
||||
var normalized = Normalize(path);
|
||||
foreach (var (from, to) in _mappings)
|
||||
{
|
||||
if (normalized.StartsWith(from, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var mapped = to + normalized[from.Length..];
|
||||
return mapped.Replace('/', Path.DirectorySeparatorChar);
|
||||
}
|
||||
}
|
||||
return path.Replace('/', Path.DirectorySeparatorChar);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace VFXReviewWorker;
|
||||
|
||||
public sealed record PreviewStageResult(
|
||||
bool Success,
|
||||
bool Retryable,
|
||||
string? ErrorMessage,
|
||||
string? MovPath,
|
||||
string? Mp4Path,
|
||||
string? ThumbnailPath,
|
||||
string? TempAep,
|
||||
PreviewBuildResult? Build,
|
||||
double Seconds);
|
||||
|
||||
/// <summary>
|
||||
/// Stage 3 — preview generation (RenderPipeline2 §9), the second half of a
|
||||
/// one-click export.
|
||||
///
|
||||
/// Runs in two steps so the unproven part stays small:
|
||||
/// 1. AfterFX.com -noui -r → builds the shot around the rendered EXRs using
|
||||
/// the studio slate/overlay template, queues the MOV + MP4 output modules
|
||||
/// and saves a throwaway AEP. Does no rendering.
|
||||
/// 2. aerender -project <that aep> (no -comp) → renders the whole queue in
|
||||
/// one launch, through the same proven runner as the EXR stage.
|
||||
/// </summary>
|
||||
public sealed class PreviewStage
|
||||
{
|
||||
private readonly WorkerOptions _options;
|
||||
private readonly AerenderRunner _runner;
|
||||
private readonly ILogger<PreviewStage> _log;
|
||||
|
||||
public PreviewStage(WorkerOptions options, AerenderRunner runner, ILogger<PreviewStage> log)
|
||||
{
|
||||
_options = options;
|
||||
_runner = runner;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
public async Task<PreviewStageResult> RunAsync(
|
||||
ClaimedJob job,
|
||||
PreviewManifest manifest,
|
||||
string machineId,
|
||||
ServerConfig config,
|
||||
CancellationToken cancelJob,
|
||||
CancellationToken shutdown)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
var mapper = new PathMapper(_options.PathMappings);
|
||||
Directory.CreateDirectory(WorkerOptions.ScratchDir);
|
||||
|
||||
var contextPath = Path.Combine(WorkerOptions.ScratchDir, $"job_{job.Id}_context.json");
|
||||
var resultPath = Path.Combine(WorkerOptions.ScratchDir, $"job_{job.Id}_result.json");
|
||||
var wrapperPath = Path.Combine(WorkerOptions.ScratchDir, $"job_{job.Id}_run.jsx");
|
||||
var tempAep = Path.Combine(WorkerOptions.ScratchDir, $"job_{job.Id}_preview.aep");
|
||||
|
||||
var movLocal = mapper.Map(manifest.MovOutput);
|
||||
var mp4Local = mapper.Map(manifest.Mp4Output);
|
||||
|
||||
// Everything the script needs, in machine-local paths.
|
||||
var context = new
|
||||
{
|
||||
jobId = job.Id,
|
||||
exportId = manifest.ExportId,
|
||||
shotCode = manifest.ShotCode,
|
||||
versionString = manifest.VersionString,
|
||||
outputDirLocal = mapper.Map(manifest.OutputDir),
|
||||
templateAep = mapper.Map(manifest.TemplateAep),
|
||||
templateComp = manifest.TemplateComp,
|
||||
overlayComp = manifest.OverlayComp,
|
||||
lutComp = manifest.LutComp,
|
||||
movTemplate = manifest.MovTemplate,
|
||||
mp4Template = manifest.Mp4Template,
|
||||
movOutput = movLocal,
|
||||
mp4Output = mp4Local,
|
||||
tempAep,
|
||||
resultPath,
|
||||
frameStart = manifest.FrameStart,
|
||||
fps = manifest.Fps,
|
||||
slateScopeProp = manifest.SlateScopeProp,
|
||||
slateSubmissionProp = manifest.SlateSubmissionProp,
|
||||
slate = manifest.Slate,
|
||||
};
|
||||
|
||||
await File.WriteAllTextAsync(contextPath, JsonSerializer.Serialize(context, WorkerOptions.JsonOpts), shutdown);
|
||||
|
||||
var scriptPath = _options.ResolvePreviewScriptPath();
|
||||
if (!File.Exists(scriptPath))
|
||||
{
|
||||
return Fail($"Preview build script not found at {scriptPath}", retryable: false, sw);
|
||||
}
|
||||
|
||||
// ExtendScript cannot read argv — hand the context path over through a
|
||||
// generated wrapper that evaluates the shared build script.
|
||||
var wrapper = new StringBuilder()
|
||||
.AppendLine("// generated by VFXReviewWorker — do not edit")
|
||||
.AppendLine($"var VFXR_CONTEXT_PATH = {JsonSerializer.Serialize(contextPath)};")
|
||||
.AppendLine($"$.evalFile({JsonSerializer.Serialize(scriptPath)});")
|
||||
.ToString();
|
||||
await File.WriteAllTextAsync(wrapperPath, wrapper, shutdown);
|
||||
|
||||
TryDelete(resultPath);
|
||||
|
||||
// ── Step 1: headless build ───────────────────────────────────────────
|
||||
var afterFx = _options.ResolveAfterFxPath();
|
||||
if (!File.Exists(afterFx))
|
||||
{
|
||||
return Fail($"AfterFX.com not found at {afterFx}", retryable: false, sw);
|
||||
}
|
||||
|
||||
// Belt and braces: the claim loop already avoids preview jobs while AE
|
||||
// is open, but an artist may have launched it in between. Never take
|
||||
// over a live session — defer instead (retryable).
|
||||
if (AeSession.InteractiveRunning())
|
||||
{
|
||||
return Fail(
|
||||
"An interactive After Effects session is open on this machine — preview build deferred to avoid taking it over",
|
||||
retryable: true, sw);
|
||||
}
|
||||
|
||||
_log.LogInformation("Building preview comp for job {JobId} via {AfterFx}", job.Id, afterFx);
|
||||
var buildExit = await RunAfterFxAsync(afterFx, wrapperPath, config, cancelJob, shutdown);
|
||||
|
||||
if (!File.Exists(resultPath))
|
||||
{
|
||||
return Fail(
|
||||
$"Headless AE produced no result file (exit {buildExit}). Check that AfterFX.com can run under this account.",
|
||||
retryable: true, sw);
|
||||
}
|
||||
|
||||
PreviewBuildResult? build;
|
||||
try
|
||||
{
|
||||
build = JsonSerializer.Deserialize<PreviewBuildResult>(
|
||||
await File.ReadAllTextAsync(resultPath, shutdown), WorkerOptions.JsonOpts);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return Fail($"Could not parse preview build result: {ex.Message}", retryable: false, sw);
|
||||
}
|
||||
|
||||
if (build is null || !build.Ok)
|
||||
{
|
||||
// A build failure is deterministic (missing template comp, missing
|
||||
// output module, bad layer name) — humans fix the template.
|
||||
return Fail($"Preview build failed: {build?.Error ?? "unknown error"}", retryable: false, sw, build);
|
||||
}
|
||||
foreach (var w in build.Warnings)
|
||||
{
|
||||
_log.LogWarning("Preview build warning (job {JobId}): {Warning}", job.Id, w);
|
||||
}
|
||||
if (string.IsNullOrWhiteSpace(build.TempAep) || !File.Exists(build.TempAep))
|
||||
{
|
||||
return Fail("Preview build reported success but the temp project is missing", retryable: false, sw, build);
|
||||
}
|
||||
|
||||
// ── Step 2: render the prepared queue ────────────────────────────────
|
||||
var render = await _runner.RunProjectAsync(
|
||||
job.Id, machineId, build.TempAep, build.FrameCount ?? manifest.TotalFrames,
|
||||
config, cancelJob, shutdown);
|
||||
|
||||
if (!render.Success)
|
||||
{
|
||||
return new PreviewStageResult(false, render.Retryable, render.ErrorMessage,
|
||||
null, null, null, build.TempAep, build, sw.Elapsed.TotalSeconds);
|
||||
}
|
||||
|
||||
// aerender can exit 0 without writing — verify both outputs exist.
|
||||
var missing = new List<string>();
|
||||
if (!File.Exists(movLocal)) missing.Add(movLocal);
|
||||
if (!File.Exists(mp4Local)) missing.Add(mp4Local);
|
||||
if (missing.Count > 0)
|
||||
{
|
||||
return new PreviewStageResult(false, false,
|
||||
"aerender exited 0 but expected preview outputs are missing: " + string.Join(", ", missing),
|
||||
null, null, null, build.TempAep, build, sw.Elapsed.TotalSeconds);
|
||||
}
|
||||
|
||||
var thumbnail = await TryMakeThumbnailAsync(job.Id, mp4Local, shutdown);
|
||||
|
||||
sw.Stop();
|
||||
_log.LogInformation("Preview complete for job {JobId} in {Sec:F0}s (mov + mp4{Thumb})",
|
||||
job.Id, sw.Elapsed.TotalSeconds, thumbnail is null ? "" : " + thumbnail");
|
||||
|
||||
return new PreviewStageResult(true, true, null, movLocal, mp4Local, thumbnail,
|
||||
build.TempAep, build, sw.Elapsed.TotalSeconds);
|
||||
}
|
||||
|
||||
private async Task<int> RunAfterFxAsync(
|
||||
string afterFx, string wrapperPath, ServerConfig config,
|
||||
CancellationToken cancelJob, CancellationToken shutdown)
|
||||
{
|
||||
var psi = new ProcessStartInfo
|
||||
{
|
||||
FileName = afterFx,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false,
|
||||
CreateNoWindow = true,
|
||||
};
|
||||
psi.ArgumentList.Add("-noui");
|
||||
psi.ArgumentList.Add("-r");
|
||||
psi.ArgumentList.Add(wrapperPath);
|
||||
|
||||
using var process = new Process { StartInfo = psi };
|
||||
process.OutputDataReceived += (_, e) => { if (!string.IsNullOrWhiteSpace(e.Data)) _log.LogDebug("[AfterFX] {Line}", e.Data); };
|
||||
process.ErrorDataReceived += (_, e) => { if (!string.IsNullOrWhiteSpace(e.Data)) _log.LogDebug("[AfterFX] {Line}", e.Data); };
|
||||
|
||||
process.Start();
|
||||
process.BeginOutputReadLine();
|
||||
process.BeginErrorReadLine();
|
||||
|
||||
// The build does no rendering, so it should finish in well under the
|
||||
// stall timeout. A hang here means AE is waiting on something (a dialog,
|
||||
// a licence prompt) and must be killed rather than left holding the job.
|
||||
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancelJob, shutdown);
|
||||
timeout.CancelAfter(TimeSpan.FromSeconds(Math.Max(120, config.StallTimeoutSeconds)));
|
||||
try
|
||||
{
|
||||
await process.WaitForExitAsync(timeout.Token);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
_log.LogError("Headless AE build timed out or was cancelled — killing process tree");
|
||||
try { process.Kill(entireProcessTree: true); } catch { }
|
||||
return -1;
|
||||
}
|
||||
return process.ExitCode;
|
||||
}
|
||||
|
||||
/// <summary>Optional 960-wide poster frame at 25% duration (§9.1). Never fatal.</summary>
|
||||
private async Task<string?> TryMakeThumbnailAsync(string jobId, string mp4Path, CancellationToken ct)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(_options.FfmpegPath) || !File.Exists(_options.FfmpegPath))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
var outPath = Path.Combine(WorkerOptions.ScratchDir, $"job_{jobId}_thumb.jpg");
|
||||
TryDelete(outPath);
|
||||
|
||||
var psi = new ProcessStartInfo
|
||||
{
|
||||
FileName = _options.FfmpegPath,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false,
|
||||
CreateNoWindow = true,
|
||||
};
|
||||
foreach (var a in new[] { "-y", "-i", mp4Path, "-vf", "thumbnail,scale=960:-1", "-frames:v", "1", outPath })
|
||||
{
|
||||
psi.ArgumentList.Add(a);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var process = Process.Start(psi);
|
||||
if (process is null) return null;
|
||||
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
timeout.CancelAfter(TimeSpan.FromMinutes(2));
|
||||
await process.WaitForExitAsync(timeout.Token);
|
||||
return File.Exists(outPath) ? outPath : null;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.LogWarning(ex, "Thumbnail generation failed (non-fatal)");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static PreviewStageResult Fail(string message, bool retryable, Stopwatch sw, PreviewBuildResult? build = null)
|
||||
{
|
||||
sw.Stop();
|
||||
return new PreviewStageResult(false, retryable, message, null, null, null, build?.TempAep, build, sw.Elapsed.TotalSeconds);
|
||||
}
|
||||
|
||||
internal static void TryDelete(string path)
|
||||
{
|
||||
try { if (File.Exists(path)) File.Delete(path); } catch { }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using VFXReviewWorker;
|
||||
|
||||
// VFXReview RenderWorker (RenderPipeline2 §7) — Windows service driving
|
||||
// aerender.exe on artist workstations / render nodes. All state flows through
|
||||
// /api/ext/* with API-key auth; the worker never touches the database.
|
||||
//
|
||||
// Run interactively for debugging: VFXReviewWorker.exe [path\to\config.json]
|
||||
// Install as a service: see RenderWorker/README.md
|
||||
|
||||
var configPath = args.FirstOrDefault(a => !a.StartsWith('-'));
|
||||
var options = WorkerOptions.Load(configPath);
|
||||
|
||||
// Positional args (config path) are consumed above and deliberately not
|
||||
// forwarded — the command-line configuration provider rejects bare tokens.
|
||||
var builder = Host.CreateApplicationBuilder();
|
||||
builder.Services.AddWindowsService(o => o.ServiceName = "VFXReviewRenderWorker");
|
||||
builder.Logging.AddProvider(new FileLoggerProvider());
|
||||
|
||||
builder.Services.AddSingleton(options);
|
||||
builder.Services.AddSingleton<ApiClient>();
|
||||
builder.Services.AddSingleton<ReportSpooler>();
|
||||
builder.Services.AddSingleton<AerenderRunner>();
|
||||
builder.Services.AddSingleton<PreviewStage>();
|
||||
builder.Services.AddSingleton<JobExecutor>();
|
||||
builder.Services.AddHostedService<WorkerService>();
|
||||
|
||||
await builder.Build().RunAsync();
|
||||
@@ -0,0 +1,80 @@
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace VFXReviewWorker;
|
||||
|
||||
/// <summary>
|
||||
/// Parses aerender stdout line-by-line (§7.3): PROGRESS lines yield the
|
||||
/// current frame; ERROR lines are collected; deterministic errors (missing
|
||||
/// footage/comp/project) mark the job non-retryable — humans fix the comp.
|
||||
/// </summary>
|
||||
public static partial class ProgressParser
|
||||
{
|
||||
// e.g. "PROGRESS: 0:00:02:03 (51): 0 Seconds" / "PROGRESS: 2 (2): ..."
|
||||
[GeneratedRegex(@"^PROGRESS:.*\((\d+)\)", RegexOptions.Compiled)]
|
||||
private static partial Regex ProgressLine();
|
||||
|
||||
[GeneratedRegex(@"aerender\s+(SYNTAX\s+)?ERROR|^ERROR:", RegexOptions.Compiled | RegexOptions.IgnoreCase)]
|
||||
private static partial Regex ErrorLine();
|
||||
|
||||
private static readonly string[] NonRetryablePatterns =
|
||||
{
|
||||
"missing footage",
|
||||
"layer source file missing",
|
||||
"file is missing",
|
||||
"no comp was found",
|
||||
"no composition",
|
||||
"project file could not be opened",
|
||||
"can not be opened",
|
||||
"cannot be opened",
|
||||
"the file format module could not parse the file",
|
||||
"no render settings template",
|
||||
"no output module template",
|
||||
"syntax error",
|
||||
"illegal argument",
|
||||
};
|
||||
|
||||
/// <summary>Returns the rendered-frame ordinal (1-based within the range) or null.</summary>
|
||||
public static int? ParseFrame(string line)
|
||||
{
|
||||
var m = ProgressLine().Match(line);
|
||||
return m.Success && int.TryParse(m.Groups[1].Value, out var f) ? f : null;
|
||||
}
|
||||
|
||||
public static bool IsErrorLine(string line) => ErrorLine().IsMatch(line);
|
||||
|
||||
/// <summary>Deterministic failures re-render identically — don't burn retries on them.</summary>
|
||||
public static bool IsNonRetryableError(string line)
|
||||
{
|
||||
foreach (var p in NonRetryablePatterns)
|
||||
{
|
||||
if (line.Contains(p, StringComparison.OrdinalIgnoreCase)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>ETA = rolling average seconds/frame × frames remaining (§7.3).</summary>
|
||||
public sealed class EtaCalculator
|
||||
{
|
||||
private readonly int _windowSize;
|
||||
private readonly Queue<double> _frameSeconds = new();
|
||||
private DateTimeOffset? _lastFrameAt;
|
||||
|
||||
public EtaCalculator(int windowSize = 10) => _windowSize = windowSize;
|
||||
|
||||
public void RecordFrame(DateTimeOffset now)
|
||||
{
|
||||
if (_lastFrameAt is { } last)
|
||||
{
|
||||
_frameSeconds.Enqueue((now - last).TotalSeconds);
|
||||
while (_frameSeconds.Count > _windowSize) _frameSeconds.Dequeue();
|
||||
}
|
||||
_lastFrameAt = now;
|
||||
}
|
||||
|
||||
public int? EstimateSeconds(int framesRemaining)
|
||||
{
|
||||
if (_frameSeconds.Count == 0 || framesRemaining <= 0) return null;
|
||||
return (int)Math.Round(_frameSeconds.Average() * framesRemaining);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace VFXReviewWorker;
|
||||
|
||||
/// <summary>
|
||||
/// Durable lifecycle reports (§7.6): fail/complete reports are written to the
|
||||
/// spool directory before the first send attempt and replayed strictly in
|
||||
/// order with exponential backoff (1 s → 60 s). Rendering continues during
|
||||
/// server downtime; a report is deleted only once the server accepts it (or
|
||||
/// permanently rejects it as contradictory).
|
||||
/// </summary>
|
||||
public sealed class ReportSpooler
|
||||
{
|
||||
private readonly ApiClient _api;
|
||||
private readonly ILogger<ReportSpooler> _log;
|
||||
private readonly string _dir;
|
||||
private readonly SemaphoreSlim _wake = new(0);
|
||||
private long _seq;
|
||||
|
||||
public ReportSpooler(ApiClient api, ILogger<ReportSpooler> log, string? dir = null)
|
||||
{
|
||||
_api = api;
|
||||
_log = log;
|
||||
_dir = dir ?? WorkerOptions.SpoolDir;
|
||||
Directory.CreateDirectory(_dir);
|
||||
}
|
||||
|
||||
public void Enqueue(string kind, string jobId, Dictionary<string, object?> payload)
|
||||
{
|
||||
var report = new SpooledReport { Kind = kind, JobId = jobId, Payload = payload };
|
||||
var name = $"{DateTimeOffset.UtcNow.UtcTicks:D20}_{Interlocked.Increment(ref _seq):D6}.json";
|
||||
var tmp = Path.Combine(_dir, name + ".tmp");
|
||||
var final = Path.Combine(_dir, name);
|
||||
File.WriteAllText(tmp, JsonSerializer.Serialize(report, WorkerOptions.JsonOpts));
|
||||
File.Move(tmp, final);
|
||||
_wake.Release();
|
||||
}
|
||||
|
||||
public bool HasPending => Directory.EnumerateFiles(_dir, "*.json").Any();
|
||||
|
||||
/// <summary>Long-running replay loop; started once by the worker service.</summary>
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
var backoff = TimeSpan.FromSeconds(1);
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
var next = Directory.EnumerateFiles(_dir, "*.json").OrderBy(f => f, StringComparer.Ordinal).FirstOrDefault();
|
||||
if (next is null)
|
||||
{
|
||||
try { await _wake.WaitAsync(TimeSpan.FromSeconds(5), ct); } catch (OperationCanceledException) { break; }
|
||||
continue;
|
||||
}
|
||||
|
||||
bool done;
|
||||
try
|
||||
{
|
||||
var report = JsonSerializer.Deserialize<SpooledReport>(File.ReadAllText(next), WorkerOptions.JsonOpts);
|
||||
if (report is null || string.IsNullOrEmpty(report.JobId))
|
||||
{
|
||||
_log.LogWarning("Discarding unreadable spool file {File}", next);
|
||||
done = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
done = await _api.SendLifecycleAsync(report.Kind, report.JobId, report.Payload, ct);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.LogWarning(ex, "Spool replay attempt failed for {File}", next);
|
||||
done = false;
|
||||
}
|
||||
|
||||
if (done)
|
||||
{
|
||||
try { File.Delete(next); } catch { /* re-sent next pass; server side is idempotent */ }
|
||||
backoff = TimeSpan.FromSeconds(1);
|
||||
}
|
||||
else
|
||||
{
|
||||
try { await Task.Delay(backoff, ct); } catch (OperationCanceledException) { break; }
|
||||
backoff = TimeSpan.FromSeconds(Math.Min(backoff.TotalSeconds * 2, 60));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk.Worker">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<RootNamespace>VFXReviewWorker</RootNamespace>
|
||||
<AssemblyName>VFXReviewWorker</AssemblyName>
|
||||
<!-- Single self-contained exe for render nodes: dotnet publish -r win-x64 (see README) -->
|
||||
<InvariantGlobalization>true</InvariantGlobalization>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.1" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="8.0.1" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="VFXReviewWorker.Tests" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<!-- Headless AE build script ships beside the exe -->
|
||||
<Content Include="..\scripts\vfxr_build_preview.jsx" Link="scripts\vfxr_build_preview.jsx">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,79 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace VFXReviewWorker;
|
||||
|
||||
/// <summary>
|
||||
/// Machine-local config (RenderPipeline2 §7.8) — deliberately minimal;
|
||||
/// everything tunable lives in server SystemConfig and arrives via E6.
|
||||
/// Default path: C:\ProgramData\VFXReviewWorker\config.json
|
||||
/// </summary>
|
||||
public sealed class WorkerOptions
|
||||
{
|
||||
public string ServerUrl { get; set; } = "";
|
||||
public string ApiKey { get; set; } = "";
|
||||
public string MachineName { get; set; } = Environment.MachineName;
|
||||
public string AerenderPath { get; set; } = @"C:\Program Files\Adobe\Adobe After Effects 2026\Support Files\aerender.exe";
|
||||
/// <summary>AfterFX.com — the console AE used for the headless preview build. Defaults beside aerender.</summary>
|
||||
public string? AfterFxPath { get; set; }
|
||||
/// <summary>vfxr_build_preview.jsx — defaults to scripts\ beside the worker exe.</summary>
|
||||
public string? PreviewScriptPath { get; set; }
|
||||
public string? FfmpegPath { get; set; }
|
||||
public string? OiiotoolPath { get; set; }
|
||||
public List<PathMapping> PathMappings { get; set; } = new();
|
||||
public string WorkerVersion { get; set; } = "1.0.0";
|
||||
public string? AeVersion { get; set; }
|
||||
|
||||
public static string DataDir =>
|
||||
Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), "VFXReviewWorker");
|
||||
|
||||
public static string DefaultConfigPath => Path.Combine(DataDir, "config.json");
|
||||
public static string LogsDir => Path.Combine(DataDir, "logs");
|
||||
public static string SpoolDir => Path.Combine(DataDir, "spool");
|
||||
public static string ScratchDir => Path.Combine(DataDir, "scratch");
|
||||
public static string StateFilePath => Path.Combine(DataDir, "current-job.json");
|
||||
|
||||
/// <summary>AfterFX.com sits beside aerender.exe in a standard AE install.</summary>
|
||||
public string ResolveAfterFxPath()
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(AfterFxPath)) return AfterFxPath;
|
||||
var dir = Path.GetDirectoryName(AerenderPath);
|
||||
return dir is null ? "AfterFX.com" : Path.Combine(dir, "AfterFX.com");
|
||||
}
|
||||
|
||||
public string ResolvePreviewScriptPath()
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(PreviewScriptPath)) return PreviewScriptPath;
|
||||
return Path.Combine(AppContext.BaseDirectory, "scripts", "vfxr_build_preview.jsx");
|
||||
}
|
||||
|
||||
public static WorkerOptions Load(string? path = null)
|
||||
{
|
||||
path ??= DefaultConfigPath;
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
throw new FileNotFoundException(
|
||||
$"Worker config not found at {path}. Create it with serverUrl, apiKey and machineName (see RenderWorker/README.md).");
|
||||
}
|
||||
var json = File.ReadAllText(path);
|
||||
var opts = JsonSerializer.Deserialize<WorkerOptions>(json, JsonOpts)
|
||||
?? throw new InvalidOperationException($"Could not parse {path}");
|
||||
if (string.IsNullOrWhiteSpace(opts.ServerUrl)) throw new InvalidOperationException("config.json: serverUrl is required");
|
||||
if (string.IsNullOrWhiteSpace(opts.ApiKey)) throw new InvalidOperationException("config.json: apiKey is required");
|
||||
if (string.IsNullOrWhiteSpace(opts.MachineName)) opts.MachineName = Environment.MachineName;
|
||||
return opts;
|
||||
}
|
||||
|
||||
public static readonly JsonSerializerOptions JsonOpts = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
}
|
||||
|
||||
public sealed class PathMapping
|
||||
{
|
||||
public string From { get; set; } = "";
|
||||
public string To { get; set; } = "";
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace VFXReviewWorker;
|
||||
|
||||
/// <summary>
|
||||
/// Main loop (§7.2): register on startup, heartbeat on its own timer (the
|
||||
/// cancel-signal channel), claim-poll only when idle, one job at a time.
|
||||
/// Workers poll dumbly — all scheduling policy (render windows, Render Now,
|
||||
/// urgent priority) lives server-side in the claim endpoint (§7.10).
|
||||
/// </summary>
|
||||
public sealed class WorkerService : BackgroundService
|
||||
{
|
||||
private readonly WorkerOptions _options;
|
||||
private readonly ApiClient _api;
|
||||
private readonly JobExecutor _executor;
|
||||
private readonly ReportSpooler _spooler;
|
||||
private readonly ILogger<WorkerService> _log;
|
||||
|
||||
private string? _machineId;
|
||||
private ServerConfig _config = new();
|
||||
private volatile string? _currentJobId;
|
||||
private CancellationTokenSource? _cancelCurrentJob;
|
||||
private bool? _lastAeOpen;
|
||||
|
||||
public WorkerService(WorkerOptions options, ApiClient api, JobExecutor executor, ReportSpooler spooler, ILogger<WorkerService> log)
|
||||
{
|
||||
_options = options;
|
||||
_api = api;
|
||||
_executor = executor;
|
||||
_spooler = spooler;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
_log.LogInformation("VFXReview RenderWorker {Version} starting as \"{Name}\" → {Server}",
|
||||
_options.WorkerVersion, _options.MachineName, _options.ServerUrl);
|
||||
|
||||
_ = Task.Run(() => _spooler.RunAsync(stoppingToken), stoppingToken);
|
||||
|
||||
await RegisterWithRetryAsync(stoppingToken);
|
||||
await _executor.ReconcileAsync(_machineId!, stoppingToken);
|
||||
|
||||
_ = Task.Run(() => HeartbeatLoopAsync(stoppingToken), stoppingToken);
|
||||
|
||||
// Claim loop — only while idle
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Preview builds drive a full AE instance, which would hijack an
|
||||
// artist's open session (§9.1). Simply don't ask for those jobs
|
||||
// while AE is running — they stay queued for a quiet moment
|
||||
// instead of being claimed and failed.
|
||||
var aeOpen = AeSession.InteractiveRunning();
|
||||
if (aeOpen != _lastAeOpen)
|
||||
{
|
||||
_log.LogInformation(aeOpen
|
||||
? "Interactive After Effects detected — not claiming preview jobs until it closes"
|
||||
: "No interactive After Effects — preview jobs enabled");
|
||||
_lastAeOpen = aeOpen;
|
||||
}
|
||||
var types = aeOpen
|
||||
? new[] { "AE_RENDER" }
|
||||
: new[] { "AE_RENDER", "PREVIEW_ONLY" };
|
||||
|
||||
var job = await _api.ClaimAsync(_machineId!, types, stoppingToken);
|
||||
if (job is not null)
|
||||
{
|
||||
_log.LogInformation("Claimed job {JobId} (attempt {Attempt}/{Max}, export {ExportId})",
|
||||
job.Id, job.Attempt, job.MaxAttempts, job.ExportId);
|
||||
_cancelCurrentJob = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken.None);
|
||||
_currentJobId = job.Id;
|
||||
try
|
||||
{
|
||||
await _executor.RunAsync(job, _machineId!, _config, _cancelCurrentJob.Token, stoppingToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_currentJobId = null;
|
||||
_cancelCurrentJob.Dispose();
|
||||
_cancelCurrentJob = null;
|
||||
}
|
||||
continue; // immediately look for the next job
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.LogWarning(ex, "Claim poll failed (server unreachable?) — retrying on normal cadence");
|
||||
}
|
||||
try { await Task.Delay(TimeSpan.FromSeconds(_config.PollSeconds), stoppingToken); }
|
||||
catch (OperationCanceledException) { break; }
|
||||
}
|
||||
_log.LogInformation("RenderWorker stopping");
|
||||
}
|
||||
|
||||
private async Task RegisterWithRetryAsync(CancellationToken ct)
|
||||
{
|
||||
var backoff = TimeSpan.FromSeconds(1);
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var reg = await _api.RegisterAsync(_options, ct);
|
||||
_machineId = reg.Machine.Id;
|
||||
_config = reg.Config;
|
||||
_log.LogInformation(
|
||||
"Registered as machine {Id} ({Name}); poll={Poll}s heartbeat={Hb}s lease={Lease}s stall={Stall}s enabled={Enabled}",
|
||||
reg.Machine.Id, reg.Machine.Name, _config.PollSeconds, _config.HeartbeatSeconds,
|
||||
_config.LeaseSeconds, _config.StallTimeoutSeconds, reg.Machine.Enabled);
|
||||
return;
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.LogWarning("Registration failed ({Error}); retrying in {Backoff}s", ex.Message, backoff.TotalSeconds);
|
||||
try { await Task.Delay(backoff, ct); } catch (OperationCanceledException) { return; }
|
||||
backoff = TimeSpan.FromSeconds(Math.Min(backoff.TotalSeconds * 2, 60));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HeartbeatLoopAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var resp = await _api.HeartbeatAsync(_machineId!, _currentJobId, ct);
|
||||
foreach (var cmd in resp.Commands)
|
||||
{
|
||||
if (cmd.Type == "CANCEL_JOB" && cmd.JobId is not null && cmd.JobId == _currentJobId)
|
||||
{
|
||||
_log.LogWarning("Heartbeat carried CANCEL_JOB for current job {JobId}", cmd.JobId);
|
||||
_cancelCurrentJob?.Cancel();
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.LogDebug(ex, "Heartbeat failed (non-fatal)");
|
||||
}
|
||||
try { await Task.Delay(TimeSpan.FromSeconds(_config.HeartbeatSeconds), ct); }
|
||||
catch (OperationCanceledException) { return; }
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user