feat(pipeline): render queue, worker service and automated preview generation

One "Queue Export" click now renders the EXR sequence, then rebuilds the shot headlessly with the studio slate/overlay template to produce the delivery MOV and review MP4. Implements RenderPipeline2 phases 1-2 plus the preview stage. Server: - New models Export, RenderJob, ExportEvent, Machine, WorkerHeartbeat, plus Project.deliveryConfig and per-submission slate fields (Export.vfxScope, Export.submissionNote, inherited from the shot's previous export). Both migrations are purely additive; no existing column is touched. - lib/render-pipeline: server-enforced state machine, transactional version increment with supersede, atomic FOR UPDATE SKIP LOCKED claim gated by machine availability windows, and a lease reaper run from instrumentation.ts. - /api/ext/* endpoints for the panel and workers; session-auth mirrors under /api/render and /api/machines for the web UI. - Pipeline pages: render queue, export detail, machine monitoring, plus an Exports tab on shot detail. RenderWorker (.NET 8 Windows service, new): - Registration, heartbeat as cancel channel, claim loop, aerender runner with progress parsing and stall watchdog, crash recovery and disk-spooled reporting that survives server downtime. - Preview stage: headless AE assembles the preview comp into a throwaway AEP with both output modules queued, then a single aerender pass renders them. Preview jobs are not claimed while an interactive AE session is open, so an artist's project is never taken over. AE panel: Queue Export with live status polling, urgent flag, retry, and the VFX Scope / Submission Note fields. Every existing panel action is unchanged. Preview chaining ships disabled behind SystemConfig preview.enabled. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-02 14:34:34 +02:00
parent 0d0f3e1a33
commit ae58dc0366
71 changed files with 11767 additions and 1 deletions
@@ -0,0 +1,91 @@
+using System.Text.Json;
+using Microsoft.Extensions.Logging;
+
+namespace VFXReviewWorker;
+
+/// <summary>
+/// Durable lifecycle reports (§7.6): fail/complete reports are written to the
+/// spool directory before the first send attempt and replayed strictly in
+/// order with exponential backoff (1 s → 60 s). Rendering continues during
+/// server downtime; a report is deleted only once the server accepts it (or
+/// permanently rejects it as contradictory).
+/// </summary>
+public sealed class ReportSpooler
+{
+    private readonly ApiClient _api;
+    private readonly ILogger<ReportSpooler> _log;
+    private readonly string _dir;
+    private readonly SemaphoreSlim _wake = new(0);
+    private long _seq;
+
+    public ReportSpooler(ApiClient api, ILogger<ReportSpooler> log, string? dir = null)
+    {
+        _api = api;
+        _log = log;
+        _dir = dir ?? WorkerOptions.SpoolDir;
+        Directory.CreateDirectory(_dir);
+    }
+
+    public void Enqueue(string kind, string jobId, Dictionary<string, object?> payload)
+    {
+        var report = new SpooledReport { Kind = kind, JobId = jobId, Payload = payload };
+        var name = $"{DateTimeOffset.UtcNow.UtcTicks:D20}_{Interlocked.Increment(ref _seq):D6}.json";
+        var tmp = Path.Combine(_dir, name + ".tmp");
+        var final = Path.Combine(_dir, name);
+        File.WriteAllText(tmp, JsonSerializer.Serialize(report, WorkerOptions.JsonOpts));
+        File.Move(tmp, final);
+        _wake.Release();
+    }
+
+    public bool HasPending => Directory.EnumerateFiles(_dir, "*.json").Any();
+
+    /// <summary>Long-running replay loop; started once by the worker service.</summary>
+    public async Task RunAsync(CancellationToken ct)
+    {
+        var backoff = TimeSpan.FromSeconds(1);
+        while (!ct.IsCancellationRequested)
+        {
+            var next = Directory.EnumerateFiles(_dir, "*.json").OrderBy(f => f, StringComparer.Ordinal).FirstOrDefault();
+            if (next is null)
+            {
+                try { await _wake.WaitAsync(TimeSpan.FromSeconds(5), ct); } catch (OperationCanceledException) { break; }
+                continue;
+            }
+
+            bool done;
+            try
+            {
+                var report = JsonSerializer.Deserialize<SpooledReport>(File.ReadAllText(next), WorkerOptions.JsonOpts);
+                if (report is null || string.IsNullOrEmpty(report.JobId))
+                {
+                    _log.LogWarning("Discarding unreadable spool file {File}", next);
+                    done = true;
+                }
+                else
+                {
+                    done = await _api.SendLifecycleAsync(report.Kind, report.JobId, report.Payload, ct);
+                }
+            }
+            catch (OperationCanceledException)
+            {
+                break;
+            }
+            catch (Exception ex)
+            {
+                _log.LogWarning(ex, "Spool replay attempt failed for {File}", next);
+                done = false;
+            }
+
+            if (done)
+            {
+                try { File.Delete(next); } catch { /* re-sent next pass; server side is idempotent */ }
+                backoff = TimeSpan.FromSeconds(1);
+            }
+            else
+            {
+                try { await Task.Delay(backoff, ct); } catch (OperationCanceledException) { break; }
+                backoff = TimeSpan.FromSeconds(Math.Min(backoff.TotalSeconds * 2, 60));
+            }
+        }
+    }
+}