using System.Text.Json; using Microsoft.Extensions.Logging; namespace VFXReviewWorker; /// /// Uploads each preview's ProRes MOV (the version's High Res download) in the /// background, after the job has already been finalized. /// /// The MOV is by far the largest upload — hundreds of MB — and on a slow or /// flaky uplink it held every preview job open for 10+ minutes, during which /// the job looked stuck in the app and a worker restart threw the whole /// preview away. Now the job completes as soon as the review MP4s are up, the /// worker moves on to the next job, and the MOV follows; an "attach-highres" /// report (through the ordered spooler, so always after its finalize) attaches /// it to the version. /// /// Pending uploads are files in highres-queue/, so a restart resumes them. The /// MOV itself lives on the SAN beside the EXRs, never in scratch. /// public sealed class HighResUploader { private const int MaxAttempts = 5; private readonly ApiClient _api; private readonly ReportSpooler _spooler; private readonly ILogger _log; private readonly string _dir; private readonly SemaphoreSlim _wake = new(0); public HighResUploader(ApiClient api, ReportSpooler spooler, ILogger log) { _api = api; _spooler = spooler; _log = log; _dir = Path.Combine(WorkerOptions.DataDir, "highres-queue"); Directory.CreateDirectory(_dir); } private sealed class Pending { public string JobId { get; set; } = ""; public string MachineId { get; set; } = ""; public string MovPath { get; set; } = ""; public int Attempts { get; set; } } public void Enqueue(string jobId, string machineId, string movPath) { var item = new Pending { JobId = jobId, MachineId = machineId, MovPath = movPath }; var name = $"{DateTimeOffset.UtcNow.UtcTicks:D20}_{jobId}.json"; var tmp = Path.Combine(_dir, name + ".tmp"); File.WriteAllText(tmp, JsonSerializer.Serialize(item, WorkerOptions.JsonOpts)); File.Move(tmp, Path.Combine(_dir, name)); _log.LogInformation("ProRes for job {JobId} queued for background upload ({MB:F0} MB)", jobId, File.Exists(movPath) ? new FileInfo(movPath).Length / 1048576.0 : 0); _wake.Release(); } /// Long-running loop; started once by the worker service. public async Task RunAsync(CancellationToken ct) { var backoff = TimeSpan.FromSeconds(30); while (!ct.IsCancellationRequested) { var next = Directory.EnumerateFiles(_dir, "*.json").OrderBy(f => f, StringComparer.Ordinal).FirstOrDefault(); if (next is null) { try { await _wake.WaitAsync(TimeSpan.FromSeconds(30), ct); } catch (OperationCanceledException) { break; } continue; } Pending? item; try { item = JsonSerializer.Deserialize(File.ReadAllText(next), WorkerOptions.JsonOpts); } catch { item = null; } if (item is null || string.IsNullOrEmpty(item.JobId)) { _log.LogWarning("Discarding unreadable ProRes queue file {File}", next); TryDelete(next); continue; } if (!File.Exists(item.MovPath)) { _log.LogWarning("ProRes for job {JobId} is gone ({Path}) — not uploading", item.JobId, item.MovPath); Report(item, null, $"MOV not found at {item.MovPath}"); TryDelete(next); continue; } string? key; try { key = await _api.UploadArtifactAsync(item.JobId, item.MachineId, "highres", item.MovPath, "video/quicktime", ct); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { _log.LogInformation("Worker stopping — ProRes upload for job {JobId} resumes on next start", item.JobId); break; } if (key is not null) { Report(item, key, null); TryDelete(next); backoff = TimeSpan.FromSeconds(30); continue; } item.Attempts++; if (item.Attempts >= MaxAttempts) { _log.LogWarning("ProRes upload for job {JobId} failed {N} times — giving up", item.JobId, item.Attempts); Report(item, null, $"upload failed {item.Attempts} times (see the worker log)"); TryDelete(next); continue; } _log.LogWarning("ProRes upload for job {JobId} failed (attempt {N}/{Max}); retrying in {Sec}s", item.JobId, item.Attempts, MaxAttempts, backoff.TotalSeconds); try { File.WriteAllText(next, JsonSerializer.Serialize(item, WorkerOptions.JsonOpts)); } catch { /* retried anyway */ } try { await Task.Delay(backoff, ct); } catch (OperationCanceledException) { break; } backoff = TimeSpan.FromSeconds(Math.Min(backoff.TotalSeconds * 2, 600)); } } private void Report(Pending item, string? key, string? error) { _spooler.Enqueue("attach-highres", item.JobId, new Dictionary { ["machineId"] = item.MachineId, ["highResKey"] = key, ["highResFileName"] = key is null ? null : Path.GetFileName(item.MovPath), ["highResSize"] = key is null ? null : new FileInfo(item.MovPath).Length, ["error"] = error, }); } private static void TryDelete(string path) { try { File.Delete(path); } catch { /* next pass */ } } }