feat(worker): upload status in the job log, ProRes uploaded in the background
Deploy / deploy (push) Successful in 3m25s

Preview jobs looked stuck after aerender finished: the worker was silently
uploading the review MP4s and the ProRes MOV over a slow uplink, and a
restart mid-upload threw the preview away.

- Worker 1.7.1 replaces the job's log tail with live upload status and logs
  each upload's size, duration and speed.
- The ProRes MOV now uploads after finalize via a persistent queue
  (HighResUploader, resumes after restart, 5 attempts) and attaches through
  the new E13b endpoint POST /api/ext/render/jobs/{id}/attach-highres.
  A MOV from a superseded preview is discarded.

Deploy the server before the 1.7.1 worker.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
twotalesanimation
2026-10-01 18:13:23 +02:00
parent f30acef101
commit 477aef7b27
11 changed files with 442 additions and 27 deletions
+29 -18
View File
@@ -18,11 +18,14 @@ public sealed class JobExecutor
private readonly DeliveryStage _delivery;
private readonly BlenderRunner _blender;
private readonly ReportSpooler _spooler;
private readonly HighResUploader _highRes;
private readonly ILogger<JobExecutor> _log;
public JobExecutor(WorkerOptions options, ApiClient api, AerenderRunner runner, PreviewStage preview,
ValidationStage validation, DeliveryStage delivery, BlenderRunner blender, ReportSpooler spooler, ILogger<JobExecutor> log)
ValidationStage validation, DeliveryStage delivery, BlenderRunner blender, ReportSpooler spooler,
HighResUploader highRes, ILogger<JobExecutor> log)
{
_highRes = highRes;
_options = options;
_api = api;
_runner = runner;
@@ -418,8 +421,15 @@ public sealed class JobExecutor
return; // temp AEP is deliberately kept for debugging
}
// From here on aerender is done and the app would keep showing its
// last frame line; this reports what the worker is doing instead,
// and renews the lease through the encode and uploads.
await using var status = new LeaseKeepAlive(_api, job.Id, machineId, TimeSpan.FromSeconds(10), shutdown);
status.Line($"Preview rendered in {result.Seconds:F0}s — uploading review media");
var previewKey = await _api.UploadArtifactAsync(
job.Id, machineId, "preview", result.Mp4Path!, "video/mp4", shutdown);
job.Id, machineId, "preview", result.Mp4Path!, "video/mp4", shutdown,
status.Track("preview MP4", result.Mp4Path!));
if (previewKey is null)
{
// The renders are on the SAN and fine — only the upload failed,
@@ -433,6 +443,7 @@ public sealed class JobExecutor
});
return;
}
status.Line("Preview MP4 uploaded");
string? thumbKey = null;
if (result.ThumbnailPath is not null)
@@ -443,28 +454,26 @@ public sealed class JobExecutor
// High quality MP4 + the MOV as the version's High Res download
// (§9.6). Both optional: a failure here costs the extras, never the
// preview, which is already safely uploaded.
// preview, which is already safely uploaded. The MOV is uploaded in
// the background after finalize (HighResUploader) — it is the big
// one and used to hold the job open for many minutes.
string? hqKey = null;
string? highResKey = null;
long? highResSize = null;
var highResPending = false;
if (manifest.HighQuality)
{
await using var keepAlive = new LeaseKeepAlive(_api, job.Id, machineId, TimeSpan.FromSeconds(60), shutdown);
status.Line("Encoding High quality MP4");
var hqPath = await _preview.TryMakeHighQualityAsync(job.Id, result.MovPath!, result.Mp4Path!, shutdown);
if (hqPath is not null)
{
hqKey = await _api.UploadArtifactAsync(job.Id, machineId, "preview", hqPath, "video/mp4", shutdown);
hqKey = await _api.UploadArtifactAsync(job.Id, machineId, "preview", hqPath, "video/mp4", shutdown,
status.Track("High quality MP4", hqPath));
if (hqKey is null) _log.LogWarning("High quality MP4 upload failed for job {JobId}", job.Id);
status.Line(hqKey is null ? "High quality MP4 upload FAILED (see worker log)" : "High quality MP4 uploaded");
}
if (result.MovPath is not null && File.Exists(result.MovPath))
{
highResSize = new FileInfo(result.MovPath).Length;
highResKey = await _api.UploadArtifactAsync(
job.Id, machineId, "highres", result.MovPath, "video/quicktime", shutdown);
if (highResKey is null) _log.LogWarning("ProRes upload failed for job {JobId}", job.Id);
}
highResPending = result.MovPath is not null && File.Exists(result.MovPath);
if (highResPending)
status.Line("ProRes MOV queued for background upload — completing the job now");
}
_spooler.Enqueue("finalize", job.Id, new Dictionary<string, object?>
@@ -476,9 +485,7 @@ public sealed class JobExecutor
["previewMovKey"] = previewKey,
["thumbnailKey"] = thumbKey,
["hqPreviewKey"] = hqKey,
["highResKey"] = highResKey,
["highResFileName"] = highResKey is null ? null : Path.GetFileName(result.MovPath),
["highResSize"] = highResKey is null ? null : highResSize,
["highResPending"] = highResPending,
},
["renderStats"] = new Dictionary<string, object?>
{
@@ -497,6 +504,10 @@ public sealed class JobExecutor
},
});
// Enqueued after finalize: its attach report must reach the spooler
// behind finalize, which creates the version it attaches to.
if (highResPending) _highRes.Enqueue(job.Id, machineId, result.MovPath!);
// Only clean up once the outputs are safely reported.
CleanScratch(job.Id, result.TempAep);
}