feat(worker): upload status in the job log, ProRes uploaded in the background
Deploy / deploy (push) Successful in 3m25s
Deploy / deploy (push) Successful in 3m25s
Preview jobs looked stuck after aerender finished: the worker was silently
uploading the review MP4s and the ProRes MOV over a slow uplink, and a
restart mid-upload threw the preview away.
- Worker 1.7.1 replaces the job's log tail with live upload status and logs
each upload's size, duration and speed.
- The ProRes MOV now uploads after finalize via a persistent queue
(HighResUploader, resumes after restart, 5 attempts) and attaches through
the new E13b endpoint POST /api/ext/render/jobs/{id}/attach-highres.
A MOV from a superseded preview is discarded.
Deploy the server before the 1.7.1 worker.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -18,11 +18,14 @@ public sealed class JobExecutor
|
||||
private readonly DeliveryStage _delivery;
|
||||
private readonly BlenderRunner _blender;
|
||||
private readonly ReportSpooler _spooler;
|
||||
private readonly HighResUploader _highRes;
|
||||
private readonly ILogger<JobExecutor> _log;
|
||||
|
||||
public JobExecutor(WorkerOptions options, ApiClient api, AerenderRunner runner, PreviewStage preview,
|
||||
ValidationStage validation, DeliveryStage delivery, BlenderRunner blender, ReportSpooler spooler, ILogger<JobExecutor> log)
|
||||
ValidationStage validation, DeliveryStage delivery, BlenderRunner blender, ReportSpooler spooler,
|
||||
HighResUploader highRes, ILogger<JobExecutor> log)
|
||||
{
|
||||
_highRes = highRes;
|
||||
_options = options;
|
||||
_api = api;
|
||||
_runner = runner;
|
||||
@@ -418,8 +421,15 @@ public sealed class JobExecutor
|
||||
return; // temp AEP is deliberately kept for debugging
|
||||
}
|
||||
|
||||
// From here on aerender is done and the app would keep showing its
|
||||
// last frame line; this reports what the worker is doing instead,
|
||||
// and renews the lease through the encode and uploads.
|
||||
await using var status = new LeaseKeepAlive(_api, job.Id, machineId, TimeSpan.FromSeconds(10), shutdown);
|
||||
status.Line($"Preview rendered in {result.Seconds:F0}s — uploading review media");
|
||||
|
||||
var previewKey = await _api.UploadArtifactAsync(
|
||||
job.Id, machineId, "preview", result.Mp4Path!, "video/mp4", shutdown);
|
||||
job.Id, machineId, "preview", result.Mp4Path!, "video/mp4", shutdown,
|
||||
status.Track("preview MP4", result.Mp4Path!));
|
||||
if (previewKey is null)
|
||||
{
|
||||
// The renders are on the SAN and fine — only the upload failed,
|
||||
@@ -433,6 +443,7 @@ public sealed class JobExecutor
|
||||
});
|
||||
return;
|
||||
}
|
||||
status.Line("Preview MP4 uploaded");
|
||||
|
||||
string? thumbKey = null;
|
||||
if (result.ThumbnailPath is not null)
|
||||
@@ -443,28 +454,26 @@ public sealed class JobExecutor
|
||||
|
||||
// High quality MP4 + the MOV as the version's High Res download
|
||||
// (§9.6). Both optional: a failure here costs the extras, never the
|
||||
// preview, which is already safely uploaded.
|
||||
// preview, which is already safely uploaded. The MOV is uploaded in
|
||||
// the background after finalize (HighResUploader) — it is the big
|
||||
// one and used to hold the job open for many minutes.
|
||||
string? hqKey = null;
|
||||
string? highResKey = null;
|
||||
long? highResSize = null;
|
||||
var highResPending = false;
|
||||
if (manifest.HighQuality)
|
||||
{
|
||||
await using var keepAlive = new LeaseKeepAlive(_api, job.Id, machineId, TimeSpan.FromSeconds(60), shutdown);
|
||||
|
||||
status.Line("Encoding High quality MP4");
|
||||
var hqPath = await _preview.TryMakeHighQualityAsync(job.Id, result.MovPath!, result.Mp4Path!, shutdown);
|
||||
if (hqPath is not null)
|
||||
{
|
||||
hqKey = await _api.UploadArtifactAsync(job.Id, machineId, "preview", hqPath, "video/mp4", shutdown);
|
||||
hqKey = await _api.UploadArtifactAsync(job.Id, machineId, "preview", hqPath, "video/mp4", shutdown,
|
||||
status.Track("High quality MP4", hqPath));
|
||||
if (hqKey is null) _log.LogWarning("High quality MP4 upload failed for job {JobId}", job.Id);
|
||||
status.Line(hqKey is null ? "High quality MP4 upload FAILED (see worker log)" : "High quality MP4 uploaded");
|
||||
}
|
||||
|
||||
if (result.MovPath is not null && File.Exists(result.MovPath))
|
||||
{
|
||||
highResSize = new FileInfo(result.MovPath).Length;
|
||||
highResKey = await _api.UploadArtifactAsync(
|
||||
job.Id, machineId, "highres", result.MovPath, "video/quicktime", shutdown);
|
||||
if (highResKey is null) _log.LogWarning("ProRes upload failed for job {JobId}", job.Id);
|
||||
}
|
||||
highResPending = result.MovPath is not null && File.Exists(result.MovPath);
|
||||
if (highResPending)
|
||||
status.Line("ProRes MOV queued for background upload — completing the job now");
|
||||
}
|
||||
|
||||
_spooler.Enqueue("finalize", job.Id, new Dictionary<string, object?>
|
||||
@@ -476,9 +485,7 @@ public sealed class JobExecutor
|
||||
["previewMovKey"] = previewKey,
|
||||
["thumbnailKey"] = thumbKey,
|
||||
["hqPreviewKey"] = hqKey,
|
||||
["highResKey"] = highResKey,
|
||||
["highResFileName"] = highResKey is null ? null : Path.GetFileName(result.MovPath),
|
||||
["highResSize"] = highResKey is null ? null : highResSize,
|
||||
["highResPending"] = highResPending,
|
||||
},
|
||||
["renderStats"] = new Dictionary<string, object?>
|
||||
{
|
||||
@@ -497,6 +504,10 @@ public sealed class JobExecutor
|
||||
},
|
||||
});
|
||||
|
||||
// Enqueued after finalize: its attach report must reach the spooler
|
||||
// behind finalize, which creates the version it attaches to.
|
||||
if (highResPending) _highRes.Enqueue(job.Id, machineId, result.MovPath!);
|
||||
|
||||
// Only clean up once the outputs are safely reported.
|
||||
CleanScratch(job.Id, result.TempAep);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user