feat(worker): upload status in the job log, ProRes uploaded in the background
Deploy / deploy (push) Successful in 3m25s

Preview jobs looked stuck after aerender finished: the worker was silently
uploading the review MP4s and the ProRes MOV over a slow uplink, and a
restart mid-upload threw the preview away.

- Worker 1.7.1 replaces the job's log tail with live upload status and logs
  each upload's size, duration and speed.
- The ProRes MOV now uploads after finalize via a persistent queue
  (HighResUploader, resumes after restart, 5 attempts) and attaches through
  the new E13b endpoint POST /api/ext/render/jobs/{id}/attach-highres.
  A MOV from a superseded preview is discarded.

Deploy the server before the 1.7.1 worker.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
twotalesanimation
2026-10-01 18:13:23 +02:00
parent f30acef101
commit 477aef7b27
11 changed files with 442 additions and 27 deletions
+16 -3
View File
@@ -124,6 +124,7 @@ public sealed class ApiClient
"validation" => $"api/ext/render/jobs/{jobId}/validation",
"delivery" => $"api/ext/render/jobs/{jobId}/delivery",
"finalize" => $"api/ext/render/jobs/{jobId}/finalize",
"attach-highres" => $"api/ext/render/jobs/{jobId}/attach-highres",
_ => throw new ArgumentOutOfRangeException(nameof(kind), kind, "unknown lifecycle report kind"),
};
var res = await _http.PostAsJsonAsync(path, payload, Json, ct);
@@ -143,8 +144,10 @@ public sealed class ApiClient
}
// E20 + PUT upload. Returns the storage key, or null when the upload failed.
// onProgress receives the bytes sent so far, for status lines in the app.
public async Task<string?> UploadArtifactAsync(
string jobId, string machineId, string kind, string filePath, string contentType, CancellationToken ct)
string jobId, string machineId, string kind, string filePath, string contentType, CancellationToken ct,
Action<long>? onProgress = null)
{
try
{
@@ -153,6 +156,13 @@ public sealed class ApiClient
_log.LogWarning("Artifact {Kind} not found at {Path}", kind, filePath);
return null;
}
var size = new FileInfo(filePath).Length;
var sw = System.Diagnostics.Stopwatch.StartNew();
// Uploads can run for many minutes on a slow uplink; without these two
// lines the log goes silent and a busy worker looks hung.
if (kind != "log")
_log.LogInformation("Uploading {Kind} for job {JobId}: {File} ({MB:F1} MB)",
kind, jobId, Path.GetFileName(filePath), size / 1048576.0);
var presignRes = await _http.PostAsJsonAsync($"api/ext/render/jobs/{jobId}/artifact-presign", new
{
@@ -173,14 +183,17 @@ public sealed class ApiClient
// presigned URL is not sent with our Authorization header.
using var upload = new HttpClient { Timeout = TimeSpan.FromMinutes(30) };
await using var stream = File.OpenRead(filePath);
var content = new StreamContent(stream);
var content = new ProgressStreamContent(stream, onProgress);
content.Headers.ContentType = new MediaTypeHeaderValue(contentType);
var putRes = await upload.PutAsync(presign.PresignedUrl, content, ct);
if (!putRes.IsSuccessStatusCode)
{
_log.LogWarning("Upload of {Kind} failed: {Status}", kind, (int)putRes.StatusCode);
_log.LogWarning("Upload of {Kind} failed after {Sec:F0}s: {Status}", kind, sw.Elapsed.TotalSeconds, (int)putRes.StatusCode);
return null;
}
if (kind != "log")
_log.LogInformation("Uploaded {Kind} for job {JobId} in {Sec:F0}s ({Speed:F1} MB/s)",
kind, jobId, sw.Elapsed.TotalSeconds, size / 1048576.0 / Math.Max(0.001, sw.Elapsed.TotalSeconds));
return presign.Key;
}
catch (Exception ex)