feat(worker): upload status in the job log, ProRes uploaded in the background
Deploy / deploy (push) Successful in 3m25s
Deploy / deploy (push) Successful in 3m25s
Preview jobs looked stuck after aerender finished: the worker was silently
uploading the review MP4s and the ProRes MOV over a slow uplink, and a
restart mid-upload threw the preview away.
- Worker 1.7.1 replaces the job's log tail with live upload status and logs
each upload's size, duration and speed.
- The ProRes MOV now uploads after finalize via a persistent queue
(HighResUploader, resumes after restart, 5 attempts) and attaches through
the new E13b endpoint POST /api/ext/render/jobs/{id}/attach-highres.
A MOV from a superseded preview is discarded.
Deploy the server before the 1.7.1 worker.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -124,6 +124,7 @@ public sealed class ApiClient
|
||||
"validation" => $"api/ext/render/jobs/{jobId}/validation",
|
||||
"delivery" => $"api/ext/render/jobs/{jobId}/delivery",
|
||||
"finalize" => $"api/ext/render/jobs/{jobId}/finalize",
|
||||
"attach-highres" => $"api/ext/render/jobs/{jobId}/attach-highres",
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(kind), kind, "unknown lifecycle report kind"),
|
||||
};
|
||||
var res = await _http.PostAsJsonAsync(path, payload, Json, ct);
|
||||
@@ -143,8 +144,10 @@ public sealed class ApiClient
|
||||
}
|
||||
|
||||
// E20 + PUT upload. Returns the storage key, or null when the upload failed.
|
||||
// onProgress receives the bytes sent so far, for status lines in the app.
|
||||
public async Task<string?> UploadArtifactAsync(
|
||||
string jobId, string machineId, string kind, string filePath, string contentType, CancellationToken ct)
|
||||
string jobId, string machineId, string kind, string filePath, string contentType, CancellationToken ct,
|
||||
Action<long>? onProgress = null)
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -153,6 +156,13 @@ public sealed class ApiClient
|
||||
_log.LogWarning("Artifact {Kind} not found at {Path}", kind, filePath);
|
||||
return null;
|
||||
}
|
||||
var size = new FileInfo(filePath).Length;
|
||||
var sw = System.Diagnostics.Stopwatch.StartNew();
|
||||
// Uploads can run for many minutes on a slow uplink; without these two
|
||||
// lines the log goes silent and a busy worker looks hung.
|
||||
if (kind != "log")
|
||||
_log.LogInformation("Uploading {Kind} for job {JobId}: {File} ({MB:F1} MB)",
|
||||
kind, jobId, Path.GetFileName(filePath), size / 1048576.0);
|
||||
|
||||
var presignRes = await _http.PostAsJsonAsync($"api/ext/render/jobs/{jobId}/artifact-presign", new
|
||||
{
|
||||
@@ -173,14 +183,17 @@ public sealed class ApiClient
|
||||
// presigned URL is not sent with our Authorization header.
|
||||
using var upload = new HttpClient { Timeout = TimeSpan.FromMinutes(30) };
|
||||
await using var stream = File.OpenRead(filePath);
|
||||
var content = new StreamContent(stream);
|
||||
var content = new ProgressStreamContent(stream, onProgress);
|
||||
content.Headers.ContentType = new MediaTypeHeaderValue(contentType);
|
||||
var putRes = await upload.PutAsync(presign.PresignedUrl, content, ct);
|
||||
if (!putRes.IsSuccessStatusCode)
|
||||
{
|
||||
_log.LogWarning("Upload of {Kind} failed: {Status}", kind, (int)putRes.StatusCode);
|
||||
_log.LogWarning("Upload of {Kind} failed after {Sec:F0}s: {Status}", kind, sw.Elapsed.TotalSeconds, (int)putRes.StatusCode);
|
||||
return null;
|
||||
}
|
||||
if (kind != "log")
|
||||
_log.LogInformation("Uploaded {Kind} for job {JobId} in {Sec:F0}s ({Speed:F1} MB/s)",
|
||||
kind, jobId, sw.Elapsed.TotalSeconds, size / 1048576.0 / Math.Max(0.001, sw.Elapsed.TotalSeconds));
|
||||
return presign.Key;
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
||||
Reference in New Issue
Block a user