feat(worker): upload status in the job log, ProRes uploaded in the background
Deploy / deploy (push) Successful in 3m25s
Deploy / deploy (push) Successful in 3m25s
Preview jobs looked stuck after aerender finished: the worker was silently
uploading the review MP4s and the ProRes MOV over a slow uplink, and a
restart mid-upload threw the preview away.
- Worker 1.7.1 replaces the job's log tail with live upload status and logs
each upload's size, duration and speed.
- The ProRes MOV now uploads after finalize via a persistent queue
(HighResUploader, resumes after restart, 5 attempts) and attaches through
the new E13b endpoint POST /api/ext/render/jobs/{id}/attach-highres.
A MOV from a superseded preview is discarded.
Deploy the server before the 1.7.1 worker.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,18 +1,35 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text;
|
||||
|
||||
namespace VFXReviewWorker;
|
||||
|
||||
/// <summary>
|
||||
/// Renews a job's lease with progress reports while the worker does something
|
||||
/// long that reports nothing on its own — the High quality encode and the
|
||||
/// ProRes upload can each outlast a 300 s lease, and an expired lease lets the
|
||||
/// uploads can each outlast a 300 s lease, and an expired lease lets the
|
||||
/// reaper hand the job to another machine mid-upload.
|
||||
///
|
||||
/// Each report also replaces the job's log tail with what the worker is doing
|
||||
/// now ("Uploading High quality MP4: 12.0 / 34.0 MB…"). Without it the app
|
||||
/// keeps showing aerender's last frame line and a busy job looks hung.
|
||||
/// </summary>
|
||||
internal sealed class LeaseKeepAlive : IAsyncDisposable
|
||||
{
|
||||
private readonly ApiClient _api;
|
||||
private readonly string _jobId;
|
||||
private readonly string _machineId;
|
||||
private readonly CancellationTokenSource _cts;
|
||||
private readonly SemaphoreSlim _wake = new(0);
|
||||
private readonly Task _loop;
|
||||
private readonly object _lock = new();
|
||||
private readonly List<string> _lines = new();
|
||||
private string? _current;
|
||||
|
||||
public LeaseKeepAlive(ApiClient api, string jobId, string machineId, TimeSpan interval, CancellationToken shutdown)
|
||||
{
|
||||
_api = api;
|
||||
_jobId = jobId;
|
||||
_machineId = machineId;
|
||||
_cts = CancellationTokenSource.CreateLinkedTokenSource(shutdown);
|
||||
var token = _cts.Token;
|
||||
_loop = Task.Run(async () =>
|
||||
@@ -21,10 +38,10 @@ internal sealed class LeaseKeepAlive : IAsyncDisposable
|
||||
{
|
||||
while (!token.IsCancellationRequested)
|
||||
{
|
||||
await Task.Delay(interval, token);
|
||||
// Best-effort by contract (ApiClient.ProgressAsync never throws
|
||||
// on transport errors); progress stays at 1.0 — the render is done.
|
||||
await api.ProgressAsync(jobId, machineId, 1.0, null, null, null, null, token);
|
||||
await _api.ProgressAsync(_jobId, _machineId, 1.0, null, null, null, Render(), token);
|
||||
await _wake.WaitAsync(interval, token);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
@@ -34,10 +51,53 @@ internal sealed class LeaseKeepAlive : IAsyncDisposable
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>Adds a finished step to the status shown in the app.</summary>
|
||||
public void Line(string text)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
_current = null;
|
||||
_lines.Add($"{DateTime.Now:HH:mm:ss} {text}");
|
||||
}
|
||||
_wake.Release();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Progress callback for <see cref="ApiClient.UploadArtifactAsync"/>: keeps
|
||||
/// one live line ("Uploading X: 12.0 / 34.0 MB (35%, 2.1 MB/s)") up to date.
|
||||
/// </summary>
|
||||
public Action<long> Track(string label, string filePath)
|
||||
{
|
||||
long total = File.Exists(filePath) ? new FileInfo(filePath).Length : 0;
|
||||
var sw = Stopwatch.StartNew();
|
||||
lock (_lock) _current = $"Uploading {label}: 0.0 / {Mb(total)} MB";
|
||||
_wake.Release();
|
||||
return sent =>
|
||||
{
|
||||
var pct = total > 0 ? 100.0 * sent / total : 0;
|
||||
var speed = sent / 1048576.0 / Math.Max(0.001, sw.Elapsed.TotalSeconds);
|
||||
lock (_lock) _current = $"Uploading {label}: {Mb(sent)} / {Mb(total)} MB ({pct:F0}%, {speed:F1} MB/s)";
|
||||
};
|
||||
}
|
||||
|
||||
private static string Mb(long bytes) => (bytes / 1048576.0).ToString("F1");
|
||||
|
||||
private string Render()
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
foreach (var l in _lines) sb.AppendLine(l);
|
||||
if (_current is not null) sb.AppendLine($"{DateTime.Now:HH:mm:ss} {_current}");
|
||||
return sb.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
_cts.Cancel();
|
||||
try { await _loop; } catch { /* already reported */ }
|
||||
_cts.Dispose();
|
||||
_wake.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user