feat(worker): parallel multipart uploads for large artifacts

From the studio a single connection to Hetzner nbg1 (188 ms away) is
capped near 0.1 MB/s, although a browser speed test shows 50 Mbps.
Measured: 1 connection 0.13 MB/s, 8 -> 0.64, 24 -> 1.69 MB/s. At the old
speed a ProRes MOV could not finish inside the 30 min request timeout.

- Worker 1.7.2 uploads files >= 16 MB as S3 multipart: 8 MB parts on
  uploadConcurrency connections (config.json, default 16), each part
  retried up to 4 times; aborts the upload if a part keeps failing.
- artifact-presign returns uploadId + per-part presigned URLs when asked
  for multipart; new E20b artifact-complete stitches or aborts.
- Old workers and old servers fall back to the single PUT.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

RenderWorker/VFXReviewWorker/ApiClient.cs

@@ -13,10 +13,16 @@ namespace VFXReviewWorker;
 public sealed class ApiClient
 {
     private readonly HttpClient _http;
+    private readonly WorkerOptions _options;
     private readonly ILogger<ApiClient> _log;
 
+    // Presigned uploads: a separate client so the URL is not sent with our
+    // Authorization header, and a generous per-request timeout for media.
+    private static readonly HttpClient _upload = new() { Timeout = TimeSpan.FromMinutes(30) };
+
     public ApiClient(WorkerOptions options, ILogger<ApiClient> log)
     {
+        _options = options;
         _log = log;
         _http = new HttpClient
         {
@@ -170,6 +176,8 @@ public sealed class ApiClient
                 kind,
                 fileName = Path.GetFileName(filePath),
                 contentType,
+                multipart = size >= MultipartThreshold,
+                size,
             }, Json, ct);
             if (!presignRes.IsSuccessStatusCode)
             {
@@ -179,17 +187,23 @@ public sealed class ApiClient
             var presign = await presignRes.Content.ReadFromJsonAsync<PresignResponse>(Json, ct);
             if (presign is null) return null;
 
-            // Media files can be large — generous timeout, separate client so the
-            // presigned URL is not sent with our Authorization header.
-            using var upload = new HttpClient { Timeout = TimeSpan.FromMinutes(30) };
-            await using var stream = File.OpenRead(filePath);
-            var content = new ProgressStreamContent(stream, onProgress);
-            content.Headers.ContentType = new MediaTypeHeaderValue(contentType);
-            var putRes = await upload.PutAsync(presign.PresignedUrl, content, ct);
-            if (!putRes.IsSuccessStatusCode)
+            if (presign.UploadId is not null && presign.PartUrls is { Count: > 0 } && presign.PartSize > 0)
             {
-                _log.LogWarning("Upload of {Kind} failed after {Sec:F0}s: {Status}", kind, sw.Elapsed.TotalSeconds, (int)putRes.StatusCode);
-                return null;
+                if (!await UploadMultipartAsync(jobId, machineId, kind, filePath, size, presign, onProgress, ct))
+                    return null;
+            }
+            else
+            {
+                // Single PUT: small files, or a server that predates multipart.
+                await using var stream = File.OpenRead(filePath);
+                var content = new ProgressStreamContent(stream, onProgress);
+                content.Headers.ContentType = new MediaTypeHeaderValue(contentType);
+                var putRes = await _upload.PutAsync(presign.PresignedUrl, content, ct);
+                if (!putRes.IsSuccessStatusCode)
+                {
+                    _log.LogWarning("Upload of {Kind} failed after {Sec:F0}s: {Status}", kind, sw.Elapsed.TotalSeconds, (int)putRes.StatusCode);
+                    return null;
+                }
             }
             if (kind != "log")
                 _log.LogInformation("Uploaded {Kind} for job {JobId} in {Sec:F0}s ({Speed:F1} MB/s)",
@@ -203,6 +217,117 @@ public sealed class ApiClient
         }
     }
 
+    // Files at least this big go up as parallel multipart parts (E20b).
+    private const long MultipartThreshold = 16L * 1024 * 1024;
+    private const int PartAttempts = 4;
+
+    /// <summary>
+    /// Uploads the parts concurrently (UploadConcurrency connections), each
+    /// retried on its own so a dropped connection costs one part, not the
+    /// file; then asks the server to stitch them. Aborts on failure so the
+    /// stored parts don't linger in the bucket.
+    /// </summary>
+    private async Task<bool> UploadMultipartAsync(string jobId, string machineId, string kind, string filePath,
+        long size, PresignResponse presign, Action<long>? onProgress, CancellationToken ct)
+    {
+        var partUrls = presign.PartUrls!;
+        var partSize = presign.PartSize;
+        var etags = new string?[partUrls.Count];
+        var sentPerPart = new long[partUrls.Count];
+        long sentTotal = 0;
+        var next = -1;
+
+        using var handle = File.OpenHandle(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.Asynchronous);
+        using var failed = CancellationTokenSource.CreateLinkedTokenSource(ct);
+
+        async Task Worker()
+        {
+            int i;
+            while ((i = Interlocked.Increment(ref next)) < partUrls.Count && !failed.IsCancellationRequested)
+            {
+                var offset = i * partSize;
+                var len = (int)Math.Min(partSize, size - offset);
+                var buffer = new byte[len];
+                var read = 0;
+                while (read < len) read += await RandomAccess.ReadAsync(handle, buffer.AsMemory(read), offset + read, failed.Token);
+
+                for (var attempt = 1; ; attempt++)
+                {
+                    var part = i;
+                    // Progress restarts from zero for a retried part.
+                    Interlocked.Add(ref sentTotal, -Interlocked.Exchange(ref sentPerPart[part], 0));
+                    var content = new ProgressStreamContent(new MemoryStream(buffer, writable: false), sent =>
+                    {
+                        var delta = sent - Interlocked.Exchange(ref sentPerPart[part], sent);
+                        onProgress?.Invoke(Interlocked.Add(ref sentTotal, delta));
+                    });
+                    try
+                    {
+                        var res = await _upload.PutAsync(partUrls[i], content, failed.Token);
+                        var etag = res.Headers.ETag?.Tag
+                                   ?? (res.Headers.TryGetValues("ETag", out var v) ? v.FirstOrDefault() : null);
+                        if (res.IsSuccessStatusCode && etag is not null)
+                        {
+                            etags[i] = etag;
+                            break;
+                        }
+                        _log.LogWarning("Part {Part}/{Count} of {Kind} for job {JobId} failed: {Status} (attempt {N}/{Max})",
+                            i + 1, partUrls.Count, kind, jobId, (int)res.StatusCode, attempt, PartAttempts);
+                    }
+                    catch (Exception ex) when (!failed.IsCancellationRequested)
+                    {
+                        _log.LogWarning("Part {Part}/{Count} of {Kind} for job {JobId} failed: {Error} (attempt {N}/{Max})",
+                            i + 1, partUrls.Count, kind, jobId, ex.Message, attempt, PartAttempts);
+                    }
+                    if (attempt >= PartAttempts)
+                    {
+                        failed.Cancel(); // no point finishing the other parts
+                        return;
+                    }
+                    await Task.Delay(TimeSpan.FromSeconds(5 * attempt), failed.Token);
+                }
+            }
+        }
+
+        var concurrency = Math.Clamp(_options.UploadConcurrency, 1, 64);
+        try
+        {
+            await Task.WhenAll(Enumerable.Range(0, Math.Min(concurrency, partUrls.Count)).Select(_ => Task.Run(Worker)));
+        }
+        catch (OperationCanceledException)
+        {
+            // a part gave up, or shutdown — either way the upload is aborted below
+        }
+
+        var complete = !failed.IsCancellationRequested && etags.All(e => e is not null);
+        if (complete)
+        {
+            var res = await _http.PostAsJsonAsync($"api/ext/render/jobs/{jobId}/artifact-complete", new
+            {
+                machineId,
+                key = presign.Key,
+                uploadId = presign.UploadId,
+                parts = etags.Select((etag, i) => new { partNumber = i + 1, etag }),
+            }, Json, ct);
+            if (res.IsSuccessStatusCode) return true;
+            _log.LogWarning("Completing multipart {Kind} for job {JobId} failed: {Status} {Body}",
+                kind, jobId, (int)res.StatusCode, await res.Content.ReadAsStringAsync(ct));
+        }
+
+        try
+        {
+            await _http.PostAsJsonAsync($"api/ext/render/jobs/{jobId}/artifact-complete", new
+            {
+                machineId,
+                key = presign.Key,
+                uploadId = presign.UploadId,
+                abort = true,
+            }, Json, CancellationToken.None);
+        }
+        catch { /* parts expire on their own */ }
+        return false;
+    }
+
     public Task<string?> UploadLogAsync(string jobId, string machineId, string logFilePath, CancellationToken ct) =>
         UploadArtifactAsync(jobId, machineId, "log", logFilePath, "text/plain", ct);