feat(worker): parallel multipart uploads for large artifacts

From the studio a single connection to Hetzner nbg1 (188 ms away) is
capped near 0.1 MB/s, although a browser speed test shows 50 Mbps.
Measured: 1 connection 0.13 MB/s, 8 -> 0.64, 24 -> 1.69 MB/s. At the old
speed a ProRes MOV could not finish inside the 30 min request timeout.

- Worker 1.7.2 uploads files >= 16 MB as S3 multipart: 8 MB parts on
  uploadConcurrency connections (config.json, default 16), each part
  retried up to 4 times; aborts the upload if a part keeps failing.
- artifact-presign returns uploadId + per-part presigned URLs when asked
  for multipart; new E20b artifact-complete stitches or aborts.
- Old workers and old servers fall back to the single PUT.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

RenderWorker/README.md

@@ -208,6 +208,9 @@ read back from the assembly, so a machine can never report a version it is not r
 `pathMappings` translate the manifest's canonical UNC paths to this machine's drive
 mappings. `plateSearchRoots` is optional: it is only consulted when a manifest carries no
 timing plate (older panel versions), to find the shot's plates for the timecode check.
+`uploadConcurrency` (default 16) is how many parallel connections a file of 16 MB or more
+uploads over. Each connection to object storage is capped near 0.1 MB/s from the studio, so
+throughput scales with it (measured 2026-10-01: 1 → 0.13 MB/s, 8 → 0.64, 24 → 1.69 MB/s).
 Everything tunable (poll interval, lease, stall timeout, max attempts, validation) lives in
 the server's SystemConfig and arrives at registration — no per-machine tuning files.
 

RenderWorker/VFXReviewWorker.Tests/MultipartUploadTests.cs

@@ -0,0 +1,205 @@
+using System.Collections.Concurrent;
+using System.Net;
+using System.Net.Sockets;
+using System.Text.Json;
+using Microsoft.Extensions.Logging.Abstractions;
+using VFXReviewWorker;
+using Xunit;
+
+namespace VFXReviewWorker.Tests;
+
+/// <summary>
+/// Large artifacts go up as parallel multipart parts: from the studio a single
+/// connection to object storage is capped near 0.1 MB/s, so a ProRes MOV could
+/// not finish inside the 30 min request timeout. These drive the real upload
+/// code against a local fake of the server + storage.
+/// </summary>
+public class MultipartUploadTests : IDisposable
+{
+    private const int PartSize = 8 * 1024 * 1024;
+
+    private readonly HttpListener _listener = new();
+    private readonly string _base;
+    private readonly ConcurrentDictionary<int, byte[]> _parts = new();
+    private readonly ConcurrentDictionary<int, int> _failuresLeft = new();
+    private readonly ConcurrentBag<string> _completions = new();
+    private int _inFlight, _maxInFlight;
+    private bool _multipartSupported = true;
+    private byte[]? _singlePut;
+
+    public MultipartUploadTests()
+    {
+        var port = FreePort();
+        _base = $"http://localhost:{port}/";
+        _listener.Prefixes.Add(_base);
+        _listener.Start();
+        _ = Task.Run(Serve);
+    }
+
+    private static int FreePort()
+    {
+        var l = new TcpListener(IPAddress.Loopback, 0);
+        l.Start();
+        var port = ((IPEndPoint)l.LocalEndpoint).Port;
+        l.Stop();
+        return port;
+    }
+
+    private async Task Serve()
+    {
+        while (_listener.IsListening)
+        {
+            HttpListenerContext ctx;
+            try { ctx = await _listener.GetContextAsync(); } catch { return; }
+            _ = Task.Run(() => Handle(ctx));
+        }
+    }
+
+    private async Task Handle(HttpListenerContext ctx)
+    {
+        var path = ctx.Request.Url!.AbsolutePath;
+        using var ms = new MemoryStream();
+        await ctx.Request.InputStream.CopyToAsync(ms);
+        var body = ms.ToArray();
+        var res = ctx.Response;
+
+        if (path.EndsWith("/artifact-presign"))
+        {
+            var req = JsonDocument.Parse(body).RootElement;
+            var size = req.GetProperty("size").GetInt64();
+            object payload = _multipartSupported && req.GetProperty("multipart").GetBoolean()
+                ? new
+                {
+                    key = "highres/x.mov",
+                    uploadId = "up1",
+                    partSize = PartSize,
+                    partUrls = Enumerable.Range(1, (int)((size + PartSize - 1) / PartSize))
+                        .Select(n => $"{_base}part/{n}").ToList(),
+                }
+                : new { key = "highres/x.mov", presignedUrl = $"{_base}single" };
+            await Json(res, payload);
+        }
+        else if (path.StartsWith("/part/"))
+        {
+            var n = int.Parse(path["/part/".Length..]);
+            var now = Interlocked.Increment(ref _inFlight);
+            InterlockedMax(ref _maxInFlight, now);
+            await Task.Delay(50);
+            Interlocked.Decrement(ref _inFlight);
+            if (_failuresLeft.TryGetValue(n, out var left) && left > 0)
+            {
+                _failuresLeft[n] = left - 1;
+                res.StatusCode = 500;
+                res.Close();
+                return;
+            }
+            _parts[n] = body;
+            res.Headers["ETag"] = $"\"etag-{n}\"";
+            res.Close();
+        }
+        else if (path == "/single")
+        {
+            _singlePut = body;
+            res.Close();
+        }
+        else if (path.EndsWith("/artifact-complete"))
+        {
+            _completions.Add(System.Text.Encoding.UTF8.GetString(body));
+            await Json(res, new { ok = true });
+        }
+        else
+        {
+            res.StatusCode = 404;
+            res.Close();
+        }
+    }
+
+    private static void InterlockedMax(ref int target, int value)
+    {
+        int seen;
+        while ((seen = Volatile.Read(ref target)) < value && Interlocked.CompareExchange(ref target, value, seen) != seen) { }
+    }
+
+    private static async Task Json(HttpListenerResponse res, object payload)
+    {
+        var bytes = JsonSerializer.SerializeToUtf8Bytes(payload, WorkerOptions.JsonOpts);
+        res.ContentType = "application/json";
+        await res.OutputStream.WriteAsync(bytes);
+        res.Close();
+    }
+
+    private ApiClient Client(int concurrency = 4) =>
+        new(new WorkerOptions { ServerUrl = _base, ApiKey = "k", UploadConcurrency = concurrency },
+            NullLogger<ApiClient>.Instance);
+
+    private static string TempFile(int bytes)
+    {
+        var path = Path.GetTempFileName();
+        var data = new byte[bytes];
+        new Random(42).NextBytes(data);
+        File.WriteAllBytes(path, data);
+        return path;
+    }
+
+    [Fact]
+    public async Task Uploads_parts_in_parallel_and_stitches_them_in_order()
+    {
+        var file = TempFile(PartSize * 5 + 12345);
+        long lastProgress = 0;
+        var key = await Client(4).UploadArtifactAsync("job1", "m1", "highres", file, "video/quicktime",
+            CancellationToken.None, sent => Interlocked.Exchange(ref lastProgress, sent));
+
+        Assert.Equal("highres/x.mov", key);
+        var stitched = Enumerable.Range(1, 6).SelectMany(n => _parts[n]).ToArray();
+        Assert.Equal(File.ReadAllBytes(file), stitched);
+        Assert.True(_maxInFlight > 1, "parts should upload concurrently");
+        Assert.True(_maxInFlight <= 4, "concurrency is capped by UploadConcurrency");
+        Assert.Equal(new FileInfo(file).Length, Interlocked.Read(ref lastProgress));
+
+        var complete = JsonDocument.Parse(Assert.Single(_completions)).RootElement;
+        Assert.False(complete.TryGetProperty("abort", out _));
+        Assert.Equal(6, complete.GetProperty("parts").GetArrayLength());
+        Assert.Equal("\"etag-1\"", complete.GetProperty("parts")[0].GetProperty("etag").GetString());
+        File.Delete(file);
+    }
+
+    [Fact]
+    public async Task A_failed_part_is_retried_on_its_own()
+    {
+        var file = TempFile(PartSize * 3);
+        _failuresLeft[2] = 2;
+        var key = await Client().UploadArtifactAsync("job1", "m1", "highres", file, "video/quicktime", CancellationToken.None);
+
+        Assert.NotNull(key);
+        Assert.Equal(File.ReadAllBytes(file), Enumerable.Range(1, 3).SelectMany(n => _parts[n]).ToArray());
+        File.Delete(file);
+    }
+
+    [Fact]
+    public async Task A_part_that_keeps_failing_aborts_the_upload()
+    {
+        var file = TempFile(PartSize * 3);
+        _failuresLeft[3] = 99;
+        var key = await Client().UploadArtifactAsync("job1", "m1", "highres", file, "video/quicktime", CancellationToken.None);
+
+        Assert.Null(key);
+        var abort = JsonDocument.Parse(Assert.Single(_completions)).RootElement;
+        Assert.True(abort.GetProperty("abort").GetBoolean());
+        File.Delete(file);
+    }
+
+    [Fact]
+    public async Task Falls_back_to_a_single_put_on_a_server_without_multipart()
+    {
+        _multipartSupported = false;
+        var file = TempFile(PartSize * 3);
+        var key = await Client().UploadArtifactAsync("job1", "m1", "highres", file, "video/quicktime", CancellationToken.None);
+
+        Assert.Equal("highres/x.mov", key);
+        Assert.Equal(File.ReadAllBytes(file), _singlePut);
+        Assert.Empty(_completions);
+        File.Delete(file);
+    }
+
+    public void Dispose() => _listener.Close();
+}

RenderWorker/VFXReviewWorker/ApiClient.cs

@@ -13,10 +13,16 @@ namespace VFXReviewWorker;
 public sealed class ApiClient
 {
     private readonly HttpClient _http;
+    private readonly WorkerOptions _options;
     private readonly ILogger<ApiClient> _log;
 
+    // Presigned uploads: a separate client so the URL is not sent with our
+    // Authorization header, and a generous per-request timeout for media.
+    private static readonly HttpClient _upload = new() { Timeout = TimeSpan.FromMinutes(30) };
+
     public ApiClient(WorkerOptions options, ILogger<ApiClient> log)
     {
+        _options = options;
         _log = log;
         _http = new HttpClient
         {
@@ -170,6 +176,8 @@ public sealed class ApiClient
                 kind,
                 fileName = Path.GetFileName(filePath),
                 contentType,
+                multipart = size >= MultipartThreshold,
+                size,
             }, Json, ct);
             if (!presignRes.IsSuccessStatusCode)
             {
@@ -179,17 +187,23 @@ public sealed class ApiClient
             var presign = await presignRes.Content.ReadFromJsonAsync<PresignResponse>(Json, ct);
             if (presign is null) return null;
 
-            // Media files can be large — generous timeout, separate client so the
-            // presigned URL is not sent with our Authorization header.
-            using var upload = new HttpClient { Timeout = TimeSpan.FromMinutes(30) };
-            await using var stream = File.OpenRead(filePath);
-            var content = new ProgressStreamContent(stream, onProgress);
-            content.Headers.ContentType = new MediaTypeHeaderValue(contentType);
-            var putRes = await upload.PutAsync(presign.PresignedUrl, content, ct);
-            if (!putRes.IsSuccessStatusCode)
+            if (presign.UploadId is not null && presign.PartUrls is { Count: > 0 } && presign.PartSize > 0)
             {
-                _log.LogWarning("Upload of {Kind} failed after {Sec:F0}s: {Status}", kind, sw.Elapsed.TotalSeconds, (int)putRes.StatusCode);
-                return null;
+                if (!await UploadMultipartAsync(jobId, machineId, kind, filePath, size, presign, onProgress, ct))
+                    return null;
+            }
+            else
+            {
+                // Single PUT: small files, or a server that predates multipart.
+                await using var stream = File.OpenRead(filePath);
+                var content = new ProgressStreamContent(stream, onProgress);
+                content.Headers.ContentType = new MediaTypeHeaderValue(contentType);
+                var putRes = await _upload.PutAsync(presign.PresignedUrl, content, ct);
+                if (!putRes.IsSuccessStatusCode)
+                {
+                    _log.LogWarning("Upload of {Kind} failed after {Sec:F0}s: {Status}", kind, sw.Elapsed.TotalSeconds, (int)putRes.StatusCode);
+                    return null;
+                }
             }
             if (kind != "log")
                 _log.LogInformation("Uploaded {Kind} for job {JobId} in {Sec:F0}s ({Speed:F1} MB/s)",
@@ -203,6 +217,117 @@ public sealed class ApiClient
         }
     }
 
+    // Files at least this big go up as parallel multipart parts (E20b).
+    private const long MultipartThreshold = 16L * 1024 * 1024;
+    private const int PartAttempts = 4;
+
+    /// <summary>
+    /// Uploads the parts concurrently (UploadConcurrency connections), each
+    /// retried on its own so a dropped connection costs one part, not the
+    /// file; then asks the server to stitch them. Aborts on failure so the
+    /// stored parts don't linger in the bucket.
+    /// </summary>
+    private async Task<bool> UploadMultipartAsync(string jobId, string machineId, string kind, string filePath,
+        long size, PresignResponse presign, Action<long>? onProgress, CancellationToken ct)
+    {
+        var partUrls = presign.PartUrls!;
+        var partSize = presign.PartSize;
+        var etags = new string?[partUrls.Count];
+        var sentPerPart = new long[partUrls.Count];
+        long sentTotal = 0;
+        var next = -1;
+
+        using var handle = File.OpenHandle(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.Asynchronous);
+        using var failed = CancellationTokenSource.CreateLinkedTokenSource(ct);
+
+        async Task Worker()
+        {
+            int i;
+            while ((i = Interlocked.Increment(ref next)) < partUrls.Count && !failed.IsCancellationRequested)
+            {
+                var offset = i * partSize;
+                var len = (int)Math.Min(partSize, size - offset);
+                var buffer = new byte[len];
+                var read = 0;
+                while (read < len) read += await RandomAccess.ReadAsync(handle, buffer.AsMemory(read), offset + read, failed.Token);
+
+                for (var attempt = 1; ; attempt++)
+                {
+                    var part = i;
+                    // Progress restarts from zero for a retried part.
+                    Interlocked.Add(ref sentTotal, -Interlocked.Exchange(ref sentPerPart[part], 0));
+                    var content = new ProgressStreamContent(new MemoryStream(buffer, writable: false), sent =>
+                    {
+                        var delta = sent - Interlocked.Exchange(ref sentPerPart[part], sent);
+                        onProgress?.Invoke(Interlocked.Add(ref sentTotal, delta));
+                    });
+                    try
+                    {
+                        var res = await _upload.PutAsync(partUrls[i], content, failed.Token);
+                        var etag = res.Headers.ETag?.Tag
+                                   ?? (res.Headers.TryGetValues("ETag", out var v) ? v.FirstOrDefault() : null);
+                        if (res.IsSuccessStatusCode && etag is not null)
+                        {
+                            etags[i] = etag;
+                            break;
+                        }
+                        _log.LogWarning("Part {Part}/{Count} of {Kind} for job {JobId} failed: {Status} (attempt {N}/{Max})",
+                            i + 1, partUrls.Count, kind, jobId, (int)res.StatusCode, attempt, PartAttempts);
+                    }
+                    catch (Exception ex) when (!failed.IsCancellationRequested)
+                    {
+                        _log.LogWarning("Part {Part}/{Count} of {Kind} for job {JobId} failed: {Error} (attempt {N}/{Max})",
+                            i + 1, partUrls.Count, kind, jobId, ex.Message, attempt, PartAttempts);
+                    }
+                    if (attempt >= PartAttempts)
+                    {
+                        failed.Cancel(); // no point finishing the other parts
+                        return;
+                    }
+                    await Task.Delay(TimeSpan.FromSeconds(5 * attempt), failed.Token);
+                }
+            }
+        }
+
+        var concurrency = Math.Clamp(_options.UploadConcurrency, 1, 64);
+        try
+        {
+            await Task.WhenAll(Enumerable.Range(0, Math.Min(concurrency, partUrls.Count)).Select(_ => Task.Run(Worker)));
+        }
+        catch (OperationCanceledException)
+        {
+            // a part gave up, or shutdown — either way the upload is aborted below
+        }
+
+        var complete = !failed.IsCancellationRequested && etags.All(e => e is not null);
+        if (complete)
+        {
+            var res = await _http.PostAsJsonAsync($"api/ext/render/jobs/{jobId}/artifact-complete", new
+            {
+                machineId,
+                key = presign.Key,
+                uploadId = presign.UploadId,
+                parts = etags.Select((etag, i) => new { partNumber = i + 1, etag }),
+            }, Json, ct);
+            if (res.IsSuccessStatusCode) return true;
+            _log.LogWarning("Completing multipart {Kind} for job {JobId} failed: {Status} {Body}",
+                kind, jobId, (int)res.StatusCode, await res.Content.ReadAsStringAsync(ct));
+        }
+
+        try
+        {
+            await _http.PostAsJsonAsync($"api/ext/render/jobs/{jobId}/artifact-complete", new
+            {
+                machineId,
+                key = presign.Key,
+                uploadId = presign.UploadId,
+                abort = true,
+            }, Json, CancellationToken.None);
+        }
+        catch { /* parts expire on their own */ }
+        return false;
+    }
+
     public Task<string?> UploadLogAsync(string jobId, string machineId, string logFilePath, CancellationToken ct) =>
         UploadArtifactAsync(jobId, machineId, "log", logFilePath, "text/plain", ct);
 

RenderWorker/VFXReviewWorker/Dtos.cs

@@ -304,6 +304,11 @@ public sealed class PresignResponse
     public string PresignedUrl { get; set; } = "";
     public string Key { get; set; } = "";
     public string Url { get; set; } = "";
+    // Multipart (server ≥ 1.7.2 contract) — absent from older servers, which
+    // only ever return PresignedUrl.
+    public string? UploadId { get; set; }
+    public long PartSize { get; set; }
+    public List<string>? PartUrls { get; set; }
 }
 
 public sealed class ExportDetailResponse

RenderWorker/VFXReviewWorker/VFXReviewWorker.csproj

@@ -15,7 +15,7 @@
       every change that gets published. publish.ps1 appends the git SHA via
       SourceRevisionId, giving "1.1.0+feb1c7b".
     -->
-    <Version>1.7.1</Version>
+    <Version>1.7.2</Version>
   </PropertyGroup>
 
   <ItemGroup>

RenderWorker/VFXReviewWorker/WorkerOptions.cs

@@ -35,6 +35,12 @@ public sealed class WorkerOptions
     /// </summary>
     public List<string> PlateSearchRoots { get; set; } = new();
     /// <summary>
+    /// Parallel connections per large upload. From the studio each connection
+    /// to object storage is capped near 0.1 MB/s, so throughput scales with
+    /// this (measured: 1 → 0.13 MB/s, 8 → 0.64, 24 → 1.69).
+    /// </summary>
+    public int UploadConcurrency { get; set; } = 16;
+    /// <summary>
     /// Read from the assembly, not from config.json: a hand-maintained version
     /// string is exactly the thing that goes stale, and a machine reporting a
     /// version it is not running is worse than no version at all.