feat(worker): parallel multipart uploads for large artifacts
Deploy / deploy (push) Successful in 3m18s

From the studio a single connection to Hetzner nbg1 (188 ms away) is
capped near 0.1 MB/s, although a browser speed test shows 50 Mbps.
Measured: 1 connection 0.13 MB/s, 8 -> 0.64, 24 -> 1.69 MB/s. At the old
speed a ProRes MOV could not finish inside the 30 min request timeout.

- Worker 1.7.2 uploads files >= 16 MB as S3 multipart: 8 MB parts on
  uploadConcurrency connections (config.json, default 16), each part
  retried up to 4 times; aborts the upload if a part keeps failing.
- artifact-presign returns uploadId + per-part presigned URLs when asked
  for multipart; new E20b artifact-complete stitches or aborts.
- Old workers and old servers fall back to the single PUT.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
twotalesanimation
2026-10-01 18:48:28 +02:00
parent 477aef7b27
commit 91e69f0bcb
11 changed files with 530 additions and 21 deletions
@@ -0,0 +1,62 @@
import { NextRequest, NextResponse } from "next/server";
import { isExtAuthorized } from "@/lib/render-pipeline/ext-auth";
import { toErrorResponse, PipelineError } from "@/lib/render-pipeline/errors";
import { isArtifactKey } from "@/lib/render-pipeline/artifact-kinds";
import { db } from "@/lib/db";
import { abortHetznerMultipartUpload, completeHetznerMultipartUpload } from "@/lib/storage";
// ── POST /api/ext/render/jobs/{jobId}/artifact-complete (E20b) ───────────────
//
// Closes a multipart upload started by artifact-presign: stitches the parts
// (by part number + ETag) into the final object, or with `abort: true` throws
// the stored parts away after the worker gave up.
export async function POST(
req: NextRequest,
{ params }: { params: Promise<{ jobId: string }> }
) {
if (!isExtAuthorized(req)) {
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
}
const { jobId } = await params;
let body: {
machineId?: string;
key?: string;
uploadId?: string;
parts?: { partNumber?: number; etag?: string }[];
abort?: boolean;
};
try {
body = await req.json();
} catch {
return NextResponse.json({ error: "Invalid JSON" }, { status: 400 });
}
try {
if (!body.machineId) throw new PipelineError(400, "machineId is required");
if (!body.key || !isArtifactKey(body.key)) throw new PipelineError(422, "key is not a worker artifact key");
if (!body.uploadId) throw new PipelineError(400, "uploadId is required");
const job = await db.renderJob.findUnique({ where: { id: jobId }, select: { machineId: true } });
if (!job) throw new PipelineError(404, "Render job not found");
if (job.machineId !== body.machineId) {
throw new PipelineError(409, "Job is not held by this machine (lease reassigned?)");
}
if (body.abort) {
await abortHetznerMultipartUpload(body.key, body.uploadId);
return NextResponse.json({ ok: true, aborted: true });
}
const parts = (body.parts ?? []).filter(
(p): p is { partNumber: number; etag: string } =>
typeof p.partNumber === "number" && typeof p.etag === "string" && p.etag.length > 0
);
if (parts.length === 0) throw new PipelineError(422, "parts are required");
await completeHetznerMultipartUpload(body.key, body.uploadId, parts);
return NextResponse.json({ ok: true, key: body.key, url: `/api/files/${body.key}` });
} catch (err) {
const { body: errBody, status } = toErrorResponse(err);
return NextResponse.json(errBody, { status });
}
}
@@ -3,7 +3,12 @@ import { randomUUID } from "crypto";
import { isExtAuthorized } from "@/lib/render-pipeline/ext-auth";
import { toErrorResponse, PipelineError } from "@/lib/render-pipeline/errors";
import { db } from "@/lib/db";
import { generateHetznerPresignedUploadUrl, sanitizeFileName } from "@/lib/storage";
import { ARTIFACT_KIND_FOLDERS } from "@/lib/render-pipeline/artifact-kinds";
import {
createHetznerMultipartUpload,
generateHetznerPresignedUploadUrl,
sanitizeFileName,
} from "@/lib/storage";
// ── POST /api/ext/render/jobs/{jobId}/artifact-presign (E20) ─────────────────
//
@@ -11,14 +16,16 @@ import { generateHetznerPresignedUploadUrl, sanitizeFileName } from "@/lib/stora
// storage folder layout: preview → videos/, thumbnail → image/,
// metadata/log → renders/, highres (the delivery MOV) → highres/ beside the
// manually uploaded High Res files.
//
// `multipart: true` + `size` (worker ≥ 1.7.2) returns an S3 multipart upload
// instead — an uploadId and one presigned URL per part — which the worker
// fills in parallel and closes with artifact-complete (E20b). A single PUT
// from the studio is capped near 0.1 MB/s; parallel parts scale with the
// number of connections. Older workers never ask and get the single PUT.
const KIND_FOLDERS: Record<string, string> = {
preview: "videos",
thumbnail: "image",
metadata: "renders",
log: "renders",
highres: "highres",
};
const KIND_FOLDERS = ARTIFACT_KIND_FOLDERS;
const MIN_PART = 8 * 1024 * 1024;
const MAX_PARTS = 10000; // S3 limit
export async function POST(
req: NextRequest,
@@ -28,7 +35,14 @@ export async function POST(
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
}
const { jobId } = await params;
let body: { machineId?: string; kind?: string; fileName?: string; contentType?: string };
let body: {
machineId?: string;
kind?: string;
fileName?: string;
contentType?: string;
multipart?: boolean;
size?: number;
};
try {
body = await req.json();
} catch {
@@ -50,6 +64,14 @@ export async function POST(
}
const key = `${folder}/${randomUUID()}-${sanitizeFileName(body.fileName)}`;
if (body.multipart && typeof body.size === "number" && body.size > MIN_PART) {
const partSize = Math.max(MIN_PART, Math.ceil(body.size / MAX_PARTS));
const partCount = Math.ceil(body.size / partSize);
const { uploadId, partUrls } = await createHetznerMultipartUpload(key, body.contentType, partCount);
return NextResponse.json({ key, url: `/api/files/${key}`, uploadId, partSize, partUrls });
}
const presignedUrl = await generateHetznerPresignedUploadUrl(key, body.contentType);
return NextResponse.json({ presignedUrl, key, url: `/api/files/${key}` });
} catch (err) {