feat(worker): parallel multipart uploads for large artifacts
Deploy / deploy (push) Successful in 3m18s
Deploy / deploy (push) Successful in 3m18s
From the studio a single connection to Hetzner nbg1 (188 ms away) is capped near 0.1 MB/s, although a browser speed test shows 50 Mbps. Measured: 1 connection 0.13 MB/s, 8 -> 0.64, 24 -> 1.69 MB/s. At the old speed a ProRes MOV could not finish inside the 30 min request timeout. - Worker 1.7.2 uploads files >= 16 MB as S3 multipart: 8 MB parts on uploadConcurrency connections (config.json, default 16), each part retried up to 4 times; aborts the upload if a part keeps failing. - artifact-presign returns uploadId + per-part presigned URLs when asked for multipart; new E20b artifact-complete stitches or aborts. - Old workers and old servers fall back to the single PUT. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,62 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { isExtAuthorized } from "@/lib/render-pipeline/ext-auth";
|
||||
import { toErrorResponse, PipelineError } from "@/lib/render-pipeline/errors";
|
||||
import { isArtifactKey } from "@/lib/render-pipeline/artifact-kinds";
|
||||
import { db } from "@/lib/db";
|
||||
import { abortHetznerMultipartUpload, completeHetznerMultipartUpload } from "@/lib/storage";
|
||||
|
||||
// ── POST /api/ext/render/jobs/{jobId}/artifact-complete (E20b) ───────────────
|
||||
//
|
||||
// Closes a multipart upload started by artifact-presign: stitches the parts
|
||||
// (by part number + ETag) into the final object, or with `abort: true` throws
|
||||
// the stored parts away after the worker gave up.
|
||||
|
||||
export async function POST(
|
||||
req: NextRequest,
|
||||
{ params }: { params: Promise<{ jobId: string }> }
|
||||
) {
|
||||
if (!isExtAuthorized(req)) {
|
||||
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
||||
}
|
||||
const { jobId } = await params;
|
||||
let body: {
|
||||
machineId?: string;
|
||||
key?: string;
|
||||
uploadId?: string;
|
||||
parts?: { partNumber?: number; etag?: string }[];
|
||||
abort?: boolean;
|
||||
};
|
||||
try {
|
||||
body = await req.json();
|
||||
} catch {
|
||||
return NextResponse.json({ error: "Invalid JSON" }, { status: 400 });
|
||||
}
|
||||
try {
|
||||
if (!body.machineId) throw new PipelineError(400, "machineId is required");
|
||||
if (!body.key || !isArtifactKey(body.key)) throw new PipelineError(422, "key is not a worker artifact key");
|
||||
if (!body.uploadId) throw new PipelineError(400, "uploadId is required");
|
||||
|
||||
const job = await db.renderJob.findUnique({ where: { id: jobId }, select: { machineId: true } });
|
||||
if (!job) throw new PipelineError(404, "Render job not found");
|
||||
if (job.machineId !== body.machineId) {
|
||||
throw new PipelineError(409, "Job is not held by this machine (lease reassigned?)");
|
||||
}
|
||||
|
||||
if (body.abort) {
|
||||
await abortHetznerMultipartUpload(body.key, body.uploadId);
|
||||
return NextResponse.json({ ok: true, aborted: true });
|
||||
}
|
||||
|
||||
const parts = (body.parts ?? []).filter(
|
||||
(p): p is { partNumber: number; etag: string } =>
|
||||
typeof p.partNumber === "number" && typeof p.etag === "string" && p.etag.length > 0
|
||||
);
|
||||
if (parts.length === 0) throw new PipelineError(422, "parts are required");
|
||||
|
||||
await completeHetznerMultipartUpload(body.key, body.uploadId, parts);
|
||||
return NextResponse.json({ ok: true, key: body.key, url: `/api/files/${body.key}` });
|
||||
} catch (err) {
|
||||
const { body: errBody, status } = toErrorResponse(err);
|
||||
return NextResponse.json(errBody, { status });
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user