feat(worker): upload status in the job log, ProRes uploaded in the background
Deploy / deploy (push) Successful in 3m25s

Preview jobs looked stuck after aerender finished: the worker was silently
uploading the review MP4s and the ProRes MOV over a slow uplink, and a
restart mid-upload threw the preview away.

- Worker 1.7.1 replaces the job's log tail with live upload status and logs
  each upload's size, duration and speed.
- The ProRes MOV now uploads after finalize via a persistent queue
  (HighResUploader, resumes after restart, 5 attempts) and attaches through
  the new E13b endpoint POST /api/ext/render/jobs/{id}/attach-highres.
  A MOV from a superseded preview is discarded.

Deploy the server before the 1.7.1 worker.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
twotalesanimation
2026-10-01 18:13:23 +02:00
parent f30acef101
commit 477aef7b27
11 changed files with 442 additions and 27 deletions
+103
View File
@@ -305,6 +305,8 @@ export interface FinalizeInput {
highResKey?: string | null;
highResFileName?: string | null;
highResSize?: number | null;
/** Worker ≥ 1.7.1 uploads the MOV after finalize and attaches it via E13b. */
highResPending?: boolean;
};
renderStats?: { renderSeconds?: number; previewSeconds?: number };
media?: {
@@ -626,12 +628,113 @@ async function finalizePreviewTx(jobId: string, machineId: string, body: Finaliz
function highResNote(artifacts: NonNullable<FinalizeInput["artifacts"]>): string {
const hq = !!artifacts.hqPreviewKey;
const mov = !!artifacts.highResKey;
if (artifacts.highResPending) {
return hq
? " — High quality MP4 attached; ProRes uploading in the background"
: " — ProRes uploading in the background (High quality encode failed or skipped)";
}
if (hq && mov) return " — High quality MP4 and ProRes attached";
if (hq) return " — High quality MP4 attached (ProRes upload failed or skipped)";
if (mov) return " — ProRes attached (High quality encode failed or skipped)";
return " — no High quality/ProRes (disabled, worker < 1.7.0, or both failed — see the worker log)";
}
export interface AttachHighResInput {
highResKey?: string | null;
highResFileName?: string | null;
highResSize?: number | null;
/** Set when the worker gave up on the upload; recorded on the timeline. */
error?: string | null;
}
/**
* E13b — the ProRes MOV for an already-finalized preview job (worker ≥ 1.7.1
* uploads it in the background so a slow uplink no longer holds the job
* open). Attaches it to the export's version as its High Res download.
*
* The upload can land minutes later, so it only attaches while this job is
* still the export's newest preview: a regenerated preview has newer media
* and its own MOV on the way, and the late file is deleted instead.
*/
export async function attachHighRes(jobId: string, machineId: string, body: AttachHighResInput) {
const outcome = await db.$transaction(async (tx) => {
const job = await tx.renderJob.findUnique({
where: { id: jobId },
include: { export: true },
});
if (!job) throw new PipelineError(404, "Render job not found");
if (job.machineId !== machineId) {
throw new PipelineError(409, "Job is not held by this machine (lease reassigned?)");
}
if (job.status !== "COMPLETED") {
throw new PipelineError(409, `Job is ${job.status} — ProRes can only attach to a finalized preview`);
}
const exportRow = job.export;
const key = body.highResKey ?? null;
const discard = key ? [key] : [];
const note = async (text: string) => {
if (!exportRow) return;
await tx.exportEvent.create({
data: {
exportId: exportRow.id,
fromStatus: exportRow.status,
toStatus: exportRow.status,
actorType: "WORKER",
actorId: machineId,
note: text,
},
});
};
if (!exportRow?.versionId) {
return { attached: false, reason: "no version", deleteKeys: discard };
}
const newer = await tx.renderJob.findFirst({
where: { exportId: exportRow.id, type: "PREVIEW_ONLY", createdAt: { gt: job.createdAt } },
select: { id: true },
});
if (newer) {
if (key) await note("ProRes from an earlier preview arrived after a newer preview — discarded");
return { attached: false, reason: "superseded", deleteKeys: discard };
}
if (!key) {
await note(`ProRes background upload failed — ${body.error ?? "no file"} (Standard and High quality unaffected)`);
return { attached: false, reason: "upload failed", deleteKeys: [] as string[] };
}
const version = await tx.version.findUnique({
where: { id: exportRow.versionId },
select: { id: true, highResKey: true },
});
if (!version) return { attached: false, reason: "version gone", deleteKeys: discard };
if (version.highResKey === key) return { attached: true, alreadyApplied: true, deleteKeys: [] as string[] };
await tx.version.update({
where: { id: version.id },
data: {
highResKey: key,
highResFilename: body.highResFileName ?? null,
highResSize: typeof body.highResSize === "number" ? BigInt(Math.round(body.highResSize)) : null,
highResPrunedAt: null,
},
});
const mb = typeof body.highResSize === "number" ? ` (${Math.round(body.highResSize / 1048576)} MB)` : "";
await note(`ProRes attached${mb}`);
return {
attached: true,
deleteKeys: version.highResKey ? [version.highResKey] : ([] as string[]),
};
});
// Storage is only touched once the transaction has committed.
const { deleteKeys, ...result } = outcome;
await deleteStorageKeys(deleteKeys);
return { ok: true, ...result };
}
/** T11 — retry the preview stage only; validated EXRs are left alone. */
export async function retryPreview(exportId: string, actorId?: string) {
const maxAttempts = await getConfigNumber("render.maxAttempts");