feat(worker): upload status in the job log, ProRes uploaded in the background
Deploy / deploy (push) Successful in 3m25s
Deploy / deploy (push) Successful in 3m25s
Preview jobs looked stuck after aerender finished: the worker was silently
uploading the review MP4s and the ProRes MOV over a slow uplink, and a
restart mid-upload threw the preview away.
- Worker 1.7.1 replaces the job's log tail with live upload status and logs
each upload's size, duration and speed.
- The ProRes MOV now uploads after finalize via a persistent queue
(HighResUploader, resumes after restart, 5 attempts) and attaches through
the new E13b endpoint POST /api/ext/render/jobs/{id}/attach-highres.
A MOV from a superseded preview is discarded.
Deploy the server before the 1.7.1 worker.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -305,6 +305,8 @@ export interface FinalizeInput {
|
||||
highResKey?: string | null;
|
||||
highResFileName?: string | null;
|
||||
highResSize?: number | null;
|
||||
/** Worker ≥ 1.7.1 uploads the MOV after finalize and attaches it via E13b. */
|
||||
highResPending?: boolean;
|
||||
};
|
||||
renderStats?: { renderSeconds?: number; previewSeconds?: number };
|
||||
media?: {
|
||||
@@ -626,12 +628,113 @@ async function finalizePreviewTx(jobId: string, machineId: string, body: Finaliz
|
||||
function highResNote(artifacts: NonNullable<FinalizeInput["artifacts"]>): string {
|
||||
const hq = !!artifacts.hqPreviewKey;
|
||||
const mov = !!artifacts.highResKey;
|
||||
if (artifacts.highResPending) {
|
||||
return hq
|
||||
? " — High quality MP4 attached; ProRes uploading in the background"
|
||||
: " — ProRes uploading in the background (High quality encode failed or skipped)";
|
||||
}
|
||||
if (hq && mov) return " — High quality MP4 and ProRes attached";
|
||||
if (hq) return " — High quality MP4 attached (ProRes upload failed or skipped)";
|
||||
if (mov) return " — ProRes attached (High quality encode failed or skipped)";
|
||||
return " — no High quality/ProRes (disabled, worker < 1.7.0, or both failed — see the worker log)";
|
||||
}
|
||||
|
||||
export interface AttachHighResInput {
|
||||
highResKey?: string | null;
|
||||
highResFileName?: string | null;
|
||||
highResSize?: number | null;
|
||||
/** Set when the worker gave up on the upload; recorded on the timeline. */
|
||||
error?: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* E13b — the ProRes MOV for an already-finalized preview job (worker ≥ 1.7.1
|
||||
* uploads it in the background so a slow uplink no longer holds the job
|
||||
* open). Attaches it to the export's version as its High Res download.
|
||||
*
|
||||
* The upload can land minutes later, so it only attaches while this job is
|
||||
* still the export's newest preview: a regenerated preview has newer media
|
||||
* and its own MOV on the way, and the late file is deleted instead.
|
||||
*/
|
||||
export async function attachHighRes(jobId: string, machineId: string, body: AttachHighResInput) {
|
||||
const outcome = await db.$transaction(async (tx) => {
|
||||
const job = await tx.renderJob.findUnique({
|
||||
where: { id: jobId },
|
||||
include: { export: true },
|
||||
});
|
||||
if (!job) throw new PipelineError(404, "Render job not found");
|
||||
if (job.machineId !== machineId) {
|
||||
throw new PipelineError(409, "Job is not held by this machine (lease reassigned?)");
|
||||
}
|
||||
if (job.status !== "COMPLETED") {
|
||||
throw new PipelineError(409, `Job is ${job.status} — ProRes can only attach to a finalized preview`);
|
||||
}
|
||||
const exportRow = job.export;
|
||||
const key = body.highResKey ?? null;
|
||||
const discard = key ? [key] : [];
|
||||
|
||||
const note = async (text: string) => {
|
||||
if (!exportRow) return;
|
||||
await tx.exportEvent.create({
|
||||
data: {
|
||||
exportId: exportRow.id,
|
||||
fromStatus: exportRow.status,
|
||||
toStatus: exportRow.status,
|
||||
actorType: "WORKER",
|
||||
actorId: machineId,
|
||||
note: text,
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
if (!exportRow?.versionId) {
|
||||
return { attached: false, reason: "no version", deleteKeys: discard };
|
||||
}
|
||||
|
||||
const newer = await tx.renderJob.findFirst({
|
||||
where: { exportId: exportRow.id, type: "PREVIEW_ONLY", createdAt: { gt: job.createdAt } },
|
||||
select: { id: true },
|
||||
});
|
||||
if (newer) {
|
||||
if (key) await note("ProRes from an earlier preview arrived after a newer preview — discarded");
|
||||
return { attached: false, reason: "superseded", deleteKeys: discard };
|
||||
}
|
||||
|
||||
if (!key) {
|
||||
await note(`ProRes background upload failed — ${body.error ?? "no file"} (Standard and High quality unaffected)`);
|
||||
return { attached: false, reason: "upload failed", deleteKeys: [] as string[] };
|
||||
}
|
||||
|
||||
const version = await tx.version.findUnique({
|
||||
where: { id: exportRow.versionId },
|
||||
select: { id: true, highResKey: true },
|
||||
});
|
||||
if (!version) return { attached: false, reason: "version gone", deleteKeys: discard };
|
||||
if (version.highResKey === key) return { attached: true, alreadyApplied: true, deleteKeys: [] as string[] };
|
||||
|
||||
await tx.version.update({
|
||||
where: { id: version.id },
|
||||
data: {
|
||||
highResKey: key,
|
||||
highResFilename: body.highResFileName ?? null,
|
||||
highResSize: typeof body.highResSize === "number" ? BigInt(Math.round(body.highResSize)) : null,
|
||||
highResPrunedAt: null,
|
||||
},
|
||||
});
|
||||
const mb = typeof body.highResSize === "number" ? ` (${Math.round(body.highResSize / 1048576)} MB)` : "";
|
||||
await note(`ProRes attached${mb}`);
|
||||
return {
|
||||
attached: true,
|
||||
deleteKeys: version.highResKey ? [version.highResKey] : ([] as string[]),
|
||||
};
|
||||
});
|
||||
|
||||
// Storage is only touched once the transaction has committed.
|
||||
const { deleteKeys, ...result } = outcome;
|
||||
await deleteStorageKeys(deleteKeys);
|
||||
return { ok: true, ...result };
|
||||
}
|
||||
|
||||
/** T11 — retry the preview stage only; validated EXRs are left alone. */
|
||||
export async function retryPreview(exportId: string, actorId?: string) {
|
||||
const maxAttempts = await getConfigNumber("render.maxAttempts");
|
||||
|
||||
Reference in New Issue
Block a user