vfxreview/lib/render-pipeline/jobs.ts

import { Prisma, RenderJobType } from "@prisma/client";
import { db } from "@/lib/db";
import { PipelineError } from "./errors";
import { applyTransition } from "./transitions";
import { getConfigBoolean, getConfigNumber } from "./config";
import { machineCanClaim, machineSupportedTypes } from "./machines";
import { reapExpiredLeases } from "./reaper";
import { buildPreviewManifest, createPreviewJob, inheritPreviewMedia } from "./preview";

const JOB_TYPES: RenderJobType[] = ["AE_RENDER", "PREVIEW_ONLY", "DELIVERY_BUILD", "BLENDER_RENDER"];

/**
 * E8 — atomically claim the next queued job (§6.4). `FOR UPDATE SKIP LOCKED`
 * makes double-claims impossible under concurrent workers. All scheduling
 * policy (availability windows, Render Now, urgent priority) is enforced
 * here server-side — workers poll dumbly (§7.10).
 */
export async function claimNextJob(machineId: string, types?: string[]) {
  const machine = await db.machine.findUnique({ where: { id: machineId } });
  if (!machine) throw new PipelineError(404, "Machine not found — register first (POST /api/ext/workers/register)");

  // Defensive sweep on each claim (studio decision 18.1-Q8)
  await reapExpiredLeases().catch((err) => console.error("[render-pipeline] claim-time reap failed", err));

  const requested = (types?.length ? types : ["AE_RENDER", "PREVIEW_ONLY", "DELIVERY_BUILD"]).filter(
    (t): t is RenderJobType => (JOB_TYPES as string[]).includes(t)
  );
  // Enforced here rather than trusted to the worker: a machine without the tool
  // installed must not be able to claim work it can only fail.
  const claimTypes = machineSupportedTypes(machine, requested) as RenderJobType[];
  if (requested.length === 0) throw new PipelineError(422, "No valid job types requested");
  if (claimTypes.length === 0) return null;

  const gate = machineCanClaim(machine, new Date());
  if (gate === "none") return null;

  const [leaseSeconds, urgentThreshold] = await Promise.all([
    getConfigNumber("render.leaseSeconds"),
    getConfigNumber("render.urgentPriorityThreshold"),
  ]);

  const urgentFilter =
    gate === "urgent-only" ? Prisma.sql`AND priority <= ${urgentThreshold}` : Prisma.empty;

  const claimed = await db.$queryRaw<{ id: string }[]>(Prisma.sql`
    UPDATE "render_jobs"
    SET status = 'CLAIMED',
        "machineId" = ${machineId},
        "claimedAt" = now(),
        "leaseExpiresAt" = now() + (${leaseSeconds}::int * interval '1 second'),
        "updatedAt" = now()
    WHERE id = (
      SELECT id FROM "render_jobs"
      WHERE status = 'QUEUED'
        AND type::text = ANY(${claimTypes}::text[])
        ${urgentFilter}
      ORDER BY priority ASC, "createdAt" ASC
      FOR UPDATE SKIP LOCKED
      LIMIT 1
    )
    RETURNING id
  `);
  if (claimed.length === 0) return null;

  const job = await db.renderJob.findUniqueOrThrow({
    where: { id: claimed[0].id },
    include: { export: { select: { id: true, status: true, versionString: true } } },
  });

  // T2: QUEUED → RENDERING
  if (job.export && job.export.status === "QUEUED") {
    await db.$transaction(async (tx) => {
      await applyTransition(tx, job.export!, "RENDERING", {
        type: "WORKER",
        id: machineId,
        note: `Claimed by ${machine.name} (attempt ${job.attempt})`,
      });
    });
  }

  // Delivery packages have their own small lifecycle on the same queue.
  if (job.deliveryId) {
    await db.deliveryPackage.updateMany({
      where: { id: job.deliveryId, status: "QUEUED" },
      data: { status: "BUILDING" },
    });
  }

  return {
    job: {
      id: job.id,
      type: job.type,
      attempt: job.attempt,
      maxAttempts: job.maxAttempts,
      priority: job.priority,
      exportId: job.exportId,
      deliveryId: job.deliveryId,
      label: job.label,
      leaseExpiresAt: job.leaseExpiresAt,
      manifest: job.manifest,
    },
  };
}

async function getJobForReport(jobId: string, machineId: string) {
  const job = await db.renderJob.findUnique({
    where: { id: jobId },
    include: { export: { select: { id: true, status: true } } },
  });
  if (!job) throw new PipelineError(404, "Render job not found");
  // Zombie-worker guard (§7.4): reports from a machine that doesn't hold the lease are rejected
  if (job.machineId !== machineId) {
    throw new PipelineError(409, "Job is not held by this machine (lease reassigned?)");
  }
  return job;
}

/** E9 — progress report; renews the lease. */
export async function reportProgress(
  jobId: string,
  machineId: string,
  body: { progress?: number; currentFrame?: number; totalFrames?: number; etaSeconds?: number; logTail?: string }
) {
  const job = await getJobForReport(jobId, machineId);

  if (job.status === "CANCELLED") {
    return { ok: true, cancelRequested: true, leaseExpiresAt: job.leaseExpiresAt };
  }
  if (["COMPLETED", "FAILED", "EXPIRED"].includes(job.status)) {
    return { ok: true, alreadyApplied: true, cancelRequested: false, leaseExpiresAt: job.leaseExpiresAt };
  }

  const leaseSeconds = await getConfigNumber("render.leaseSeconds");
  const leaseExpiresAt = new Date(Date.now() + leaseSeconds * 1000);
  await db.renderJob.update({
    where: { id: job.id },
    data: {
      status: "RUNNING",
      startedAt: job.startedAt ?? new Date(),
      leaseExpiresAt,
      ...(body.progress != null ? { progress: Math.min(1, Math.max(0, body.progress)) } : {}),
      ...(body.currentFrame != null ? { currentFrame: body.currentFrame } : {}),
      ...(body.totalFrames != null ? { totalFrames: body.totalFrames } : {}),
      ...(body.etaSeconds != null ? { etaSeconds: body.etaSeconds } : {}),
      ...(body.logTail != null ? { logTail: body.logTail.slice(-20000) } : {}),
    },
  });
  return { ok: true, leaseExpiresAt, cancelRequested: false };
}

/** E10 — failure report. Auto-requeues while attempts remain and the error is retryable. */
export async function reportFail(
  jobId: string,
  machineId: string,
  body: {
    stage?: string;
    exitCode?: number;
    errorMessage?: string;
    logTail?: string;
    logFileKey?: string;
    retryable?: boolean;
  }
) {
  return db.$transaction(async (tx) => {
    const job = await tx.renderJob.findUnique({
      where: { id: jobId },
      include: { export: { select: { id: true, status: true } } },
    });
    if (!job) throw new PipelineError(404, "Render job not found");
    if (job.machineId !== machineId) throw new PipelineError(409, "Job is not held by this machine (lease reassigned?)");

    if (job.status === "FAILED") {
      return { job: { id: job.id, status: job.status }, export: job.export ? { id: job.export.id, status: job.export.status } : null, autoRequeued: false, alreadyApplied: true };
    }
    if (job.status === "CANCELLED") {
      // worker acknowledging a cancel — record the wind-down, stay CANCELLED
      await tx.renderJob.update({ where: { id: job.id }, data: { finishedAt: job.finishedAt ?? new Date(), logTail: body.logTail?.slice(-20000) ?? job.logTail } });
      return { job: { id: job.id, status: "CANCELLED" }, export: job.export ? { id: job.export.id, status: job.export.status } : null, autoRequeued: false, alreadyApplied: true };
    }
    if (job.status === "COMPLETED") {
      throw new PipelineError(409, "Job already reported complete — contradictory fail report rejected");
    }

    await tx.renderJob.update({
      where: { id: job.id },
      data: {
        status: "FAILED",
        finishedAt: new Date(),
        exitCode: body.exitCode ?? null,
        errorMessage: body.errorMessage?.slice(0, 2000) ?? null,
        logTail: body.logTail?.slice(-20000) ?? job.logTail,
        logFileKey: body.logFileKey ?? job.logFileKey,
      },
    });

    if (!job.export) {
      // A standalone job (a Blender render) has no Export to move, but a
      // transient failure should still be retried like any other work.
      // Delivery builds are deliberately excluded: a half-written package
      // folder wants a human to look at it before it is rebuilt.
      const retryableStandalone =
        !job.deliveryId && body.retryable !== false && job.attempt < job.maxAttempts;
      if (retryableStandalone) {
        const next = await tx.renderJob.create({
          data: {
            type: job.type,
            projectId: job.projectId,
            label: job.label,
            submittedBy: job.submittedBy,
            deliveryId: job.deliveryId,
            attempt: job.attempt + 1,
            maxAttempts: job.maxAttempts,
            priority: job.priority,
            manifest: job.manifest as Prisma.InputJsonValue,
          },
        });
        return {
          job: { id: job.id, status: "FAILED" },
          export: null,
          autoRequeued: true,
          nextJob: { id: next.id, attempt: next.attempt },
        };
      }
      return { job: { id: job.id, status: "FAILED" }, export: null, autoRequeued: false };
    }

    const retryable = body.retryable !== false;
    const autoRequeue = retryable && job.attempt < job.maxAttempts;
    const isPreview = job.type === "PREVIEW_ONLY";

    if (autoRequeue) {
      const next = await tx.renderJob.create({
        data: {
          type: job.type,
          exportId: job.exportId,
          attempt: job.attempt + 1,
          maxAttempts: job.maxAttempts,
          priority: job.priority,
          manifest: job.manifest as Prisma.InputJsonValue,
        },
      });
      const note = `${body.stage ?? "RENDER"} failed (${body.errorMessage ?? "no message"}) — auto-requeued as attempt ${next.attempt}`;

      if (isPreview) {
        // The export stays in GENERATING_PREVIEW across preview attempts —
        // the EXRs are untouched, only the preview build is repeated.
        await tx.exportEvent.create({
          data: {
            exportId: job.export.id,
            fromStatus: job.export.status,
            toStatus: job.export.status,
            actorType: "SYSTEM",
            actorId: machineId,
            note,
          },
        });
        return { job: { id: job.id, status: "FAILED" }, export: { id: job.export.id, status: job.export.status }, autoRequeued: true, nextJob: { id: next.id, attempt: next.attempt } };
      }

      const exp = await applyTransition(tx, job.export, "QUEUED", { type: "SYSTEM", id: machineId, note });
      return { job: { id: job.id, status: "FAILED" }, export: { id: exp.id, status: exp.status }, autoRequeued: true, nextJob: { id: next.id, attempt: next.attempt } };
    }

    const exp = await applyTransition(tx, job.export, isPreview ? "PREVIEW_FAILED" : "RENDER_FAILED", {
      type: "WORKER",
      id: machineId,
      note: `${body.stage ?? "RENDER"} failed: ${body.errorMessage ?? "no message"}${retryable ? " (max attempts reached)" : " (not retryable)"}`,
    });
    return { job: { id: job.id, status: "FAILED" }, export: { id: exp.id, status: exp.status }, autoRequeued: false };
  });
}

/**
 * E11 — aerender finished OK.
 *
 * Hands off to validation (§8) when it is enabled: the worker then runs the
 * checks and reports them via E12, which queues the preview stage. With
 * validation disabled the old direct handoff to the preview stage is used, so
 * the kill switch restores the pre-Phase-3 flow exactly.
 */
export async function completeRender(
  jobId: string,
  machineId: string,
  body: { renderSeconds?: number; logFileKey?: string; logTail?: string; exrFileCount?: number; exrTotalBytes?: number }
) {
  return db.$transaction(async (tx) => {
    const job = await tx.renderJob.findUnique({
      where: { id: jobId },
      include: { export: { select: { id: true, status: true } } },
    });
    if (!job) throw new PipelineError(404, "Render job not found");
    if (job.machineId !== machineId) throw new PipelineError(409, "Job is not held by this machine (lease reassigned?)");

    if (job.status === "COMPLETED") {
      return { job: { id: job.id, status: job.status }, export: job.export ? { id: job.export.id, status: job.export.status } : null, alreadyApplied: true };
    }
    if (["FAILED", "CANCELLED", "EXPIRED"].includes(job.status)) {
      throw new PipelineError(409, `Job is ${job.status} — contradictory complete report rejected`);
    }

    // With validation to follow, the job deliberately stays open and keeps its
    // lease: a worker that dies mid-validation is then reaped and retried like
    // any other stalled job, instead of leaving the Export stuck in VALIDATING.
    const validationEnabled = job.export ? await getConfigBoolean("validation.enabled") : false;

    await tx.renderJob.update({
      where: { id: job.id },
      data: {
        status: validationEnabled ? "RUNNING" : "COMPLETED",
        progress: 1,
        ...(validationEnabled ? {} : { finishedAt: new Date() }),
        exitCode: 0,
        logFileKey: body.logFileKey ?? job.logFileKey,
        logTail: body.logTail?.slice(-20000) ?? job.logTail,
      },
    });

    let exportResult: { id: string; status: string } | null = null;
    let previewJob: { id: string; type: string } | null = null;

    if (job.export) {
      const exportRow = await tx.export.findUniqueOrThrow({ where: { id: job.export.id } });
      const stats = {
        ...(body.exrFileCount != null ? { exrFileCount: body.exrFileCount } : {}),
        ...(body.exrTotalBytes != null ? { exrTotalBytes: BigInt(Math.round(body.exrTotalBytes)) } : {}),
      };

      // The EXR render is only the first stage. Validation comes next when
      // enabled; the worker reports its results via E12 on the same job.
      if (validationEnabled) {
        const exp = await applyTransition(tx, job.export, "VALIDATING", {
          type: "WORKER",
          id: machineId,
          note: `Render complete in ${body.renderSeconds ?? "?"}s — validating`,
        }, stats);
        return {
          job: { id: job.id, status: "RUNNING" },
          export: { id: exp.id, status: exp.status },
          previewJob: null,
          validationRequired: true,
          alreadyApplied: false,
        };
      }

      // Validation disabled: hand off to the preview stage (headless AE rebuild
      // → delivery MOV + review MP4) exactly as before Phase 3.
      const previewEnabled = await getConfigBoolean("preview.enabled");
      if (previewEnabled && !exportRow.skipPreview) {
        const manifest = await buildPreviewManifest(exportRow);
        const created = await createPreviewJob(tx, exportRow, manifest, job.maxAttempts, job.priority);
        previewJob = { id: created.id, type: created.type };
        const exp = await applyTransition(tx, job.export, "GENERATING_PREVIEW", {
          type: "WORKER",
          id: machineId,
          note: `Render complete in ${body.renderSeconds ?? "?"}s — queued preview build`,
        }, stats);
        exportResult = { id: exp.id, status: exp.status };
      } else {
        const inherited = exportRow.skipPreview
          ? await inheritPreviewMedia(tx, exportRow)
          : { data: {}, note: "preview stage disabled" };
        const exp = await applyTransition(tx, job.export, "READY_FOR_QC", {
          type: "WORKER",
          id: machineId,
          note: `Render complete in ${body.renderSeconds ?? "?"}s — ${inherited.note}`,
        }, { ...stats, ...inherited.data });
        exportResult = { id: exp.id, status: exp.status };
      }
    }
    return { job: { id: job.id, status: "COMPLETED" }, export: exportResult, previewJob, alreadyApplied: false };
  });
}

/** E15 — cancel a single job from the web UI (delegates to export-level cancel when linked). */
export async function getJobDetail(jobId: string) {
  const job = await db.renderJob.findUnique({
    where: { id: jobId },
    include: {
      machine: { select: { id: true, name: true } },
      export: { include: { shot: { select: { shotCode: true } } } },
    },
  });
  if (!job) throw new PipelineError(404, "Render job not found");
  return job;
}