vfxreview/lib/dashboard/ops.ts

import { db } from "@/lib/db";
import { getConfigNumber } from "@/lib/render-pipeline/config";
import { listMachines } from "@/lib/render-pipeline/machines";
import type { ExportStatus, Prisma } from "@prisma/client";

/**
 * Data for the operations dashboard.
 *
 * Everything here is derived from state the pipeline maintains on its own —
 * export transitions, validation results, QC reviews, worker heartbeats — and
 * never from a field a human has to remember to fill in. The dashboard this
 * replaced counted `Task.dueDate`, which nothing in the pipeline writes, so
 * its headline numbers were structurally always zero.
 */

/** Terminal-until-someone-acts: these never move on their own. */
const FAILED_STATUSES: ExportStatus[] = [
  "RENDER_FAILED",
  "VALIDATION_FAILED",
  "PREVIEW_FAILED",
  "QC_FAILED",
];

/** In flight: fine until they stop progressing, then they are stuck. */
const IN_FLIGHT_STATUSES: ExportStatus[] = [
  "QUEUED",
  "RENDERING",
  "VALIDATING",
  "GENERATING_PREVIEW",
];

export type BlockedSeverity = "critical" | "warning";

export interface BlockedExport {
  exportId: string;
  shotId: string;
  shotCode: string;
  versionString: string;
  projectCode: string;
  episode: string | null;
  status: ExportStatus;
  severity: BlockedSeverity;
  /** When it entered the status it is stuck in. */
  since: Date;
  /** What a human has to do something about, in one line. */
  reason: string;
  artistName: string | null;
}

/**
 * Everything that has stopped moving and needs a person.
 *
 * Two kinds: exports in a failed state (which stay there forever — nothing
 * retries a VALIDATION_FAILED on its own), and exports sitting in a live state
 * far longer than that state should take, which is how a dead worker or a
 * forgotten QC pass shows up. A failed export is currently only visible if
 * someone thinks to open the render queue and filter for it.
 */
export async function getBlockedExports(limit = 25): Promise<BlockedExport[]> {
  const [qcStaleHours, stalledHours] = await Promise.all([
    getConfigNumber("dashboard.qcStaleHours"),
    getConfigNumber("dashboard.stalledJobHours"),
  ]);
  const now = Date.now();
  const qcCutoff = new Date(now - qcStaleHours * 3600_000);
  const stalledCutoff = new Date(now - stalledHours * 3600_000);

  const rows = await db.export.findMany({
    where: {
      // Finished shows would otherwise fill the list with failures nobody
      // intends to fix. Export carries a projectId but no project relation, so
      // the scope is reached through the shot.
      shot: { project: { status: { in: ["ACTIVE", "ON_HOLD"] } } },
      OR: [
        { status: { in: FAILED_STATUSES } },
        { status: "READY_FOR_QC", statusChangedAt: { lt: qcCutoff } },
        { status: { in: IN_FLIGHT_STATUSES }, statusChangedAt: { lt: stalledCutoff } },
      ],
    },
    orderBy: { statusChangedAt: "asc" },
    take: limit,
    select: {
      id: true,
      shotId: true,
      versionString: true,
      status: true,
      statusChangedAt: true,
      shot: {
        select: {
          shotCode: true,
          episode: true,
          artist: { select: { name: true, email: true } },
          project: { select: { code: true } },
        },
      },
      // Only the failures matter for the reason line, and only a couple of
      // them: the point is to say what to look at, not to reproduce the
      // validation report.
      validations: {
        where: { status: "FAIL" },
        orderBy: { createdAt: "desc" },
        take: 3,
        select: { checkName: true, expected: true, actual: true, message: true },
      },
      qcReviews: {
        orderBy: { createdAt: "desc" },
        take: 1,
        select: { result: true, notes: true, reviewerName: true },
      },
      renderJobs: {
        orderBy: { createdAt: "desc" },
        take: 1,
        select: {
          errorMessage: true,
          status: true,
          attempt: true,
          maxAttempts: true,
          machine: { select: { name: true } },
        },
      },
    },
  });

  const blocked = rows.map((row): BlockedExport => {
    const job = row.renderJobs[0];
    const qc = row.qcReviews[0];
    let reason: string;

    switch (row.status) {
      case "VALIDATION_FAILED": {
        const first = row.validations[0];
        const extra = row.validations.length > 1 ? ` (+${row.validations.length - 1} more)` : "";
        reason = first
          ? `${first.checkName} — ${first.message ?? `expected ${first.expected ?? "?"}, got ${first.actual ?? "?"}`}${extra}`
          : "Validation failed";
        break;
      }
      case "QC_FAILED":
        reason = qc?.notes?.trim()
          ? `${qc.notes.trim()}${qc.reviewerName ? ` — ${qc.reviewerName}` : ""}`
          : "Failed technical QC";
        break;
      case "RENDER_FAILED":
      case "PREVIEW_FAILED":
        reason =
          job?.errorMessage?.trim() ||
          (row.status === "RENDER_FAILED" ? "Render failed" : "Preview build failed");
        if (job && job.attempt >= job.maxAttempts) reason += ` (attempt ${job.attempt}/${job.maxAttempts})`;
        break;
      case "READY_FOR_QC":
        reason = "Waiting on a QC pass";
        break;
      default:
        // Still nominally in flight, but past the point where that is normal.
        reason = job?.machine?.name
          ? `No progress since claim on ${job.machine.name}`
          : "Queued with no machine picking it up";
    }

    return {
      exportId: row.id,
      shotId: row.shotId,
      shotCode: row.shot.shotCode,
      versionString: row.versionString,
      projectCode: row.shot.project.code,
      episode: row.shot.episode,
      status: row.status,
      severity: FAILED_STATUSES.includes(row.status) ? "critical" : "warning",
      since: row.statusChangedAt,
      reason,
      artistName: row.shot.artist?.name ?? row.shot.artist?.email ?? null,
    };
  });

  // Failures first, then whatever has been waiting longest. Both orderings
  // matter: severity decides what to fix, age decides what to fix first.
  return blocked.sort((a, b) => {
    if (a.severity !== b.severity) return a.severity === "critical" ? -1 : 1;
    return a.since.getTime() - b.since.getTime();
  });
}

export interface OpsCounts {
  blocked: number;
  inQc: number;
  readyToDeliver: number;
  awaitingClientReview: number;
  activeProjects: number;
}

/**
 * The headline numbers, all scoped to live productions.
 *
 * The counters these replaced were unscoped and cumulative — "approved" across
 * every project the studio has ever finished only ever goes up, so it never
 * told anyone anything. Each of these can go down, which is what makes it
 * worth looking at twice in a day.
 */
export async function getOpsCounts(): Promise<OpsCounts> {
  // Export has a projectId column but no project relation, so live-production
  // scope is reached through the shot.
  const liveProject: Prisma.ExportWhereInput = {
    shot: { project: { status: { in: ["ACTIVE", "ON_HOLD"] } } },
  };
  const [qcStaleHours, stalledHours] = await Promise.all([
    getConfigNumber("dashboard.qcStaleHours"),
    getConfigNumber("dashboard.stalledJobHours"),
  ]);
  const now = Date.now();

  const [blocked, inQc, readyToDeliver, awaitingClientReview, activeProjects] = await Promise.all([
    db.export.count({
      where: {
        ...liveProject,
        OR: [
          { status: { in: FAILED_STATUSES } },
          { status: "READY_FOR_QC", statusChangedAt: { lt: new Date(now - qcStaleHours * 3600_000) } },
          {
            status: { in: IN_FLIGHT_STATUSES },
            statusChangedAt: { lt: new Date(now - stalledHours * 3600_000) },
          },
        ],
      },
    }),
    db.export.count({ where: { ...liveProject, status: "READY_FOR_QC" } }),
    db.export.count({ where: { ...liveProject, status: "READY_FOR_DELIVERY" } }),
    // Sent to the client and still unanswered — the wait that is not ours to
    // end, as opposed to notes we have not acted on yet.
    db.version.count({
      where: {
        approvalStatus: "PENDING_REVIEW",
        isClientVisible: true,
        isLatest: true,
        shot: { project: { status: { in: ["ACTIVE", "ON_HOLD"] } } },
      },
    }),
    db.project.count({ where: { status: "ACTIVE" } }),
  ]);

  return { blocked, inQc, readyToDeliver, awaitingClientReview, activeProjects };
}

export interface FarmStatus {
  machines: Awaited<ReturnType<typeof listMachines>>;
  online: number;
  total: number;
  queued: { render: number; preview: number; delivery: number; blender: number; total: number };
  running: number;
  /** Age of the oldest job still waiting to be claimed, in ms. */
  oldestQueuedMs: number | null;
  /** Highest worker version any machine reports, and who is behind it. */
  latestWorkerVersion: string | null;
  behindMachines: { name: string; workerVersion: string | null }[];
}

/**
 * Farm health in one strip: who is up, what is waiting, and — because a
 * worker's version is what decides whether it has a given fix — which
 * machines are running something older than the newest one out there.
 */
export async function getFarmStatus(): Promise<FarmStatus> {
  const [machines, queuedByType, running, oldest] = await Promise.all([
    listMachines(),
    db.renderJob.groupBy({ by: ["type"], where: { status: "QUEUED" }, _count: { _all: true } }),
    db.renderJob.count({ where: { status: { in: ["CLAIMED", "RUNNING"] } } }),
    db.renderJob.findFirst({
      where: { status: "QUEUED" },
      orderBy: { createdAt: "asc" },
      select: { createdAt: true },
    }),
  ]);

  const countOf = (type: string) =>
    queuedByType.find((q) => q.type === type)?._count._all ?? 0;

  const versions = machines
    .map((m) => m.workerVersion)
    .filter((v): v is string => Boolean(v));
  const latest = versions.length ? versions.reduce((a, b) => (compareVersions(a, b) >= 0 ? a : b)) : null;

  return {
    machines,
    online: machines.filter((m) => m.status === "ONLINE").length,
    total: machines.length,
    queued: {
      render: countOf("AE_RENDER"),
      preview: countOf("PREVIEW_ONLY"),
      delivery: countOf("DELIVERY_BUILD"),
      blender: countOf("BLENDER_RENDER"),
      total: queuedByType.reduce((sum, q) => sum + q._count._all, 0),
    },
    running,
    oldestQueuedMs: oldest ? Date.now() - oldest.createdAt.getTime() : null,
    latestWorkerVersion: latest,
    behindMachines: latest
      ? machines
          .filter((m) => m.workerVersion !== latest)
          .map((m) => ({ name: m.name, workerVersion: m.workerVersion }))
      : [],
  };
}

/**
 * Compares "1.6.1+fe420b9" style versions numerically, ignoring the build
 * stamp publish.ps1 appends. A string compare would rank 1.10.0 below 1.9.0.
 */
export function compareVersions(a: string, b: string): number {
  const parts = (v: string) =>
    v.split("+")[0].split(".").map((n) => Number.parseInt(n, 10) || 0);
  const [pa, pb] = [parts(a), parts(b)];
  for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
    const diff = (pa[i] ?? 0) - (pb[i] ?? 0);
    if (diff !== 0) return diff;
  }
  return 0;
}