vfxreview/lib/dashboard/ops.ts

import { db } from "@/lib/db";
import { getConfigNumber } from "@/lib/render-pipeline/config";
import { listMachines } from "@/lib/render-pipeline/machines";
import type { ExportStatus, Prisma } from "@prisma/client";

/**
 * Data for the operations dashboard.
 *
 * Everything here is derived from state the pipeline maintains on its own —
 * export transitions, validation results, QC reviews, worker heartbeats — and
 * never from a field a human has to remember to fill in. The dashboard this
 * replaced counted `Task.dueDate`, which nothing in the pipeline writes, so
 * its headline numbers were structurally always zero.
 */

/** Terminal-until-someone-acts: these never move on their own. */
const FAILED_STATUSES: ExportStatus[] = [
  "RENDER_FAILED",
  "VALIDATION_FAILED",
  "PREVIEW_FAILED",
  "QC_FAILED",
];

/** In flight: fine until they stop progressing, then they are stuck. */
const IN_FLIGHT_STATUSES: ExportStatus[] = [
  "QUEUED",
  "RENDERING",
  "VALIDATING",
  "GENERATING_PREVIEW",
];

export type BlockedSeverity = "critical" | "warning";

export interface BlockedExport {
  exportId: string;
  shotId: string;
  shotCode: string;
  versionString: string;
  projectCode: string;
  episode: string | null;
  status: ExportStatus;
  severity: BlockedSeverity;
  /** When it entered the status it is stuck in. */
  since: Date;
  /** What a human has to do something about, in one line. */
  reason: string;
  artistName: string | null;
}

/**
 * Everything that has stopped moving and needs a person.
 *
 * Two kinds: exports in a failed state (which stay there forever — nothing
 * retries a VALIDATION_FAILED on its own), and exports sitting in a live state
 * far longer than that state should take, which is how a dead worker or a
 * forgotten QC pass shows up. A failed export is currently only visible if
 * someone thinks to open the render queue and filter for it.
 */
export async function getBlockedExports(limit = 25): Promise<BlockedExport[]> {
  const [qcStaleHours, stalledHours] = await Promise.all([
    getConfigNumber("dashboard.qcStaleHours"),
    getConfigNumber("dashboard.stalledJobHours"),
  ]);
  const now = Date.now();
  const qcCutoff = new Date(now - qcStaleHours * 3600_000);
  const stalledCutoff = new Date(now - stalledHours * 3600_000);

  const rows = await db.export.findMany({
    where: {
      // Finished shows would otherwise fill the list with failures nobody
      // intends to fix. Export carries a projectId but no project relation, so
      // the scope is reached through the shot.
      shot: { project: { status: { in: ["ACTIVE", "ON_HOLD"] } } },
      OR: [
        { status: { in: FAILED_STATUSES } },
        { status: "READY_FOR_QC", statusChangedAt: { lt: qcCutoff } },
        { status: { in: IN_FLIGHT_STATUSES }, statusChangedAt: { lt: stalledCutoff } },
      ],
    },
    orderBy: { statusChangedAt: "asc" },
    take: limit,
    select: {
      id: true,
      shotId: true,
      versionString: true,
      status: true,
      statusChangedAt: true,
      shot: {
        select: {
          shotCode: true,
          episode: true,
          artist: { select: { name: true, email: true } },
          project: { select: { code: true } },
        },
      },
      // Only the failures matter for the reason line, and only a couple of
      // them: the point is to say what to look at, not to reproduce the
      // validation report.
      validations: {
        where: { status: "FAIL" },
        orderBy: { createdAt: "desc" },
        take: 3,
        select: { checkName: true, expected: true, actual: true, message: true },
      },
      qcReviews: {
        orderBy: { createdAt: "desc" },
        take: 1,
        select: { result: true, notes: true, reviewerName: true },
      },
      renderJobs: {
        orderBy: { createdAt: "desc" },
        take: 1,
        select: {
          errorMessage: true,
          status: true,
          attempt: true,
          maxAttempts: true,
          machine: { select: { name: true } },
        },
      },
    },
  });

  const blocked = rows.map((row): BlockedExport => {
    const job = row.renderJobs[0];
    const qc = row.qcReviews[0];
    let reason: string;

    switch (row.status) {
      case "VALIDATION_FAILED": {
        const first = row.validations[0];
        const extra = row.validations.length > 1 ? ` (+${row.validations.length - 1} more)` : "";
        reason = first
          ? `${first.checkName} — ${first.message ?? `expected ${first.expected ?? "?"}, got ${first.actual ?? "?"}`}${extra}`
          : "Validation failed";
        break;
      }
      case "QC_FAILED":
        reason = qc?.notes?.trim()
          ? `${qc.notes.trim()}${qc.reviewerName ? ` — ${qc.reviewerName}` : ""}`
          : "Failed technical QC";
        break;
      case "RENDER_FAILED":
      case "PREVIEW_FAILED":
        reason =
          job?.errorMessage?.trim() ||
          (row.status === "RENDER_FAILED" ? "Render failed" : "Preview build failed");
        if (job && job.attempt >= job.maxAttempts) reason += ` (attempt ${job.attempt}/${job.maxAttempts})`;
        break;
      case "READY_FOR_QC":
        reason = "Waiting on a QC pass";
        break;
      default:
        // Still nominally in flight, but past the point where that is normal.
        reason = job?.machine?.name
          ? `No progress since claim on ${job.machine.name}`
          : "Queued with no machine picking it up";
    }

    return {
      exportId: row.id,
      shotId: row.shotId,
      shotCode: row.shot.shotCode,
      versionString: row.versionString,
      projectCode: row.shot.project.code,
      episode: row.shot.episode,
      status: row.status,
      severity: FAILED_STATUSES.includes(row.status) ? "critical" : "warning",
      since: row.statusChangedAt,
      reason,
      artistName: row.shot.artist?.name ?? row.shot.artist?.email ?? null,
    };
  });

  // Failures first, then whatever has been waiting longest. Both orderings
  // matter: severity decides what to fix, age decides what to fix first.
  return blocked.sort((a, b) => {
    if (a.severity !== b.severity) return a.severity === "critical" ? -1 : 1;
    return a.since.getTime() - b.since.getTime();
  });
}

export type CountdownVerdict = "complete" | "overdue" | "stalled" | "behind" | "tight" | "ahead";

export interface EpisodeCountdown {
  projectId: string;
  projectCode: string;
  projectName: string;
  /** Null on a standard (non-episodic) show, which has one date of its own. */
  episode: string | null;
  dueDate: Date;
  /** Calendar days to the due date; negative once it has passed. */
  daysLeft: number;
  total: number;
  delivered: number;
  inQc: number;
  inProgress: number;
  notStarted: number;
  /** Shots per day still needed to land the rest by the due date. */
  requiredPerDay: number;
  /** Shots per day actually delivered over the last week. */
  actualPerDay: number;
  verdict: CountdownVerdict;
  /**
   * How late this is: the days the current rate overruns by when `verdict` is
   * "behind", or the days the date has already passed when it is "overdue".
   */
  daysBehind: number | null;
}

const DAY_MS = 86_400_000;
/** Shots past QC but not yet in a delivered package. */
const POST_QC_STATUSES: ExportStatus[] = ["READY_FOR_QC", "READY_FOR_DELIVERY", "PACKAGED"];

/**
 * Turns "23 shots left, 14 days, 8 delivered last week" into a verdict.
 *
 * Kept free of the database so the arithmetic that decides whether a producer
 * panics can be tested directly. The trailing week is the rate we have
 * evidence for; anything else is a hope. A run that only just lands on the due
 * date is reported as tight rather than fine — finishing on the last available
 * day is luck, not schedule.
 */
export function projectPace(
  remaining: number,
  daysLeft: number,
  deliveredThisWeek: number
): Pick<EpisodeCountdown, "requiredPerDay" | "actualPerDay" | "verdict" | "daysBehind"> {
  const actualPerDay = deliveredThisWeek / 7;
  const requiredPerDay = remaining <= 0 ? 0 : remaining / Math.max(daysLeft, 1);

  if (remaining <= 0) {
    return { requiredPerDay, actualPerDay, verdict: "complete", daysBehind: null };
  }
  if (daysLeft < 0) {
    return { requiredPerDay, actualPerDay, verdict: "overdue", daysBehind: -daysLeft };
  }
  if (actualPerDay === 0) {
    return { requiredPerDay, actualPerDay, verdict: "stalled", daysBehind: null };
  }

  const projectedDays = remaining / actualPerDay;
  if (projectedDays > daysLeft) {
    return {
      requiredPerDay,
      actualPerDay,
      verdict: "behind",
      daysBehind: Math.ceil(projectedDays - daysLeft),
    };
  }
  return {
    requiredPerDay,
    actualPerDay,
    verdict: projectedDays > daysLeft * 0.9 ? "tight" : "ahead",
    daysBehind: null,
  };
}

/**
 * Are we going to make the date, per episode?
 *
 * A shot counts as delivered when it appears in a package that has actually
 * gone out — a DeliveryItem in a DELIVERED EXR_DELIVERY package. Shot.status
 * is deliberately not the measure: it says where the work is, not whether the
 * client has the frames, and a review package (MOV_REVIEW) ships nothing.
 *
 * The run rate compares what is left per remaining day against what the studio
 * actually managed per day over the last week, which is the only honest way to
 * turn "23 shots to go" into "you are not going to make it".
 */
export async function getDeliveryCountdown(limit = 6): Promise<EpisodeCountdown[]> {
  const projects = await db.project.findMany({
    where: { status: "ACTIVE" },
    select: {
      id: true,
      code: true,
      name: true,
      dueDate: true,
      projectType: true,
      episodeDueDates: { select: { episode: true, dueDate: true } },
    },
  });
  if (projects.length === 0) return [];

  const projectIds = projects.map((p) => p.id);
  const weekAgo = new Date(Date.now() - 7 * DAY_MS);

  const [shots, postQc, deliveredItems] = await Promise.all([
    db.shot.findMany({
      where: { projectId: { in: projectIds } },
      select: { id: true, projectId: true, episode: true, status: true },
    }),
    db.export.findMany({
      where: { status: { in: POST_QC_STATUSES }, shot: { projectId: { in: projectIds } } },
      select: { shotId: true },
    }),
    db.deliveryItem.findMany({
      where: {
        package: {
          projectId: { in: projectIds },
          kind: "EXR_DELIVERY",
          status: "DELIVERED",
        },
      },
      select: { shotId: true, package: { select: { deliveredAt: true } } },
    }),
  ]);

  const postQcShots = new Set(postQc.map((e) => e.shotId));
  // A shot can be in several packages (a redelivery); the first one out is the
  // one that counts towards the rate.
  const deliveredAt = new Map<string, Date>();
  for (const item of deliveredItems) {
    const when = item.package.deliveredAt;
    if (!when) continue;
    const seen = deliveredAt.get(item.shotId);
    if (!seen || when < seen) deliveredAt.set(item.shotId, when);
  }

  const now = Date.now();
  const rows: EpisodeCountdown[] = [];

  for (const project of projects) {
    // Episodic shows carry a date per episode; a standard show has one date on
    // the project itself, and is treated as a single unnamed target.
    const targets: { episode: string | null; dueDate: Date }[] =
      project.projectType === "EPISODIC"
        ? project.episodeDueDates.map((e) => ({ episode: e.episode, dueDate: e.dueDate }))
        : project.dueDate
          ? [{ episode: null, dueDate: project.dueDate }]
          : [];

    for (const target of targets) {
      const targetShots = shots.filter(
        (s) =>
          s.projectId === project.id &&
          (target.episode === null || (s.episode ?? "") === target.episode)
      );
      if (targetShots.length === 0) continue;

      let delivered = 0;
      let inQc = 0;
      let inProgress = 0;
      let notStarted = 0;
      let deliveredThisWeek = 0;

      for (const shot of targetShots) {
        const when = deliveredAt.get(shot.id);
        if (when) {
          delivered++;
          if (when >= weekAgo) deliveredThisWeek++;
        } else if (postQcShots.has(shot.id)) {
          inQc++;
        } else if (shot.status === "WAITING") {
          notStarted++;
        } else {
          inProgress++;
        }
      }

      const total = targetShots.length;
      const daysLeft = Math.ceil((target.dueDate.getTime() - now) / DAY_MS);
      const pace = projectPace(total - delivered, daysLeft, deliveredThisWeek);

      // A finished episode whose date has passed is history, not a countdown.
      if (pace.verdict === "complete" && daysLeft < 0) continue;

      rows.push({
        projectId: project.id,
        projectCode: project.code,
        projectName: project.name,
        episode: target.episode,
        dueDate: target.dueDate,
        daysLeft,
        total,
        delivered,
        inQc,
        inProgress,
        notStarted,
        ...pace,
      });
    }
  }

  return rows.sort((a, b) => a.dueDate.getTime() - b.dueDate.getTime()).slice(0, limit);
}

export interface OpsCounts {
  blocked: number;
  inQc: number;
  readyToDeliver: number;
  awaitingClientReview: number;
  activeProjects: number;
}

/**
 * The headline numbers, all scoped to live productions.
 *
 * The counters these replaced were unscoped and cumulative — "approved" across
 * every project the studio has ever finished only ever goes up, so it never
 * told anyone anything. Each of these can go down, which is what makes it
 * worth looking at twice in a day.
 */
export async function getOpsCounts(): Promise<OpsCounts> {
  // Export has a projectId column but no project relation, so live-production
  // scope is reached through the shot.
  const liveProject: Prisma.ExportWhereInput = {
    shot: { project: { status: { in: ["ACTIVE", "ON_HOLD"] } } },
  };
  const [qcStaleHours, stalledHours] = await Promise.all([
    getConfigNumber("dashboard.qcStaleHours"),
    getConfigNumber("dashboard.stalledJobHours"),
  ]);
  const now = Date.now();

  const [blocked, inQc, readyToDeliver, awaitingClientReview, activeProjects] = await Promise.all([
    db.export.count({
      where: {
        ...liveProject,
        OR: [
          { status: { in: FAILED_STATUSES } },
          { status: "READY_FOR_QC", statusChangedAt: { lt: new Date(now - qcStaleHours * 3600_000) } },
          {
            status: { in: IN_FLIGHT_STATUSES },
            statusChangedAt: { lt: new Date(now - stalledHours * 3600_000) },
          },
        ],
      },
    }),
    db.export.count({ where: { ...liveProject, status: "READY_FOR_QC" } }),
    db.export.count({ where: { ...liveProject, status: "READY_FOR_DELIVERY" } }),
    // Sent to the client and still unanswered — the wait that is not ours to
    // end, as opposed to notes we have not acted on yet.
    db.version.count({
      where: {
        approvalStatus: "PENDING_REVIEW",
        isClientVisible: true,
        isLatest: true,
        shot: { project: { status: { in: ["ACTIVE", "ON_HOLD"] } } },
      },
    }),
    db.project.count({ where: { status: "ACTIVE" } }),
  ]);

  return { blocked, inQc, readyToDeliver, awaitingClientReview, activeProjects };
}

export interface FarmStatus {
  machines: Awaited<ReturnType<typeof listMachines>>;
  online: number;
  total: number;
  queued: { render: number; preview: number; delivery: number; blender: number; total: number };
  running: number;
  /** Age of the oldest job still waiting to be claimed, in ms. */
  oldestQueuedMs: number | null;
  /** Highest worker version any machine reports, and who is behind it. */
  latestWorkerVersion: string | null;
  behindMachines: { name: string; workerVersion: string | null }[];
}

/**
 * Farm health in one strip: who is up, what is waiting, and — because a
 * worker's version is what decides whether it has a given fix — which
 * machines are running something older than the newest one out there.
 */
export async function getFarmStatus(): Promise<FarmStatus> {
  const [machines, queuedByType, running, oldest] = await Promise.all([
    listMachines(),
    db.renderJob.groupBy({ by: ["type"], where: { status: "QUEUED" }, _count: { _all: true } }),
    db.renderJob.count({ where: { status: { in: ["CLAIMED", "RUNNING"] } } }),
    db.renderJob.findFirst({
      where: { status: "QUEUED" },
      orderBy: { createdAt: "asc" },
      select: { createdAt: true },
    }),
  ]);

  const countOf = (type: string) =>
    queuedByType.find((q) => q.type === type)?._count._all ?? 0;

  const versions = machines
    .map((m) => m.workerVersion)
    .filter((v): v is string => Boolean(v));
  const latest = versions.length ? versions.reduce((a, b) => (compareVersions(a, b) >= 0 ? a : b)) : null;

  return {
    machines,
    online: machines.filter((m) => m.status === "ONLINE").length,
    total: machines.length,
    queued: {
      render: countOf("AE_RENDER"),
      preview: countOf("PREVIEW_ONLY"),
      delivery: countOf("DELIVERY_BUILD"),
      blender: countOf("BLENDER_RENDER"),
      total: queuedByType.reduce((sum, q) => sum + q._count._all, 0),
    },
    running,
    oldestQueuedMs: oldest ? Date.now() - oldest.createdAt.getTime() : null,
    latestWorkerVersion: latest,
    behindMachines: latest
      ? machines
          .filter((m) => m.workerVersion !== latest)
          .map((m) => ({ name: m.name, workerVersion: m.workerVersion }))
      : [],
  };
}

/**
 * Compares "1.6.1+fe420b9" style versions numerically, ignoring the build
 * stamp publish.ps1 appends. A string compare would rank 1.10.0 below 1.9.0.
 */
export function compareVersions(a: string, b: string): number {
  const parts = (v: string) =>
    v.split("+")[0].split(".").map((n) => Number.parseInt(n, 10) || 0);
  const [pa, pb] = [parts(a), parts(b)];
  for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
    const diff = (pa[i] ?? 0) - (pb[i] ?? 0);
    if (diff !== 0) return diff;
  }
  return 0;
}