feat(render-pipeline): validation, QC and delivery builder

Completes RenderPipeline2 phases 3-5. Additive throughout: with
validation.enabled=false the pipeline behaves exactly as before.

Validation (§8) runs on the worker straight after a render, while it
still holds the job and the files are local. Everything is read from the
EXR header — one 64 KB read per sampled frame, no pixel decode — so the
engine needs no oiiotool. The render job deliberately stays open through
the stage, so a worker that dies there is reaped and retried instead of
stranding the export in VALIDATING.

Timecode is the primary check and is anchored on the plate, never the
API: measured on the live show, Shot.timecodeStart is the editorial
in-point and sits one handle-length (8f) ahead of the plate, so checking
against it would pass the defective exports and fail the correct ones.
Reference precedence is manifest timing plate -> plate found on disk ->
API value, and the API rank is WARN-only. The panel now resolves plate
provenance by walking the comp's own footage layers, so the worker never
guesses which element carries the comp's timing.

Verified against real studio renders via the new --validate mode: it
catches both defect classes seen in shipped work — the +8 handle offset
and frame-derived timecode (00:00:41:17 = frame 1001 at 24fps).

QC (§10) is one service behind two front doors, API-key for the panel
and session for the web. PASS means ready to upload and nothing else;
FAIL sets the comp task to CHANGES. The panel gains a QC section that
imports the real EXR sequence at full res rather than the H.264.

Delivery (§11) rides the existing queue as a DELIVERY_BUILD job: hard
links with a copy fallback, verification by re-hashing delivered bytes
against the validation-time digest, manifest.json and the CSV report.
fsutil hardlink list is unavailable over SMB, so link identity is never
queried — reading the bytes back covers links and copies alike.

Also fixes ApiClient.ProgressAsync throwing on transport failure, which
would have aborted work in flight during a brief server outage; progress
reporting is best-effort by contract.

Removes the stale RenderWorker/ rule from the root .gitignore. The
worker's source is tracked like the rest of the repo and its own
.gitignore already covers bin/, obj/ and publish/; the rule silently
excluded new files while the existing 20 stayed tracked.

Not yet applied: migration 20260813000000 was hand-written because the
dev database was unavailable — run prisma migrate deploy and verify
before relying on it. None of this has run against a live server.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
twotalesanimation
2026-08-13 13:49:51 +02:00
parent 7f2f5e1005
commit feb1c7b3e9
48 changed files with 5786 additions and 238 deletions
@@ -0,0 +1,110 @@
-- RenderPipeline2 Phase 3 (validation), Phase 4 QC half, Phase 5 (delivery).
-- Purely additive: no existing table is altered, no backfill needed.
-- CreateTable
CREATE TABLE "validation_results" (
"id" TEXT NOT NULL,
"exportId" TEXT NOT NULL,
"renderJobId" TEXT NOT NULL,
"checkName" TEXT NOT NULL,
"status" "ValidationStatus" NOT NULL,
"expected" TEXT,
"actual" TEXT,
"message" TEXT,
"details" JSONB,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "validation_results_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "qc_reviews" (
"id" TEXT NOT NULL,
"exportId" TEXT NOT NULL,
"result" "QCResult" NOT NULL,
"notes" TEXT,
"reviewedById" TEXT,
"reviewerName" TEXT,
"source" TEXT NOT NULL,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "qc_reviews_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "delivery_packages" (
"id" TEXT NOT NULL,
"projectId" TEXT NOT NULL,
"episode" TEXT NOT NULL,
"name" TEXT NOT NULL,
"packageNumber" INTEGER NOT NULL,
"status" "DeliveryStatus" NOT NULL DEFAULT 'DRAFT',
"rootPath" TEXT,
"configSnapshot" JSONB NOT NULL,
"manifest" JSONB,
"reportKey" TEXT,
"errorMessage" TEXT,
"createdById" TEXT NOT NULL,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"builtAt" TIMESTAMP(3),
"deliveredAt" TIMESTAMP(3),
"deliveredById" TEXT,
CONSTRAINT "delivery_packages_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "delivery_items" (
"id" TEXT NOT NULL,
"packageId" TEXT NOT NULL,
"exportId" TEXT NOT NULL,
"shotId" TEXT NOT NULL,
"shotCode" TEXT NOT NULL,
"versionString" TEXT NOT NULL,
"exrPath" TEXT,
"movPath" TEXT,
"fileCount" INTEGER,
"totalBytes" BIGINT,
"checksum" TEXT,
"linkMode" TEXT,
"status" TEXT NOT NULL DEFAULT 'PENDING',
"errorMessage" TEXT,
CONSTRAINT "delivery_items_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE INDEX "validation_results_exportId_renderJobId_idx" ON "validation_results"("exportId", "renderJobId");
-- CreateIndex
CREATE INDEX "qc_reviews_exportId_idx" ON "qc_reviews"("exportId");
-- CreateIndex
CREATE INDEX "delivery_packages_projectId_episode_idx" ON "delivery_packages"("projectId", "episode");
-- CreateIndex
CREATE INDEX "delivery_packages_status_idx" ON "delivery_packages"("status");
-- CreateIndex
CREATE UNIQUE INDEX "delivery_packages_projectId_packageNumber_key" ON "delivery_packages"("projectId", "packageNumber");
-- CreateIndex
CREATE INDEX "delivery_items_exportId_idx" ON "delivery_items"("exportId");
-- CreateIndex
CREATE UNIQUE INDEX "delivery_items_packageId_exportId_key" ON "delivery_items"("packageId", "exportId");
-- AddForeignKey
ALTER TABLE "validation_results" ADD CONSTRAINT "validation_results_exportId_fkey" FOREIGN KEY ("exportId") REFERENCES "exports"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "qc_reviews" ADD CONSTRAINT "qc_reviews_exportId_fkey" FOREIGN KEY ("exportId") REFERENCES "exports"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "delivery_packages" ADD CONSTRAINT "delivery_packages_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "projects"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "delivery_items" ADD CONSTRAINT "delivery_items_packageId_fkey" FOREIGN KEY ("packageId") REFERENCES "delivery_packages"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "delivery_items" ADD CONSTRAINT "delivery_items_exportId_fkey" FOREIGN KEY ("exportId") REFERENCES "exports"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
+181 -87
View File
@@ -215,40 +215,40 @@ enum AttachmentCategory {
// ─────────────────────────────────────────────
model User {
id String @id @default(cuid())
name String?
email String @unique
emailVerified DateTime?
image String?
id String @id @default(cuid())
name String?
email String @unique
emailVerified DateTime?
image String?
passwordHash String?
role Role @default(ARTIST)
isActive Boolean @default(true)
mustChangePassword Boolean @default(false)
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
// NextAuth relations
accounts Account[]
sessions Session[]
// App relations
producedProjects Project[] @relation("ProducerProjects")
supervisedProjects Project[] @relation("SupervisorProjects")
assignedShots Shot[] @relation("ArtistShots")
uploadedVersions Version[]
comments Comment[]
commentReplies CommentReply[]
annotations Annotation[]
approvals Approval[]
notifications Notification[]
clientAccess ClientAccess[]
assignedTasks Task[] @relation("TaskArtist")
createdTasks Task[] @relation("TaskCreator")
sharedVersions Version[] @relation("VersionSharedBy")
ledAssets Asset[] @relation("AssetLead")
createdShootDays ShootDay[] @relation("ShootDayCreator")
createdTakes Take[] @relation("TakeCreator")
uploadedTakeAttachments TakeAttachment[] @relation("TakeAttachmentUploader")
producedProjects Project[] @relation("ProducerProjects")
supervisedProjects Project[] @relation("SupervisorProjects")
assignedShots Shot[] @relation("ArtistShots")
uploadedVersions Version[]
comments Comment[]
commentReplies CommentReply[]
annotations Annotation[]
approvals Approval[]
notifications Notification[]
clientAccess ClientAccess[]
assignedTasks Task[] @relation("TaskArtist")
createdTasks Task[] @relation("TaskCreator")
sharedVersions Version[] @relation("VersionSharedBy")
ledAssets Asset[] @relation("AssetLead")
createdShootDays ShootDay[] @relation("ShootDayCreator")
createdTakes Take[] @relation("TakeCreator")
uploadedTakeAttachments TakeAttachment[] @relation("TakeAttachmentUploader")
@@map("users")
}
@@ -360,7 +360,8 @@ model Project {
/// Per-production delivery naming/layout templates (RenderPipeline2 §11.2);
/// falls back to SystemConfig defaults when null
deliveryConfig Json?
deliveryConfig Json?
deliveryPackages DeliveryPackage[]
@@map("projects")
}
@@ -380,60 +381,60 @@ model EpisodeDueDate {
}
model Shot {
id String @id @default(cuid())
shotCode String
scene String @default("")
episode String?
shotNumber Int @default(0)
sequence String?
description String? @db.Text
notes String? @db.Text
status ShotStatus @default(WAITING)
priority ShotPriority @default(NORMAL)
artistId String?
projectId String
frameStart Int?
frameEnd Int?
fps Float @default(24)
dueDate DateTime?
thumbnailUrl String?
originalFootageUrl String?
originalFootageKey String?
shotGroupId String?
shotApprovalStatus ShotApprovalStatus @default(PENDING)
sharedWithClient Boolean @default(false)
id String @id @default(cuid())
shotCode String
scene String @default("")
episode String?
shotNumber Int @default(0)
sequence String?
description String? @db.Text
notes String? @db.Text
status ShotStatus @default(WAITING)
priority ShotPriority @default(NORMAL)
artistId String?
projectId String
frameStart Int?
frameEnd Int?
fps Float @default(24)
dueDate DateTime?
thumbnailUrl String?
originalFootageUrl String?
originalFootageKey String?
shotGroupId String?
shotApprovalStatus ShotApprovalStatus @default(PENDING)
sharedWithClient Boolean @default(false)
// EDL / pull CSV fields
sourceClip String?
timecodeStart String?
timecodeEnd String?
clipDuration String?
exrOutput String?
sourceClip String?
timecodeStart String?
timecodeEnd String?
clipDuration String?
exrOutput String?
// Sequence / picture lock timecodes
seqTimecodeStart String?
seqTimecodeEnd String?
seqTimecodeStart String?
seqTimecodeEnd String?
// High-res deliverable (stored in Hetzner Object Storage)
highResKey String?
highResFilename String?
highResKey String?
highResFilename String?
// Shot-level version tracking (e.g. v001, v002)
shotVersion String @default("v001")
shotVersion String @default("v001")
// Key shot flag — highlighted for prioritisation
isKeyShot Boolean @default(false)
isKeyShot Boolean @default(false)
// Netflix slate fields — managed via the Slate tab, used by the preview pipeline
slateType String?
slateDescription String? @db.Text
slateVfxScope String? @db.Text
slateSubmissionNote String? @db.Text
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
slateType String?
slateDescription String? @db.Text
slateVfxScope String? @db.Text
slateSubmissionNote String? @db.Text
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
artist User? @relation("ArtistShots", fields: [artistId], references: [id])
shotGroup ShotGroup? @relation(fields: [shotGroupId], references: [id])
versions Version[]
tasks Task[]
project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
artist User? @relation("ArtistShots", fields: [artistId], references: [id])
shotGroup ShotGroup? @relation(fields: [shotGroupId], references: [id])
versions Version[]
tasks Task[]
footagePlates FootagePlate[]
references ShotReference[]
loggedTakes Take[] @relation("TakeToShot")
loggedTakes Take[] @relation("TakeToShot")
exports Export[]
@@unique([projectId, shotCode])
@@ -565,7 +566,7 @@ model Annotation {
commentId String?
authorId String?
frameNumber Int
drawingData Json // Array of AnnotationShape objects
drawingData Json // Array of AnnotationShape objects
color String @default("#ef4444")
isVisible Boolean @default(true)
createdAt DateTime @default(now())
@@ -597,7 +598,7 @@ model Notification {
type NotificationType
title String
message String @db.Text
data Json? // Extra context (versionId, shotCode, etc.)
data Json? // Extra context (versionId, shotCode, etc.)
isRead Boolean @default(false)
createdAt DateTime @default(now())
@@ -647,13 +648,13 @@ model Task {
scheduledStartDate DateTime?
scheduledEndDate DateTime?
scheduleNotes String? @db.Text
scheduleNotes String? @db.Text
shot Shot? @relation(fields: [shotId], references: [id], onDelete: Cascade)
asset Asset? @relation(fields: [assetId], references: [id], onDelete: Cascade)
assignedArtist User? @relation("TaskArtist", fields: [assignedArtistId], references: [id])
createdBy User? @relation("TaskCreator", fields: [createdById], references: [id], onDelete: SetNull)
project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
shot Shot? @relation(fields: [shotId], references: [id], onDelete: Cascade)
asset Asset? @relation(fields: [assetId], references: [id], onDelete: Cascade)
assignedArtist User? @relation("TaskArtist", fields: [assignedArtistId], references: [id])
createdBy User? @relation("TaskCreator", fields: [createdById], references: [id], onDelete: SetNull)
project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
versions Version[]
@@index([scheduledStartDate])
@@ -692,9 +693,9 @@ model SystemConfig {
/// Physical display event log — polled by ESP32 dot-matrix display
model DisplayEvent {
id Int @id @default(autoincrement())
type String // APPROVED | CHANGES | COMMENT | REJECTED
type String // APPROVED | CHANGES | COMMENT | REJECTED
shotCode String
by String // actor name (reviewer / commenter)
by String // actor name (reviewer / commenter)
createdAt DateTime @default(now())
@@index([createdAt])
@@ -716,8 +717,8 @@ model ShootDay {
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
createdBy User? @relation("ShootDayCreator", fields: [createdById], references: [id], onDelete: SetNull)
project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
createdBy User? @relation("ShootDayCreator", fields: [createdById], references: [id], onDelete: SetNull)
setups Setup[]
@@map("shoot_days")
@@ -747,9 +748,9 @@ model Take {
pipelineShotId String?
// General
scene String?
shotLabel String?
unitLabel String?
scene String?
shotLabel String?
unitLabel String?
// Camera
cameraLetter String?
@@ -880,8 +881,11 @@ model Export {
submittedByName String? // free text from panel config as fallback
supersededById String? // newer Export that replaced this one
renderJobs RenderJob[]
events ExportEvent[]
renderJobs RenderJob[]
events ExportEvent[]
validations ValidationResult[]
qcReviews QCReview[]
deliveryItems DeliveryItem[]
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
@@ -984,6 +988,96 @@ model WorkerHeartbeat {
@@map("worker_heartbeats")
}
/// One row per check per validation run, so the UI shows exactly what failed
/// and history survives retries (§5.4).
model ValidationResult {
id String @id @default(cuid())
exportId String
export Export @relation(fields: [exportId], references: [id])
renderJobId String // which attempt produced this run
checkName String // "timecode_plate_match", "missing_frames", ...
status ValidationStatus
expected String?
actual String?
message String?
details Json? // e.g. list of missing frame numbers, or the timecode reference used
createdAt DateTime @default(now())
@@index([exportId, renderJobId])
@@map("validation_results")
}
/// Human technical QC on an Export (§5.5). Multiple rows allowed — the latest
/// is authoritative and drives T13/T14.
model QCReview {
id String @id @default(cuid())
exportId String
export Export @relation(fields: [exportId], references: [id])
result QCResult
notes String? @db.Text
reviewedById String? // resolved User where possible
reviewerName String? // fallback for API-key reviews from the panel
source String // "AE_PANEL" | "WEB"
createdAt DateTime @default(now())
@@index([exportId])
@@map("qc_reviews")
}
/// A dated delivery folder built from READY_FOR_DELIVERY exports (§5.7).
/// Never deleted — a mistaken package is CANCELLED or superseded.
model DeliveryPackage {
id String @id @default(cuid())
projectId String
project Project @relation(fields: [projectId], references: [id])
episode String // "101" — empty string on standard (non-episodic) shows
name String // "260805_Delivery"
packageNumber Int // sequential per project → "Package #15"
status DeliveryStatus @default(DRAFT)
rootPath String? // final path on the delivery volume
configSnapshot Json // naming template + options at build time (immutable record)
manifest Json? // built manifest: files, sizes, checksums
reportKey String? // delivery report in object storage; also written into the folder
errorMessage String?
createdById String
createdAt DateTime @default(now())
builtAt DateTime?
deliveredAt DateTime?
deliveredById String?
items DeliveryItem[]
@@unique([projectId, packageNumber])
@@index([projectId, episode])
@@index([status])
@@map("delivery_packages")
}
/// Version lock: the exact Export delivered. shotCode/versionString are
/// denormalised so a later shot rename or re-render never mutates history.
model DeliveryItem {
id String @id @default(cuid())
packageId String
package DeliveryPackage @relation(fields: [packageId], references: [id], onDelete: Cascade)
exportId String
export Export @relation(fields: [exportId], references: [id])
shotId String
shotCode String
versionString String
exrPath String? // path inside the package
movPath String?
fileCount Int?
totalBytes BigInt?
checksum String?
linkMode String? // "hardlink" | "copy" — what the builder actually did
status String @default("PENDING") // PENDING | COPIED | VERIFIED | FAILED
errorMessage String?
@@unique([packageId, exportId])
@@index([exportId])
@@map("delivery_items")
}
model SketchTemplate {
id String @id @default(cuid())
name String