perf(delivery): stop re-reading hard links to verify them
Deploy / deploy (push) Successful in 6m25s
Deploy / deploy (push) Successful in 6m25s
A 25-shot episode took about an hour to package when the linking itself is instant. All of it was verification: every delivered file was read back and hashed, and at ~100 MB/s over SMB one 2.8 GB shot costs 30 s. Re-hashing a hard link cannot disagree with validation. The entry is a second name for the same data, so it re-reads the exact bytes that were already hashed — half an hour per episode to prove something that cannot be false. Verification is now proportional to what was actually done: copies are read back in full, links carry over the validation-time digest. Linking has one gap hashing covered incidentally: a source changed after it passed QC, which a link would carry faithfully. Source timestamps are now checked against the export's validation time, which costs a stat per file rather than a full read. "verifyMode": "always-hash" restores unconditional read-back for a production that requires it. Worker 1.3.0. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -25,6 +25,13 @@ export interface DeliveryConfig {
|
||||
checksumAlgo: string;
|
||||
reportFormats: string[];
|
||||
copyMode: "auto" | "always-copy";
|
||||
/**
|
||||
* "auto" verifies copies by hash and hard links by identity — a link is the
|
||||
* same data as the source, so re-reading it cannot disagree with what
|
||||
* validation already hashed, and at SAN speeds costs ~30 s per shot to prove
|
||||
* nothing. "always-hash" reads every delivered file back regardless.
|
||||
*/
|
||||
verifyMode: "auto" | "always-hash";
|
||||
}
|
||||
|
||||
export const DELIVERY_CONFIG_DEFAULT: DeliveryConfig = {
|
||||
@@ -39,6 +46,7 @@ export const DELIVERY_CONFIG_DEFAULT: DeliveryConfig = {
|
||||
checksumAlgo: "xxh64",
|
||||
reportFormats: ["csv", "json"],
|
||||
copyMode: "auto",
|
||||
verifyMode: "auto",
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -333,6 +341,8 @@ export interface DeliveryManifestItem {
|
||||
checksum: string | null;
|
||||
deliveryMovPath: string | null;
|
||||
shotDir: string;
|
||||
/** When validation last ran for this export; null when it never did. */
|
||||
validatedAt: string | null;
|
||||
}
|
||||
|
||||
export interface DeliveryManifest {
|
||||
@@ -345,6 +355,7 @@ export interface DeliveryManifest {
|
||||
reportFile: string;
|
||||
includeMovs: boolean;
|
||||
copyMode: string;
|
||||
verifyMode: string;
|
||||
checksumAlgo: string;
|
||||
reportFormats: string[];
|
||||
items: DeliveryManifestItem[];
|
||||
@@ -357,7 +368,17 @@ export async function buildDeliveryJobManifest(
|
||||
const pkg = await tx.deliveryPackage.findUniqueOrThrow({
|
||||
where: { id: packageId },
|
||||
include: {
|
||||
items: { include: { export: true } },
|
||||
items: {
|
||||
include: {
|
||||
export: {
|
||||
include: {
|
||||
// Newest validation run: a linked package uses it to detect a
|
||||
// source file changed after it passed QC.
|
||||
validations: { orderBy: { createdAt: "desc" }, take: 1, select: { createdAt: true } },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
project: { select: { code: true, showId: true } },
|
||||
},
|
||||
});
|
||||
@@ -380,6 +401,7 @@ export async function buildDeliveryJobManifest(
|
||||
reportFile: applyTokens(snapshot.layout.reportFile, baseTokens),
|
||||
includeMovs: snapshot.includeMovs,
|
||||
copyMode: snapshot.copyMode ?? "auto",
|
||||
verifyMode: snapshot.verifyMode ?? "auto",
|
||||
checksumAlgo: snapshot.checksumAlgo,
|
||||
reportFormats: snapshot.reportFormats,
|
||||
items: pkg.items.map((item) => ({
|
||||
@@ -394,6 +416,7 @@ export async function buildDeliveryJobManifest(
|
||||
fileCount: item.export.exrFileCount,
|
||||
checksum: item.export.checksum,
|
||||
deliveryMovPath: snapshot.includeMovs ? item.export.deliveryMovPath : null,
|
||||
validatedAt: item.export.validations[0]?.createdAt.toISOString() ?? null,
|
||||
shotDir: applyTokens(snapshot.layout.shotDir, {
|
||||
...baseTokens,
|
||||
shotCode: item.shotCode,
|
||||
|
||||
Reference in New Issue
Block a user