perf(delivery): stop re-reading hard links to verify them
Deploy / deploy (push) Successful in 6m25s

A 25-shot episode took about an hour to package when the linking itself
is instant. All of it was verification: every delivered file was read
back and hashed, and at ~100 MB/s over SMB one 2.8 GB shot costs 30 s.

Re-hashing a hard link cannot disagree with validation. The entry is a
second name for the same data, so it re-reads the exact bytes that were
already hashed — half an hour per episode to prove something that cannot
be false. Verification is now proportional to what was actually done:
copies are read back in full, links carry over the validation-time digest.

Linking has one gap hashing covered incidentally: a source changed after
it passed QC, which a link would carry faithfully. Source timestamps are
now checked against the export's validation time, which costs a stat per
file rather than a full read.

"verifyMode": "always-hash" restores unconditional read-back for a
production that requires it.

Worker 1.3.0.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
twotalesanimation
2026-08-14 09:29:04 +02:00
parent 991abe072c
commit af810258e6
6 changed files with 165 additions and 24 deletions
@@ -123,17 +123,80 @@ public class DeliveryStageTests : IDisposable
}
[Fact]
public async Task ChecksumMismatchFailsTheItem()
public async Task ChecksumMismatchFailsACopiedItem()
{
WriteFrames(3);
var result = await NewStage().RunAsync(new ClaimedJob { Id = "job1" },
Manifest(checksum: "deadbeefdeadbeef"), "machine1", CancellationToken.None, CancellationToken.None);
var manifest = Manifest(checksum: "deadbeefdeadbeef", copyMode: "always-copy");
var result = await NewStage().RunAsync(new ClaimedJob { Id = "job1" }, manifest, "machine1",
CancellationToken.None, CancellationToken.None);
Assert.False(result.Ok);
Assert.Equal("FAILED", result.Items[0].Status);
Assert.Contains("Checksum mismatch", result.Items[0].ErrorMessage!);
}
[Fact]
public async Task HardLinkedItemsAreNotReadBack()
{
// A hard link is the same data as the source, so re-hashing it re-reads
// the exact bytes validation already hashed — half an hour per episode
// over SMB to prove something that cannot be false. A deliberately wrong
// recorded checksum is therefore not consulted on the linked path, and
// the validation-time digest carries over unchanged.
WriteFrames(3);
var result = await NewStage().RunAsync(new ClaimedJob { Id = "job1" },
Manifest(checksum: "deadbeefdeadbeef"), "machine1", CancellationToken.None, CancellationToken.None);
Assert.True(result.Ok);
Assert.Equal("VERIFIED", result.Items[0].Status);
Assert.Equal("hardlink", result.Items[0].LinkMode);
Assert.Equal("deadbeefdeadbeef", result.Items[0].Checksum);
}
[Fact]
public async Task AlwaysHashForcesReadBackEvenWhenLinked()
{
WriteFrames(3);
var manifest = Manifest(checksum: "deadbeefdeadbeef");
manifest.VerifyMode = "always-hash";
var result = await NewStage().RunAsync(new ClaimedJob { Id = "job1" }, manifest, "machine1",
CancellationToken.None, CancellationToken.None);
Assert.False(result.Ok);
Assert.Contains("Checksum mismatch", result.Items[0].ErrorMessage!);
}
[Fact]
public async Task SourceTouchedAfterValidationFailsTheLinkedItem()
{
// The one thing linking cannot rule out: the source changing after it
// passed QC. The link would faithfully carry the new bytes.
WriteFrames(3);
var manifest = Manifest();
manifest.Items[0].ValidatedAt = DateTimeOffset.UtcNow.AddHours(-2);
File.SetLastWriteTimeUtc(
Path.Combine(_sourceDir, "SHOT_001_cmp_TT_v001.01002.exr"), DateTime.UtcNow);
var result = await NewStage().RunAsync(new ClaimedJob { Id = "job1" }, manifest, "machine1",
CancellationToken.None, CancellationToken.None);
Assert.False(result.Ok);
Assert.Contains("modified after validation", result.Items[0].ErrorMessage!);
}
[Fact]
public async Task UntouchedSourcesPassTheValidationTimestampCheck()
{
WriteFrames(3);
var manifest = Manifest();
manifest.Items[0].ValidatedAt = DateTimeOffset.UtcNow.AddHours(1);
var result = await NewStage().RunAsync(new ClaimedJob { Id = "job1" }, manifest, "machine1",
CancellationToken.None, CancellationToken.None);
Assert.True(result.Ok);
}
[Fact]
public async Task FrameCountDriftSinceQcFailsTheItem()
{
@@ -163,14 +226,15 @@ public class DeliveryStageTests : IDisposable
public async Task ChecksumMatchesTheValidationTimeDigestForIdenticalContent()
{
WriteFrames(3);
var first = await NewStage().RunAsync(new ClaimedJob { Id = "job1" }, Manifest(), "machine1",
CancellationToken.None, CancellationToken.None);
var first = await NewStage().RunAsync(new ClaimedJob { Id = "job1" },
Manifest(copyMode: "always-copy"), "machine1", CancellationToken.None, CancellationToken.None);
// Feeding the recorded digest back in must verify, which is exactly what
// the delivery builder does against Export.checksum.
// the delivery builder does against Export.checksum on the copy path.
Directory.Delete(_packageDir, recursive: true);
var second = await NewStage().RunAsync(new ClaimedJob { Id = "job1" },
Manifest(checksum: first.Items[0].Checksum), "machine1", CancellationToken.None, CancellationToken.None);
Manifest(checksum: first.Items[0].Checksum, copyMode: "always-copy"), "machine1",
CancellationToken.None, CancellationToken.None);
Assert.True(second.Ok);
}
+46 -9
View File
@@ -37,6 +37,7 @@ public sealed class DeliveryStage
var results = new List<DeliveryItemResult>();
var manifestFiles = new List<object>();
var alwaysCopy = string.Equals(manifest.CopyMode, "always-copy", StringComparison.OrdinalIgnoreCase);
var alwaysHash = string.Equals(manifest.VerifyMode, "always-hash", StringComparison.OrdinalIgnoreCase);
_log.LogInformation("Building delivery package {Name} ({Count} shots) at {Dir}",
manifest.PackageName, manifest.Items.Count, packageDir);
@@ -50,7 +51,7 @@ public sealed class DeliveryStage
await _api.ProgressAsync(job.Id, machineId, (double)i / manifest.Items.Count, i, manifest.Items.Count, null,
$"Packaging {item.ShotCode} {item.VersionString} ({i + 1}/{manifest.Items.Count})", shutdown);
var result = BuildItem(item, mapper, alwaysCopy, manifestFiles);
var result = BuildItem(item, mapper, alwaysCopy, alwaysHash, manifestFiles);
results.Add(result);
if (result.Status == "FAILED")
{
@@ -77,7 +78,7 @@ public sealed class DeliveryStage
}
private DeliveryItemResult BuildItem(
DeliveryManifestItem item, PathMapper mapper, bool alwaysCopy, List<object> manifestFiles)
DeliveryManifestItem item, PathMapper mapper, bool alwaysCopy, bool alwaysHash, List<object> manifestFiles)
{
var sourceDir = mapper.Map(item.SourceDir);
var shotDir = mapper.Map(item.ShotDir);
@@ -121,15 +122,51 @@ public sealed class DeliveryStage
totalBytes += new FileInfo(dest).Length;
}
// Verification reads the delivered bytes back, which works the same
// whether the entry is a link or a copy — `fsutil hardlink list` is
// not available over SMB, so link identity cannot be queried.
var digest = SequenceChecksum(sources.Select(s => Path.Combine(shotDir, Path.GetFileName(s))));
if (item.Checksum is { Length: > 0 } && !string.Equals(digest, item.Checksum, StringComparison.OrdinalIgnoreCase))
// Verification is proportional to what was actually done.
//
// A hard link is a second name for the same data, so hashing it
// re-reads the exact bytes validation already hashed — it cannot
// disagree, and at ~100 MB/s over SMB it costs half an hour for an
// episode to prove nothing. A copy is different: it can be
// truncated or corrupted, so it is read back in full.
string? digest;
string verifiedBy;
if (usedCopy || alwaysHash)
{
return DeliveryItemResult.Failed(item,
$"Checksum mismatch after packaging: expected {item.Checksum}, got {digest}");
digest = SequenceChecksum(sources.Select(s => Path.Combine(shotDir, Path.GetFileName(s))));
if (item.Checksum is { Length: > 0 } && !string.Equals(digest, item.Checksum, StringComparison.OrdinalIgnoreCase))
{
return DeliveryItemResult.Failed(item,
$"Checksum mismatch after packaging: expected {item.Checksum}, got {digest}");
}
verifiedBy = "hash";
}
else
{
// The one thing linking cannot rule out is the source having
// changed since it passed QC — the link would faithfully carry
// the new bytes. Timestamps answer that for the price of a stat.
if (item.ValidatedAt is DateTimeOffset validatedAt)
{
var touched = sources
.Select(s => new FileInfo(s))
.Where(f => f.LastWriteTimeUtc > validatedAt.UtcDateTime.AddSeconds(1))
.ToList();
if (touched.Count > 0)
{
return DeliveryItemResult.Failed(item,
$"{touched.Count} source file(s) were modified after validation " +
$"({touched[0].Name} at {touched[0].LastWriteTimeUtc:u}, validated {validatedAt.UtcDateTime:u}) — " +
"the render changed after it passed QC");
}
}
// Same data as the source, so the validation-time digest describes
// the delivered files exactly.
digest = item.Checksum;
verifiedBy = "link identity";
}
_log.LogDebug("{Shot}: verified by {How}", item.ShotCode, verifiedBy);
string? movDest = null;
if (!string.IsNullOrWhiteSpace(item.DeliveryMovPath))
+4
View File
@@ -79,6 +79,8 @@ public sealed class DeliveryManifest
public string ReportFile { get; set; } = "";
public bool IncludeMovs { get; set; } = true;
public string CopyMode { get; set; } = "auto";
/// <summary>"auto" verifies copies by hash and links by identity; "always-hash" reads everything back.</summary>
public string VerifyMode { get; set; } = "auto";
public string ChecksumAlgo { get; set; } = "xxh64";
public List<string> ReportFormats { get; set; } = new();
public List<DeliveryManifestItem> Items { get; set; } = new();
@@ -98,6 +100,8 @@ public sealed class DeliveryManifestItem
public string? Checksum { get; set; }
public string? DeliveryMovPath { get; set; }
public string ShotDir { get; set; } = "";
/// <summary>When this export's validation ran — lets a linked package detect a source changed since QC.</summary>
public DateTimeOffset? ValidatedAt { get; set; }
}
/// <summary>PREVIEW_ONLY manifest (server-built, RenderPipeline2 §9).</summary>
@@ -15,7 +15,7 @@
every change that gets published. publish.ps1 appends the git SHA via
SourceRevisionId, giving "1.1.0+feb1c7b".
-->
<Version>1.2.1</Version>
<Version>1.3.0</Version>
</PropertyGroup>
<ItemGroup>