Honour write backpressure when reassembling uploads

createWriteStream.write() returns false once its internal buffer is full;
both reassembly loops ignored that and kept queueing chunks, so the queue
grew in memory instead of flushing to disk. Measured over a 1.5 GiB
reassembly in 10 MB chunks, this buffered 170 MB versus 10 MB when the
drain event is awaited.

Also fail loudly on a missing chunk in the v1 loop, matching v2, rather
than silently writing a corrupt file.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
twotalesanimation
2026-08-08 09:15:32 +02:00
parent 2e688c8e52
commit 2d0c33a45b
2 changed files with 25 additions and 4 deletions
+12 -2
View File
@@ -291,9 +291,19 @@ async function handleChunkComplete(
const writeStream = fs.createWriteStream(finalPath);
// Honour backpressure: createWriteStream queues in memory whenever write()
// returns false, so a multi-GB reassembly that ignores the return value
// grows the internal buffer instead of flushing to disk.
const write = (buf: Buffer): Promise<void> =>
new Promise((resolve, reject) => {
if (writeStream.write(buf)) return resolve();
writeStream.once('drain', resolve);
writeStream.once('error', reject);
});
// Write the format header: magic (8) + salt (16). The per-chunk IVs live
// in the frames themselves, so nothing needs storing in the DB.
writeStream.write(buildHeader(session.salt));
await write(buildHeader(session.salt));
// Assemble encrypted frames in order, with progress tracking
let chunksAssembled = 0;
@@ -320,7 +330,7 @@ async function handleChunkComplete(
throw new Error(`Chunk ${i} missing during reassembly`);
}
writeStream.write(chunkData);
await write(chunkData);
chunksAssembled++;
if (chunksAssembled % 10 === 0 || chunksAssembled === session.totalChunks) {
+13 -2
View File
@@ -265,6 +265,15 @@ async function handleChunkComplete(
const writeStream = fs.createWriteStream(finalPath);
// Honour backpressure so a multi-GB reassembly flushes to disk instead of
// queueing in the stream's internal buffer.
const write = (buf: Buffer): Promise<void> =>
new Promise((resolve, reject) => {
if (writeStream.write(buf)) return resolve();
writeStream.once('drain', resolve);
writeStream.once('error', reject);
});
for (let i = 0; i < session.totalChunks; i++) {
const chunkPath = path.join(chunkDir, `chunk-${i}`);
// Retry reading with exponential backoff to handle file locks
@@ -281,9 +290,11 @@ async function handleChunkComplete(
}
}
}
if (chunkData) {
writeStream.write(chunkData);
// A silently skipped chunk would corrupt the file, so fail loudly.
if (!chunkData) {
throw new Error(`Chunk ${i} missing during reassembly`);
}
await write(chunkData);
}
await new Promise<void>((resolve, reject) => {