pinto/Vault
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stream video uploads to disk instead of buffering in memory

Browse Source 
req.formData() parsed the entire multipart body into RAM, so large
uploads (1.6GB+) OOM-killed the server. Bridge the web Request body to
a Node stream and parse it with formidable, which writes the file to
disk as it arrives:

- Temp dir is /uploads/tmp (same filesystem as the destination) so the
  post-parse move is an atomic rename, not a second multi-GB write
- Mark the bridged request as chunked when a proxy strips
  content-length, otherwise formidable assumes an empty body
- Track formidable temp files via fileBegin so aborted uploads get
  their partial files cleaned up
- Remove the dead formidable branch gated on (req as any).req, which
  is always null in the App Router

Also fix the nginx example in VPS_UPLOAD_CONFIG.md: client_max_body_size
was 1024M, below the app's 2.5GB limit, and would 413 large uploads at
the proxy before they reached the app.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
twotalesanimation
2026-08-01 15:54:57 +02:00
parent 81ad7e4ea9
commit 4cd8986e69
2 changed files with 106 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VPS_UPLOAD_CONFIG.md
+4 -3
View File
@@ -63,9 +63,10 @@ If running with a proxy (nginx/Apache), ensure:
location /api/admin/upload {
proxy_pass http://next-server;
proxy_connect_timeout 60s;
proxy_send_timeout 300s; # 5 minutes for large uploads
proxy_read_timeout 300s; # 5 minutes for large uploads
client_max_body_size 1024M; # Adjust based on your max video size
proxy_send_timeout 2700s; # 45 minutes, matches maxDuration in the route
proxy_read_timeout 2700s; # 45 minutes, matches maxDuration in the route
client_max_body_size 3G; # Must exceed the 2.5GB app-level limit
proxy_request_buffering off; # Stream to the app instead of spooling to nginx disk
}
```
app/api/admin/upload/route.ts
+94 -147
View File
@@ -2,10 +2,10 @@
import { NextResponse } from "next/server";
import fs from "fs";
import path from "path";
import { Readable } from "stream";
import { IncomingMessage } from "http";
import { spawn } from "child_process";
import { pipeline } from "stream/promises";
import formidable, { File as FormidableFile } from "formidable";
import formidable, { Fields, Files, File as FormidableFile } from "formidable";
import { prisma } from "@/lib/prisma";
import { getServerSession } from "next-auth";
import { authOptions } from "@/lib/auth-options";
@@ -16,6 +16,10 @@ export const maxDuration = 2700; // 45 minutes for very large uploads over slow
// Use ConfigurableUPLOADS_DIR from environment or default to /uploads
const UPLOADS_DIR = process.env.UPLOADS_DIR || "/uploads";
// Temp dir must live on the same filesystem as the final destination so the
// post-parse rename is a cheap atomic move instead of a multi-GB copy
const TMP_DIR = path.join(UPLOADS_DIR, "tmp");
// Helper to extract video duration using FFprobe
async function getVideoDurationFromFile(filePath: string): Promise<number | null> {
return new Promise((resolve) => {
@@ -65,23 +69,69 @@ async function getVideoDurationFromFile(filePath: string): Promise<number | null
});
}
function parseForm(req: IncomingMessage): Promise<{ fields: any; files: any }> {
// Formidable expects a Node IncomingMessage, but App Router route handlers get
// a web Request. Bridging the body stream lets formidable write the multipart
// payload to disk as it arrives — req.formData() would buffer the whole file
// (1.6GB+) in memory and OOM the process.
function toNodeRequest(req: Request): IncomingMessage {
if (!req.body) throw { status: 400, message: "empty request body" };
const nodeStream = Readable.fromWeb(req.body as any) as any;
const headers = Object.fromEntries(req.headers.entries());
// Formidable treats a missing content-length without transfer-encoding as an
// empty body (DummyParser); proxies that re-chunk uploads strip content-length
if (!headers["content-length"] && headers["transfer-encoding"] === undefined) {
headers["transfer-encoding"] = "chunked";
}
nodeStream.headers = headers;
nodeStream.method = "POST";
return nodeStream as IncomingMessage;
}
// formidable v3 returns every field/file as an array
function first<T>(value: T | T[] | undefined): T | undefined {
return Array.isArray(value) ? value[0] : value;
}
async function parseForm(req: Request, tempPaths: string[]): Promise<{ fields: Fields; files: Files }> {
await fs.promises.mkdir(TMP_DIR, { recursive: true });
const form = formidable({
multiples: false,
uploadDir: TMP_DIR,
maxFileSize: 2.5 * 1024 * 1024 * 1024, // 2.5GB max file size
maxFieldsSize: 10 * 1024 * 1024, // 10MB for all fields combined
maxFields: 50,
keepExtensions: true,
});
// record temp paths as files start writing, so aborted uploads still get cleaned up
form.on("fileBegin", (_name, file) => {
if (file.filepath) tempPaths.push(file.filepath);
});
return new Promise((resolve, reject) => {
form.parse(req, (err, fields, files) => {
form.parse(toNodeRequest(req), (err, fields, files) => {
if (err) {
console.error('[Upload] Formidable parse error:', err.code, err.message);
console.error("[Upload] Formidable parse error:", err.code, err.message);
reject(err);
} else resolve({ fields, files });
});
});
}
// rename() is atomic when TMP_DIR shares a filesystem with dest; fall back to
// copy+unlink if the volumes are ever split across devices
async function moveFile(src: string, dest: string) {
await fs.promises.mkdir(path.dirname(dest), { recursive: true });
try {
await fs.promises.rename(src, dest);
} catch (err: any) {
if (err?.code === "EXDEV") {
await fs.promises.copyFile(src, dest);
await fs.promises.unlink(src).catch(() => {});
} else {
throw err;
}
else resolve({ fields, files });
});
});
}
// Ensure the file is readable by all processes (mode 644)
await fs.promises.chmod(dest, 0o644);
}
async function checkAdmin() {
@@ -100,175 +150,68 @@ export async function POST(req: Request) {
// track saved paths so we can cleanup on error
let savedVideoDest: string | undefined;
let savedThumbDest: string | undefined;
// formidable temp files not yet moved to their final location
const tempPaths: string[] = [];
try {
console.log('[Upload] Request received, content-type:', req.headers.get("content-type"));
const nodeReq = (req as any).req ?? (globalThis as any).__NEXT_INIT?.req ?? null;
const session = await checkAdmin();
console.log('[Upload] Admin check passed for:', session.user?.email);
// helper to stream a File directly to disk (handles large files efficiently)
async function saveVideoStream(file: File, origName?: string) {
const filename = `${Date.now()}-${String(origName ?? "upload.mp4")}`;
const dest = path.join(UPLOADS_DIR, "videos", filename);
const { fields, files } = await parseForm(req, tempPaths);
console.log('[Upload] Parsing complete, fields:', Object.keys(fields), 'files:', Object.keys(files));
// Ensure directory exists
await fs.promises.mkdir(path.dirname(dest), { recursive: true });
const videoFile = first(files.file) as FormidableFile | undefined;
const thumbFile = first(files.thumbnail) as FormidableFile | undefined;
// Create write stream to destination
const writeStream = fs.createWriteStream(dest);
const isManualCopy = first(fields.manualFileCopy) === "true";
const title = (first(fields.title) as string) || undefined;
const playlistId = (first(fields.playlistId) as string) || undefined;
try {
// Stream the file directly to disk without buffering
// file.stream() returns a Web ReadableStream, convert to Node.js stream
const nodeStream = file.stream() as any;
await pipeline(nodeStream, writeStream);
// Ensure the file is readable by all processes (mode 644)
await fs.promises.chmod(dest, 0o644);
return { filename, dest };
} catch (err) {
// Clean up the partially written file if stream fails
try {
await fs.promises.unlink(dest);
} catch (_) {}
throw err;
}
}
// helper to save a thumbnail buffer to UPLOADS_DIR/thumbnails
async function saveThumbBuffer(buffer: Buffer, ext = "jpg") {
const filename = `${Date.now()}-thumb.${ext.replace(/^\./, "")}`;
const dest = path.join(UPLOADS_DIR, "thumbnails", filename);
await fs.promises.mkdir(path.dirname(dest), { recursive: true });
await fs.promises.writeFile(dest, buffer);
// Ensure the file is readable by all processes (mode 644)
await fs.promises.chmod(dest, 0o644);
return { filename, dest };
}
let title: string | undefined;
let playlistId: string | undefined;
let durationSec: number | null = null;
let savedFilename: string | undefined;
// **IMPORTANT**: single thumbnailUrl used in both branches
let thumbnailUrl: string | null = null;
if (!nodeReq) {
// Request.formData() flow (some Next.js environments)
let formData: FormData | null = null;
try {
formData = await req.formData();
} catch (e: any) {
console.error("Failed to parse body as FormData.", e, "content-type=", req.headers.get("content-type"));
return NextResponse.json(
{ error: "Failed to parse body as FormData. Ensure request is sent with multipart/form-data." },
{ status: 400 }
);
}
const isManualCopy = formData.get("manualFileCopy") === "true";
const file = isManualCopy ? null : (formData.get("file") as Blob | null);
title = (formData.get("title") as string) || undefined;
playlistId = (formData.get("playlistId") as string) || undefined;
const durationField = formData.get("durationSec") as string | null;
const durationField = first(fields.durationSec) ?? first(fields.duration);
if (durationField) {
const n = Number(durationField);
if (!isNaN(n)) durationSec = Math.round(n);
}
// thumbnail (optional)
const thumb = formData.get("thumbnail") as Blob | null;
if (thumb) {
let savedFilename: string | undefined;
let thumbnailUrl: string | null = null;
// thumbnail (optional, non-fatal on failure)
if (thumbFile) {
try {
const thumbArrayBuffer = await thumb.arrayBuffer();
const thumbBuffer = Buffer.from(thumbArrayBuffer);
// try to infer extension from name, fallback to jpg
const fName = (thumb as any).name ?? "";
const extMatch = fName.match(/\.([a-z0-9]+)$/i);
const ext = extMatch ? extMatch[1] : "jpg";
const saved = await saveThumbBuffer(thumbBuffer, ext);
thumbnailUrl = `/api/thumbnails/${saved.filename}`;
savedThumbDest = saved.dest;
const originalThumbName = thumbFile.originalFilename || path.basename(thumbFile.filepath);
const tExt = path.extname(originalThumbName) || ".jpg";
const thumbFilename = `${Date.now()}-thumb${tExt}`;
const thumbDest = path.join(UPLOADS_DIR, "thumbnails", thumbFilename);
await moveFile(thumbFile.filepath, thumbDest);
thumbnailUrl = `/api/thumbnails/${thumbFilename}`;
savedThumbDest = thumbDest;
} catch (err) {
console.warn("thumbnail save failed (formData)", err);
// not fatal — we simply leave thumbnailUrl null
console.warn("thumbnail save failed", err);
}
}
if (!isManualCopy) {
if (!file) return NextResponse.json({ error: "no file" }, { status: 400 });
if (!videoFile) return NextResponse.json({ error: "no file" }, { status: 400 });
// Stream the large file directly to disk without buffering
const origName = (file as any).name ?? `upload-${Date.now()}.mp4`;
const saved = await saveVideoStream(file as File, origName);
savedFilename = saved.filename;
savedVideoDest = saved.dest;
} else {
// Manual copy mode: don't save file, just mark for manual copy
console.log('[Upload] Manual file copy mode enabled');
savedFilename = undefined;
savedVideoDest = undefined;
}
} else {
// formidable flow (Node IncomingMessage available)
console.log('[Upload] Using formidable flow for file upload');
const { fields, files } = await parseForm(nodeReq as IncomingMessage);
console.log('[Upload] Formidable parsing complete, fields:', Object.keys(fields), 'files:', Object.keys(files));
const f: FormidableFile | undefined =
(files && (files.file as FormidableFile)) || (files && Object.values(files)[0]);
if (!f) return NextResponse.json({ error: "no file found" }, { status: 400 });
// handle thumbnail file if present in formidable files
const thumbFile = (files && (files.thumbnail as FormidableFile)) || undefined;
if (thumbFile) {
try {
const tPath = (thumbFile as any).filepath || (thumbFile as any).path;
const originalThumbName = (thumbFile as any).originalFilename || path.basename(tPath);
const tExt = path.extname(originalThumbName) || ".jpg";
const thumbFilename = `${Date.now()}-thumb${tExt}`;
const thumbDest = path.join(UPLOADS_DIR, "thumbnails", thumbFilename);
await fs.promises.mkdir(path.dirname(thumbDest), { recursive: true });
await fs.promises.copyFile(tPath, thumbDest);
// Ensure the file is readable by all processes (mode 644)
await fs.promises.chmod(thumbDest, 0o644);
thumbnailUrl = `/api/thumbnails/${thumbFilename}`;
savedThumbDest = thumbDest;
} catch (err) {
console.warn("thumbnail save failed (formidable)", err);
}
}
// copy video file to UPLOADS_DIR/videos
const filePath = (f as any).filepath || (f as any).path;
const originalFilename = (f as any).originalFilename || (f as any).name || path.basename(filePath);
const originalFilename = videoFile.originalFilename || path.basename(videoFile.filepath);
const filename = `${Date.now()}-${originalFilename}`;
const dest = path.join(UPLOADS_DIR, "videos", filename);
console.log('[Upload] Copying video file', { size: (f as any).size, path: filePath, dest });
await fs.promises.mkdir(path.dirname(dest), { recursive: true });
await fs.promises.copyFile(filePath, dest);
// Ensure the file is readable by all processes (mode 644)
await fs.promises.chmod(dest, 0o644);
console.log('[Upload] Video file copied successfully');
title = fields.title ?? originalFilename;
playlistId = fields.playlistId;
const durationField = fields.durationSec ?? fields.duration ?? null;
if (durationField) {
const n = Number(durationField);
if (!isNaN(n)) durationSec = Math.round(n);
}
console.log('[Upload] Moving video file', { size: videoFile.size, path: videoFile.filepath, dest });
await moveFile(videoFile.filepath, dest);
console.log('[Upload] Video file moved successfully');
savedFilename = filename;
savedVideoDest = dest;
} else {
// Manual copy mode: don't save file, just mark for manual copy
console.log('[Upload] Manual file copy mode enabled');
if (videoFile) await fs.promises.unlink(videoFile.filepath).catch(() => {});
}
// validate playlist: avoid FK errors
@@ -386,6 +329,10 @@ export async function POST(req: Request) {
await fs.promises.unlink(savedThumbDest);
} catch (_) {}
}
// cleanup any formidable temp files that never got moved
for (const p of tempPaths) {
await fs.promises.unlink(p).catch(() => {});
}
const status = err?.status ?? 500;
const message = err?.message ?? (err?.toString ? err.toString() : "server error");