From 4cd8986e69c6ad30e03df5499ea7afd9c4e671af Mon Sep 17 00:00:00 2001 From: twotalesanimation <80506065+twotalesanimation@users.noreply.github.com> Date: Sat, 1 Aug 2026 15:54:57 +0200 Subject: [PATCH] Stream video uploads to disk instead of buffering in memory req.formData() parsed the entire multipart body into RAM, so large uploads (1.6GB+) OOM-killed the server. Bridge the web Request body to a Node stream and parse it with formidable, which writes the file to disk as it arrives: - Temp dir is /uploads/tmp (same filesystem as the destination) so the post-parse move is an atomic rename, not a second multi-GB write - Mark the bridged request as chunked when a proxy strips content-length, otherwise formidable assumes an empty body - Track formidable temp files via fileBegin so aborted uploads get their partial files cleaned up - Remove the dead formidable branch gated on (req as any).req, which is always null in the App Router Also fix the nginx example in VPS_UPLOAD_CONFIG.md: client_max_body_size was 1024M, below the app's 2.5GB limit, and would 413 large uploads at the proxy before they reached the app. Co-Authored-By: Claude Fable 5 --- VPS_UPLOAD_CONFIG.md | 7 +- app/api/admin/upload/route.ts | 257 ++++++++++++++-------------------- 2 files changed, 106 insertions(+), 158 deletions(-) diff --git a/VPS_UPLOAD_CONFIG.md b/VPS_UPLOAD_CONFIG.md index a732a07..f75da84 100644 --- a/VPS_UPLOAD_CONFIG.md +++ b/VPS_UPLOAD_CONFIG.md @@ -63,9 +63,10 @@ If running with a proxy (nginx/Apache), ensure: location /api/admin/upload { proxy_pass http://next-server; proxy_connect_timeout 60s; - proxy_send_timeout 300s; # 5 minutes for large uploads - proxy_read_timeout 300s; # 5 minutes for large uploads - client_max_body_size 1024M; # Adjust based on your max video size + proxy_send_timeout 2700s; # 45 minutes, matches maxDuration in the route + proxy_read_timeout 2700s; # 45 minutes, matches maxDuration in the route + client_max_body_size 3G; # Must exceed the 2.5GB app-level limit + proxy_request_buffering off; # Stream to the app instead of spooling to nginx disk } ``` diff --git a/app/api/admin/upload/route.ts b/app/api/admin/upload/route.ts index 7e73a5a..9f4c11e 100644 --- a/app/api/admin/upload/route.ts +++ b/app/api/admin/upload/route.ts @@ -2,10 +2,10 @@ import { NextResponse } from "next/server"; import fs from "fs"; import path from "path"; +import { Readable } from "stream"; import { IncomingMessage } from "http"; import { spawn } from "child_process"; -import { pipeline } from "stream/promises"; -import formidable, { File as FormidableFile } from "formidable"; +import formidable, { Fields, Files, File as FormidableFile } from "formidable"; import { prisma } from "@/lib/prisma"; import { getServerSession } from "next-auth"; import { authOptions } from "@/lib/auth-options"; @@ -16,6 +16,10 @@ export const maxDuration = 2700; // 45 minutes for very large uploads over slow // Use ConfigurableUPLOADS_DIR from environment or default to /uploads const UPLOADS_DIR = process.env.UPLOADS_DIR || "/uploads"; +// Temp dir must live on the same filesystem as the final destination so the +// post-parse rename is a cheap atomic move instead of a multi-GB copy +const TMP_DIR = path.join(UPLOADS_DIR, "tmp"); + // Helper to extract video duration using FFprobe async function getVideoDurationFromFile(filePath: string): Promise { return new Promise((resolve) => { @@ -65,25 +69,71 @@ async function getVideoDurationFromFile(filePath: string): Promise { - const form = formidable({ +// Formidable expects a Node IncomingMessage, but App Router route handlers get +// a web Request. Bridging the body stream lets formidable write the multipart +// payload to disk as it arrives — req.formData() would buffer the whole file +// (1.6GB+) in memory and OOM the process. +function toNodeRequest(req: Request): IncomingMessage { + if (!req.body) throw { status: 400, message: "empty request body" }; + const nodeStream = Readable.fromWeb(req.body as any) as any; + const headers = Object.fromEntries(req.headers.entries()); + // Formidable treats a missing content-length without transfer-encoding as an + // empty body (DummyParser); proxies that re-chunk uploads strip content-length + if (!headers["content-length"] && headers["transfer-encoding"] === undefined) { + headers["transfer-encoding"] = "chunked"; + } + nodeStream.headers = headers; + nodeStream.method = "POST"; + return nodeStream as IncomingMessage; +} + +// formidable v3 returns every field/file as an array +function first(value: T | T[] | undefined): T | undefined { + return Array.isArray(value) ? value[0] : value; +} + +async function parseForm(req: Request, tempPaths: string[]): Promise<{ fields: Fields; files: Files }> { + await fs.promises.mkdir(TMP_DIR, { recursive: true }); + const form = formidable({ multiples: false, + uploadDir: TMP_DIR, maxFileSize: 2.5 * 1024 * 1024 * 1024, // 2.5GB max file size maxFieldsSize: 10 * 1024 * 1024, // 10MB for all fields combined maxFields: 50, keepExtensions: true, }); + // record temp paths as files start writing, so aborted uploads still get cleaned up + form.on("fileBegin", (_name, file) => { + if (file.filepath) tempPaths.push(file.filepath); + }); return new Promise((resolve, reject) => { - form.parse(req, (err, fields, files) => { + form.parse(toNodeRequest(req), (err, fields, files) => { if (err) { - console.error('[Upload] Formidable parse error:', err.code, err.message); + console.error("[Upload] Formidable parse error:", err.code, err.message); reject(err); - } - else resolve({ fields, files }); + } else resolve({ fields, files }); }); }); } +// rename() is atomic when TMP_DIR shares a filesystem with dest; fall back to +// copy+unlink if the volumes are ever split across devices +async function moveFile(src: string, dest: string) { + await fs.promises.mkdir(path.dirname(dest), { recursive: true }); + try { + await fs.promises.rename(src, dest); + } catch (err: any) { + if (err?.code === "EXDEV") { + await fs.promises.copyFile(src, dest); + await fs.promises.unlink(src).catch(() => {}); + } else { + throw err; + } + } + // Ensure the file is readable by all processes (mode 644) + await fs.promises.chmod(dest, 0o644); +} + async function checkAdmin() { const session = await getServerSession(authOptions); if (!session?.user?.email) throw { status: 401, message: "Unauthorized" }; @@ -100,175 +150,68 @@ export async function POST(req: Request) { // track saved paths so we can cleanup on error let savedVideoDest: string | undefined; let savedThumbDest: string | undefined; + // formidable temp files not yet moved to their final location + const tempPaths: string[] = []; try { console.log('[Upload] Request received, content-type:', req.headers.get("content-type")); - - const nodeReq = (req as any).req ?? (globalThis as any).__NEXT_INIT?.req ?? null; + const session = await checkAdmin(); - + console.log('[Upload] Admin check passed for:', session.user?.email); - // helper to stream a File directly to disk (handles large files efficiently) - async function saveVideoStream(file: File, origName?: string) { - const filename = `${Date.now()}-${String(origName ?? "upload.mp4")}`; - const dest = path.join(UPLOADS_DIR, "videos", filename); - - // Ensure directory exists - await fs.promises.mkdir(path.dirname(dest), { recursive: true }); - - // Create write stream to destination - const writeStream = fs.createWriteStream(dest); - - try { - // Stream the file directly to disk without buffering - // file.stream() returns a Web ReadableStream, convert to Node.js stream - const nodeStream = file.stream() as any; - await pipeline(nodeStream, writeStream); - - // Ensure the file is readable by all processes (mode 644) - await fs.promises.chmod(dest, 0o644); - - return { filename, dest }; - } catch (err) { - // Clean up the partially written file if stream fails - try { - await fs.promises.unlink(dest); - } catch (_) {} - throw err; - } - } + const { fields, files } = await parseForm(req, tempPaths); + console.log('[Upload] Parsing complete, fields:', Object.keys(fields), 'files:', Object.keys(files)); - // helper to save a thumbnail buffer to UPLOADS_DIR/thumbnails - async function saveThumbBuffer(buffer: Buffer, ext = "jpg") { - const filename = `${Date.now()}-thumb.${ext.replace(/^\./, "")}`; - const dest = path.join(UPLOADS_DIR, "thumbnails", filename); - await fs.promises.mkdir(path.dirname(dest), { recursive: true }); - await fs.promises.writeFile(dest, buffer); - // Ensure the file is readable by all processes (mode 644) - await fs.promises.chmod(dest, 0o644); - return { filename, dest }; - } + const videoFile = first(files.file) as FormidableFile | undefined; + const thumbFile = first(files.thumbnail) as FormidableFile | undefined; + + const isManualCopy = first(fields.manualFileCopy) === "true"; + const title = (first(fields.title) as string) || undefined; + const playlistId = (first(fields.playlistId) as string) || undefined; - let title: string | undefined; - let playlistId: string | undefined; let durationSec: number | null = null; - let savedFilename: string | undefined; + const durationField = first(fields.durationSec) ?? first(fields.duration); + if (durationField) { + const n = Number(durationField); + if (!isNaN(n)) durationSec = Math.round(n); + } - // **IMPORTANT**: single thumbnailUrl used in both branches + let savedFilename: string | undefined; let thumbnailUrl: string | null = null; - if (!nodeReq) { - // Request.formData() flow (some Next.js environments) - let formData: FormData | null = null; + // thumbnail (optional, non-fatal on failure) + if (thumbFile) { try { - formData = await req.formData(); - } catch (e: any) { - console.error("Failed to parse body as FormData.", e, "content-type=", req.headers.get("content-type")); - return NextResponse.json( - { error: "Failed to parse body as FormData. Ensure request is sent with multipart/form-data." }, - { status: 400 } - ); + const originalThumbName = thumbFile.originalFilename || path.basename(thumbFile.filepath); + const tExt = path.extname(originalThumbName) || ".jpg"; + const thumbFilename = `${Date.now()}-thumb${tExt}`; + const thumbDest = path.join(UPLOADS_DIR, "thumbnails", thumbFilename); + await moveFile(thumbFile.filepath, thumbDest); + thumbnailUrl = `/api/thumbnails/${thumbFilename}`; + savedThumbDest = thumbDest; + } catch (err) { + console.warn("thumbnail save failed", err); } + } - const isManualCopy = formData.get("manualFileCopy") === "true"; - const file = isManualCopy ? null : (formData.get("file") as Blob | null); - title = (formData.get("title") as string) || undefined; - playlistId = (formData.get("playlistId") as string) || undefined; - const durationField = formData.get("durationSec") as string | null; - if (durationField) { - const n = Number(durationField); - if (!isNaN(n)) durationSec = Math.round(n); - } + if (!isManualCopy) { + if (!videoFile) return NextResponse.json({ error: "no file" }, { status: 400 }); - // thumbnail (optional) - const thumb = formData.get("thumbnail") as Blob | null; - if (thumb) { - try { - const thumbArrayBuffer = await thumb.arrayBuffer(); - const thumbBuffer = Buffer.from(thumbArrayBuffer); - // try to infer extension from name, fallback to jpg - const fName = (thumb as any).name ?? ""; - const extMatch = fName.match(/\.([a-z0-9]+)$/i); - const ext = extMatch ? extMatch[1] : "jpg"; - const saved = await saveThumbBuffer(thumbBuffer, ext); - thumbnailUrl = `/api/thumbnails/${saved.filename}`; - savedThumbDest = saved.dest; - } catch (err) { - console.warn("thumbnail save failed (formData)", err); - // not fatal — we simply leave thumbnailUrl null - } - } - - if (!isManualCopy) { - if (!file) return NextResponse.json({ error: "no file" }, { status: 400 }); - - // Stream the large file directly to disk without buffering - const origName = (file as any).name ?? `upload-${Date.now()}.mp4`; - const saved = await saveVideoStream(file as File, origName); - savedFilename = saved.filename; - savedVideoDest = saved.dest; - } else { - // Manual copy mode: don't save file, just mark for manual copy - console.log('[Upload] Manual file copy mode enabled'); - savedFilename = undefined; - savedVideoDest = undefined; - } - } else { - // formidable flow (Node IncomingMessage available) - console.log('[Upload] Using formidable flow for file upload'); - const { fields, files } = await parseForm(nodeReq as IncomingMessage); - console.log('[Upload] Formidable parsing complete, fields:', Object.keys(fields), 'files:', Object.keys(files)); - - const f: FormidableFile | undefined = - (files && (files.file as FormidableFile)) || (files && Object.values(files)[0]); - if (!f) return NextResponse.json({ error: "no file found" }, { status: 400 }); - - // handle thumbnail file if present in formidable files - const thumbFile = (files && (files.thumbnail as FormidableFile)) || undefined; - if (thumbFile) { - try { - const tPath = (thumbFile as any).filepath || (thumbFile as any).path; - const originalThumbName = (thumbFile as any).originalFilename || path.basename(tPath); - const tExt = path.extname(originalThumbName) || ".jpg"; - const thumbFilename = `${Date.now()}-thumb${tExt}`; - const thumbDest = path.join(UPLOADS_DIR, "thumbnails", thumbFilename); - await fs.promises.mkdir(path.dirname(thumbDest), { recursive: true }); - await fs.promises.copyFile(tPath, thumbDest); - // Ensure the file is readable by all processes (mode 644) - await fs.promises.chmod(thumbDest, 0o644); - thumbnailUrl = `/api/thumbnails/${thumbFilename}`; - savedThumbDest = thumbDest; - } catch (err) { - console.warn("thumbnail save failed (formidable)", err); - } - } - - // copy video file to UPLOADS_DIR/videos - const filePath = (f as any).filepath || (f as any).path; - const originalFilename = (f as any).originalFilename || (f as any).name || path.basename(filePath); + const originalFilename = videoFile.originalFilename || path.basename(videoFile.filepath); const filename = `${Date.now()}-${originalFilename}`; const dest = path.join(UPLOADS_DIR, "videos", filename); - - console.log('[Upload] Copying video file', { size: (f as any).size, path: filePath, dest }); - - await fs.promises.mkdir(path.dirname(dest), { recursive: true }); - await fs.promises.copyFile(filePath, dest); - // Ensure the file is readable by all processes (mode 644) - await fs.promises.chmod(dest, 0o644); - - console.log('[Upload] Video file copied successfully'); - title = fields.title ?? originalFilename; - playlistId = fields.playlistId; - const durationField = fields.durationSec ?? fields.duration ?? null; - if (durationField) { - const n = Number(durationField); - if (!isNaN(n)) durationSec = Math.round(n); - } + console.log('[Upload] Moving video file', { size: videoFile.size, path: videoFile.filepath, dest }); + await moveFile(videoFile.filepath, dest); + console.log('[Upload] Video file moved successfully'); savedFilename = filename; savedVideoDest = dest; + } else { + // Manual copy mode: don't save file, just mark for manual copy + console.log('[Upload] Manual file copy mode enabled'); + if (videoFile) await fs.promises.unlink(videoFile.filepath).catch(() => {}); } // validate playlist: avoid FK errors @@ -386,6 +329,10 @@ export async function POST(req: Request) { await fs.promises.unlink(savedThumbDest); } catch (_) {} } + // cleanup any formidable temp files that never got moved + for (const p of tempPaths) { + await fs.promises.unlink(p).catch(() => {}); + } const status = err?.status ?? 500; const message = err?.message ?? (err?.toString ? err.toString() : "server error");