Files
twotalesanimation 4cd8986e69 Stream video uploads to disk instead of buffering in memory
req.formData() parsed the entire multipart body into RAM, so large
uploads (1.6GB+) OOM-killed the server. Bridge the web Request body to
a Node stream and parse it with formidable, which writes the file to
disk as it arrives:

- Temp dir is /uploads/tmp (same filesystem as the destination) so the
  post-parse move is an atomic rename, not a second multi-GB write
- Mark the bridged request as chunked when a proxy strips
  content-length, otherwise formidable assumes an empty body
- Track formidable temp files via fileBegin so aborted uploads get
  their partial files cleaned up
- Remove the dead formidable branch gated on (req as any).req, which
  is always null in the App Router

Also fix the nginx example in VPS_UPLOAD_CONFIG.md: client_max_body_size
was 1024M, below the app's 2.5GB limit, and would 413 large uploads at
the proxy before they reached the app.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-01 15:54:57 +02:00

342 lines
12 KiB
TypeScript

// app/api/admin/upload/route.ts
import { NextResponse } from "next/server";
import fs from "fs";
import path from "path";
import { Readable } from "stream";
import { IncomingMessage } from "http";
import { spawn } from "child_process";
import formidable, { Fields, Files, File as FormidableFile } from "formidable";
import { prisma } from "@/lib/prisma";
import { getServerSession } from "next-auth";
import { authOptions } from "@/lib/auth-options";
// For 2GB uploads over Tailscale (~10-50 Mbps), need 30-45 minutes
export const maxDuration = 2700; // 45 minutes for very large uploads over slow connections
// Use ConfigurableUPLOADS_DIR from environment or default to /uploads
const UPLOADS_DIR = process.env.UPLOADS_DIR || "/uploads";
// Temp dir must live on the same filesystem as the final destination so the
// post-parse rename is a cheap atomic move instead of a multi-GB copy
const TMP_DIR = path.join(UPLOADS_DIR, "tmp");
// Helper to extract video duration using FFprobe
async function getVideoDurationFromFile(filePath: string): Promise<number | null> {
return new Promise((resolve) => {
try {
const ffprobe = spawn("ffprobe", [
"-v", "error",
"-show_entries", "format=duration",
"-of", "default=noprint_wrappers=1:nokey=1:nokey=1",
filePath,
]);
let output = "";
let timedOut = false;
// Large 2GB files may take longer to scan; timeout after 30s
const timeoutId = setTimeout(() => {
timedOut = true;
ffprobe.kill();
resolve(null); // Return null instead of blocking
}, 30000);
ffprobe.stdout.on("data", (data) => {
output += data.toString();
});
ffprobe.on("close", (code) => {
clearTimeout(timeoutId);
if (!timedOut && code === 0) {
const duration = parseFloat(output.trim());
if (!isNaN(duration) && isFinite(duration) && duration > 0) {
resolve(Math.round(duration));
} else {
resolve(null);
}
} else {
resolve(null);
}
});
ffprobe.on("error", () => {
clearTimeout(timeoutId);
resolve(null);
});
} catch (err) {
resolve(null);
}
});
}
// Formidable expects a Node IncomingMessage, but App Router route handlers get
// a web Request. Bridging the body stream lets formidable write the multipart
// payload to disk as it arrives — req.formData() would buffer the whole file
// (1.6GB+) in memory and OOM the process.
function toNodeRequest(req: Request): IncomingMessage {
if (!req.body) throw { status: 400, message: "empty request body" };
const nodeStream = Readable.fromWeb(req.body as any) as any;
const headers = Object.fromEntries(req.headers.entries());
// Formidable treats a missing content-length without transfer-encoding as an
// empty body (DummyParser); proxies that re-chunk uploads strip content-length
if (!headers["content-length"] && headers["transfer-encoding"] === undefined) {
headers["transfer-encoding"] = "chunked";
}
nodeStream.headers = headers;
nodeStream.method = "POST";
return nodeStream as IncomingMessage;
}
// formidable v3 returns every field/file as an array
function first<T>(value: T | T[] | undefined): T | undefined {
return Array.isArray(value) ? value[0] : value;
}
async function parseForm(req: Request, tempPaths: string[]): Promise<{ fields: Fields; files: Files }> {
await fs.promises.mkdir(TMP_DIR, { recursive: true });
const form = formidable({
multiples: false,
uploadDir: TMP_DIR,
maxFileSize: 2.5 * 1024 * 1024 * 1024, // 2.5GB max file size
maxFieldsSize: 10 * 1024 * 1024, // 10MB for all fields combined
maxFields: 50,
keepExtensions: true,
});
// record temp paths as files start writing, so aborted uploads still get cleaned up
form.on("fileBegin", (_name, file) => {
if (file.filepath) tempPaths.push(file.filepath);
});
return new Promise((resolve, reject) => {
form.parse(toNodeRequest(req), (err, fields, files) => {
if (err) {
console.error("[Upload] Formidable parse error:", err.code, err.message);
reject(err);
} else resolve({ fields, files });
});
});
}
// rename() is atomic when TMP_DIR shares a filesystem with dest; fall back to
// copy+unlink if the volumes are ever split across devices
async function moveFile(src: string, dest: string) {
await fs.promises.mkdir(path.dirname(dest), { recursive: true });
try {
await fs.promises.rename(src, dest);
} catch (err: any) {
if (err?.code === "EXDEV") {
await fs.promises.copyFile(src, dest);
await fs.promises.unlink(src).catch(() => {});
} else {
throw err;
}
}
// Ensure the file is readable by all processes (mode 644)
await fs.promises.chmod(dest, 0o644);
}
async function checkAdmin() {
const session = await getServerSession(authOptions);
if (!session?.user?.email) throw { status: 401, message: "Unauthorized" };
const allowed = (process.env.ALLOWED_ADMINS || "")
.split(",")
.map((s) => s.trim())
.filter(Boolean);
if (allowed.length && !allowed.includes(session.user.email))
throw { status: 403, message: "Forbidden" };
return session;
}
export async function POST(req: Request) {
// track saved paths so we can cleanup on error
let savedVideoDest: string | undefined;
let savedThumbDest: string | undefined;
// formidable temp files not yet moved to their final location
const tempPaths: string[] = [];
try {
console.log('[Upload] Request received, content-type:', req.headers.get("content-type"));
const session = await checkAdmin();
console.log('[Upload] Admin check passed for:', session.user?.email);
const { fields, files } = await parseForm(req, tempPaths);
console.log('[Upload] Parsing complete, fields:', Object.keys(fields), 'files:', Object.keys(files));
const videoFile = first(files.file) as FormidableFile | undefined;
const thumbFile = first(files.thumbnail) as FormidableFile | undefined;
const isManualCopy = first(fields.manualFileCopy) === "true";
const title = (first(fields.title) as string) || undefined;
const playlistId = (first(fields.playlistId) as string) || undefined;
let durationSec: number | null = null;
const durationField = first(fields.durationSec) ?? first(fields.duration);
if (durationField) {
const n = Number(durationField);
if (!isNaN(n)) durationSec = Math.round(n);
}
let savedFilename: string | undefined;
let thumbnailUrl: string | null = null;
// thumbnail (optional, non-fatal on failure)
if (thumbFile) {
try {
const originalThumbName = thumbFile.originalFilename || path.basename(thumbFile.filepath);
const tExt = path.extname(originalThumbName) || ".jpg";
const thumbFilename = `${Date.now()}-thumb${tExt}`;
const thumbDest = path.join(UPLOADS_DIR, "thumbnails", thumbFilename);
await moveFile(thumbFile.filepath, thumbDest);
thumbnailUrl = `/api/thumbnails/${thumbFilename}`;
savedThumbDest = thumbDest;
} catch (err) {
console.warn("thumbnail save failed", err);
}
}
if (!isManualCopy) {
if (!videoFile) return NextResponse.json({ error: "no file" }, { status: 400 });
const originalFilename = videoFile.originalFilename || path.basename(videoFile.filepath);
const filename = `${Date.now()}-${originalFilename}`;
const dest = path.join(UPLOADS_DIR, "videos", filename);
console.log('[Upload] Moving video file', { size: videoFile.size, path: videoFile.filepath, dest });
await moveFile(videoFile.filepath, dest);
console.log('[Upload] Video file moved successfully');
savedFilename = filename;
savedVideoDest = dest;
} else {
// Manual copy mode: don't save file, just mark for manual copy
console.log('[Upload] Manual file copy mode enabled');
if (videoFile) await fs.promises.unlink(videoFile.filepath).catch(() => {});
}
// validate playlist: avoid FK errors
if (!playlistId) {
// cleanup if needed
if (savedVideoDest) await fs.promises.unlink(savedVideoDest).catch(() => {});
if (savedThumbDest) await fs.promises.unlink(savedThumbDest).catch(() => {});
return NextResponse.json({ error: "playlistId required" }, { status: 400 });
}
const playlist = await prisma.playlist.findUnique({ where: { id: playlistId } });
if (!playlist) {
if (savedVideoDest) await fs.promises.unlink(savedVideoDest).catch(() => {});
if (savedThumbDest) await fs.promises.unlink(savedThumbDest).catch(() => {});
return NextResponse.json({ error: "playlist not found" }, { status: 400 });
}
// Get current user
const user = await prisma.user.findUnique({
where: { email: session.user?.email ?? "" },
select: { id: true },
});
if (!user) {
if (savedVideoDest) await fs.promises.unlink(savedVideoDest).catch(() => {});
if (savedThumbDest) await fs.promises.unlink(savedThumbDest).catch(() => {});
return NextResponse.json({ error: "user not found" }, { status: 400 });
}
// If duration not provided by client, try to extract it from the saved video file
if (durationSec === null && savedVideoDest) {
try {
const extractedDuration = await getVideoDurationFromFile(savedVideoDest);
if (extractedDuration !== null) {
durationSec = extractedDuration;
console.log(`[Upload] Extracted duration: ${durationSec}s from ${savedVideoDest}`);
} else {
console.warn(`[Upload] Could not extract duration from ${savedVideoDest}`);
}
} catch (err) {
console.warn(`[Upload] Duration extraction error:`, err);
}
}
// For manual copy mode, check if we have a file
const isManualMode = !savedVideoDest;
// Calculate index: find max index in playlist and add 1
const maxIndexVideo = await prisma.video.findFirst({
where: { playlistId },
orderBy: { index: 'desc' },
select: { index: true },
});
const desiredIndex = (maxIndexVideo?.index ?? -1) + 1;
// First, create the video record to get its ID
const video = await prisma.video.create({
data: {
title: title ?? savedFilename ?? 'Untitled',
url: '', // Will be set below or after rename
thumbnail: thumbnailUrl,
index: desiredIndex,
playlistId,
userId: user.id,
transcodingStatus: isManualMode ? 'pending_manual_file' : 'uploaded',
...(durationSec !== null && { durationSec }),
},
});
// If manual copy mode, set the URL and return
if (isManualMode) {
const videoUrl = `/uploads/videos/${video.id}.mp4`;
await prisma.video.update({
where: { id: video.id },
data: { url: videoUrl },
});
console.log('[Upload] Manual copy mode: video created with ID', video.id, 'URL:', videoUrl);
return NextResponse.json({ video: { ...video, url: videoUrl } }, { status: 201 });
}
// Now rename the uploaded file to use the video ID
const finalFilename = `${video.id}.mp4`;
const finalDest = path.join(UPLOADS_DIR, "videos", finalFilename);
try {
if (savedVideoDest) {
await fs.promises.rename(savedVideoDest, finalDest);
}
// Update the URL in the database to reflect the final filename
await prisma.video.update({
where: { id: video.id },
data: { url: `/uploads/videos/${finalFilename}` },
});
} catch (err) {
console.error("Error renaming video file:", err);
// If rename fails, cleanup and delete the record
await prisma.video.delete({ where: { id: video.id } });
throw err;
}
return NextResponse.json({ video: { ...video, url: `/uploads/videos/${finalFilename}` } }, { status: 201 });
} catch (err: any) {
console.error("upload error", {
code: err?.code,
message: err?.message,
errno: err?.errno,
}, err);
// cleanup saved files if something failed
if (savedVideoDest) {
try {
await fs.promises.unlink(savedVideoDest);
} catch (_) {}
}
if (savedThumbDest) {
try {
await fs.promises.unlink(savedThumbDest);
} catch (_) {}
}
// cleanup any formidable temp files that never got moved
for (const p of tempPaths) {
await fs.promises.unlink(p).catch(() => {});
}
const status = err?.status ?? 500;
const message = err?.message ?? (err?.toString ? err.toString() : "server error");
return NextResponse.json({ error: message }, { status });
}
}