diff --git a/app/(dashboard)/settings/migration/page.tsx b/app/(dashboard)/settings/migration/page.tsx new file mode 100644 index 0000000..189936c --- /dev/null +++ b/app/(dashboard)/settings/migration/page.tsx @@ -0,0 +1,404 @@ +"use client"; + +import { useState, useEffect, useCallback, useMemo } from "react"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Badge } from "@/components/ui/badge"; +import { cn } from "@/lib/utils"; +import { + CheckCircle2, + AlertCircle, + Loader2, + RefreshCw, + Upload, + Search, + ChevronDown, +} from "lucide-react"; + +// ── Types ────────────────────────────────────────────────────────────────────── + +interface FileEntry { + key: string; + filename: string; + folder: string; + sizeBytes: number; + hetznerExists: boolean; +} + +type MigrateStatus = "idle" | "uploading" | "done" | "error"; + +function formatBytes(bytes: number) { + if (bytes >= 1024 ** 3) return `${(bytes / 1024 ** 3).toFixed(1)} GB`; + if (bytes >= 1024 ** 2) return `${(bytes / 1024 ** 2).toFixed(1)} MB`; + return `${(bytes / 1024).toFixed(1)} KB`; +} + +// ── Page ─────────────────────────────────────────────────────────────────────── + +export default function MigrationPage() { + const [files, setFiles] = useState([]); + const [loading, setLoading] = useState(true); + const [hetznerConfigured, setHetznerConfigured] = useState(true); + const [fetchError, setFetchError] = useState(null); + + // Selection: Set of keys + const [selected, setSelected] = useState>(new Set()); + + // Per-file migration status + const [statuses, setStatuses] = useState>(new Map()); + const [errors, setErrors] = useState>(new Map()); + + // Filters + const [search, setSearch] = useState(""); + const [folderFilter, setFolderFilter] = useState("all"); + const [folderMenuOpen, setFolderMenuOpen] = useState(false); + + const [migrating, setMigrating] = useState(false); + + // ── Load file list ────────────────────────────────────────────────────────── + + const loadFiles = useCallback(async () => { + setLoading(true); + setFetchError(null); + try { + const res = await fetch("/api/admin/migration"); + const data = await res.json(); + if (!res.ok) throw new Error(data.error ?? "Failed to load"); + setFiles(data.files); + setHetznerConfigured(data.hetznerConfigured); + // Pre-select all files NOT yet on Hetzner + const preSelected = new Set( + (data.files as FileEntry[]) + .filter((f) => !f.hetznerExists) + .map((f) => f.key) + ); + setSelected(preSelected); + setStatuses(new Map()); + setErrors(new Map()); + } catch (e: unknown) { + setFetchError(e instanceof Error ? e.message : "Failed to load file list"); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { loadFiles(); }, [loadFiles]); + + // ── Derived data ──────────────────────────────────────────────────────────── + + const folders = useMemo( + () => ["all", ...Array.from(new Set(files.map((f) => f.folder))).sort()], + [files] + ); + + const filtered = useMemo(() => { + return files.filter((f) => { + const matchFolder = folderFilter === "all" || f.folder === folderFilter; + const matchSearch = + !search || f.filename.toLowerCase().includes(search.toLowerCase()); + return matchFolder && matchSearch; + }); + }, [files, folderFilter, search]); + + const stats = useMemo(() => { + const synced = files.filter((f) => f.hetznerExists).length; + const total = files.length; + const totalBytes = files.reduce((a, f) => a + f.sizeBytes, 0); + const selectedBytes = files + .filter((f) => selected.has(f.key)) + .reduce((a, f) => a + f.sizeBytes, 0); + return { synced, total, totalBytes, selectedBytes }; + }, [files, selected]); + + // ── Selection helpers ─────────────────────────────────────────────────────── + + const toggleOne = (key: string) => { + setSelected((prev) => { + const next = new Set(prev); + next.has(key) ? next.delete(key) : next.add(key); + return next; + }); + }; + + const selectAll = () => + setSelected(new Set(filtered.map((f) => f.key))); + + const deselectAll = () => + setSelected((prev) => { + const next = new Set(prev); + filtered.forEach((f) => next.delete(f.key)); + return next; + }); + + const selectUnsynced = () => + setSelected(new Set(filtered.filter((f) => !f.hetznerExists).map((f) => f.key))); + + // ── Migration ─────────────────────────────────────────────────────────────── + + const handleMigrate = async () => { + const toMigrate = files.filter((f) => selected.has(f.key)); + if (!toMigrate.length) return; + setMigrating(true); + + for (const file of toMigrate) { + setStatuses((prev) => new Map(prev).set(file.key, "uploading")); + try { + const res = await fetch("/api/admin/migration", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ key: file.key }), + }); + const data = await res.json(); + if (!res.ok) throw new Error(data.error ?? "Upload failed"); + setStatuses((prev) => new Map(prev).set(file.key, "done")); + // Mark as synced in the file list + setFiles((prev) => + prev.map((f) => (f.key === file.key ? { ...f, hetznerExists: true } : f)) + ); + } catch (e: unknown) { + setStatuses((prev) => new Map(prev).set(file.key, "error")); + setErrors((prev) => + new Map(prev).set(file.key, e instanceof Error ? e.message : "Failed") + ); + } + } + + setMigrating(false); + }; + + // ── Render ────────────────────────────────────────────────────────────────── + + if (loading) { + return ( +
+ + Scanning files… +
+ ); + } + + if (fetchError) { + return ( +
+
+ + {fetchError} +
+ +
+ ); + } + + if (!hetznerConfigured) { + return ( +
+
+ + Hetzner object storage is not configured. Set{" "} + HETZNER_ENDPOINT,{" "} + HETZNER_ACCESS_KEY,{" "} + HETZNER_SECRET_KEY, and{" "} + HETZNER_BUCKET_NAME in your environment, + or configure them in Settings. +
+
+ ); + } + + const allFilteredSelected = filtered.length > 0 && filtered.every((f) => selected.has(f.key)); + + return ( +
+ {/* Header */} +
+
+

Storage Migration

+

+ {stats.total} local files · {formatBytes(stats.totalBytes)} total ·{" "} + {stats.synced} already on Hetzner +

+
+ +
+ + {/* Toolbar */} +
+ {/* Search */} +
+ + setSearch(e.target.value)} + className="pl-8 h-8 text-sm" + /> +
+ + {/* Folder filter */} +
+ + {folderMenuOpen && ( +
+ {folders.map((f) => ( + + ))} +
+ )} +
+ + {/* Bulk actions */} + + +
+ + {/* Table */} +
+ + + + + + + + + + + + {filtered.length === 0 && ( + + + + )} + {filtered.map((file) => { + const status = statuses.get(file.key); + const err = errors.get(file.key); + const isSelected = selected.has(file.key); + + return ( + !migrating && toggleOne(file.key)} + className={cn( + "transition-colors cursor-pointer", + isSelected ? "bg-primary/5 hover:bg-primary/10" : "hover:bg-muted/40" + )} + > + {/* Checkbox */} + + + {/* Filename */} + + + {/* Folder */} + + + {/* Size */} + + + {/* Status */} + + + ); + })} + +
+ + FilenameFolderSizeStatus
+ No files match your filters. +
e.stopPropagation()}> + !migrating && toggleOne(file.key)} + disabled={migrating} + className="rounded accent-primary" + /> + + {file.filename} + {err && ( + {err} + )} + {file.folder} + {formatBytes(file.sizeBytes)} + + {status === "uploading" ? ( + + + Uploading… + + ) : status === "done" || (status === undefined && file.hetznerExists) ? ( + + + Synced + + ) : status === "error" ? ( + + + Failed + + ) : ( + + Local only + + )} +
+
+ + {/* Footer action bar */} + {selected.size > 0 && ( +
+ + {selected.size} files selected ·{" "} + {formatBytes(stats.selectedBytes)} + + +
+ )} +
+ ); +} diff --git a/app/api/admin/migration/route.ts b/app/api/admin/migration/route.ts new file mode 100644 index 0000000..ca5ef9b --- /dev/null +++ b/app/api/admin/migration/route.ts @@ -0,0 +1,116 @@ +import { NextRequest, NextResponse } from "next/server"; +import { auth } from "@/auth"; +import fs from "fs"; +import path from "path"; +import { listHetznerKeys, uploadToHetznerWithKey } from "@/lib/storage"; + +export const maxDuration = 300; // 5 min — large video uploads + +const MIME_MAP: Record = { + ".mp4": "video/mp4", + ".mov": "video/quicktime", + ".avi": "video/x-msvideo", + ".mxf": "application/mxf", + ".webm": "video/webm", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".png": "image/png", + ".gif": "image/gif", + ".webp": "image/webp", +}; + +function walkDir( + dir: string, + base: string +): Array<{ key: string; sizeBytes: number }> { + const results: Array<{ key: string; sizeBytes: number }> = []; + if (!fs.existsSync(dir)) return results; + + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const abs = path.join(dir, entry.name); + if (entry.isDirectory()) { + results.push(...walkDir(abs, base)); + } else if (entry.isFile()) { + const key = path.relative(base, abs).replace(/\\/g, "/"); + results.push({ key, sizeBytes: fs.statSync(abs).size }); + } + } + return results; +} + +/** GET /api/admin/migration — list local files with Hetzner sync status */ +export async function GET() { + const session = await auth(); + if (!session?.user) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const uploadDir = path.resolve(process.env.LOCAL_UPLOAD_DIR ?? "./uploads"); + const localFiles = walkDir(uploadDir, uploadDir); + + let hetznerKeys = new Set(); + let hetznerConfigured = true; + try { + hetznerKeys = await listHetznerKeys(); + } catch { + hetznerConfigured = false; + } + + const files = localFiles.map(({ key, sizeBytes }) => ({ + key, + filename: path.basename(key), + folder: path.dirname(key).replace(/\\/g, "/") || "root", + sizeBytes, + hetznerExists: hetznerKeys.has(key), + })); + + return NextResponse.json({ files, hetznerConfigured }); +} + +/** POST /api/admin/migration — migrate a single file to Hetzner */ +export async function POST(req: NextRequest) { + const session = await auth(); + if (!session?.user) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const body = await req.json(); + const key = body?.key; + if (!key || typeof key !== "string") { + return NextResponse.json({ error: "key is required" }, { status: 400 }); + } + + const uploadDir = path.resolve(process.env.LOCAL_UPLOAD_DIR ?? "./uploads"); + const absPath = path.resolve(path.join(uploadDir, key)); + + // Path traversal guard + if (!absPath.startsWith(uploadDir + path.sep)) { + return NextResponse.json({ error: "Invalid key" }, { status: 403 }); + } + + let stat: fs.Stats; + try { + stat = fs.statSync(absPath); + } catch { + return NextResponse.json({ error: "File not found on disk" }, { status: 404 }); + } + + if (!stat.isFile()) { + return NextResponse.json({ error: "Not a file" }, { status: 400 }); + } + + const ext = path.extname(key).toLowerCase(); + const contentType = MIME_MAP[ext] ?? "application/octet-stream"; + + const start = Date.now(); + try { + const stream = fs.createReadStream(absPath); + await uploadToHetznerWithKey(stream, key, contentType, stat.size); + return NextResponse.json({ success: true, key, ms: Date.now() - start }); + } catch (e: unknown) { + return NextResponse.json( + { error: e instanceof Error ? e.message : "Upload failed", key }, + { status: 500 } + ); + } +} diff --git a/app/api/batch-upload/upload/route.ts b/app/api/batch-upload/upload/route.ts index 6789ca5..947c1f5 100644 --- a/app/api/batch-upload/upload/route.ts +++ b/app/api/batch-upload/upload/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from "next/server"; import { auth } from "@/auth"; import { db } from "@/lib/db"; -import { uploadFile, uploadToHetzner, deleteFromHetzner } from "@/lib/storage"; +import { uploadToHetzner, deleteFromHetzner } from "@/lib/storage"; import { recalcShotStatus } from "@/lib/shot-status"; export const maxDuration = 120; @@ -133,8 +133,9 @@ export async function POST( ); } - // Upload the video to the configured storage backend - const result = await uploadFile(buffer, file.name, file.type, "videos"); + // Upload the video to Hetzner object storage + const { key: videoKey } = await uploadToHetzner(buffer, file.name, file.type, "videos"); + const result = { url: `/api/files/${videoKey}`, key: videoKey }; // Mark all existing versions for this task as no longer latest await db.version.updateMany({ diff --git a/app/api/ext/shots/route.ts b/app/api/ext/shots/route.ts index 1f11755..0c7fcb0 100644 --- a/app/api/ext/shots/route.ts +++ b/app/api/ext/shots/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from "next/server"; import { db } from "@/lib/db"; -import { uploadFile } from "@/lib/storage"; +import { uploadToHetzner } from "@/lib/storage"; import { ShotPriority } from "@prisma/client"; import { z } from "zod"; @@ -169,8 +169,8 @@ export async function POST(req: NextRequest) { ); } const buffer = Buffer.from(await thumbnailFile.arrayBuffer()); - const result = await uploadFile(buffer, thumbnailFile.name, thumbnailFile.type, "image"); - thumbnailUrl = result.url; + const { key: thumbKey } = await uploadToHetzner(buffer, thumbnailFile.name, thumbnailFile.type, "image"); + thumbnailUrl = `/api/files/${thumbKey}`; } // Resolve shot group diff --git a/app/api/files/[...key]/route.ts b/app/api/files/[...key]/route.ts index ea99102..809b207 100644 --- a/app/api/files/[...key]/route.ts +++ b/app/api/files/[...key]/route.ts @@ -1,6 +1,7 @@ import { NextRequest, NextResponse } from "next/server"; import fs from "fs"; import path from "path"; +import { generateHetznerStreamUrl } from "@/lib/storage"; // ── Tuning constants ────────────────────────────────────────────────────────── // @@ -50,7 +51,15 @@ export async function GET( try { stat = fs.statSync(filePath); } catch { - return new NextResponse("Not found", { status: 404 }); + // File not on local disk — attempt a redirect to Hetzner object storage. + // This covers files uploaded after the migration and any migrated files + // whose local copies have been removed. + try { + const hetznerUrl = await generateHetznerStreamUrl(relativePath, 3600); + return NextResponse.redirect(hetznerUrl, 302); + } catch { + return new NextResponse("Not found", { status: 404 }); + } } // Guard against directory traversal that resolves to a directory diff --git a/app/api/shots/[shotId]/references/route.ts b/app/api/shots/[shotId]/references/route.ts index 708d99f..687c92c 100644 --- a/app/api/shots/[shotId]/references/route.ts +++ b/app/api/shots/[shotId]/references/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from "next/server"; import { auth } from "@/auth"; import { db } from "@/lib/db"; -import { uploadFile } from "@/lib/storage"; +import { uploadToHetzner } from "@/lib/storage"; function requireAuth(role: string) { return ["ADMIN", "PRODUCER", "SUPERVISOR"].includes(role); @@ -58,7 +58,8 @@ export async function POST( return NextResponse.json({ error: "File too large (max 50 MB)" }, { status: 413 }); const buffer = Buffer.from(await file.arrayBuffer()); - const uploaded = await uploadFile(buffer, file.name, file.type, "image"); + const { key: refKey } = await uploadToHetzner(buffer, file.name, file.type, "image"); + const uploaded = { url: `/api/files/${refKey}`, key: refKey }; const maxOrder = await db.shotReference.aggregate({ where: { shotId }, diff --git a/app/api/sketch-templates/route.ts b/app/api/sketch-templates/route.ts index a94c2b1..65183e0 100644 --- a/app/api/sketch-templates/route.ts +++ b/app/api/sketch-templates/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from "next/server"; import { auth } from "@/auth"; import { db } from "@/lib/db"; -import { uploadFile } from "@/lib/storage"; +import { uploadToHetzner } from "@/lib/storage"; function canManage(role: string) { return ["ADMIN", "PRODUCER", "SUPERVISOR"].includes(role); @@ -44,7 +44,8 @@ export async function POST(req: NextRequest) { return NextResponse.json({ error: "File too large (max 20 MB)" }, { status: 413 }); const buffer = Buffer.from(await file.arrayBuffer()); - const uploaded = await uploadFile(buffer, file.name, file.type, "image"); + const { key: tmplKey } = await uploadToHetzner(buffer, file.name, file.type, "image"); + const uploaded = { url: `/api/files/${tmplKey}`, key: tmplKey }; const maxOrder = await db.sketchTemplate.aggregate({ _max: { sortOrder: true } }); const sortOrder = (maxOrder._max.sortOrder ?? -1) + 1; diff --git a/app/api/upload/local/route.ts b/app/api/upload/local/route.ts index df79be0..4ca09ef 100644 --- a/app/api/upload/local/route.ts +++ b/app/api/upload/local/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from "next/server"; import { auth } from "@/auth"; -import { uploadFile } from "@/lib/storage"; +import { uploadToHetzner } from "@/lib/storage"; export const config = { api: { bodyParser: false } }; @@ -30,9 +30,9 @@ export async function POST(req: NextRequest) { } const buffer = Buffer.from(await file.arrayBuffer()); - const result = await uploadFile(buffer, file.name, file.type, "videos"); + const { key } = await uploadToHetzner(buffer, file.name, file.type, "videos"); - return NextResponse.json({ url: result.url, key: result.key }); + return NextResponse.json({ url: `/api/files/${key}`, key }); } catch (err) { console.error("[local-upload]", err); return NextResponse.json({ error: "Upload failed" }, { status: 500 }); diff --git a/app/api/upload/route.ts b/app/api/upload/route.ts index d7788d3..b9d83fe 100644 --- a/app/api/upload/route.ts +++ b/app/api/upload/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from "next/server"; import { auth } from "@/auth"; -import { uploadFile } from "@/lib/storage"; +import { uploadToHetzner } from "@/lib/storage"; export const config = { api: { bodyParser: false } }; @@ -41,9 +41,9 @@ export async function POST(req: NextRequest) { } const buffer = Buffer.from(await file.arrayBuffer()); - const result = await uploadFile(buffer, file.name, file.type, type); + const { key } = await uploadToHetzner(buffer, file.name, file.type, type); - return NextResponse.json({ url: result.url, key: result.key }); + return NextResponse.json({ url: `/api/files/${key}`, key }); } catch (err) { console.error("[upload]", err); return NextResponse.json({ error: "Upload failed" }, { status: 500 }); diff --git a/lib/storage.ts b/lib/storage.ts index 20908b6..53cddd3 100644 --- a/lib/storage.ts +++ b/lib/storage.ts @@ -15,6 +15,7 @@ import { PutObjectCommand, DeleteObjectCommand, GetObjectCommand, + ListObjectsV2Command, } from "@aws-sdk/client-s3"; import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; import fs from "fs"; @@ -277,6 +278,50 @@ export async function uploadToHetzner( return { key }; } +/** + * List all object keys currently stored in the Hetzner bucket. + * Paginates automatically. Used by the migration tool. + */ +export async function listHetznerKeys(): Promise> { + const { client, bucket } = await buildHetznerClient(); + const keys = new Set(); + let continuationToken: string | undefined; + + do { + const resp = await client.send( + new ListObjectsV2Command({ Bucket: bucket, ContinuationToken: continuationToken }) + ); + for (const obj of resp.Contents ?? []) { + if (obj.Key) keys.add(obj.Key); + } + continuationToken = resp.NextContinuationToken; + } while (continuationToken); + + return keys; +} + +/** + * Upload a stream directly to Hetzner under an exact key (no UUID prefix). + * Used by the migration tool to preserve existing file paths. + */ +export async function uploadToHetznerWithKey( + body: NodeJS.ReadableStream, + key: string, + contentType: string, + contentLength: number +): Promise { + const { client, bucket } = await buildHetznerClient(); + await client.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body as unknown as Blob, + ContentType: contentType, + ContentLength: contentLength, + }) + ); +} + /** * Generate a presigned URL for streaming a Hetzner object directly in the browser. * No Content-Disposition header — suitable for