BIG MOOOOVE to Object Storage
Deploy / deploy (push) Successful in 2m43s

This commit is contained in:
twotalesanimation
2026-07-16 22:15:58 +02:00
parent 3264abb432
commit 745634f1b6
10 changed files with 594 additions and 17 deletions
+404
View File
@@ -0,0 +1,404 @@
"use client";
import { useState, useEffect, useCallback, useMemo } from "react";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Badge } from "@/components/ui/badge";
import { cn } from "@/lib/utils";
import {
CheckCircle2,
AlertCircle,
Loader2,
RefreshCw,
Upload,
Search,
ChevronDown,
} from "lucide-react";
// ── Types ──────────────────────────────────────────────────────────────────────
interface FileEntry {
key: string;
filename: string;
folder: string;
sizeBytes: number;
hetznerExists: boolean;
}
type MigrateStatus = "idle" | "uploading" | "done" | "error";
function formatBytes(bytes: number) {
if (bytes >= 1024 ** 3) return `${(bytes / 1024 ** 3).toFixed(1)} GB`;
if (bytes >= 1024 ** 2) return `${(bytes / 1024 ** 2).toFixed(1)} MB`;
return `${(bytes / 1024).toFixed(1)} KB`;
}
// ── Page ───────────────────────────────────────────────────────────────────────
export default function MigrationPage() {
const [files, setFiles] = useState<FileEntry[]>([]);
const [loading, setLoading] = useState(true);
const [hetznerConfigured, setHetznerConfigured] = useState(true);
const [fetchError, setFetchError] = useState<string | null>(null);
// Selection: Set of keys
const [selected, setSelected] = useState<Set<string>>(new Set());
// Per-file migration status
const [statuses, setStatuses] = useState<Map<string, MigrateStatus>>(new Map());
const [errors, setErrors] = useState<Map<string, string>>(new Map());
// Filters
const [search, setSearch] = useState("");
const [folderFilter, setFolderFilter] = useState("all");
const [folderMenuOpen, setFolderMenuOpen] = useState(false);
const [migrating, setMigrating] = useState(false);
// ── Load file list ──────────────────────────────────────────────────────────
const loadFiles = useCallback(async () => {
setLoading(true);
setFetchError(null);
try {
const res = await fetch("/api/admin/migration");
const data = await res.json();
if (!res.ok) throw new Error(data.error ?? "Failed to load");
setFiles(data.files);
setHetznerConfigured(data.hetznerConfigured);
// Pre-select all files NOT yet on Hetzner
const preSelected = new Set<string>(
(data.files as FileEntry[])
.filter((f) => !f.hetznerExists)
.map((f) => f.key)
);
setSelected(preSelected);
setStatuses(new Map());
setErrors(new Map());
} catch (e: unknown) {
setFetchError(e instanceof Error ? e.message : "Failed to load file list");
} finally {
setLoading(false);
}
}, []);
useEffect(() => { loadFiles(); }, [loadFiles]);
// ── Derived data ────────────────────────────────────────────────────────────
const folders = useMemo(
() => ["all", ...Array.from(new Set(files.map((f) => f.folder))).sort()],
[files]
);
const filtered = useMemo(() => {
return files.filter((f) => {
const matchFolder = folderFilter === "all" || f.folder === folderFilter;
const matchSearch =
!search || f.filename.toLowerCase().includes(search.toLowerCase());
return matchFolder && matchSearch;
});
}, [files, folderFilter, search]);
const stats = useMemo(() => {
const synced = files.filter((f) => f.hetznerExists).length;
const total = files.length;
const totalBytes = files.reduce((a, f) => a + f.sizeBytes, 0);
const selectedBytes = files
.filter((f) => selected.has(f.key))
.reduce((a, f) => a + f.sizeBytes, 0);
return { synced, total, totalBytes, selectedBytes };
}, [files, selected]);
// ── Selection helpers ───────────────────────────────────────────────────────
const toggleOne = (key: string) => {
setSelected((prev) => {
const next = new Set(prev);
next.has(key) ? next.delete(key) : next.add(key);
return next;
});
};
const selectAll = () =>
setSelected(new Set(filtered.map((f) => f.key)));
const deselectAll = () =>
setSelected((prev) => {
const next = new Set(prev);
filtered.forEach((f) => next.delete(f.key));
return next;
});
const selectUnsynced = () =>
setSelected(new Set(filtered.filter((f) => !f.hetznerExists).map((f) => f.key)));
// ── Migration ───────────────────────────────────────────────────────────────
const handleMigrate = async () => {
const toMigrate = files.filter((f) => selected.has(f.key));
if (!toMigrate.length) return;
setMigrating(true);
for (const file of toMigrate) {
setStatuses((prev) => new Map(prev).set(file.key, "uploading"));
try {
const res = await fetch("/api/admin/migration", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ key: file.key }),
});
const data = await res.json();
if (!res.ok) throw new Error(data.error ?? "Upload failed");
setStatuses((prev) => new Map(prev).set(file.key, "done"));
// Mark as synced in the file list
setFiles((prev) =>
prev.map((f) => (f.key === file.key ? { ...f, hetznerExists: true } : f))
);
} catch (e: unknown) {
setStatuses((prev) => new Map(prev).set(file.key, "error"));
setErrors((prev) =>
new Map(prev).set(file.key, e instanceof Error ? e.message : "Failed")
);
}
}
setMigrating(false);
};
// ── Render ──────────────────────────────────────────────────────────────────
if (loading) {
return (
<div className="flex items-center justify-center h-64">
<Loader2 className="h-6 w-6 animate-spin text-muted-foreground" />
<span className="ml-2 text-sm text-muted-foreground">Scanning files</span>
</div>
);
}
if (fetchError) {
return (
<div className="p-6 max-w-2xl">
<div className="flex items-center gap-2 text-sm text-red-400 rounded-md border border-red-500/20 bg-red-500/10 px-3 py-2">
<AlertCircle className="h-4 w-4 shrink-0" />
{fetchError}
</div>
<Button variant="outline" size="sm" className="mt-3" onClick={loadFiles}>
Retry
</Button>
</div>
);
}
if (!hetznerConfigured) {
return (
<div className="p-6 max-w-2xl">
<div className="flex items-center gap-2 text-sm text-amber-400 rounded-md border border-amber-500/20 bg-amber-500/10 px-3 py-2">
<AlertCircle className="h-4 w-4 shrink-0" />
Hetzner object storage is not configured. Set{" "}
<code className="font-mono text-xs">HETZNER_ENDPOINT</code>,{" "}
<code className="font-mono text-xs">HETZNER_ACCESS_KEY</code>,{" "}
<code className="font-mono text-xs">HETZNER_SECRET_KEY</code>, and{" "}
<code className="font-mono text-xs">HETZNER_BUCKET_NAME</code> in your environment,
or configure them in Settings.
</div>
</div>
);
}
const allFilteredSelected = filtered.length > 0 && filtered.every((f) => selected.has(f.key));
return (
<div className="p-6 space-y-4 max-w-5xl">
{/* Header */}
<div className="flex items-start justify-between gap-4">
<div>
<h1 className="text-xl font-semibold">Storage Migration</h1>
<p className="text-sm text-muted-foreground mt-0.5">
{stats.total} local files · {formatBytes(stats.totalBytes)} total ·{" "}
<span className="text-emerald-400">{stats.synced} already on Hetzner</span>
</p>
</div>
<Button variant="outline" size="sm" onClick={loadFiles} disabled={migrating} className="gap-1.5 shrink-0">
<RefreshCw className="h-3.5 w-3.5" />
Refresh
</Button>
</div>
{/* Toolbar */}
<div className="flex flex-wrap items-center gap-2">
{/* Search */}
<div className="relative flex-1 min-w-[180px]">
<Search className="absolute left-2.5 top-1/2 -translate-y-1/2 h-3.5 w-3.5 text-muted-foreground pointer-events-none" />
<Input
placeholder="Search filenames…"
value={search}
onChange={(e) => setSearch(e.target.value)}
className="pl-8 h-8 text-sm"
/>
</div>
{/* Folder filter */}
<div className="relative">
<button
onClick={() => setFolderMenuOpen((o) => !o)}
className="flex items-center gap-1.5 h-8 px-3 text-sm rounded-md border border-input bg-background hover:bg-muted transition-colors"
>
{folderFilter === "all" ? "All folders" : folderFilter}
<ChevronDown className="h-3.5 w-3.5 text-muted-foreground" />
</button>
{folderMenuOpen && (
<div className="absolute top-full left-0 mt-1 z-10 rounded-md border border-border bg-popover shadow-md py-1 min-w-[140px]">
{folders.map((f) => (
<button
key={f}
onClick={() => { setFolderFilter(f); setFolderMenuOpen(false); }}
className={cn(
"w-full text-left px-3 py-1.5 text-sm hover:bg-muted transition-colors",
folderFilter === f && "text-primary font-medium"
)}
>
{f === "all" ? "All folders" : f}
</button>
))}
</div>
)}
</div>
{/* Bulk actions */}
<Button variant="outline" size="sm" className="h-8" onClick={allFilteredSelected ? deselectAll : selectAll}>
{allFilteredSelected ? "Deselect all" : "Select all"}
</Button>
<Button variant="outline" size="sm" className="h-8" onClick={selectUnsynced}>
Select unsynced
</Button>
</div>
{/* Table */}
<div className="rounded-md border border-border overflow-hidden">
<table className="w-full text-sm">
<thead className="bg-muted/50">
<tr>
<th className="w-10 px-3 py-2">
<input
type="checkbox"
checked={allFilteredSelected}
onChange={allFilteredSelected ? deselectAll : selectAll}
className="rounded accent-primary"
/>
</th>
<th className="text-left px-3 py-2 font-medium text-muted-foreground">Filename</th>
<th className="text-left px-3 py-2 font-medium text-muted-foreground w-24">Folder</th>
<th className="text-right px-3 py-2 font-medium text-muted-foreground w-24">Size</th>
<th className="text-right px-3 py-2 font-medium text-muted-foreground w-28">Status</th>
</tr>
</thead>
<tbody className="divide-y divide-border">
{filtered.length === 0 && (
<tr>
<td colSpan={5} className="text-center py-10 text-muted-foreground text-sm">
No files match your filters.
</td>
</tr>
)}
{filtered.map((file) => {
const status = statuses.get(file.key);
const err = errors.get(file.key);
const isSelected = selected.has(file.key);
return (
<tr
key={file.key}
onClick={() => !migrating && toggleOne(file.key)}
className={cn(
"transition-colors cursor-pointer",
isSelected ? "bg-primary/5 hover:bg-primary/10" : "hover:bg-muted/40"
)}
>
{/* Checkbox */}
<td className="px-3 py-2" onClick={(e) => e.stopPropagation()}>
<input
type="checkbox"
checked={isSelected}
onChange={() => !migrating && toggleOne(file.key)}
disabled={migrating}
className="rounded accent-primary"
/>
</td>
{/* Filename */}
<td className="px-3 py-2 font-mono text-xs text-foreground max-w-xs truncate">
{file.filename}
{err && (
<span className="ml-2 text-red-400 font-sans not-italic">{err}</span>
)}
</td>
{/* Folder */}
<td className="px-3 py-2 text-muted-foreground">{file.folder}</td>
{/* Size */}
<td className="px-3 py-2 text-right text-muted-foreground tabular-nums">
{formatBytes(file.sizeBytes)}
</td>
{/* Status */}
<td className="px-3 py-2 text-right">
{status === "uploading" ? (
<span className="inline-flex items-center gap-1 text-amber-400 text-xs">
<Loader2 className="h-3 w-3 animate-spin" />
Uploading
</span>
) : status === "done" || (status === undefined && file.hetznerExists) ? (
<span className="inline-flex items-center gap-1 text-emerald-400 text-xs">
<CheckCircle2 className="h-3 w-3" />
Synced
</span>
) : status === "error" ? (
<span className="inline-flex items-center gap-1 text-red-400 text-xs">
<AlertCircle className="h-3 w-3" />
Failed
</span>
) : (
<Badge variant="outline" className="text-xs text-muted-foreground">
Local only
</Badge>
)}
</td>
</tr>
);
})}
</tbody>
</table>
</div>
{/* Footer action bar */}
{selected.size > 0 && (
<div className="sticky bottom-4 flex items-center justify-between gap-4 rounded-lg border border-border bg-background/95 backdrop-blur px-4 py-3 shadow-lg">
<span className="text-sm text-muted-foreground">
<span className="font-semibold text-foreground">{selected.size} files</span> selected ·{" "}
{formatBytes(stats.selectedBytes)}
</span>
<Button
onClick={handleMigrate}
disabled={migrating}
className="gap-2"
>
{migrating ? (
<>
<Loader2 className="h-4 w-4 animate-spin" />
Migrating
</>
) : (
<>
<Upload className="h-4 w-4" />
Migrate {selected.size} {selected.size === 1 ? "file" : "files"} to Hetzner
</>
)}
</Button>
</div>
)}
</div>
);
}
+116
View File
@@ -0,0 +1,116 @@
import { NextRequest, NextResponse } from "next/server";
import { auth } from "@/auth";
import fs from "fs";
import path from "path";
import { listHetznerKeys, uploadToHetznerWithKey } from "@/lib/storage";
export const maxDuration = 300; // 5 min — large video uploads
const MIME_MAP: Record<string, string> = {
".mp4": "video/mp4",
".mov": "video/quicktime",
".avi": "video/x-msvideo",
".mxf": "application/mxf",
".webm": "video/webm",
".jpg": "image/jpeg",
".jpeg": "image/jpeg",
".png": "image/png",
".gif": "image/gif",
".webp": "image/webp",
};
function walkDir(
dir: string,
base: string
): Array<{ key: string; sizeBytes: number }> {
const results: Array<{ key: string; sizeBytes: number }> = [];
if (!fs.existsSync(dir)) return results;
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
const abs = path.join(dir, entry.name);
if (entry.isDirectory()) {
results.push(...walkDir(abs, base));
} else if (entry.isFile()) {
const key = path.relative(base, abs).replace(/\\/g, "/");
results.push({ key, sizeBytes: fs.statSync(abs).size });
}
}
return results;
}
/** GET /api/admin/migration — list local files with Hetzner sync status */
export async function GET() {
const session = await auth();
if (!session?.user) {
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
}
const uploadDir = path.resolve(process.env.LOCAL_UPLOAD_DIR ?? "./uploads");
const localFiles = walkDir(uploadDir, uploadDir);
let hetznerKeys = new Set<string>();
let hetznerConfigured = true;
try {
hetznerKeys = await listHetznerKeys();
} catch {
hetznerConfigured = false;
}
const files = localFiles.map(({ key, sizeBytes }) => ({
key,
filename: path.basename(key),
folder: path.dirname(key).replace(/\\/g, "/") || "root",
sizeBytes,
hetznerExists: hetznerKeys.has(key),
}));
return NextResponse.json({ files, hetznerConfigured });
}
/** POST /api/admin/migration — migrate a single file to Hetzner */
export async function POST(req: NextRequest) {
const session = await auth();
if (!session?.user) {
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
}
const body = await req.json();
const key = body?.key;
if (!key || typeof key !== "string") {
return NextResponse.json({ error: "key is required" }, { status: 400 });
}
const uploadDir = path.resolve(process.env.LOCAL_UPLOAD_DIR ?? "./uploads");
const absPath = path.resolve(path.join(uploadDir, key));
// Path traversal guard
if (!absPath.startsWith(uploadDir + path.sep)) {
return NextResponse.json({ error: "Invalid key" }, { status: 403 });
}
let stat: fs.Stats;
try {
stat = fs.statSync(absPath);
} catch {
return NextResponse.json({ error: "File not found on disk" }, { status: 404 });
}
if (!stat.isFile()) {
return NextResponse.json({ error: "Not a file" }, { status: 400 });
}
const ext = path.extname(key).toLowerCase();
const contentType = MIME_MAP[ext] ?? "application/octet-stream";
const start = Date.now();
try {
const stream = fs.createReadStream(absPath);
await uploadToHetznerWithKey(stream, key, contentType, stat.size);
return NextResponse.json({ success: true, key, ms: Date.now() - start });
} catch (e: unknown) {
return NextResponse.json(
{ error: e instanceof Error ? e.message : "Upload failed", key },
{ status: 500 }
);
}
}
+4 -3
View File
@@ -1,7 +1,7 @@
import { NextRequest, NextResponse } from "next/server";
import { auth } from "@/auth";
import { db } from "@/lib/db";
import { uploadFile, uploadToHetzner, deleteFromHetzner } from "@/lib/storage";
import { uploadToHetzner, deleteFromHetzner } from "@/lib/storage";
import { recalcShotStatus } from "@/lib/shot-status";
export const maxDuration = 120;
@@ -133,8 +133,9 @@ export async function POST(
);
}
// Upload the video to the configured storage backend
const result = await uploadFile(buffer, file.name, file.type, "videos");
// Upload the video to Hetzner object storage
const { key: videoKey } = await uploadToHetzner(buffer, file.name, file.type, "videos");
const result = { url: `/api/files/${videoKey}`, key: videoKey };
// Mark all existing versions for this task as no longer latest
await db.version.updateMany({
+3 -3
View File
@@ -1,6 +1,6 @@
import { NextRequest, NextResponse } from "next/server";
import { db } from "@/lib/db";
import { uploadFile } from "@/lib/storage";
import { uploadToHetzner } from "@/lib/storage";
import { ShotPriority } from "@prisma/client";
import { z } from "zod";
@@ -169,8 +169,8 @@ export async function POST(req: NextRequest) {
);
}
const buffer = Buffer.from(await thumbnailFile.arrayBuffer());
const result = await uploadFile(buffer, thumbnailFile.name, thumbnailFile.type, "image");
thumbnailUrl = result.url;
const { key: thumbKey } = await uploadToHetzner(buffer, thumbnailFile.name, thumbnailFile.type, "image");
thumbnailUrl = `/api/files/${thumbKey}`;
}
// Resolve shot group
+10 -1
View File
@@ -1,6 +1,7 @@
import { NextRequest, NextResponse } from "next/server";
import fs from "fs";
import path from "path";
import { generateHetznerStreamUrl } from "@/lib/storage";
// ── Tuning constants ──────────────────────────────────────────────────────────
//
@@ -50,7 +51,15 @@ export async function GET(
try {
stat = fs.statSync(filePath);
} catch {
return new NextResponse("Not found", { status: 404 });
// File not on local disk — attempt a redirect to Hetzner object storage.
// This covers files uploaded after the migration and any migrated files
// whose local copies have been removed.
try {
const hetznerUrl = await generateHetznerStreamUrl(relativePath, 3600);
return NextResponse.redirect(hetznerUrl, 302);
} catch {
return new NextResponse("Not found", { status: 404 });
}
}
// Guard against directory traversal that resolves to a directory
+3 -2
View File
@@ -1,7 +1,7 @@
import { NextRequest, NextResponse } from "next/server";
import { auth } from "@/auth";
import { db } from "@/lib/db";
import { uploadFile } from "@/lib/storage";
import { uploadToHetzner } from "@/lib/storage";
function requireAuth(role: string) {
return ["ADMIN", "PRODUCER", "SUPERVISOR"].includes(role);
@@ -58,7 +58,8 @@ export async function POST(
return NextResponse.json({ error: "File too large (max 50 MB)" }, { status: 413 });
const buffer = Buffer.from(await file.arrayBuffer());
const uploaded = await uploadFile(buffer, file.name, file.type, "image");
const { key: refKey } = await uploadToHetzner(buffer, file.name, file.type, "image");
const uploaded = { url: `/api/files/${refKey}`, key: refKey };
const maxOrder = await db.shotReference.aggregate({
where: { shotId },
+3 -2
View File
@@ -1,7 +1,7 @@
import { NextRequest, NextResponse } from "next/server";
import { auth } from "@/auth";
import { db } from "@/lib/db";
import { uploadFile } from "@/lib/storage";
import { uploadToHetzner } from "@/lib/storage";
function canManage(role: string) {
return ["ADMIN", "PRODUCER", "SUPERVISOR"].includes(role);
@@ -44,7 +44,8 @@ export async function POST(req: NextRequest) {
return NextResponse.json({ error: "File too large (max 20 MB)" }, { status: 413 });
const buffer = Buffer.from(await file.arrayBuffer());
const uploaded = await uploadFile(buffer, file.name, file.type, "image");
const { key: tmplKey } = await uploadToHetzner(buffer, file.name, file.type, "image");
const uploaded = { url: `/api/files/${tmplKey}`, key: tmplKey };
const maxOrder = await db.sketchTemplate.aggregate({ _max: { sortOrder: true } });
const sortOrder = (maxOrder._max.sortOrder ?? -1) + 1;
+3 -3
View File
@@ -1,6 +1,6 @@
import { NextRequest, NextResponse } from "next/server";
import { auth } from "@/auth";
import { uploadFile } from "@/lib/storage";
import { uploadToHetzner } from "@/lib/storage";
export const config = { api: { bodyParser: false } };
@@ -30,9 +30,9 @@ export async function POST(req: NextRequest) {
}
const buffer = Buffer.from(await file.arrayBuffer());
const result = await uploadFile(buffer, file.name, file.type, "videos");
const { key } = await uploadToHetzner(buffer, file.name, file.type, "videos");
return NextResponse.json({ url: result.url, key: result.key });
return NextResponse.json({ url: `/api/files/${key}`, key });
} catch (err) {
console.error("[local-upload]", err);
return NextResponse.json({ error: "Upload failed" }, { status: 500 });
+3 -3
View File
@@ -1,6 +1,6 @@
import { NextRequest, NextResponse } from "next/server";
import { auth } from "@/auth";
import { uploadFile } from "@/lib/storage";
import { uploadToHetzner } from "@/lib/storage";
export const config = { api: { bodyParser: false } };
@@ -41,9 +41,9 @@ export async function POST(req: NextRequest) {
}
const buffer = Buffer.from(await file.arrayBuffer());
const result = await uploadFile(buffer, file.name, file.type, type);
const { key } = await uploadToHetzner(buffer, file.name, file.type, type);
return NextResponse.json({ url: result.url, key: result.key });
return NextResponse.json({ url: `/api/files/${key}`, key });
} catch (err) {
console.error("[upload]", err);
return NextResponse.json({ error: "Upload failed" }, { status: 500 });
+45
View File
@@ -15,6 +15,7 @@ import {
PutObjectCommand,
DeleteObjectCommand,
GetObjectCommand,
ListObjectsV2Command,
} from "@aws-sdk/client-s3";
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
import fs from "fs";
@@ -277,6 +278,50 @@ export async function uploadToHetzner(
return { key };
}
/**
* List all object keys currently stored in the Hetzner bucket.
* Paginates automatically. Used by the migration tool.
*/
export async function listHetznerKeys(): Promise<Set<string>> {
const { client, bucket } = await buildHetznerClient();
const keys = new Set<string>();
let continuationToken: string | undefined;
do {
const resp = await client.send(
new ListObjectsV2Command({ Bucket: bucket, ContinuationToken: continuationToken })
);
for (const obj of resp.Contents ?? []) {
if (obj.Key) keys.add(obj.Key);
}
continuationToken = resp.NextContinuationToken;
} while (continuationToken);
return keys;
}
/**
* Upload a stream directly to Hetzner under an exact key (no UUID prefix).
* Used by the migration tool to preserve existing file paths.
*/
export async function uploadToHetznerWithKey(
body: NodeJS.ReadableStream,
key: string,
contentType: string,
contentLength: number
): Promise<void> {
const { client, bucket } = await buildHetznerClient();
await client.send(
new PutObjectCommand({
Bucket: bucket,
Key: key,
Body: body as unknown as Blob,
ContentType: contentType,
ContentLength: contentLength,
})
);
}
/**
* Generate a presigned URL for streaming a Hetzner object directly in the browser.
* No Content-Disposition header — suitable for <video src="..."> playback.