Files
vfxreview/lib/storage.ts
T
twotalesanimation 745634f1b6
Deploy / deploy (push) Successful in 2m43s
BIG MOOOOVE to Object Storage
2026-07-16 22:15:58 +02:00

365 lines
11 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* S3-compatible storage abstraction.
*
* STORAGE_PROVIDER env controls which backend is used:
* local local filesystem (dev only, no CDN)
* uploadthing UploadThing managed (default for quick start)
* s3 AWS S3
* r2 Cloudflare R2
* b2 Backblaze B2 (S3-compatible endpoint)
* minio Self-hosted MinIO
*/
import {
S3Client,
PutObjectCommand,
DeleteObjectCommand,
GetObjectCommand,
ListObjectsV2Command,
} from "@aws-sdk/client-s3";
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
import fs from "fs";
import path from "path";
import { randomUUID } from "crypto";
import { db } from "@/lib/db";
export type StorageProvider = "local" | "uploadthing" | "s3" | "r2" | "b2" | "minio";
function getProvider(): StorageProvider {
return (process.env.STORAGE_PROVIDER as StorageProvider) ?? "uploadthing";
}
// ── S3 Client factory ────────────────────────────────────────────────────────
function buildS3Client(): S3Client {
const provider = getProvider();
if (provider === "r2") {
return new S3Client({
region: "auto",
endpoint: `https://${process.env.R2_ACCOUNT_ID!}.r2.cloudflarestorage.com`,
credentials: {
accessKeyId: process.env.R2_ACCESS_KEY_ID!,
secretAccessKey: process.env.R2_SECRET_ACCESS_KEY!,
},
});
}
if (provider === "b2") {
return new S3Client({
region: "auto",
endpoint: process.env.B2_ENDPOINT!,
credentials: {
accessKeyId: process.env.B2_APPLICATION_KEY_ID!,
secretAccessKey: process.env.B2_APPLICATION_KEY!,
},
});
}
if (provider === "minio") {
return new S3Client({
region: "us-east-1",
endpoint: process.env.MINIO_ENDPOINT ?? "http://localhost:9000",
forcePathStyle: true,
credentials: {
accessKeyId: process.env.MINIO_ACCESS_KEY!,
secretAccessKey: process.env.MINIO_SECRET_KEY!,
},
});
}
// Default: AWS S3
return new S3Client({
region: process.env.AWS_REGION ?? "us-east-1",
credentials: {
accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
},
});
}
function getBucketName(): string {
const provider = getProvider();
const map: Record<string, string | undefined> = {
s3: process.env.AWS_BUCKET_NAME,
r2: process.env.R2_BUCKET_NAME,
b2: process.env.B2_BUCKET_NAME,
minio: process.env.MINIO_BUCKET_NAME,
};
return map[provider] ?? "vfx-review";
}
// ── Public API ───────────────────────────────────────────────────────────────
export interface UploadResult {
url: string;
key: string;
provider: StorageProvider;
}
/**
* Upload a file buffer to the configured storage backend.
* Returns the public URL and storage key.
*/
export async function uploadFile(
buffer: Buffer,
fileName: string,
contentType: string,
folder: string = "uploads"
): Promise<UploadResult> {
const provider = getProvider();
const key = `${folder}/${randomUUID()}-${fileName}`;
if (provider === "local") {
return uploadLocal(buffer, key);
}
// All S3-compatible providers share the same logic
const client = buildS3Client();
const bucket = getBucketName();
await client.send(
new PutObjectCommand({
Bucket: bucket,
Key: key,
Body: buffer,
ContentType: contentType,
})
);
const url = getPublicUrl(key, provider);
return { url, key, provider };
}
/**
* Generate a presigned URL for direct browser-to-storage upload.
* Expires in 1 hour by default.
*/
export async function generatePresignedUploadUrl(
key: string,
contentType: string,
expiresIn: number = 3600
): Promise<string> {
const provider = getProvider();
if (provider === "local" || provider === "uploadthing") {
// Not applicable for local / uploadthing (use their own flow)
throw new Error(`Presigned URLs not supported for provider: ${provider}`);
}
const client = buildS3Client();
const command = new PutObjectCommand({
Bucket: getBucketName(),
Key: key,
ContentType: contentType,
});
return getSignedUrl(client, command, { expiresIn });
}
/**
* Generate a presigned download URL for a private object.
*/
export async function generatePresignedDownloadUrl(
key: string,
expiresIn: number = 3600
): Promise<string> {
const provider = getProvider();
if (provider === "local") {
return `/api/files/${encodeURIComponent(key)}`;
}
const client = buildS3Client();
const command = new GetObjectCommand({
Bucket: getBucketName(),
Key: key,
});
return getSignedUrl(client, command, { expiresIn });
}
/**
* Delete a file from storage by key.
*/
export async function deleteFile(key: string): Promise<void> {
const provider = getProvider();
if (provider === "local") {
const filePath = path.join(process.env.LOCAL_UPLOAD_DIR ?? "./uploads", key);
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
return;
}
const client = buildS3Client();
await client.send(
new DeleteObjectCommand({ Bucket: getBucketName(), Key: key })
);
}
// ── Private helpers ──────────────────────────────────────────────────────────
function uploadLocal(buffer: Buffer, key: string): UploadResult {
const uploadDir = process.env.LOCAL_UPLOAD_DIR ?? "./uploads";
const filePath = path.join(uploadDir, key);
const dir = path.dirname(filePath);
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(filePath, buffer);
const url = `/api/files/${encodeURIComponent(key)}`;
return { url, key, provider: "local" };
}
function getPublicUrl(key: string, provider: StorageProvider): string {
if (provider === "r2" && process.env.R2_PUBLIC_URL) {
return `${process.env.R2_PUBLIC_URL}/${key}`;
}
if (provider === "s3") {
return `https://${process.env.AWS_BUCKET_NAME}.s3.${process.env.AWS_REGION ?? "us-east-1"}.amazonaws.com/${key}`;
}
if (provider === "minio") {
return `${process.env.MINIO_ENDPOINT}/${getBucketName()}/${key}`;
}
// Fallback
return `/${key}`;
}
// ── Hetzner Object Storage (dedicated high-res bucket) ───────────────────────
async function getHetznerConfig(): Promise<{
endpoint: string;
accessKey: string;
secretKey: string;
bucket: string;
}> {
const rows = await db.systemConfig.findMany({
where: { key: { in: ["hetzner_endpoint", "hetzner_access_key", "hetzner_secret_key", "hetzner_bucket_name"] } },
});
const map = Object.fromEntries(rows.map((r) => [r.key, r.value]));
return {
endpoint: map["hetzner_endpoint"] ?? process.env.HETZNER_ENDPOINT ?? "",
accessKey: map["hetzner_access_key"] ?? process.env.HETZNER_ACCESS_KEY ?? "",
secretKey: map["hetzner_secret_key"] ?? process.env.HETZNER_SECRET_KEY ?? "",
bucket: map["hetzner_bucket_name"] ?? process.env.HETZNER_BUCKET_NAME ?? "vfx-review",
};
}
async function buildHetznerClient(): Promise<{ client: S3Client; bucket: string }> {
const cfg = await getHetznerConfig();
const client = new S3Client({
region: "auto",
endpoint: cfg.endpoint,
credentials: {
accessKeyId: cfg.accessKey,
secretAccessKey: cfg.secretKey,
},
});
return { client, bucket: cfg.bucket };
}
/**
* Upload a buffer directly to the Hetzner bucket (always uses Hetzner credentials,
* regardless of the global STORAGE_PROVIDER setting).
*/
export async function uploadToHetzner(
buffer: Buffer,
fileName: string,
contentType: string,
folder: string = "highres"
): Promise<{ key: string }> {
const key = `${folder}/${randomUUID()}-${fileName}`;
const { client, bucket } = await buildHetznerClient();
await client.send(
new PutObjectCommand({
Bucket: bucket,
Key: key,
Body: buffer,
ContentType: contentType,
})
);
return { key };
}
/**
* List all object keys currently stored in the Hetzner bucket.
* Paginates automatically. Used by the migration tool.
*/
export async function listHetznerKeys(): Promise<Set<string>> {
const { client, bucket } = await buildHetznerClient();
const keys = new Set<string>();
let continuationToken: string | undefined;
do {
const resp = await client.send(
new ListObjectsV2Command({ Bucket: bucket, ContinuationToken: continuationToken })
);
for (const obj of resp.Contents ?? []) {
if (obj.Key) keys.add(obj.Key);
}
continuationToken = resp.NextContinuationToken;
} while (continuationToken);
return keys;
}
/**
* Upload a stream directly to Hetzner under an exact key (no UUID prefix).
* Used by the migration tool to preserve existing file paths.
*/
export async function uploadToHetznerWithKey(
body: NodeJS.ReadableStream,
key: string,
contentType: string,
contentLength: number
): Promise<void> {
const { client, bucket } = await buildHetznerClient();
await client.send(
new PutObjectCommand({
Bucket: bucket,
Key: key,
Body: body as unknown as Blob,
ContentType: contentType,
ContentLength: contentLength,
})
);
}
/**
* Generate a presigned URL for streaming a Hetzner object directly in the browser.
* No Content-Disposition header — suitable for <video src="..."> playback.
*/
export async function generateHetznerStreamUrl(
key: string,
expiresIn: number = 3600
): Promise<string> {
const { client, bucket } = await buildHetznerClient();
const command = new GetObjectCommand({ Bucket: bucket, Key: key });
return getSignedUrl(client, command, { expiresIn });
}
/**
* Generate a presigned download URL from Hetzner that forces a file download
* with the original filename preserved via Content-Disposition.
*/
export async function generateHetznerDownloadUrl(
key: string,
originalFilename: string,
expiresIn: number = 3600
): Promise<string> {
const { client, bucket } = await buildHetznerClient();
const command = new GetObjectCommand({
Bucket: bucket,
Key: key,
ResponseContentDisposition: `attachment; filename="${originalFilename}"`,
});
return getSignedUrl(client, command, { expiresIn });
}
/**
* Delete a file from the Hetzner bucket by key.
*/
export async function deleteFromHetzner(key: string): Promise<void> {
const { client, bucket } = await buildHetznerClient();
await client.send(
new DeleteObjectCommand({ Bucket: bucket, Key: key })
);
}