"use client"; import { useCallback, useEffect, useRef, useState } from "react"; import Link from "next/link"; // On-set plate comp: a background plate that's already been shot, with the live camera // comped over it through a simple polygon mask (where the foreground goes). For lining up // a shot against its replacement background. type Mode = "comp" | "plate" | "mask"; type Pt = [number, number]; // Plate placement in frame units: centre (cx, cy) as a fraction of the frame, scale 1 = fit to width type Place = { cx: number; cy: number; scale: number }; // aspect: framing crop (width / height), 0 = the camera's full picture type Settings = { aspect: number; place: Place; mask: Pt[]; outline: boolean }; const MODES: [Mode, string][] = [ ["comp", "Comp"], ["plate", "Move plate"], ["mask", "Draw mask"], ]; const ASPECTS: [number, string][] = [ [0, "Full"], [16 / 9, "16:9"], [1.85, "1.85:1"], [2.39, "2.39:1"], ]; const FIT: Place = { cx: 0.5, cy: 0.5, scale: 1 }; const DEFAULTS: Settings = { aspect: 0, place: FIT, mask: [], outline: false }; const SETTINGS_KEY = "ttvfx-plate-settings-v1"; const PLATE_KEY = "ttvfx-plate-image-v1"; // Must match CACHE in public/set/sw.js const OFFLINE_CACHE = "ttvfx-set-v1"; // Kept small enough to remember between visits (localStorage holds ~5 MB) const PLATE_MAX = 2048; // Camera opacity outside the mask while moving the plate or drawing the mask const GHOST = 0.5; // Touch target around a mask point, in CSS px const HIT = 22; // The centred part of the source inside the chosen frame aspect (the shot's framing) function cropOf(w: number, h: number, aspect: number) { if (!aspect || Math.abs(w / h - aspect) < 0.01) return { x: 0, y: 0, w, h }; if (w / h > aspect) { const cw = h * aspect; return { x: (w - cw) / 2, y: 0, w: cw, h }; } const ch = w / aspect; return { x: 0, y: (h - ch) / 2, w, h: ch }; } // Where the frame sits inside the canvas box (letterboxed, "contain") function fitRect(boxW: number, boxH: number, srcW: number, srcH: number) { const scale = Math.min(boxW / srcW, boxH / srcH); const w = srcW * scale, h = srcH * scale; return { scale, w, h, x: (boxW - w) / 2, y: (boxH - h) / 2 }; } // Nearest edge of the closed polygon to p, as the index to insert a new point at function insertAt(pts: Pt[], p: Pt, fw: number, fh: number) { let best = pts.length, bestD = Infinity; for (let i = 0; i < pts.length; i++) { const a = pts[i], b = pts[(i + 1) % pts.length]; const ax = a[0] * fw, ay = a[1] * fh, bx = b[0] * fw, by = b[1] * fh, px = p[0] * fw, py = p[1] * fh; const dx = bx - ax, dy = by - ay; const t = Math.max(0, Math.min(1, ((px - ax) * dx + (py - ay) * dy) / (dx * dx + dy * dy || 1))); const d = Math.hypot(px - (ax + t * dx), py - (ay + t * dy)); if (d < bestD) { bestD = d; best = i + 1; } } return best; } type Status = "idle" | "starting" | "live" | "error"; type Gesture = | { kind: "point"; index: number } | { kind: "pan"; x: number; y: number; place: Place } | { kind: "pinch"; d: number; mx: number; my: number; place: Place } | null; export default function PlateComp() { const [status, setStatus] = useState("idle"); const [error, setError] = useState(""); const [frozen, setFrozen] = useState(false); const [mode, setMode] = useState("plate"); const [settings, setSettings] = useState(DEFAULTS); const [plate, setPlate] = useState(null); const [history, setHistory] = useState([]); // The mask point last added or moved, for Delete point const [sel, setSel] = useState(-1); const [offline, setOffline] = useState(false); const canvasRef = useRef(null); const videoRef = useRef(null); const plateFileRef = useRef(null); const photoFileRef = useRef(null); const frozenCanvas = useRef(null); const stream = useRef(null); const pointers = useRef(new Map()); const gesture = useRef(null); const live = useRef({ settings: DEFAULTS, mode: "plate" as Mode, frozen: false, plate: null as HTMLCanvasElement | null, sel: -1 }); const showing = status === "live" || frozen; useEffect(() => { live.current = { settings, mode, frozen, plate, sel }; }, [settings, mode, frozen, plate, sel]); // Remember the plate, its placement and the mask between visits useEffect(() => { try { const saved = JSON.parse(localStorage.getItem(SETTINGS_KEY) ?? "null"); if (saved) requestAnimationFrame(() => setSettings((s) => ({ ...s, ...saved }))); const url = localStorage.getItem(PLATE_KEY); if (url) { const img = new Image(); img.onload = () => setPlate(toCanvas(img)); img.src = url; } } catch {} }, []); useEffect(() => { try { localStorage.setItem(SETTINGS_KEY, JSON.stringify(settings)); } catch {} }, [settings]); const source = useCallback((): { el: CanvasImageSource; w: number; h: number } | null => { if (live.current.frozen && frozenCanvas.current) { const c = frozenCanvas.current; return { el: c, w: c.width, h: c.height }; } const v = videoRef.current; if (v && v.videoWidth) return { el: v, w: v.videoWidth, h: v.videoHeight }; return null; }, []); // The frame (camera crop) inside the canvas box, in CSS px const frameRect = useCallback( (bw: number, bh: number) => { const src = source(); if (!src) return null; const cr = cropOf(src.w, src.h, live.current.settings.aspect); return { src, cr, fit: fitRect(bw, bh, cr.w, cr.h) }; }, [source], ); // Render loop: plate, then the camera clipped to the mask useEffect(() => { if (!showing) return; const canvas = canvasRef.current; if (!canvas) return; const ctx = canvas.getContext("2d")!; let raf = 0; const draw = () => { raf = requestAnimationFrame(draw); const dpr = Math.min(window.devicePixelRatio || 1, 2); const bw = canvas.clientWidth, bh = canvas.clientHeight; if (canvas.width !== Math.round(bw * dpr) || canvas.height !== Math.round(bh * dpr)) { canvas.width = Math.round(bw * dpr); canvas.height = Math.round(bh * dpr); } ctx.setTransform(dpr, 0, 0, dpr, 0, 0); ctx.fillStyle = "#000"; ctx.fillRect(0, 0, bw, bh); const fr = frameRect(bw, bh); if (!fr) return; const { src, cr, fit: f } = fr; const { settings: s, mode: m, plate: pl, sel: sl } = live.current; const hasMask = s.mask.length >= 3; ctx.save(); ctx.beginPath(); ctx.rect(f.x, f.y, f.w, f.h); ctx.clip(); if (pl) { const w = f.w * s.place.scale, h = (w * pl.height) / pl.width; ctx.drawImage(pl, f.x + s.place.cx * f.w - w / 2, f.y + s.place.cy * f.h - h / 2, w, h); } const camera = (alpha: number) => { ctx.globalAlpha = alpha; ctx.drawImage(src.el, cr.x, cr.y, cr.w, cr.h, f.x, f.y, f.w, f.h); ctx.globalAlpha = 1; }; const path = () => { ctx.beginPath(); s.mask.forEach(([x, y], i) => (i ? ctx.lineTo(f.x + x * f.w, f.y + y * f.h) : ctx.moveTo(f.x + x * f.w, f.y + y * f.h))); ctx.closePath(); }; // Lining up: the whole camera picture shows faintly over the plate if (m !== "comp" && pl) camera(GHOST); if (hasMask) { ctx.save(); path(); ctx.clip(); camera(1); ctx.restore(); } else if (!pl || m === "mask") { // No mask yet: nothing to comp, so show the camera camera(1); } // Mask outline and points if (s.mask.length && (m === "mask" || s.outline)) { path(); if (s.mask.length < 3 || m === "mask") { ctx.lineWidth = 3; ctx.strokeStyle = "#000"; ctx.stroke(); } ctx.lineWidth = 1.5; ctx.setLineDash(m === "mask" ? [] : [6, 4]); ctx.strokeStyle = "#fff"; ctx.stroke(); ctx.setLineDash([]); if (m === "mask") { s.mask.forEach(([x, y], i) => { ctx.beginPath(); ctx.arc(f.x + x * f.w, f.y + y * f.h, i === sl ? 9 : 6, 0, Math.PI * 2); ctx.fillStyle = i === sl ? "#ffd600" : "#fff"; ctx.fill(); ctx.lineWidth = 2; ctx.strokeStyle = "#000"; ctx.stroke(); }); } } ctx.restore(); }; draw(); return () => cancelAnimationFrame(raf); }, [showing, frameRect]); // Keep the screen awake while lining up useEffect(() => { if (status !== "live") return; let wake: WakeLockSentinel | null = null; const acquire = async () => { try { wake = (await navigator.wakeLock?.request("screen")) ?? null; } catch {} }; const onVisible = () => document.visibilityState === "visible" && acquire(); acquire(); document.addEventListener("visibilitychange", onVisible); return () => { document.removeEventListener("visibilitychange", onVisible); wake?.release().catch(() => {}); }; }, [status]); // Stop the camera when leaving the page useEffect(() => () => stream.current?.getTracks().forEach((t) => t.stop()), []); // Offline: same service worker as /set; save this page and its assets on first visit useEffect(() => { if (process.env.NODE_ENV !== "production" || !("serviceWorker" in navigator) || !("caches" in window)) return; let cancelled = false; (async () => { try { await navigator.serviceWorker.register("/set/sw.js", { scope: "/set/" }); await navigator.serviceWorker.ready; const assets = performance .getEntriesByType("resource") .map((e) => new URL(e.name)) .filter((u) => u.origin === location.origin && /^\/(_next\/|icon|apple-icon)/.test(u.pathname)) .map((u) => u.pathname + u.search); const cache = await caches.open(OFFLINE_CACHE); await cache.addAll(["/set/plate/", ...new Set(assets)]); if (!cancelled) setOffline(true); } catch {} })(); return () => { cancelled = true; }; }, []); const startCamera = async () => { setError(""); if (!navigator.mediaDevices?.getUserMedia) { setError(window.isSecureContext ? "This browser has no camera access." : "The camera needs a secure (https) page."); setStatus("error"); return; } setStatus("starting"); try { const s = await navigator.mediaDevices.getUserMedia({ audio: false, video: { facingMode: { ideal: "environment" }, width: { ideal: 1920 }, height: { ideal: 1080 } }, }); stream.current = s; const v = videoRef.current!; v.srcObject = s; await v.play(); setFrozen(false); setStatus("live"); } catch (e) { const name = (e as DOMException)?.name; setError( name === "NotAllowedError" ? "Camera permission was refused. Allow camera access for this site in your browser settings, then try again." : "Couldn't start the camera.", ); setStatus("error"); } }; const freeze = () => { const v = videoRef.current; if (!v?.videoWidth) return; const c = (frozenCanvas.current ??= document.createElement("canvas")); c.width = v.videoWidth; c.height = v.videoHeight; c.getContext("2d")!.drawImage(v, 0, 0); setFrozen(true); }; const loadImage = (file: File, done: (img: HTMLImageElement) => void) => { const url = URL.createObjectURL(file); const img = new Image(); img.onload = () => { done(img); URL.revokeObjectURL(url); }; img.src = url; }; const openPlate = (file: File) => loadImage(file, (img) => { const c = toCanvas(img); setPlate(c); setSettings((s) => ({ ...s, place: FIT })); try { localStorage.setItem(PLATE_KEY, c.toDataURL("image/jpeg", 0.85)); } catch { // Too big to remember; it still works for this visit try { localStorage.removeItem(PLATE_KEY); } catch {} } }); // A still in place of the camera, for planning off set const openPhoto = (file: File) => loadImage(file, (img) => { frozenCanvas.current = toCanvas(img); setFrozen(true); }); const set = (k: K, v: Settings[K]) => setSettings((s) => ({ ...s, [k]: v })); const setMask = (mask: Pt[], undoable = true) => { if (undoable) setHistory((h) => [...h.slice(-49), settings.mask]); set("mask", mask); }; const setPlace = (p: Place) => set("place", { ...p, scale: Math.max(0.1, Math.min(10, p.scale)) }); // Pointer position as frame units (0..1 across the frame) const toFrame = (e: { clientX: number; clientY: number }, el: HTMLElement) => { const rect = el.getBoundingClientRect(); const fr = frameRect(rect.width, rect.height); if (!fr) return null; const { fit: f } = fr; return { p: [(e.clientX - rect.left - f.x) / f.w, (e.clientY - rect.top - f.y) / f.h] as Pt, f }; }; const startPinch = () => { const [a, b] = [...pointers.current.values()]; gesture.current = { kind: "pinch", d: Math.hypot(a.x - b.x, a.y - b.y), mx: (a.x + b.x) / 2, my: (a.y + b.y) / 2, place: settings.place }; }; const onDown = (e: React.PointerEvent) => { e.currentTarget.setPointerCapture(e.pointerId); pointers.current.set(e.pointerId, { x: e.clientX, y: e.clientY }); if (mode === "plate") { if (pointers.current.size === 2) startPinch(); else gesture.current = { kind: "pan", x: e.clientX, y: e.clientY, place: settings.place }; return; } if (mode !== "mask" || pointers.current.size > 1) return; const hit = toFrame(e, e.currentTarget); if (!hit) return; const { p, f } = hit; const near = settings.mask.findIndex(([x, y]) => Math.hypot((x - p[0]) * f.w, (y - p[1]) * f.h) < HIT); if (near >= 0) { setHistory((h) => [...h.slice(-49), settings.mask]); gesture.current = { kind: "point", index: near }; setSel(near); return; } const q: Pt = [Math.max(0, Math.min(1, p[0])), Math.max(0, Math.min(1, p[1]))]; const i = settings.mask.length < 3 ? settings.mask.length : insertAt(settings.mask, q, f.w, f.h); setMask([...settings.mask.slice(0, i), q, ...settings.mask.slice(i)]); gesture.current = { kind: "point", index: i }; setSel(i); }; const onMove = (e: React.PointerEvent) => { if (!pointers.current.has(e.pointerId)) return; pointers.current.set(e.pointerId, { x: e.clientX, y: e.clientY }); const g = gesture.current; if (!g) return; const rect = e.currentTarget.getBoundingClientRect(); const fr = frameRect(rect.width, rect.height); if (!fr) return; const { fit: f } = fr; if (g.kind === "point") { const hit = toFrame(e, e.currentTarget); if (!hit) return; const q: Pt = [Math.max(0, Math.min(1, hit.p[0])), Math.max(0, Math.min(1, hit.p[1]))]; setSettings((s) => ({ ...s, mask: s.mask.map((pt, i) => (i === g.index ? q : pt)) })); } else if (g.kind === "pan") { setPlace({ ...g.place, cx: g.place.cx + (e.clientX - g.x) / f.w, cy: g.place.cy + (e.clientY - g.y) / f.h }); } else if (g.kind === "pinch" && pointers.current.size >= 2) { const [a, b] = [...pointers.current.values()]; const ratio = Math.hypot(a.x - b.x, a.y - b.y) / (g.d || 1); const mx = (a.x + b.x) / 2, my = (a.y + b.y) / 2; // Scale about where the fingers started, and follow them as they move const ox = (g.mx - rect.left - f.x) / f.w, oy = (g.my - rect.top - f.y) / f.h; setPlace({ scale: g.place.scale * ratio, cx: ox + (g.place.cx - ox) * ratio + (mx - g.mx) / f.w, cy: oy + (g.place.cy - oy) * ratio + (my - g.my) / f.h, }); } }; const onUp = (e: React.PointerEvent) => { pointers.current.delete(e.pointerId); const left = [...pointers.current.values()][0]; // Lifting one finger of a pinch carries on as a pan with the other gesture.current = mode === "plate" && left ? { kind: "pan", x: left.x, y: left.y, place: settings.place } : null; }; // Desktop: scroll wheel scales the plate about the cursor useEffect(() => { const canvas = canvasRef.current; if (!canvas || mode !== "plate") return; const onWheel = (e: WheelEvent) => { e.preventDefault(); const rect = canvas.getBoundingClientRect(); const fr = frameRect(rect.width, rect.height); if (!fr) return; const { fit: f } = fr; const ratio = Math.exp(-e.deltaY * 0.0015); const ox = (e.clientX - rect.left - f.x) / f.w, oy = (e.clientY - rect.top - f.y) / f.h; setSettings((s) => ({ ...s, place: { scale: Math.max(0.1, Math.min(10, s.place.scale * ratio)), cx: ox + (s.place.cx - ox) * ratio, cy: oy + (s.place.cy - oy) * ratio }, })); }; canvas.addEventListener("wheel", onWheel, { passive: false }); return () => canvas.removeEventListener("wheel", onWheel); }, [mode, showing, frameRect]); const fitHeight = () => { const src = source(); if (!src || !plate) return; const cr = cropOf(src.w, src.h, settings.aspect); // Scale is relative to the frame width; fit height makes the plate as tall as the frame setPlace({ cx: 0.5, cy: 0.5, scale: ((plate.width / plate.height) * cr.h) / cr.w }); }; const undo = () => { const prev = history[history.length - 1]; if (!prev) return; setHistory((h) => h.slice(0, -1)); set("mask", prev); }; const cycleAspect = () => { const i = ASPECTS.findIndex(([a]) => a === settings.aspect); set("aspect", ASPECTS[(i + 1) % ASPECTS.length][0]); }; const hint = !plate ? "Load the background plate to start" : mode === "plate" ? "Drag to move the plate · pinch or scroll to scale" : mode === "mask" && settings.mask.length < 3 ? "Tap around the foreground to draw the mask" : mode === "mask" ? "Drag points to adjust · tap an edge to add a point" : null; return (
); } // Downscale to a size that's quick to draw every frame and small enough to remember function toCanvas(img: HTMLImageElement) { const scale = Math.min(1, PLATE_MAX / Math.max(img.width, img.height)); const c = document.createElement("canvas"); c.width = Math.round(img.width * scale); c.height = Math.round(img.height * scale); c.getContext("2d")!.drawImage(img, 0, 0, c.width, c.height); return c; } function FileInput({ inputRef, onFile }: { inputRef: React.RefObject; onFile: (f: File) => void }) { return ( { const f = e.target.files?.[0]; if (f) onFile(f); e.target.value = ""; }} /> ); } function StartScreen({ status, error, plate, onCamera, onPlate, onPhoto, }: { status: Status; error: string; plate: HTMLCanvasElement | null; onCamera: () => void; onPlate: () => void; onPhoto: () => void; }) { return (

Line up the shot against its background plate. The plate sits behind, the camera shows through a mask you draw around the foreground.

  1. Load the background plate (a still or frame grab).
  2. Start the camera. Move and scale the plate to match.
  3. Draw a mask around where the foreground goes.
{error &&

{error}

}
); } function Action({ children, onClick, active, disabled }: { children: React.ReactNode; onClick: () => void; active?: boolean; disabled?: boolean }) { return ( ); }