diff --git a/README.md b/README.md index 66a38cb..cec1b6e 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,15 @@ live key with despill) and Evenness (screen brightness bands). "Pick white" sets neutral reference, like a white balance pick. The code is in `components/set/SpillChecker.tsx`; thresholds are tuned on a real Alexa car-interior plate. The camera needs https. +## On-set plate comp (`/set/plate`) + +For shooting a foreground against a background plate that's already been shot. Load the +plate (fit to width, then drag / pinch / scroll to place it), start the camera, and draw a +polygon mask around where the foreground goes: the camera shows inside the mask, the plate +everywhere else. "Move plate" and "Draw mask" show the camera faintly over the whole plate +for lining up. The plate, its placement and the mask are remembered between visits. The +code is in `components/set/PlateComp.tsx`. + ## Structure ``` diff --git a/app/set/plate/page.tsx b/app/set/plate/page.tsx new file mode 100644 index 0000000..b07a817 --- /dev/null +++ b/app/set/plate/page.tsx @@ -0,0 +1,25 @@ +import type { Metadata, Viewport } from "next"; +import PlateComp from "@/components/set/PlateComp"; + +// On-set plate comp: line up a shot against its background plate. Like /set, not linked +// from the site and kept out of search results and the sitemap. +export const metadata: Metadata = { + title: "Plate comp", + description: "Two Tales VFX on-set background plate comp.", + robots: { index: false, follow: false }, + manifest: "/set/manifest.webmanifest", + appleWebApp: { capable: true, title: "Plate", statusBarStyle: "black-translucent" }, +}; + +export const viewport: Viewport = { + themeColor: "#000000", + viewportFit: "cover", +}; + +export default function PlatePage() { + return ( +
+ +
+ ); +} diff --git a/components/set/MarkerPicker.tsx b/components/set/MarkerPicker.tsx index 5de4e03..a89d3a4 100644 --- a/components/set/MarkerPicker.tsx +++ b/components/set/MarkerPicker.tsx @@ -373,6 +373,16 @@ export default function MarkerPicker({ screens, extras }: { screens: MarkerScree → + + + Plate comp + Line up the shot against its background plate + + → + aspect) { + const cw = h * aspect; + return { x: (w - cw) / 2, y: 0, w: cw, h }; + } + const ch = w / aspect; + return { x: 0, y: (h - ch) / 2, w, h: ch }; +} + +// Where the frame sits inside the canvas box (letterboxed, "contain") +function fitRect(boxW: number, boxH: number, srcW: number, srcH: number) { + const scale = Math.min(boxW / srcW, boxH / srcH); + const w = srcW * scale, h = srcH * scale; + return { scale, w, h, x: (boxW - w) / 2, y: (boxH - h) / 2 }; +} + +// Nearest edge of the closed polygon to p, as the index to insert a new point at +function insertAt(pts: Pt[], p: Pt, fw: number, fh: number) { + let best = pts.length, bestD = Infinity; + for (let i = 0; i < pts.length; i++) { + const a = pts[i], b = pts[(i + 1) % pts.length]; + const ax = a[0] * fw, ay = a[1] * fh, bx = b[0] * fw, by = b[1] * fh, px = p[0] * fw, py = p[1] * fh; + const dx = bx - ax, dy = by - ay; + const t = Math.max(0, Math.min(1, ((px - ax) * dx + (py - ay) * dy) / (dx * dx + dy * dy || 1))); + const d = Math.hypot(px - (ax + t * dx), py - (ay + t * dy)); + if (d < bestD) { + bestD = d; + best = i + 1; + } + } + return best; +} + +type Status = "idle" | "starting" | "live" | "error"; +type Gesture = + | { kind: "point"; index: number } + | { kind: "pan"; x: number; y: number; place: Place } + | { kind: "pinch"; d: number; mx: number; my: number; place: Place } + | null; + +export default function PlateComp() { + const [status, setStatus] = useState("idle"); + const [error, setError] = useState(""); + const [frozen, setFrozen] = useState(false); + const [mode, setMode] = useState("plate"); + const [settings, setSettings] = useState(DEFAULTS); + const [plate, setPlate] = useState(null); + const [history, setHistory] = useState([]); + // The mask point last added or moved, for Delete point + const [sel, setSel] = useState(-1); + const [offline, setOffline] = useState(false); + + const canvasRef = useRef(null); + const videoRef = useRef(null); + const plateFileRef = useRef(null); + const photoFileRef = useRef(null); + const frozenCanvas = useRef(null); + const stream = useRef(null); + const pointers = useRef(new Map()); + const gesture = useRef(null); + const live = useRef({ settings: DEFAULTS, mode: "plate" as Mode, frozen: false, plate: null as HTMLCanvasElement | null, sel: -1 }); + + const showing = status === "live" || frozen; + + useEffect(() => { + live.current = { settings, mode, frozen, plate, sel }; + }, [settings, mode, frozen, plate, sel]); + + // Remember the plate, its placement and the mask between visits + useEffect(() => { + try { + const saved = JSON.parse(localStorage.getItem(SETTINGS_KEY) ?? "null"); + if (saved) requestAnimationFrame(() => setSettings((s) => ({ ...s, ...saved }))); + const url = localStorage.getItem(PLATE_KEY); + if (url) { + const img = new Image(); + img.onload = () => setPlate(toCanvas(img)); + img.src = url; + } + } catch {} + }, []); + useEffect(() => { + try { + localStorage.setItem(SETTINGS_KEY, JSON.stringify(settings)); + } catch {} + }, [settings]); + + const source = useCallback((): { el: CanvasImageSource; w: number; h: number } | null => { + if (live.current.frozen && frozenCanvas.current) { + const c = frozenCanvas.current; + return { el: c, w: c.width, h: c.height }; + } + const v = videoRef.current; + if (v && v.videoWidth) return { el: v, w: v.videoWidth, h: v.videoHeight }; + return null; + }, []); + + // The frame (camera crop) inside the canvas box, in CSS px + const frameRect = useCallback( + (bw: number, bh: number) => { + const src = source(); + if (!src) return null; + const cr = cropOf(src.w, src.h, live.current.settings.aspect); + return { src, cr, fit: fitRect(bw, bh, cr.w, cr.h) }; + }, + [source], + ); + + // Render loop: plate, then the camera clipped to the mask + useEffect(() => { + if (!showing) return; + const canvas = canvasRef.current; + if (!canvas) return; + const ctx = canvas.getContext("2d")!; + let raf = 0; + const draw = () => { + raf = requestAnimationFrame(draw); + const dpr = Math.min(window.devicePixelRatio || 1, 2); + const bw = canvas.clientWidth, bh = canvas.clientHeight; + if (canvas.width !== Math.round(bw * dpr) || canvas.height !== Math.round(bh * dpr)) { + canvas.width = Math.round(bw * dpr); + canvas.height = Math.round(bh * dpr); + } + ctx.setTransform(dpr, 0, 0, dpr, 0, 0); + ctx.fillStyle = "#000"; + ctx.fillRect(0, 0, bw, bh); + const fr = frameRect(bw, bh); + if (!fr) return; + const { src, cr, fit: f } = fr; + const { settings: s, mode: m, plate: pl, sel: sl } = live.current; + const hasMask = s.mask.length >= 3; + + ctx.save(); + ctx.beginPath(); + ctx.rect(f.x, f.y, f.w, f.h); + ctx.clip(); + + if (pl) { + const w = f.w * s.place.scale, h = (w * pl.height) / pl.width; + ctx.drawImage(pl, f.x + s.place.cx * f.w - w / 2, f.y + s.place.cy * f.h - h / 2, w, h); + } + const camera = (alpha: number) => { + ctx.globalAlpha = alpha; + ctx.drawImage(src.el, cr.x, cr.y, cr.w, cr.h, f.x, f.y, f.w, f.h); + ctx.globalAlpha = 1; + }; + const path = () => { + ctx.beginPath(); + s.mask.forEach(([x, y], i) => (i ? ctx.lineTo(f.x + x * f.w, f.y + y * f.h) : ctx.moveTo(f.x + x * f.w, f.y + y * f.h))); + ctx.closePath(); + }; + + // Lining up: the whole camera picture shows faintly over the plate + if (m !== "comp" && pl) camera(GHOST); + if (hasMask) { + ctx.save(); + path(); + ctx.clip(); + camera(1); + ctx.restore(); + } else if (!pl || m === "mask") { + // No mask yet: nothing to comp, so show the camera + camera(1); + } + + // Mask outline and points + if (s.mask.length && (m === "mask" || s.outline)) { + path(); + if (s.mask.length < 3 || m === "mask") { + ctx.lineWidth = 3; + ctx.strokeStyle = "#000"; + ctx.stroke(); + } + ctx.lineWidth = 1.5; + ctx.setLineDash(m === "mask" ? [] : [6, 4]); + ctx.strokeStyle = "#fff"; + ctx.stroke(); + ctx.setLineDash([]); + if (m === "mask") { + s.mask.forEach(([x, y], i) => { + ctx.beginPath(); + ctx.arc(f.x + x * f.w, f.y + y * f.h, i === sl ? 9 : 6, 0, Math.PI * 2); + ctx.fillStyle = i === sl ? "#ffd600" : "#fff"; + ctx.fill(); + ctx.lineWidth = 2; + ctx.strokeStyle = "#000"; + ctx.stroke(); + }); + } + } + ctx.restore(); + }; + draw(); + return () => cancelAnimationFrame(raf); + }, [showing, frameRect]); + + // Keep the screen awake while lining up + useEffect(() => { + if (status !== "live") return; + let wake: WakeLockSentinel | null = null; + const acquire = async () => { + try { + wake = (await navigator.wakeLock?.request("screen")) ?? null; + } catch {} + }; + const onVisible = () => document.visibilityState === "visible" && acquire(); + acquire(); + document.addEventListener("visibilitychange", onVisible); + return () => { + document.removeEventListener("visibilitychange", onVisible); + wake?.release().catch(() => {}); + }; + }, [status]); + + // Stop the camera when leaving the page + useEffect(() => () => stream.current?.getTracks().forEach((t) => t.stop()), []); + + // Offline: same service worker as /set; save this page and its assets on first visit + useEffect(() => { + if (process.env.NODE_ENV !== "production" || !("serviceWorker" in navigator) || !("caches" in window)) return; + let cancelled = false; + (async () => { + try { + await navigator.serviceWorker.register("/set/sw.js", { scope: "/set/" }); + await navigator.serviceWorker.ready; + const assets = performance + .getEntriesByType("resource") + .map((e) => new URL(e.name)) + .filter((u) => u.origin === location.origin && /^\/(_next\/|icon|apple-icon)/.test(u.pathname)) + .map((u) => u.pathname + u.search); + const cache = await caches.open(OFFLINE_CACHE); + await cache.addAll(["/set/plate/", ...new Set(assets)]); + if (!cancelled) setOffline(true); + } catch {} + })(); + return () => { + cancelled = true; + }; + }, []); + + const startCamera = async () => { + setError(""); + if (!navigator.mediaDevices?.getUserMedia) { + setError(window.isSecureContext ? "This browser has no camera access." : "The camera needs a secure (https) page."); + setStatus("error"); + return; + } + setStatus("starting"); + try { + const s = await navigator.mediaDevices.getUserMedia({ + audio: false, + video: { facingMode: { ideal: "environment" }, width: { ideal: 1920 }, height: { ideal: 1080 } }, + }); + stream.current = s; + const v = videoRef.current!; + v.srcObject = s; + await v.play(); + setFrozen(false); + setStatus("live"); + } catch (e) { + const name = (e as DOMException)?.name; + setError( + name === "NotAllowedError" + ? "Camera permission was refused. Allow camera access for this site in your browser settings, then try again." + : "Couldn't start the camera.", + ); + setStatus("error"); + } + }; + + const freeze = () => { + const v = videoRef.current; + if (!v?.videoWidth) return; + const c = (frozenCanvas.current ??= document.createElement("canvas")); + c.width = v.videoWidth; + c.height = v.videoHeight; + c.getContext("2d")!.drawImage(v, 0, 0); + setFrozen(true); + }; + + const loadImage = (file: File, done: (img: HTMLImageElement) => void) => { + const url = URL.createObjectURL(file); + const img = new Image(); + img.onload = () => { + done(img); + URL.revokeObjectURL(url); + }; + img.src = url; + }; + + const openPlate = (file: File) => + loadImage(file, (img) => { + const c = toCanvas(img); + setPlate(c); + setSettings((s) => ({ ...s, place: FIT })); + try { + localStorage.setItem(PLATE_KEY, c.toDataURL("image/jpeg", 0.85)); + } catch { + // Too big to remember; it still works for this visit + try { + localStorage.removeItem(PLATE_KEY); + } catch {} + } + }); + + // A still in place of the camera, for planning off set + const openPhoto = (file: File) => + loadImage(file, (img) => { + frozenCanvas.current = toCanvas(img); + setFrozen(true); + }); + + const set = (k: K, v: Settings[K]) => setSettings((s) => ({ ...s, [k]: v })); + const setMask = (mask: Pt[], undoable = true) => { + if (undoable) setHistory((h) => [...h.slice(-49), settings.mask]); + set("mask", mask); + }; + const setPlace = (p: Place) => set("place", { ...p, scale: Math.max(0.1, Math.min(10, p.scale)) }); + + // Pointer position as frame units (0..1 across the frame) + const toFrame = (e: { clientX: number; clientY: number }, el: HTMLElement) => { + const rect = el.getBoundingClientRect(); + const fr = frameRect(rect.width, rect.height); + if (!fr) return null; + const { fit: f } = fr; + return { p: [(e.clientX - rect.left - f.x) / f.w, (e.clientY - rect.top - f.y) / f.h] as Pt, f }; + }; + + const startPinch = () => { + const [a, b] = [...pointers.current.values()]; + gesture.current = { kind: "pinch", d: Math.hypot(a.x - b.x, a.y - b.y), mx: (a.x + b.x) / 2, my: (a.y + b.y) / 2, place: settings.place }; + }; + + const onDown = (e: React.PointerEvent) => { + e.currentTarget.setPointerCapture(e.pointerId); + pointers.current.set(e.pointerId, { x: e.clientX, y: e.clientY }); + if (mode === "plate") { + if (pointers.current.size === 2) startPinch(); + else gesture.current = { kind: "pan", x: e.clientX, y: e.clientY, place: settings.place }; + return; + } + if (mode !== "mask" || pointers.current.size > 1) return; + const hit = toFrame(e, e.currentTarget); + if (!hit) return; + const { p, f } = hit; + const near = settings.mask.findIndex(([x, y]) => Math.hypot((x - p[0]) * f.w, (y - p[1]) * f.h) < HIT); + if (near >= 0) { + setHistory((h) => [...h.slice(-49), settings.mask]); + gesture.current = { kind: "point", index: near }; + setSel(near); + return; + } + const q: Pt = [Math.max(0, Math.min(1, p[0])), Math.max(0, Math.min(1, p[1]))]; + const i = settings.mask.length < 3 ? settings.mask.length : insertAt(settings.mask, q, f.w, f.h); + setMask([...settings.mask.slice(0, i), q, ...settings.mask.slice(i)]); + gesture.current = { kind: "point", index: i }; + setSel(i); + }; + + const onMove = (e: React.PointerEvent) => { + if (!pointers.current.has(e.pointerId)) return; + pointers.current.set(e.pointerId, { x: e.clientX, y: e.clientY }); + const g = gesture.current; + if (!g) return; + const rect = e.currentTarget.getBoundingClientRect(); + const fr = frameRect(rect.width, rect.height); + if (!fr) return; + const { fit: f } = fr; + if (g.kind === "point") { + const hit = toFrame(e, e.currentTarget); + if (!hit) return; + const q: Pt = [Math.max(0, Math.min(1, hit.p[0])), Math.max(0, Math.min(1, hit.p[1]))]; + setSettings((s) => ({ ...s, mask: s.mask.map((pt, i) => (i === g.index ? q : pt)) })); + } else if (g.kind === "pan") { + setPlace({ ...g.place, cx: g.place.cx + (e.clientX - g.x) / f.w, cy: g.place.cy + (e.clientY - g.y) / f.h }); + } else if (g.kind === "pinch" && pointers.current.size >= 2) { + const [a, b] = [...pointers.current.values()]; + const ratio = Math.hypot(a.x - b.x, a.y - b.y) / (g.d || 1); + const mx = (a.x + b.x) / 2, my = (a.y + b.y) / 2; + // Scale about where the fingers started, and follow them as they move + const ox = (g.mx - rect.left - f.x) / f.w, oy = (g.my - rect.top - f.y) / f.h; + setPlace({ + scale: g.place.scale * ratio, + cx: ox + (g.place.cx - ox) * ratio + (mx - g.mx) / f.w, + cy: oy + (g.place.cy - oy) * ratio + (my - g.my) / f.h, + }); + } + }; + + const onUp = (e: React.PointerEvent) => { + pointers.current.delete(e.pointerId); + const left = [...pointers.current.values()][0]; + // Lifting one finger of a pinch carries on as a pan with the other + gesture.current = mode === "plate" && left ? { kind: "pan", x: left.x, y: left.y, place: settings.place } : null; + }; + + // Desktop: scroll wheel scales the plate about the cursor + useEffect(() => { + const canvas = canvasRef.current; + if (!canvas || mode !== "plate") return; + const onWheel = (e: WheelEvent) => { + e.preventDefault(); + const rect = canvas.getBoundingClientRect(); + const fr = frameRect(rect.width, rect.height); + if (!fr) return; + const { fit: f } = fr; + const ratio = Math.exp(-e.deltaY * 0.0015); + const ox = (e.clientX - rect.left - f.x) / f.w, oy = (e.clientY - rect.top - f.y) / f.h; + setSettings((s) => ({ + ...s, + place: { scale: Math.max(0.1, Math.min(10, s.place.scale * ratio)), cx: ox + (s.place.cx - ox) * ratio, cy: oy + (s.place.cy - oy) * ratio }, + })); + }; + canvas.addEventListener("wheel", onWheel, { passive: false }); + return () => canvas.removeEventListener("wheel", onWheel); + }, [mode, showing, frameRect]); + + const fitHeight = () => { + const src = source(); + if (!src || !plate) return; + const cr = cropOf(src.w, src.h, settings.aspect); + // Scale is relative to the frame width; fit height makes the plate as tall as the frame + setPlace({ cx: 0.5, cy: 0.5, scale: ((plate.width / plate.height) * cr.h) / cr.w }); + }; + + const undo = () => { + const prev = history[history.length - 1]; + if (!prev) return; + setHistory((h) => h.slice(0, -1)); + set("mask", prev); + }; + + const cycleAspect = () => { + const i = ASPECTS.findIndex(([a]) => a === settings.aspect); + set("aspect", ASPECTS[(i + 1) % ASPECTS.length][0]); + }; + + const hint = + !plate + ? "Load the background plate to start" + : mode === "plate" + ? "Drag to move the plate · pinch or scroll to scale" + : mode === "mask" && settings.mask.length < 3 + ? "Tap around the foreground to draw the mask" + : mode === "mask" + ? "Drag points to adjust · tap an edge to add a point" + : null; + + return ( +
+
+ ); +} + +// Downscale to a size that's quick to draw every frame and small enough to remember +function toCanvas(img: HTMLImageElement) { + const scale = Math.min(1, PLATE_MAX / Math.max(img.width, img.height)); + const c = document.createElement("canvas"); + c.width = Math.round(img.width * scale); + c.height = Math.round(img.height * scale); + c.getContext("2d")!.drawImage(img, 0, 0, c.width, c.height); + return c; +} + +function FileInput({ inputRef, onFile }: { inputRef: React.RefObject; onFile: (f: File) => void }) { + return ( + { + const f = e.target.files?.[0]; + if (f) onFile(f); + e.target.value = ""; + }} + /> + ); +} + +function StartScreen({ + status, + error, + plate, + onCamera, + onPlate, + onPhoto, +}: { + status: Status; + error: string; + plate: HTMLCanvasElement | null; + onCamera: () => void; + onPlate: () => void; + onPhoto: () => void; +}) { + return ( +
+

+ Line up the shot against its background plate. The plate sits behind, the camera shows through a mask you draw + around the foreground. +

+
    +
  1. Load the background plate (a still or frame grab).
    2. +
  2. Start the camera. Move and scale the plate to match.
    3. +
  3. Draw a mask around where the foreground goes.
    4. +
+ {error &&

{error}

} + + + +
+ ); +} + +function Action({ children, onClick, active, disabled }: { children: React.ReactNode; onClick: () => void; active?: boolean; disabled?: boolean }) { + return ( + + ); +} diff --git a/public/set/sw.js b/public/set/sw.js index a4b4256..caac9e7 100644 --- a/public/set/sw.js +++ b/public/set/sw.js @@ -1,4 +1,4 @@ -// Offline cache for /set (screen markers, spill checker, VFX report). Crews often have no signal on set: +// Offline cache for /set (screen markers, spill checker, plate comp, VFX report). Crews often have no signal on set: // once the page has loaded once, it and every marker image work offline. // Must match OFFLINE_CACHE in components/set/MarkerPicker.tsx const CACHE = "ttvfx-set-v1"; @@ -22,7 +22,7 @@ self.addEventListener("fetch", (event) => { const url = new URL(req.url); if (url.origin !== self.location.origin) return; - // Pages (/set/, /set/spill/): network first so updates arrive, cached copy when offline. + // Pages (/set/, /set/spill/, /set/plate/): network first so updates arrive, cached copy when offline. // Each page is cached under its own path so one never replaces the other. if (req.mode === "navigate") { const page = url.pathname.endsWith("/") ? url.pathname : url.pathname + "/";