From 2817c7e815eb367473234cf3bec9fc3cecfa2f40 Mon Sep 17 00:00:00 2001 From: twotalesanimation <80506065+twotalesanimation@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:34:57 +0200 Subject: [PATCH] fix(annotations): lock annotations to the picture at any player size Points were normalised to the whole player box, but the video is drawn object-contain inside it, so any change in the player's shape (window or panel resize, fullscreen, phone rotation, the annotation toolbar opening) moved the picture under the marks. The canvas now sits exactly over the displayed picture (same letterbox measurement as AspectOverlay), so points are fractions of the image and new saves are tagged version "2.0". Legacy "1.0" annotations are mapped onto the picture at load time from their stored canvasWidth/canvasHeight and the video's intrinsic size; no data migration. The save action now accepts both versions. Co-Authored-By: Claude Opus 5 --- actions/annotations.ts | 2 +- components/annotations/AnnotationCanvas.tsx | 102 ++++++++++++++++++-- components/player/ReviewPlayer.tsx | 1 + types/index.ts | 11 ++- 4 files changed, 103 insertions(+), 13 deletions(-) diff --git a/actions/annotations.ts b/actions/annotations.ts index 917a4b2..466b85f 100644 --- a/actions/annotations.ts +++ b/actions/annotations.ts @@ -14,7 +14,7 @@ const saveAnnotationSchema = z.object({ shapes: z.array(z.any()), canvasWidth: z.number(), canvasHeight: z.number(), - version: z.literal("1.0"), + version: z.enum(["1.0", "2.0"]), }), color: z.string().default("#ef4444"), }); diff --git a/components/annotations/AnnotationCanvas.tsx b/components/annotations/AnnotationCanvas.tsx index d343d2c..7fc3a3c 100644 --- a/components/annotations/AnnotationCanvas.tsx +++ b/components/annotations/AnnotationCanvas.tsx @@ -6,11 +6,13 @@ import { useState, useCallback, MouseEvent, + type RefObject, } from "react"; import { useReviewStore } from "@/hooks/use-review-player"; import { saveAnnotation } from "@/actions/annotations"; import { addComment } from "@/actions/comments"; import { useToast } from "@/components/ui/use-toast"; +import { letterboxRect } from "@/lib/aspect-overlays"; import type { AnnotationShape, AnnotationDrawingData, AnnotationPoint } from "@/types"; import { v4 as uuidv4 } from "uuid"; @@ -30,6 +32,8 @@ interface AddCommentData { interface AnnotationCanvasProps { versionId: string; + /** The video under the canvas — the canvas is sized to its displayed picture. */ + videoRef: RefObject; frameNumber: number; fps: number; isAnnotating: boolean; @@ -42,6 +46,20 @@ interface AnnotationCanvasProps { addCommentFn?: (data: AddCommentData) => Promise; } +/** + * A shape as held in memory. Legacy (v1.0) shapes keep their points relative to + * the player box they were drawn in; `legacyBox` is that box, used to map them + * onto the picture once the video's intrinsic size is known. + */ +type LoadedShape = AnnotationShape & { legacyBox?: { w: number; h: number } }; + +function shapesOf(drawingData: AnnotationDrawingData | undefined): LoadedShape[] { + const shapes = drawingData?.shapes ?? []; + if (!drawingData || drawingData.version === "2.0") return shapes; + const legacyBox = { w: drawingData.canvasWidth, h: drawingData.canvasHeight }; + return shapes.map((s) => ({ ...s, legacyBox })); +} + type DrawingState = { isDrawing: boolean; currentShape: AnnotationShape | null; @@ -50,6 +68,7 @@ type DrawingState = { export function AnnotationCanvas({ versionId, + videoRef, frameNumber, fps, isAnnotating, @@ -73,10 +92,10 @@ export function AnnotationCanvas({ // Per-frame saved shapes — persists across frame navigation so annotations // remain visible without depending on a parent API refetch. - const [savedShapesByFrame, setSavedShapesByFrame] = useState>(() => { - const map: Record = {}; + const [savedShapesByFrame, setSavedShapesByFrame] = useState>(() => { + const map: Record = {}; for (const a of (existingAnnotations as { frameNumber: number; drawingData: AnnotationDrawingData }[])) { - const shapes = a.drawingData?.shapes ?? []; + const shapes = shapesOf(a.drawingData); map[a.frameNumber] = [...(map[a.frameNumber] ?? []), ...shapes]; } return map; @@ -114,10 +133,10 @@ export function AnnotationCanvas({ setSavedShapesByFrame((prev) => { // Build map from server data - const serverMap: Record = {}; + const serverMap: Record = {}; const serverShapeIds = new Set(); for (const a of (existingAnnotations as { frameNumber: number; drawingData: AnnotationDrawingData }[])) { - const shapes = a.drawingData?.shapes ?? []; + const shapes = shapesOf(a.drawingData); serverMap[a.frameNumber] = [...(serverMap[a.frameNumber] ?? []), ...shapes]; for (const s of shapes) serverShapeIds.add(s.id); } @@ -145,6 +164,65 @@ export function AnnotationCanvas({ }, [existingAnnotations]); + // ── Picture rect ───────────────────────────────────────────────────────── + // The canvas is laid exactly over the displayed picture (not the letterboxed + // player box), so normalized points are fractions of the image itself and + // stay locked to it at any player size. Same measurement as AspectOverlay; + // clientWidth/Height ignore the zoom transform, which the canvas shares. + const [pictureRect, setPictureRect] = useState<{ + left: number; + top: number; + width: number; + height: number; + } | null>(null); + + useEffect(() => { + const video = videoRef.current; + if (!video) return; + const measure = () => { + const rect = letterboxRect( + video.clientWidth, + video.clientHeight, + video.videoWidth, + video.videoHeight + ); + setPictureRect((prev) => + prev && + rect && + prev.left === rect.left && + prev.top === rect.top && + prev.width === rect.width && + prev.height === rect.height + ? prev + : rect + ); + }; + measure(); + const observer = new ResizeObserver(measure); + observer.observe(video); + video.addEventListener("loadedmetadata", measure); + return () => { + observer.disconnect(); + video.removeEventListener("loadedmetadata", measure); + }; + }, [videoRef]); + + /** A shape's points as fractions of the picture (maps legacy box-relative ones). */ + const pictureSpacePoints = useCallback( + (shape: LoadedShape): AnnotationPoint[] => { + const box = shape.legacyBox; + const video = videoRef.current; + if (!box || !video) return shape.points; + const r = letterboxRect(box.w, box.h, video.videoWidth, video.videoHeight); + if (!r) return shape.points; // no metadata yet — best effort + return shape.points.map((p) => ({ + x: (p.x * box.w - r.left) / r.width, + y: (p.y * box.h - r.top) / r.height, + })); + }, + [videoRef] + ); + // ── Coordinate normalization ───────────────────────────────────────────── const toNormalized = (e: MouseEvent): AnnotationPoint => { const canvas = canvasRef.current!; @@ -164,7 +242,7 @@ export function AnnotationCanvas({ const renderShape = useCallback( ( ctx: CanvasRenderingContext2D, - shape: AnnotationShape, + shape: LoadedShape, w: number, h: number ) => { @@ -176,7 +254,7 @@ export function AnnotationCanvas({ ctx.lineJoin = "round"; ctx.globalAlpha = 0.85; - const pts = shape.points.map((p) => toPixel(p, w, h)); + const pts = pictureSpacePoints(shape).map((p) => toPixel(p, w, h)); switch (shape.tool) { case "freehand": { @@ -244,7 +322,7 @@ export function AnnotationCanvas({ } ctx.globalAlpha = 1; }, - [] + [pictureSpacePoints] ); const redraw = useCallback(() => { @@ -352,7 +430,7 @@ export function AnnotationCanvas({ shapes: [finishedShape], canvasWidth: canvas?.offsetWidth ?? 1920, canvasHeight: canvas?.offsetHeight ?? 1080, - version: "1.0", + version: "2.0", }; const doSaveAnnotation = saveAnnotationFn @@ -447,6 +525,12 @@ export function AnnotationCanvas({ ref={canvasRef} className={`annotation-canvas-overlay ${isAnnotating ? "is-annotating" : ""}`} style={{ + ...(pictureRect && { + left: pictureRect.left, + top: pictureRect.top, + width: pictureRect.width, + height: pictureRect.height, + }), cursor: isAnnotating ? "crosshair" : "default", touchAction: isAnnotating ? "none" : "auto", }} diff --git a/components/player/ReviewPlayer.tsx b/components/player/ReviewPlayer.tsx index 1696747..7cc908f 100644 --- a/components/player/ReviewPlayer.tsx +++ b/components/player/ReviewPlayer.tsx @@ -338,6 +338,7 @@ export const ReviewPlayer = forwardRef( {/* Annotation canvas overlay */}