fix(annotations): lock annotations to the picture at any player size

Points were normalised to the whole player box, but the video is drawn
object-contain inside it, so any change in the player's shape (window or
panel resize, fullscreen, phone rotation, the annotation toolbar opening)
moved the picture under the marks.

The canvas now sits exactly over the displayed picture (same letterbox
measurement as AspectOverlay), so points are fractions of the image and
new saves are tagged version "2.0". Legacy "1.0" annotations are mapped
onto the picture at load time from their stored canvasWidth/canvasHeight
and the video's intrinsic size; no data migration. The save action now
accepts both versions.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

components/annotations/AnnotationCanvas.tsx

@@ -6,11 +6,13 @@ import {
   useState,
   useCallback,
   MouseEvent,
+  type RefObject,
 } from "react";
 import { useReviewStore } from "@/hooks/use-review-player";
 import { saveAnnotation } from "@/actions/annotations";
 import { addComment } from "@/actions/comments";
 import { useToast } from "@/components/ui/use-toast";
+import { letterboxRect } from "@/lib/aspect-overlays";
 import type { AnnotationShape, AnnotationDrawingData, AnnotationPoint } from "@/types";
 import { v4 as uuidv4 } from "uuid";
 
@@ -30,6 +32,8 @@ interface AddCommentData {
 
 interface AnnotationCanvasProps {
   versionId: string;
+  /** The video under the canvas — the canvas is sized to its displayed picture. */
+  videoRef: RefObject<HTMLVideoElement | null>;
   frameNumber: number;
   fps: number;
   isAnnotating: boolean;
@@ -42,6 +46,20 @@ interface AnnotationCanvasProps {
   addCommentFn?: (data: AddCommentData) => Promise<void>;
 }
 
+/**
+ * A shape as held in memory. Legacy (v1.0) shapes keep their points relative to
+ * the player box they were drawn in; `legacyBox` is that box, used to map them
+ * onto the picture once the video's intrinsic size is known.
+ */
+type LoadedShape = AnnotationShape & { legacyBox?: { w: number; h: number } };
+
+function shapesOf(drawingData: AnnotationDrawingData | undefined): LoadedShape[] {
+  const shapes = drawingData?.shapes ?? [];
+  if (!drawingData || drawingData.version === "2.0") return shapes;
+  const legacyBox = { w: drawingData.canvasWidth, h: drawingData.canvasHeight };
+  return shapes.map((s) => ({ ...s, legacyBox }));
+}
+
 type DrawingState = {
   isDrawing: boolean;
   currentShape: AnnotationShape | null;
@@ -50,6 +68,7 @@ type DrawingState = {
 
 export function AnnotationCanvas({
   versionId,
+  videoRef,
   frameNumber,
   fps,
   isAnnotating,
@@ -73,10 +92,10 @@ export function AnnotationCanvas({
 
   // Per-frame saved shapes — persists across frame navigation so annotations
   // remain visible without depending on a parent API refetch.
-  const [savedShapesByFrame, setSavedShapesByFrame] = useState<Record<number, AnnotationShape[]>>(() => {
-    const map: Record<number, AnnotationShape[]> = {};
+  const [savedShapesByFrame, setSavedShapesByFrame] = useState<Record<number, LoadedShape[]>>(() => {
+    const map: Record<number, LoadedShape[]> = {};
     for (const a of (existingAnnotations as { frameNumber: number; drawingData: AnnotationDrawingData }[])) {
-      const shapes = a.drawingData?.shapes ?? [];
+      const shapes = shapesOf(a.drawingData);
       map[a.frameNumber] = [...(map[a.frameNumber] ?? []), ...shapes];
     }
     return map;
@@ -114,10 +133,10 @@ export function AnnotationCanvas({
 
     setSavedShapesByFrame((prev) => {
       // Build map from server data
-      const serverMap: Record<number, AnnotationShape[]> = {};
+      const serverMap: Record<number, LoadedShape[]> = {};
       const serverShapeIds = new Set<string>();
       for (const a of (existingAnnotations as { frameNumber: number; drawingData: AnnotationDrawingData }[])) {
-        const shapes = a.drawingData?.shapes ?? [];
+        const shapes = shapesOf(a.drawingData);
         serverMap[a.frameNumber] = [...(serverMap[a.frameNumber] ?? []), ...shapes];
         for (const s of shapes) serverShapeIds.add(s.id);
       }
@@ -145,6 +164,65 @@ export function AnnotationCanvas({
   }, [existingAnnotations]);
 
 
+  // ── Picture rect ─────────────────────────────────────────────────────────
+  // The canvas is laid exactly over the displayed picture (not the letterboxed
+  // player box), so normalized points are fractions of the image itself and
+  // stay locked to it at any player size. Same measurement as AspectOverlay;
+  // clientWidth/Height ignore the zoom transform, which the canvas shares.
+  const [pictureRect, setPictureRect] = useState<{
+    left: number;
+    top: number;
+    width: number;
+    height: number;
+  } | null>(null);
+
+  useEffect(() => {
+    const video = videoRef.current;
+    if (!video) return;
+    const measure = () => {
+      const rect = letterboxRect(
+        video.clientWidth,
+        video.clientHeight,
+        video.videoWidth,
+        video.videoHeight
+      );
+      setPictureRect((prev) =>
+        prev &&
+        rect &&
+        prev.left === rect.left &&
+        prev.top === rect.top &&
+        prev.width === rect.width &&
+        prev.height === rect.height
+          ? prev
+          : rect
+      );
+    };
+    measure();
+    const observer = new ResizeObserver(measure);
+    observer.observe(video);
+    video.addEventListener("loadedmetadata", measure);
+    return () => {
+      observer.disconnect();
+      video.removeEventListener("loadedmetadata", measure);
+    };
+  }, [videoRef]);
+
+  /** A shape's points as fractions of the picture (maps legacy box-relative ones). */
+  const pictureSpacePoints = useCallback(
+    (shape: LoadedShape): AnnotationPoint[] => {
+      const box = shape.legacyBox;
+      const video = videoRef.current;
+      if (!box || !video) return shape.points;
+      const r = letterboxRect(box.w, box.h, video.videoWidth, video.videoHeight);
+      if (!r) return shape.points; // no metadata yet — best effort
+      return shape.points.map((p) => ({
+        x: (p.x * box.w - r.left) / r.width,
+        y: (p.y * box.h - r.top) / r.height,
+      }));
+    },
+    [videoRef]
+  );
+
   // ── Coordinate normalization ─────────────────────────────────────────────
   const toNormalized = (e: MouseEvent): AnnotationPoint => {
     const canvas = canvasRef.current!;
@@ -164,7 +242,7 @@ export function AnnotationCanvas({
   const renderShape = useCallback(
     (
       ctx: CanvasRenderingContext2D,
-      shape: AnnotationShape,
+      shape: LoadedShape,
       w: number,
       h: number
     ) => {
@@ -176,7 +254,7 @@ export function AnnotationCanvas({
       ctx.lineJoin = "round";
       ctx.globalAlpha = 0.85;
 
-      const pts = shape.points.map((p) => toPixel(p, w, h));
+      const pts = pictureSpacePoints(shape).map((p) => toPixel(p, w, h));
 
       switch (shape.tool) {
         case "freehand": {
@@ -244,7 +322,7 @@ export function AnnotationCanvas({
       }
       ctx.globalAlpha = 1;
     },
-    []
+    [pictureSpacePoints]
   );
 
   const redraw = useCallback(() => {
@@ -352,7 +430,7 @@ export function AnnotationCanvas({
         shapes: [finishedShape],
         canvasWidth: canvas?.offsetWidth ?? 1920,
         canvasHeight: canvas?.offsetHeight ?? 1080,
-        version: "1.0",
+        version: "2.0",
       };
 
       const doSaveAnnotation = saveAnnotationFn
@@ -447,6 +525,12 @@ export function AnnotationCanvas({
       ref={canvasRef}
       className={`annotation-canvas-overlay ${isAnnotating ? "is-annotating" : ""}`}
       style={{
+        ...(pictureRect && {
+          left: pictureRect.left,
+          top: pictureRect.top,
+          width: pictureRect.width,
+          height: pictureRect.height,
+        }),
         cursor: isAnnotating ? "crosshair" : "default",
         touchAction: isAnnotating ? "none" : "auto",
       }}