Show a cut into the middle of a clip without its old picture (#404)

A switched-off driven video waits at the video time it has where it
is, before its first key mostly the start value. Switched on at
another spot, its player has to seek first. Until the seek is done the
texture still holds the old picture, so the start of the clip flashed
for a frame or two at the cut.

- A view holds its last picture while a driven video that just came
  into sight is still seeking, for at most 0.3 s. At a cut the old clip
  stays one frame longer and the right picture of the new one follows.
- A seek counts as done only when the browser hands in the next video
  frame (requestVideoFrameCallback), and the texture is uploaded right
  then. The seeked event alone was too early, one red frame was left.
- A seek that takes longer than 0.3 s leaves the sprite out until its
  picture is there. The wrong picture does not show either way.

Tests:
- e2e 66q switches a sprite off at 0.2 s (red) and on at 1.2 s (blue)
  and reads every frame in the browser. No red may show, blue must
  stand at the end.
- Against the old code 66q fails with 2 red frames before the blue.
  With only the seeked event it still fails with 1 red frame.
- e2e 66 on melektaus-atx: ALL PASSED, 28 checks, run twice.
- Unit video-library, video-follow: 19 passed.
- Gate on melektaus-atx: CHECK EXIT 0, 297 files, 4562 tests.
- tsc, eslint, Prettier clean.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

README.md

@@ -47,7 +47,7 @@ npm run build        # writes dist/ and regenerates index.html; the folder is wh
 The browser suite and the three example games need a machine with a GPU:
 
 ```bash
-node tools/seed-e2e.mjs && node tools/e2e.mjs      # the editor, 2007 checks (GM_STEPS=20 for one step)
+node tools/seed-e2e.mjs && node tools/e2e.mjs      # the editor, 2008 checks (GM_STEPS=20 for one step)
                                                    #   every tool signs in first – see the Accounts page
 node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs
 node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs

src/viewport/VideoLibrary.ts

@@ -57,6 +57,8 @@ interface Drive {
   reading: SpeedReading | null;
   /** The newest seek asked for while one was still running: only that one follows. */
   wanted: number | null;
+  /** Set from a seek until the picture it went to is in the texture, see `seek`. */
+  landing: boolean;
 }
 
 interface Player {
@@ -130,6 +132,15 @@ export class VideoLibrary {
     return this.players.get(key)?.texture != null;
   }
 
+  /**
+   * Whether the player behind `key` shows the picture its wearer asked for last: it has one and no
+   * seek is under way. Until a seek has landed, its texture still holds the picture it left.
+   */
+  showsAsked(key: string): boolean {
+    const player = this.players.get(key);
+    return player?.texture != null && !player.drive?.landing;
+  }
+
   /**
    * Whether a driven player (#404) the view `view` wears (see `wear`) is still without its first
    * picture. A run waits for them at its start: the Game tab opens players of its own at Play.
@@ -180,7 +191,7 @@ export class VideoLibrary {
       // A video that ran to the end of its file starts over from its first picture on play().
       end: element.ended ? element.currentTime : Math.min(play.end, element.duration),
     });
-    if (step.seek !== null) this.seek(element, drive, step.seek);
+    if (step.seek !== null) this.seek(player, drive, step.seek);
     if (step.kind === 'hold') {
       element.muted = true;
       if (!element.paused) element.pause();
@@ -192,10 +203,28 @@ export class VideoLibrary {
     if (element.paused) void element.play().catch(() => {});
   }
 
-  /** Seeks, or, while a seek is still running, remembers the newest wish for when it is done. */
-  private seek(element: HTMLVideoElement, drive: Drive, time: number): void {
+  /**
+   * Seeks, or, while a seek is still running, remembers the newest wish for when it is done. The
+   * seek has landed with the first frame Chrome hands over once all of them are done: at `seeked`
+   * the picture was not there yet, and Pixi, which loads a paused video's picture only then, kept
+   * the one before it for a frame.
+   */
+  private seek(player: Player, drive: Drive, time: number): void {
+    const { element } = player;
     if (element.seeking) drive.wanted = time;
     else element.currentTime = time;
+    if (drive.landing) return;
+    drive.landing = true;
+    const land = () => {
+      if (player.gone) return;
+      if (element.seeking || drive.wanted !== null) {
+        element.requestVideoFrameCallback(land);
+        return;
+      }
+      player.texture?.source.update();
+      drive.landing = false;
+    };
+    element.requestVideoFrameCallback(land);
   }
 
   /** Lets every player go: a session that ends takes its videos along. */
@@ -255,7 +284,7 @@ export class VideoLibrary {
     const play = parseVideoKey(key);
     if (!play) return null;
     const drive: Drive | null = isDrivenVideoKey(key)
-      ? { time, heard: false, reading: null, wanted: null }
+      ? { time, heard: false, reading: null, wanted: null, landing: false }
       : null;
     const at = drive ? drivenTimeOf(play, time, NaN) : play.start;
     const player: Player = {

src/viewport/WorldViewport.ts

@@ -26,11 +26,14 @@ import type { ActiveCamera } from './activeCamera';
 import { CameraShade, shadedPicture } from './cameraShade';
 import { snapPlacement } from './pixelGrid';
 import { cameraSees } from '../core/model/renderLayers';
-import { isVideoKey } from '../core/model/video';
+import { drivenVideoKey, isVideoKey } from '../core/model/video';
+import { videoDrivenOf } from '../core/properties/videos';
 import { videoLibrary } from './VideoLibrary';
 
 /** How strongly the cut-off part of an object shows while the scene view opens its clip area (#260). */
 const CLIP_GHOST_ALPHA = 0.3;
+/** How long a view keeps its last frame at most while a driven video it brings in seeks (#404). */
+const VIDEO_HOLD_MS = 300;
 
 let views = 0;
 
@@ -100,6 +103,10 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
   private readonly animateKey = `animate:${(views += 1)}`;
   /** What a driven video (#404) knows this view by: each view drives its own players. */
   private readonly wearer = `view${views}`;
+  /** The driven player (#404) each sprite showed in the last redraw, by object id. */
+  private readonly videosShown = new Map<string, string>();
+  /** Since when the view keeps its last frame for a driven video it brings in, see `holdsForVideo`. */
+  private heldSince: number | null = null;
   private cameraMoves = 0;
   private readonly stopFiles: (() => void)[];
   /** One view per overlay camera (#302), by camera id, laid over this one in camera order. */
@@ -369,6 +376,14 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
 
   override redraw(): void {
     const view = this.currentView();
+    const frame = view && this.collectFrame();
+    const camera = frame ? this.mainCamera() : null;
+    const items = frame ? (camera ? seenBy(camera, frame.items) : frame.items) : [];
+    const shaders = shaderLibrary.source(this.state.project);
+    if (view && this.holdsForVideo(view, items, shaders)) {
+      this.animated = true;
+      return;
+    }
     this.bg.clear().rect(0, 0, this.w, this.h).fill(this.baseColor);
     this.beforeItems(view);
     this.contentMask.clear();
@@ -376,17 +391,14 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
     this.visibleCount = 0;
     this.animated = false;
     this.drawn = null;
-    if (!view) {
+    if (!view || !frame) {
       this.unshadeCamera();
       this.syncOverlays(null, null, null);
       return;
     }
 
     this.redraws++;
-    const frame = this.collectFrame();
     const { background } = frame;
-    const camera = this.mainCamera();
-    const items = camera ? seenBy(camera, frame.items) : frame.items;
     this.lastItems = items;
     const { rect } = view;
     if (this.paintsBackground()) this.bg.rect(rect.x, rect.y, rect.w, rect.h).fill(background);
@@ -394,7 +406,6 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
 
     this.placeWorld(view);
     const bounds = worldBounds(view);
-    const shaders = shaderLibrary.source(this.state.project);
     const shading = this.shadingCamera(camera, items);
     const grid = this.objectGrid(view, shading, shaders);
     const culled: { node: Graphics; extent: Extent | null }[] = [];
@@ -405,11 +416,8 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
       const size = footprintOf(item.type, item.props, item.frame);
       const extent = footprintExtent(item.type, item.props, item.m, item.frame);
       // A switched-off object is not drawn at all, in the scene view as in the game; it is
-      // selected and switched back on from the hierarchy panel. A source is never culled by its
-      // own little square: its particles may be anywhere on screen while it is not. Nor is a wearer
-      // of a shader that pads it (#69) or moves its corners (#79): both draw past its rectangle.
-      const worn = wornShader(item.props, shaders);
-      const unbound = item.type === 'particles' || (worn?.padding ?? 0) > 0 || Boolean(worn?.vertex);
+      // selected and switched back on from the hierarchy panel.
+      const unbound = isUnbound(item, shaders);
       const onScreen = item.enabled && (unbound || rectsIntersect(extent, bounds));
       let node = this.nodes.get(item.id);
       if (!node) {
@@ -451,6 +459,7 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
       const sprite = this.sprites.get(id);
       if (sprite) disposeSpriteMesh(sprite);
       this.sprites.delete(id);
+      this.videosShown.delete(id);
       this.fields.get(id)?.destroy();
       this.fields.delete(id);
       this.tilemaps.get(id)?.destroy();
@@ -466,6 +475,33 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
     this.syncOverlays(frame, view, camera);
   }
 
+  /**
+   * Whether the view keeps its last frame for a driven video (#404) it brings in: a sprite switched
+   * on, or come into view, at another picture than its player holds. Its texture shows the held one
+   * until the seek is done, and a cut into the middle of a clip flashed the picture it was held at.
+   * At most `VIDEO_HOLD_MS`; after that such a sprite is left out until its picture is there.
+   */
+  private holdsForVideo(view: View, items: DrawItem[], shaders: ShaderSource): boolean {
+    const bounds = worldBounds(view);
+    const waits = items.some((item) => {
+      const file = item.texture?.file;
+      if (item.type !== 'sprite' || !item.enabled || !file || !isVideoKey(file)) return false;
+      if (!videoDrivenOf(item.props)) return false;
+      const key = drivenVideoKey(file, this.videoWearer(item));
+      if (this.videosShown.get(item.id) === key) return false;
+      const extent = footprintExtent(item.type, item.props, item.m, item.frame);
+      if (!isUnbound(item, shaders) && !rectsIntersect(extent, bounds)) return false;
+      // Worn here already, so the seek to the asked picture starts with the frame held.
+      this.wearVideo(item);
+      // One still without a picture is left out instead (`syncSprite`).
+      return videoLibrary.hasPicture(key) && !videoLibrary.showsAsked(key);
+    });
+    const now = performance.now();
+    if (!waits) this.heldSince = null;
+    else this.heldSince ??= now;
+    return waits && now - this.heldSince! < VIDEO_HOLD_MS;
+  }
+
   /**
    * The steps per unit the objects are drawn on, or null to draw them as they are: through a pixel
    * perfect camera the view's grid, or with its shader laid over the grid of the shader's picture.
@@ -754,6 +790,7 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
       const ghost = this.ghostSprites.get(item.id);
       if (ghost) disposeSpriteMesh(ghost);
       this.ghostSprites.delete(item.id);
+      this.videosShown.delete(item.id);
       // A driven video (#404) is kept ready at its Video time while its sprite is switched off or out
       // of view: switched on, it showed its plain colour until the video had loaded.
       if (item.type === 'sprite' && this.wearVideo(item, false) !== item) this.animated = true;
@@ -771,13 +808,24 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
     const drawn = this.wearVideo(item);
     let ghost = this.ghostSprites.get(item.id);
     // A driven video (#404) is left out until it has its first picture: its plain colour flashed
-    // white at the start of a run and at a cut.
-    if (drawn !== item && !videoLibrary.hasPicture(drawn.texture?.file ?? '')) {
+    // white at the start of a run and at a cut. So is one brought in while it still seeks, once the
+    // view has held its last frame long enough (`holdsForVideo`).
+    const key = drawn !== item ? (drawn.texture?.file ?? '') : null;
+    if (
+      key !== null &&
+      !(
+        videoLibrary.hasPicture(key) &&
+        (this.videosShown.get(item.id) === key || videoLibrary.showsAsked(key))
+      )
+    ) {
       sprite.visible = false;
       if (ghost) disposeSpriteMesh(ghost);
       this.ghostSprites.delete(item.id);
+      this.videosShown.delete(item.id);
       return;
     }
+    if (key !== null) this.videosShown.set(item.id, key);
+    else this.videosShown.delete(item.id);
     syncSpriteMesh(sprite, drawn, size, textureLibrary, shaders);
     if (!this.ghosted(item, onScreen)) {
       if (ghost) disposeSpriteMesh(ghost);
@@ -801,12 +849,16 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
     const texture = videoLibrary.wear(
       item.texture,
       item.props,
-      `${this.wearer}:${item.id}`,
+      this.videoWearer(item),
       seen && this.hearsVideo(),
     );
     return texture === item.texture ? item : { ...item, texture };
   }
 
+  private videoWearer(item: DrawItem): string {
+    return `${this.wearer}:${item.id}`;
+  }
+
   /** Whether a driven video (#404) this view wears is still without its first picture. */
   videosLoading(): boolean {
     return videoLibrary.loading(this.wearer);
@@ -868,6 +920,16 @@ function seenBy(camera: ActiveCamera, items: DrawItem[]): DrawItem[] {
   return items.filter((item) => cameraSees(camera.props, item.renderLayer));
 }
 
+/**
+ * Whether an item is never culled by its own rectangle. A source is not: its particles may be
+ * anywhere on screen while it is not. Nor is a wearer of a shader that pads it (#69) or moves its
+ * corners (#79): both draw past its rectangle.
+ */
+function isUnbound(item: DrawItem, shaders: ShaderSource): boolean {
+  const worn = wornShader(item.props, shaders);
+  return item.type === 'particles' || (worn?.padding ?? 0) > 0 || Boolean(worn?.vertex);
+}
+
 /**
  * What one overlay camera sees (#302), drawn like any view but handed its view and frame by the view
  * it lies on, which also redraws it. See-through, and never hit: the mouse belongs to the main

tools/e2e/66-video.mjs

@@ -498,6 +498,41 @@ export async function video(t) {
     }),
   );
 
+  // 66q. Switched on at another picture than the one it was held at, a cut into the middle of a
+  //      clip, a driven sprite showed the held one until its seek was done: red before blue. Every
+  //      frame is read in the page.
+  await showTab('scene');
+  const cutLate = (on, time) =>
+    mutate(`
+      const late = __gm.state.activeScene.objects.find((o) => o.id === '${LATE}');
+      late.overrides = { ...late.overrides, videoTime: ${time}, enabled: ${on} };
+      __gm.lookAt(late.position.x, late.position.y);`);
+  await cutLate(true, 1.2);
+  const lateRect = await waitFor(
+    () => json(`__gm.objectRect('${LATE}')`),
+    (r) => r !== null,
+  );
+  await waitFor(
+    () => colorAt(lateRect),
+    (color) => color === COLORS.blue,
+  );
+  await cutLate(false, 0.2);
+  await sleep(800);
+  await t.evaluate(`window.__vd66q = [];
+    (function read() {
+      window.__vd66q.push(__gm.pixelAt(${Math.round(lateRect.x)}, ${Math.round(lateRect.y)}));
+      if (window.__vd66q.length < 60) requestAnimationFrame(read);
+    })();`);
+  await sleep(100);
+  await cutLate(true, 1.2);
+  await sleep(1200);
+  const cutFrames = await json('window.__vd66q.splice(0)');
+  check(
+    'a driven sprite switched on at another picture never shows the one it was held at',
+    cutFrames.length >= 30 && !cutFrames.includes(COLORS.red) && cutFrames.at(-1) === COLORS.blue,
+    JSON.stringify(cutFrames),
+  );
+
   await mutate(`
     const p = __gm.state.project;
     const gone = ['${OBJECT}', '${EARLY}', '${LATE}'];