Let free videos wait while a run waits for its driven videos (#437)

A run that waits for its driven videos does not tick its world, so the
clock of every free video sprite stands. Its player played on anyway.
The picture ran ahead of the clock by the length of the wait, so On
video finished came late and a Pause video sent the picture back.

- VideoLibrary.setHeld holds the free players next to the toolbar
  pause. PlaySession and Player switch it with the wait at the start,
  after a Load scene and while a recording sets up its encoders.
- The hold starts in waitForVideos, not on the next tick. Otherwise a
  free video started at Play would be a frame ahead.
- A free player without its first picture plays on until it has one.
  Held at once, it would never get one.
- startRun and endRun clear the hold, so a run that ends while it waits
  leaves no held preview behind.

Found on the way:

- Resuming called play() on a clip without Loop that already stood at
  its end, and play() starts such a video over. This happened after
  the toolbar pause already and would have happened after every waiting
  Load scene. Such a clip now stays at its end.

Tests:

- Unit video-library 14/14, 2 of them new. 7 deliberate breaks, all
  red.
- e2e 66 (atx): ALL PASSED, 40 checks. New 66t: the driven video came
  1.5 s late, the wait took 1686 ms. The free video stayed red all
  along, blue came at 0.983 s, white at 1.483 s and On video finished
  at 2.017 s. Without the hold the free video was white in the first
  frame and only 66t was red.
- Gate (atx): CHECK EXIT 0, 339 files, 5253 tests. The guard for the
  clip end came in while the gate ran. It is checked with tsc, lint and
  its test file with 3 deliberate breaks.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/editor/PlaySession.ts

@@ -317,6 +317,8 @@ export class PlaySession {
   /** The Game tab has drawn the new world once by now, which opened the players it shows. */
   private waitForVideos(): void {
     this.videoWait = this.host.gameView.videosLoading() ? performance.now() + VIDEO_WAIT_MS : null;
+    // From now, not from the next tick: a free video started at Play would be a frame ahead.
+    videoLibrary.setHeld(this.videoWait !== null);
   }
 
   /** Whether the run still waits for its videos; the frame it stops waiting, the notice says so. */
@@ -395,12 +397,10 @@ export class PlaySession {
     if (!run) return;
     const world = run.world;
     this.tellPace();
-    if (
-      (this.state.paused && !this.stepping) ||
-      this.waitsForVideos() ||
-      this.recorder.isStarting ||
-      this.recordStep === 0
-    ) {
+    // The world does not tick while it waits, and the free videos wait with it (#437).
+    const waits = this.waitsForVideos() || this.recorder.isStarting;
+    videoLibrary.setHeld(waits);
+    if ((this.state.paused && !this.stepping) || waits || this.recordStep === 0) {
       // Keys and clicks that happen while paused are dropped instead of piling up – otherwise
       // resuming would deliver every one of them in a single frame.
       this.input.endFrame();

src/player/Player.ts

@@ -112,6 +112,8 @@ export class Player {
     // A recording that sets up its encoders holds the run too: its frames meanwhile would be lost.
     // One not in real time (#423) moves it by one of its frames, or holds it until it has taken one.
     const waits = this.waitsForVideos() || this.recorder.isStarting;
+    // The world does not tick while it waits, and the free videos wait with it (#437).
+    videoLibrary.setHeld(waits);
     const recorded = waits ? null : this.recorder.step();
     if (waits || recorded === 0) {
       this.input.endFrame();
@@ -143,6 +145,8 @@ export class Player {
   /** The screen has drawn the new world once by now, which opened the players it shows. */
   private waitForVideos(): void {
     this.videoWait = this.screen.videosLoading() ? performance.now() + VIDEO_WAIT_MS : null;
+    // From now, not from the next tick: a free video started at Play would be a frame ahead.
+    videoLibrary.setHeld(this.videoWait !== null);
   }
 
   private waitsForVideos(): boolean {

src/viewport/VideoLibrary.ts

@@ -4,7 +4,8 @@
  * same clip share a player and show the same picture.
  *
  * In the editor a video plays without sound, as a preview. A run starts every player over at its
- * clip's start and lets it be heard along the master routing of its video; a pause holds them, and
+ * clip's start and lets it be heard along the master routing of its video; a pause holds them, as
+ * does the run waiting for its driven videos or its recording (see `setHeld`), and
  * the end of the run makes them quiet previews again.
  *
  * A sprite whose Video clock is driven (#404) has a player of its own instead, which shows the
@@ -118,6 +119,8 @@ export class VideoLibrary {
   private readonly listeners = new Set<() => void>();
   private heard = false;
   private paused = false;
+  /** The free players wait with a run that waits for its driven videos or its recording (#437). */
+  private held = false;
   /** Driven players hold at every picture asked rather than play: a recording takes each (#423). */
   private stepped = false;
   private sweeper: ReturnType<typeof setInterval> | null = null;
@@ -201,6 +204,7 @@ export class VideoLibrary {
   startRun(): void {
     this.heard = true;
     this.paused = false;
+    this.held = false;
     for (const player of this.freePlayers()) {
       if (!player.hushed) this.hear(player);
       this.rewind(player);
@@ -211,6 +215,7 @@ export class VideoLibrary {
   endRun(): void {
     this.heard = false;
     this.paused = false;
+    this.held = false;
     for (const player of this.freePlayers()) {
       player.element.muted = true;
       this.rewind(player);
@@ -219,9 +224,34 @@ export class VideoLibrary {
 
   setPaused(paused: boolean): void {
     this.paused = paused;
+    this.settle();
+  }
+
+  /**
+   * While a run waits at its start or after a Load scene (#437), its world does not tick, and the
+   * clock of every free sprite stands. Its player holds too, so the picture keeps to that clock:
+   * else On video finished came late by the wait, and a Pause video sent the picture back.
+   */
+  setHeld(held: boolean): void {
+    if (this.held === held) return;
+    this.held = held;
+    this.settle();
+  }
+
+  /** Whether the free players stand: paused by the toolbar or held by a waiting run. */
+  private get still(): boolean {
+    return this.paused || this.held;
+  }
+
+  /**
+   * Holds or plays the free players as `still` says; one without its first picture plays on to get
+   * it. One held at the end of a clip without Loop stays there: play() would start it over.
+   */
+  private settle(): void {
     for (const player of this.freePlayers())
-      if (paused) player.element.pause();
-      else if (!player.hushed) void player.element.play().catch(() => {});
+      if (this.still) {
+        if (player.texture) player.element.pause();
+      } else if (!player.hushed && !atEnd(player)) void player.element.play().catch(() => {});
   }
 
   private freePlayers(): Player[] {
@@ -320,7 +350,7 @@ export class VideoLibrary {
   private unhush(player: Player): void {
     player.hushed = false;
     if (this.heard) this.hear(player);
-    if (!this.paused) void player.element.play().catch(() => {});
+    if (!this.still) void player.element.play().catch(() => {});
   }
 
   private stopSweeping(): void {
@@ -371,8 +401,8 @@ export class VideoLibrary {
         if (player.gone) return texture.destroy(true);
         player.texture = texture;
         if (drive) this.follow(player, drive);
-        // Started even in a paused run or when nothing wears it, and held once there is a picture.
-        else if (this.paused || player.hushed) player.element.pause();
+        // Started even in a paused or waiting run or when nothing wears it, and held once there is a picture.
+        else if (this.still || player.hushed) player.element.pause();
         this.notify();
       }),
       texture: null,
@@ -449,7 +479,7 @@ export class VideoLibrary {
 
   private rewind(player: Player): void {
     player.element.currentTime = player.play.start;
-    if (!this.paused && !player.hushed) void player.element.play().catch(() => {});
+    if (!this.still && !player.hushed) void player.element.play().catch(() => {});
   }
 
   private drop(player: Player): void {
@@ -482,6 +512,11 @@ export class VideoLibrary {
 
 export const videoLibrary = new VideoLibrary();
 
+/** Whether a free player stands at the end of a clip without Loop, as `watch` holds it. */
+function atEnd({ element, play }: Player): boolean {
+  return !play.loop && (element.ended || element.currentTime >= play.end - END_SLACK);
+}
+
 /**
  * A `<video>` of `url`, started without sound at `at` seconds, and its picture as a texture once it
  * has shown its first frame. Chrome has no picture of a video that never played, and a Pixi source

tests/viewport/video-library.test.ts

@@ -354,6 +354,102 @@ describe('VideoLibrary driven players (#404)', () => {
     vi.useRealTimers();
   });
 
+  it('holds the free players while a run waits, alongside a pause of the toolbar (#437)', async () => {
+    const events = new Map<string, () => void>();
+    let frames: (() => void)[] = [];
+    const element = {
+      ...fakeVideo(),
+      play() {
+        this.paused = false;
+        return Promise.resolve();
+      },
+      pause() {
+        this.paused = true;
+      },
+      addEventListener: (type: string, listener: () => void) => events.set(type, listener),
+      requestVideoFrameCallback: (callback: () => void) => frames.push(callback),
+    };
+    vi.stubGlobal('document', { hidden: false, addEventListener: () => {}, createElement: () => element });
+    const { soundLibrary } = await import('../../src/viewport/SoundLibrary');
+    vi.spyOn(soundLibrary, 'mediaInput').mockReturnValue(null);
+    const videos = await freshLibrary();
+    videos.startRun();
+    videos.get(KEY);
+    events.get('loadeddata')!();
+    // Without its first picture it plays on to get one, and holds at it.
+    videos.setHeld(true);
+    expect(element.paused).toBe(false);
+    [frames] = [[], ...frames.map((frame) => frame())];
+    expect(element.paused).toBe(true);
+    // The wait is over: it plays, from where it was held.
+    element.currentTime = 0.02;
+    videos.setHeld(false);
+    expect([element.paused, element.currentTime]).toEqual([false, 0.02]);
+    // A wait after a Load scene holds it again; paused meanwhile, it stays held until both end.
+    videos.setHeld(true);
+    videos.setPaused(true);
+    videos.setHeld(false);
+    expect(element.paused).toBe(true);
+    videos.setHeld(true);
+    videos.setPaused(false);
+    expect(element.paused).toBe(true);
+    videos.setHeld(false);
+    expect(element.paused).toBe(false);
+    // A run stopped while it waits lets go of the wait: the preview plays again.
+    videos.setHeld(true);
+    videos.endRun();
+    expect(element.paused).toBe(false);
+  });
+
+  it('leaves a free clip without Loop at its end after a wait or a pause, as play() starts it over', async () => {
+    const events = new Map<string, () => void>();
+    type Frame = (now?: number, frame?: { mediaTime: number }) => void;
+    let frames: Frame[] = [];
+    const element = {
+      ...fakeVideo(),
+      ended: false,
+      play() {
+        this.paused = false;
+        return Promise.resolve();
+      },
+      pause() {
+        this.paused = true;
+      },
+      addEventListener: (type: string, listener: () => void) => events.set(type, listener),
+      requestVideoFrameCallback: (callback: Frame) => frames.push(callback),
+    };
+    vi.stubGlobal('document', { hidden: false, addEventListener: () => {}, createElement: () => element });
+    const { soundLibrary } = await import('../../src/viewport/SoundLibrary');
+    vi.spyOn(soundLibrary, 'mediaInput').mockReturnValue(null);
+    const videos = await freshLibrary();
+    videos.startRun();
+    videos.get(KEY.replace('#0~0~1~1', '#0~0~0~1'));
+    events.get('loadeddata')!();
+    [frames] = [[], ...frames.map((frame) => frame())];
+    Object.assign(element, { ended: true, paused: true });
+    videos.setHeld(true);
+    videos.setHeld(false);
+    videos.setPaused(true);
+    videos.setPaused(false);
+    expect(element.paused).toBe(true);
+    // Short of its end it plays on.
+    element.ended = false;
+    videos.setHeld(true);
+    videos.setHeld(false);
+    expect(element.paused).toBe(false);
+    // A clip that ends at 1 s, inside the video, is held there by `watch`: it stays there too.
+    const clip = await freshLibrary();
+    clip.startRun();
+    clip.get(KEY.replace('#0~0~1~1', '#0~1~0~1'));
+    events.get('loadeddata')!();
+    // Its first picture, and the check of `watch` at 0 s, short of the end.
+    [frames] = [[], ...frames.map((frame) => frame(0, { mediaTime: 0 }))];
+    Object.assign(element, { currentTime: 1, paused: true });
+    clip.setHeld(true);
+    clip.setHeld(false);
+    expect(element.paused).toBe(true);
+  });
+
   it('tells a view whether one of its driven players still waits for its first picture', async () => {
     const videos = await freshLibrary();
     videos.get(KEY);

tools/e2e/66-video.mjs

@@ -29,6 +29,7 @@ const STEER = 'vd66-steer';
 const STEERED = 'vd66-steered';
 const STEER_LOGIC = 'vd66-steer-logic';
 const STEER_BRAIN = 'vd66-steer-brain';
+const SLOW = 'vd66-slow';
 const LANE = LANES.video;
 const COLORS = { red: '#fe0000', green: '#00ff01', blue: '#0000ff', white: '#ffffff' };
 
@@ -710,7 +711,7 @@ export async function video(t) {
   //      video sends it to 0.5 s (green) and it stays held there, Play video plays on from there:
   //      blue, then white held at the end of a video without Loop, and On video finished is heard
   //      once. Read from the Game tab, the frames after Play in the page. No driven sprite is left
-  //      in the scene: the run would wait for its picture, free videos playing meanwhile, keys lost.
+  //      in the scene: the run would wait for its picture, and keys pressed meanwhile are lost.
   await mutate(`
     const p = __gm.state.project;
     const scene = __gm.state.activeScene;
@@ -824,9 +825,94 @@ export async function video(t) {
     JSON.stringify({ finished, frames: playedFrames.join(' ') }),
   );
 
+  // 66t. A driven sprite whose video comes 1.5 s late holds the run at its start (#437). The free
+  //      video beside it waits too, at its first picture: red all through the wait, then its colours
+  //      at their game times, and On video finished (it logs its Time) with the end of its last
+  //      picture, not 1.5 s after it. The video elements opened at Play get their file 1.5 s late.
   await mutate(`
     const p = __gm.state.project;
-    const gone = ['${OBJECT}', '${EARLY}', '${LATE}', '${STEERED}', '${STEER_BRAIN}'];
+    p.assets.find((a) => a.id === '${STEER_LOGIC}').graph = ${JSON.stringify({
+      nodes: [
+        node('vd66t-end', 'OnVideoFinished', 0, 0),
+        node('vd66t-time', 'Time', 0, 160),
+        node('vd66t-log', 'Log', 260, 0),
+      ],
+      edges: [
+        edge('vd66t-e1', 'vd66t-end', 'exec', 'vd66t-log', 'exec'),
+        edge('vd66t-e2', 'vd66t-time', 'time', 'vd66t-log', 'message'),
+      ],
+    })};
+    ${MAKE_OBJECT}
+    const scene = __gm.state.activeScene;
+    const cam = scene.objects.find((o) => o.id === 'o3').position;
+    const slow = make('${SLOW}', '${DRIVEN}', '', 0, '${SLOW}');
+    Object.assign(slow.position, { x: cam.x - 200, y: cam.y - 100, z: 900 });
+    scene.objects.push(slow);`);
+  await t.evaluate(`(() => {
+    const make = document.createElement;
+    const src = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'src');
+    document.createElement = function (tag, ...rest) {
+      const element = make.call(document, tag, ...rest);
+      if (String(tag).toLowerCase() === 'video')
+        Object.defineProperty(element, 'src', {
+          configurable: true,
+          get: () => src.get.call(element),
+          set: (url) => setTimeout(() => src.set.call(element, url), 1500),
+        });
+      return element;
+    };
+    setTimeout(() => (document.createElement = make), 1000);
+    window.__vd66t = [];
+    (function read() {
+      const info = __gm.playInfo();
+      const r = info.playing && __gm.gameRect('${STEERED}');
+      if (r) window.__vd66t.push({ color: __gm.pixelAt(Math.round(r.x), Math.round(r.y)), waiting: info.waiting, time: info.time, at: performance.now() });
+      if (window.__vd66t.length < 600 && !(info.time > 2.4)) requestAnimationFrame(read);
+    })();
+  })()`);
+  const loggedFrom = t.consoleLog.length;
+  await play();
+  await waitFor(
+    () => json('__gm.playInfo().time'),
+    (time) => time > 2.4,
+    40,
+  );
+  await sleep(200);
+  const slowFrames = await json('window.__vd66t.splice(0)');
+  const logged = t.consoleLog
+    .slice(loggedFrom)
+    .filter((line) => line.includes(`[behavior ${STEER_BRAIN}]`))
+    .map((line) => Number(line.split(' ').at(-1)));
+  await stop();
+  const slowWait = slowFrames.filter((f) => f.waiting);
+  const slowRun = slowFrames.filter((f) => !f.waiting && f.time > 0);
+  const firstAt = (color) => slowRun.find((f) => f.color === color)?.time ?? null;
+  const [blueAt, whiteAt] = [firstAt(COLORS.blue), firstAt(COLORS.white)];
+  check(
+    'a free video waits with a run that waits for a driven one, and On video finished comes with its last picture',
+    slowWait.length > 0 &&
+      slowWait.at(-1).at - slowWait[0].at > 1000 &&
+      slowWait.every((f) => f.color === COLORS.red) &&
+      slowRun[0]?.color === COLORS.red &&
+      blueAt !== null &&
+      Math.abs(blueAt - 1) < 0.15 &&
+      whiteAt !== null &&
+      Math.abs(whiteAt - 1.5) < 0.15 &&
+      logged.length === 1 &&
+      Math.abs(logged[0] - 2) < 0.15,
+    JSON.stringify({
+      waited: slowWait.length && Math.round(slowWait.at(-1).at - slowWait[0].at),
+      waitColors: [...new Set(slowWait.map((f) => f.color))],
+      blueAt,
+      whiteAt,
+      logged,
+      frames: slowRun.map((f) => `${f.color}@${f.time.toFixed(2)}`).join(' '),
+    }),
+  );
+
+  await mutate(`
+    const p = __gm.state.project;
+    const gone = ['${OBJECT}', '${EARLY}', '${LATE}', '${STEERED}', '${STEER_BRAIN}', '${SLOW}'];
     for (const holder of p.assets) if (holder.objects) holder.objects = holder.objects.filter((o) => !gone.includes(o.id));
     p.assets = p.assets.filter((a) => ![ '${SPRITE}', '${DRIVEN}', '${CUT}', '${BUS}', '${STEER}', '${STEER_LOGIC}', ${JSON.stringify(assetId)} ].includes(a.id));
     __gm.state.select(null);