Keep a video's loading out of recordings and first frames (#417)

A Start recording right after Play composition still began with
empty frames. The composition's clock waited for its video, but the
recording went on and took every frame while the video loaded. A
clip that starts later in the video also slipped through, because
its player already had one picture while it still sought to the
clip's start.

- The view now counts every video sprite it leaves out as missing,
  not only one without any first picture.
- GameRecorder takes no frame while the game view misses a video,
  for at most VIDEO_WAIT_MS (3 s) at a time. The recorded time
  stands and the sound of that time is dropped, so it stays in sync.
- RecordingClock.start forgets a hold from before the first frame.
  Without that, the first real frames all got the time 0.
- A followed player starts at 1x until the first speed reading.
  Before, it paused and sought every frame for about 100 ms, so the
  first pictures after a wait stuck and then jumped. A step faster
  than 4x and a time running backwards do not count as a start.
- Test hooks: __gm.gameVideo(id) reads the game view's own player,
  TakeInfo.lastFrameAt shows when the last frame was taken.

Tests:
- Unit recording (clock after an early hold), video-follow (2 new,
  1 adapted): 449 green in tests/core/recording, tests/viewport and
  tests/core/composition*
- e2e 68e (a clip from 0.8 s shows that picture first and then
  plays), 68f (a recording that already runs takes no frame while
  the composition's video loads)
- 68e is red without the start at 1x, 68f without the hold
- README: 2038 editor checks

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/core/recording/recording.ts

@@ -126,8 +126,10 @@ export class RecordingClock {
     return this.origin !== null;
   }
 
+  /** A pause before the first frame is no part of the recording's time. */
   start(now: number): void {
     this.origin = now;
+    this.paused = 0;
   }
 
   pause(now: number): void {

src/editor/debug/sceneHooks.ts

@@ -113,6 +113,8 @@ export function sceneHooks({ state, editor }: HookContext) {
     gameScreen: (id: string) => gameView.objectScreen(id),
     gameRect: (id: string) => gameView.objectRect(id),
     gameDrawnRect: (id: string) => gameView.drawnRect(id),
+    /** The player of a driven video the Game tab draws for this object, not another view's. */
+    gameVideo: (id: string) => gameView.videoOf(id),
     gameCenter: () => centreOf(gameView),
     /** What the main camera draws in the Game tab, back to front (#302). */
     gameDrawOrder: () => gameView.drawOrder(),

src/runtime/systems/CompositionSystem.ts

@@ -38,13 +38,13 @@ interface Run {
   id: number;
   /** By item id. An item a behavior destroyed stays here until it ends, so it is not built again. */
   live: Map<string, Spawned>;
-  /** Seconds the clock has stood so far for a video without its first picture, see `waits`. */
+  /** Seconds the clock has stood so far for a video without its picture, see `waits`. */
   waited: number;
 }
 
 /** What the view that shows a run knows of its driven videos (#404). */
 export interface VideoPictures {
-  /** Whether it left the sprite `objectId` out, its video still without a first picture. */
+  /** Whether it did not show the sprite `objectId`, its video still without the picture asked for. */
   missing(objectId: string): boolean;
 }
 
@@ -55,7 +55,7 @@ export interface VideoPictures {
  * frame's `finished`. The editor run plays none: it writes into the saved scene, where the objects of
  * a cut would stay behind.
  *
- * While the view still waits for the first picture of a video an item brings in, the composition's
+ * While the view still waits for the picture of a video an item brings in, the composition's
  * clock stands, so the video starts at its first frame and not where the clock has got to meanwhile.
  */
 export class CompositionSystem implements System {

src/viewport/GameRecorder.ts

@@ -1,4 +1,5 @@
 import { Matrix, type RenderTexture } from 'pixi.js';
+import { VIDEO_WAIT_MS } from '../core/model/video';
 import { preSkipOf } from '../core/music/ogg';
 import { AVC_CODEC, colourOf, MatroskaWriter, VP9_CODEC } from '../core/recording/matroska';
 import {
@@ -34,6 +35,8 @@ const LAST_FRAMES_MS = 500;
 export interface RecordingView {
   picture(): { shown: PictureArea; whole: PictureArea } | null;
   drawPicture(transform: Matrix, target: RenderTexture): void;
+  /** Whether its last redraw left out a video sprite, the video still without its picture. */
+  missesVideo(): boolean;
 }
 
 /** A recording once it is made. */
@@ -73,6 +76,8 @@ export interface TakeInfo {
   soundFrames: number;
   /** From the first frame to the end of the last, in ms. */
   durationMs: number;
+  /** `performance.now()` when the last frame was taken, null before the first. */
+  lastFrameAt: number | null;
   /** What the video encoder said its colour is, null while it said nothing. */
   colour: VideoColorSpaceInit | null;
   bytes: number;
@@ -165,6 +170,10 @@ class Take {
   private soundEnded: (() => void) | null = null;
   private stopAsked = false;
   private paused = false;
+  private held = false;
+  /** Since when the view lacks a video, see `holdForVideo`; null while it lacks none. */
+  private heldSince: number | null = null;
+  private lastFrameAt: number | null = null;
   private failure: string | null = null;
 
   constructor(
@@ -278,13 +287,37 @@ class Take {
   }
 
   setPaused(paused: boolean): void {
-    if (paused === this.paused) return;
-    this.paused = paused;
+    this.stand(() => (this.paused = paused));
+  }
+
+  /**
+   * A frame that lacks a video is not taken (#417): a composition's clock stands until its video has
+   * its picture, and the frames meanwhile were empty where the video goes. Like a pause, but the game
+   * plays on, so what it sounds then is left out too. At most `VIDEO_WAIT_MS` at a stretch, as long
+   * as the composition waits.
+   */
+  private holdForVideo(view: RecordingView): void {
     const now = performance.now();
-    if (paused) this.clock.pause(now);
+    if (view.missesVideo()) this.heldSince ??= now;
+    else this.heldSince = null;
+    const held = this.heldSince !== null && now - this.heldSince < VIDEO_WAIT_MS;
+    this.stand(() => (this.held = held));
+  }
+
+  /** The clock stands while paused or held, whichever came first and whichever ends last. */
+  private stand(change: () => void): void {
+    const was = this.standing;
+    change();
+    if (this.standing === was) return;
+    const now = performance.now();
+    if (this.standing) this.clock.pause(now);
     else this.clock.resume(now);
   }
 
+  private get standing(): boolean {
+    return this.paused || this.held;
+  }
+
   capture(view: RecordingView): void {
     if (this.state === 'waiting') {
       const picture = view.picture();
@@ -302,7 +335,9 @@ class Take {
     if (this.state !== 'recording' || !video || !reader) return;
     reader.collect((bytes, ms) => this.encode(bytes, ms));
     // A frame taken may have reached the most a file holds and ended the recording.
-    if (!this.taking || this.paused) return;
+    if (!this.taking) return;
+    this.holdForVideo(view);
+    if (this.standing) return;
     const now = performance.now();
     if (!this.clock.started) this.clock.start(now);
     const ms = this.clock.at(now);
@@ -318,6 +353,7 @@ class Take {
     );
     view.drawPicture(new Matrix(m.a, 0, 0, m.d, m.tx, m.ty), reader.texture);
     reader.read(ms);
+    this.lastFrameAt = now;
   }
 
   private get taking(): boolean {
@@ -352,8 +388,9 @@ class Take {
       return;
     }
     // Before the first frame there is no time to put it at. Pieces keep coming during a pause only
-    // as long as they were on their way, so they belong before it and are kept.
-    if (!audio || !this.tap || this.state !== 'recording' || !this.clock.started) return;
+    // as long as they were on their way, so they belong before it and are kept. A hold leaves them
+    // out: the game plays on meanwhile.
+    if (!audio || !this.tap || this.state !== 'recording' || !this.clock.started || this.held) return;
     const rate = this.tap.rate;
     this.soundStart ??= Math.max(0, this.clock.at(performance.now()) - (frames / rate) * 1000);
     const planar = new Float32Array(frames * 2);
@@ -511,6 +548,7 @@ class Take {
       dropped: this.dropped,
       soundFrames: this.soundFrames,
       durationMs: this.frames > 0 ? Math.round(this.lastMs + 1000 / this.settings.fps) : 0,
+      lastFrameAt: this.lastFrameAt,
       colour: this.colour,
       bytes: this.writer?.bytes ?? 0,
       failure: this.failure,

src/viewport/WorldViewport.ts

@@ -125,7 +125,7 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
   private readonly videosShown = new Map<string, string>();
   /** Since when the view keeps its last frame for a driven video it brings in, see `holdsForVideo`. */
   private heldSince: number | null = null;
-  /** The sprites the last redraw left out, their driven video still without its first picture. */
+  /** The sprites the last redraw did not show, their driven video still without the picture asked for. */
   private readonly videosLeftOut = new Set<string>();
   private cameraMoves = 0;
   private readonly stopFiles: (() => void)[];
@@ -506,7 +506,7 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
    */
   private holdsForVideo(view: View, items: DrawItem[], shaders: ShaderSource): boolean {
     const bounds = worldBounds(view);
-    const waits = items.some((item) => {
+    const waiting = items.filter((item) => {
       const file = item.texture?.file;
       if (item.type !== 'sprite' || !item.enabled || !file || !isVideoKey(file)) return false;
       if (!videoDrivenOf(item.props)) return false;
@@ -520,9 +520,12 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
       return videoLibrary.hasPicture(key) && !videoLibrary.showsAsked(key);
     });
     const now = performance.now();
-    if (!waits) this.heldSince = null;
+    if (waiting.length === 0) this.heldSince = null;
     else this.heldSince ??= now;
-    return waits && now - this.heldSince! < VIDEO_HOLD_MS;
+    const holds = waiting.length > 0 && now - this.heldSince! < VIDEO_HOLD_MS;
+    // The held frame does not show them either, so a composition waits for them meanwhile.
+    if (holds) for (const item of waiting) this.videosLeftOut.add(item.id);
+    return holds;
   }
 
   /**
@@ -888,7 +891,7 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
       if (ghost) disposeSpriteMesh(ghost);
       this.ghostSprites.delete(item.id);
       this.videosShown.delete(item.id);
-      if (!videoLibrary.hasPicture(key)) this.videosLeftOut.add(item.id);
+      this.videosLeftOut.add(item.id);
       return;
     }
     if (key !== null) this.videosShown.set(item.id, key);
@@ -926,14 +929,26 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
     return `${this.wearer}:${item.id}`;
   }
 
+  /**
+   * What the player of a driven video drawn here for `objectId` is doing (test hook): the one it
+   * showed last, else the newest it opened; null for none.
+   */
+  videoOf(objectId: string): ReturnType<typeof videoLibrary.debugInfo>[number] | null {
+    const mark = `@${this.videoWearer({ id: objectId })}`;
+    const players = videoLibrary.debugInfo().filter((player) => player.key.endsWith(mark));
+    const shown = this.videosShown.get(objectId);
+    return players.find((player) => player.key === shown) ?? players[players.length - 1] ?? null;
+  }
+
   /** Whether a driven video (#404) this view wears is still without its first picture. */
   videosLoading(): boolean {
     return videoLibrary.loading(this.wearer);
   }
 
   /**
-   * Whether the last redraw left out a driven video (#404) it should show, its player still without
-   * a first picture. Unlike `videosLoading`, one opened ahead or switched off does not count.
+   * Whether the last redraw did not show a driven video (#404) it should show: its player has no
+   * picture yet, or still seeks to the one asked for, as after loading a clip that starts later.
+   * Unlike `videosLoading`, one opened ahead or switched off does not count.
    */
   missesVideo(): boolean {
     return this.videosLeftOut.size > 0;

src/viewport/videoFollow.ts

@@ -42,7 +42,8 @@ export interface SpeedReading {
  * The next reading after the wearer asked for `time` at `now`. A jump (a loop going round, a key
  * set to another time) starts the window over and keeps the speed: playing on through a loop is
  * right, and a still picture stays still. While the time stands, the window waits at its latest
- * frame, so a start is measured from where it began and not across the wait before it.
+ * frame, so a start is measured from where it began and not across the wait before it. Until then
+ * a start counts as 1x.
  */
 export function readSpeed(reading: SpeedReading | null, time: number, now: number): SpeedReading {
   if (!reading) return { samples: [{ time, at: now }], speed: 0 };
@@ -58,8 +59,13 @@ export function readSpeed(reading: SpeedReading | null, time: number, now: numbe
   while (samples.length > 2 && now - samples[1]!.at >= SPEED_SPAN_MS) samples.shift();
   const first = samples[0]!;
   const span = now - first.at;
-  if (span < SPEED_WINDOW_MS) return { samples, speed: reading.speed };
-  return { samples, speed: ((time - first.time) * 1000) / span };
+  const pace = ((time - first.time) * 1000) / Math.max(span, 1);
+  if (span >= SPEED_WINDOW_MS) return { samples, speed: pace };
+  // A start from standing plays at 1x until the window has its reading: held and sought frame by
+  // frame instead, the first pictures of a cut stuck and were skipped. A step faster than a video
+  // plays is a jump, not a start.
+  const starts = reading.speed === 0 && pace >= MIN_PLAY_SPEED && pace <= MAX_PLAY_SPEED;
+  return { samples, speed: starts ? 1 : reading.speed };
 }
 
 export interface FollowState {