Keep a video's loading out of recordings and first frames (#417)

A Start recording right after Play composition still began with
empty frames. The composition's clock waited for its video, but the
recording went on and took every frame while the video loaded. A
clip that starts later in the video also slipped through, because
its player already had one picture while it still sought to the
clip's start.

- The view now counts every video sprite it leaves out as missing,
  not only one without any first picture.
- GameRecorder takes no frame while the game view misses a video,
  for at most VIDEO_WAIT_MS (3 s) at a time. The recorded time
  stands and the sound of that time is dropped, so it stays in sync.
- RecordingClock.start forgets a hold from before the first frame.
  Without that, the first real frames all got the time 0.
- A followed player starts at 1x until the first speed reading.
  Before, it paused and sought every frame for about 100 ms, so the
  first pictures after a wait stuck and then jumped. A step faster
  than 4x and a time running backwards do not count as a start.
- Test hooks: __gm.gameVideo(id) reads the game view's own player,
  TakeInfo.lastFrameAt shows when the last frame was taken.

Tests:
- Unit recording (clock after an early hold), video-follow (2 new,
  1 adapted): 449 green in tests/core/recording, tests/viewport and
  tests/core/composition*
- e2e 68e (a clip from 0.8 s shows that picture first and then
  plays), 68f (a recording that already runs takes no frame while
  the composition's video loads)
- 68e is red without the start at 1x, 68f without the hold
- README: 2038 editor checks

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

README.md

@@ -47,7 +47,7 @@ npm run build        # writes dist/ and regenerates index.html; the folder is wh
 The browser suite and the three example games need a machine with a GPU:
 
 ```bash
-node tools/seed-e2e.mjs && node tools/e2e.mjs      # the editor, 2036 checks (GM_STEPS=20 for one step)
+node tools/seed-e2e.mjs && node tools/e2e.mjs      # the editor, 2038 checks (GM_STEPS=20 for one step)
                                                    #   every tool signs in first – see the Accounts page
 node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs
 node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs

src/core/recording/recording.ts

@@ -126,8 +126,10 @@ export class RecordingClock {
     return this.origin !== null;
   }
 
+  /** A pause before the first frame is no part of the recording's time. */
   start(now: number): void {
     this.origin = now;
+    this.paused = 0;
   }
 
   pause(now: number): void {

src/editor/debug/sceneHooks.ts

@@ -113,6 +113,8 @@ export function sceneHooks({ state, editor }: HookContext) {
     gameScreen: (id: string) => gameView.objectScreen(id),
     gameRect: (id: string) => gameView.objectRect(id),
     gameDrawnRect: (id: string) => gameView.drawnRect(id),
+    /** The player of a driven video the Game tab draws for this object, not another view's. */
+    gameVideo: (id: string) => gameView.videoOf(id),
     gameCenter: () => centreOf(gameView),
     /** What the main camera draws in the Game tab, back to front (#302). */
     gameDrawOrder: () => gameView.drawOrder(),

src/runtime/systems/CompositionSystem.ts

@@ -38,13 +38,13 @@ interface Run {
   id: number;
   /** By item id. An item a behavior destroyed stays here until it ends, so it is not built again. */
   live: Map<string, Spawned>;
-  /** Seconds the clock has stood so far for a video without its first picture, see `waits`. */
+  /** Seconds the clock has stood so far for a video without its picture, see `waits`. */
   waited: number;
 }
 
 /** What the view that shows a run knows of its driven videos (#404). */
 export interface VideoPictures {
-  /** Whether it left the sprite `objectId` out, its video still without a first picture. */
+  /** Whether it did not show the sprite `objectId`, its video still without the picture asked for. */
   missing(objectId: string): boolean;
 }
 
@@ -55,7 +55,7 @@ export interface VideoPictures {
  * frame's `finished`. The editor run plays none: it writes into the saved scene, where the objects of
  * a cut would stay behind.
  *
- * While the view still waits for the first picture of a video an item brings in, the composition's
+ * While the view still waits for the picture of a video an item brings in, the composition's
  * clock stands, so the video starts at its first frame and not where the clock has got to meanwhile.
  */
 export class CompositionSystem implements System {

src/viewport/GameRecorder.ts

@@ -1,4 +1,5 @@
 import { Matrix, type RenderTexture } from 'pixi.js';
+import { VIDEO_WAIT_MS } from '../core/model/video';
 import { preSkipOf } from '../core/music/ogg';
 import { AVC_CODEC, colourOf, MatroskaWriter, VP9_CODEC } from '../core/recording/matroska';
 import {
@@ -34,6 +35,8 @@ const LAST_FRAMES_MS = 500;
 export interface RecordingView {
   picture(): { shown: PictureArea; whole: PictureArea } | null;
   drawPicture(transform: Matrix, target: RenderTexture): void;
+  /** Whether its last redraw left out a video sprite, the video still without its picture. */
+  missesVideo(): boolean;
 }
 
 /** A recording once it is made. */
@@ -73,6 +76,8 @@ export interface TakeInfo {
   soundFrames: number;
   /** From the first frame to the end of the last, in ms. */
   durationMs: number;
+  /** `performance.now()` when the last frame was taken, null before the first. */
+  lastFrameAt: number | null;
   /** What the video encoder said its colour is, null while it said nothing. */
   colour: VideoColorSpaceInit | null;
   bytes: number;
@@ -165,6 +170,10 @@ class Take {
   private soundEnded: (() => void) | null = null;
   private stopAsked = false;
   private paused = false;
+  private held = false;
+  /** Since when the view lacks a video, see `holdForVideo`; null while it lacks none. */
+  private heldSince: number | null = null;
+  private lastFrameAt: number | null = null;
   private failure: string | null = null;
 
   constructor(
@@ -278,13 +287,37 @@ class Take {
   }
 
   setPaused(paused: boolean): void {
-    if (paused === this.paused) return;
-    this.paused = paused;
+    this.stand(() => (this.paused = paused));
+  }
+
+  /**
+   * A frame that lacks a video is not taken (#417): a composition's clock stands until its video has
+   * its picture, and the frames meanwhile were empty where the video goes. Like a pause, but the game
+   * plays on, so what it sounds then is left out too. At most `VIDEO_WAIT_MS` at a stretch, as long
+   * as the composition waits.
+   */
+  private holdForVideo(view: RecordingView): void {
     const now = performance.now();
-    if (paused) this.clock.pause(now);
+    if (view.missesVideo()) this.heldSince ??= now;
+    else this.heldSince = null;
+    const held = this.heldSince !== null && now - this.heldSince < VIDEO_WAIT_MS;
+    this.stand(() => (this.held = held));
+  }
+
+  /** The clock stands while paused or held, whichever came first and whichever ends last. */
+  private stand(change: () => void): void {
+    const was = this.standing;
+    change();
+    if (this.standing === was) return;
+    const now = performance.now();
+    if (this.standing) this.clock.pause(now);
     else this.clock.resume(now);
   }
 
+  private get standing(): boolean {
+    return this.paused || this.held;
+  }
+
   capture(view: RecordingView): void {
     if (this.state === 'waiting') {
       const picture = view.picture();
@@ -302,7 +335,9 @@ class Take {
     if (this.state !== 'recording' || !video || !reader) return;
     reader.collect((bytes, ms) => this.encode(bytes, ms));
     // A frame taken may have reached the most a file holds and ended the recording.
-    if (!this.taking || this.paused) return;
+    if (!this.taking) return;
+    this.holdForVideo(view);
+    if (this.standing) return;
     const now = performance.now();
     if (!this.clock.started) this.clock.start(now);
     const ms = this.clock.at(now);
@@ -318,6 +353,7 @@ class Take {
     );
     view.drawPicture(new Matrix(m.a, 0, 0, m.d, m.tx, m.ty), reader.texture);
     reader.read(ms);
+    this.lastFrameAt = now;
   }
 
   private get taking(): boolean {
@@ -352,8 +388,9 @@ class Take {
       return;
     }
     // Before the first frame there is no time to put it at. Pieces keep coming during a pause only
-    // as long as they were on their way, so they belong before it and are kept.
-    if (!audio || !this.tap || this.state !== 'recording' || !this.clock.started) return;
+    // as long as they were on their way, so they belong before it and are kept. A hold leaves them
+    // out: the game plays on meanwhile.
+    if (!audio || !this.tap || this.state !== 'recording' || !this.clock.started || this.held) return;
     const rate = this.tap.rate;
     this.soundStart ??= Math.max(0, this.clock.at(performance.now()) - (frames / rate) * 1000);
     const planar = new Float32Array(frames * 2);
@@ -511,6 +548,7 @@ class Take {
       dropped: this.dropped,
       soundFrames: this.soundFrames,
       durationMs: this.frames > 0 ? Math.round(this.lastMs + 1000 / this.settings.fps) : 0,
+      lastFrameAt: this.lastFrameAt,
       colour: this.colour,
       bytes: this.writer?.bytes ?? 0,
       failure: this.failure,

src/viewport/WorldViewport.ts

@@ -125,7 +125,7 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
   private readonly videosShown = new Map<string, string>();
   /** Since when the view keeps its last frame for a driven video it brings in, see `holdsForVideo`. */
   private heldSince: number | null = null;
-  /** The sprites the last redraw left out, their driven video still without its first picture. */
+  /** The sprites the last redraw did not show, their driven video still without the picture asked for. */
   private readonly videosLeftOut = new Set<string>();
   private cameraMoves = 0;
   private readonly stopFiles: (() => void)[];
@@ -506,7 +506,7 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
    */
   private holdsForVideo(view: View, items: DrawItem[], shaders: ShaderSource): boolean {
     const bounds = worldBounds(view);
-    const waits = items.some((item) => {
+    const waiting = items.filter((item) => {
       const file = item.texture?.file;
       if (item.type !== 'sprite' || !item.enabled || !file || !isVideoKey(file)) return false;
       if (!videoDrivenOf(item.props)) return false;
@@ -520,9 +520,12 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
       return videoLibrary.hasPicture(key) && !videoLibrary.showsAsked(key);
     });
     const now = performance.now();
-    if (!waits) this.heldSince = null;
+    if (waiting.length === 0) this.heldSince = null;
     else this.heldSince ??= now;
-    return waits && now - this.heldSince! < VIDEO_HOLD_MS;
+    const holds = waiting.length > 0 && now - this.heldSince! < VIDEO_HOLD_MS;
+    // The held frame does not show them either, so a composition waits for them meanwhile.
+    if (holds) for (const item of waiting) this.videosLeftOut.add(item.id);
+    return holds;
   }
 
   /**
@@ -888,7 +891,7 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
       if (ghost) disposeSpriteMesh(ghost);
       this.ghostSprites.delete(item.id);
       this.videosShown.delete(item.id);
-      if (!videoLibrary.hasPicture(key)) this.videosLeftOut.add(item.id);
+      this.videosLeftOut.add(item.id);
       return;
     }
     if (key !== null) this.videosShown.set(item.id, key);
@@ -926,14 +929,26 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
     return `${this.wearer}:${item.id}`;
   }
 
+  /**
+   * What the player of a driven video drawn here for `objectId` is doing (test hook): the one it
+   * showed last, else the newest it opened; null for none.
+   */
+  videoOf(objectId: string): ReturnType<typeof videoLibrary.debugInfo>[number] | null {
+    const mark = `@${this.videoWearer({ id: objectId })}`;
+    const players = videoLibrary.debugInfo().filter((player) => player.key.endsWith(mark));
+    const shown = this.videosShown.get(objectId);
+    return players.find((player) => player.key === shown) ?? players[players.length - 1] ?? null;
+  }
+
   /** Whether a driven video (#404) this view wears is still without its first picture. */
   videosLoading(): boolean {
     return videoLibrary.loading(this.wearer);
   }
 
   /**
-   * Whether the last redraw left out a driven video (#404) it should show, its player still without
-   * a first picture. Unlike `videosLoading`, one opened ahead or switched off does not count.
+   * Whether the last redraw did not show a driven video (#404) it should show: its player has no
+   * picture yet, or still seeks to the one asked for, as after loading a clip that starts later.
+   * Unlike `videosLoading`, one opened ahead or switched off does not count.
    */
   missesVideo(): boolean {
     return this.videosLeftOut.size > 0;

src/viewport/videoFollow.ts

@@ -42,7 +42,8 @@ export interface SpeedReading {
  * The next reading after the wearer asked for `time` at `now`. A jump (a loop going round, a key
  * set to another time) starts the window over and keeps the speed: playing on through a loop is
  * right, and a still picture stays still. While the time stands, the window waits at its latest
- * frame, so a start is measured from where it began and not across the wait before it.
+ * frame, so a start is measured from where it began and not across the wait before it. Until then
+ * a start counts as 1x.
  */
 export function readSpeed(reading: SpeedReading | null, time: number, now: number): SpeedReading {
   if (!reading) return { samples: [{ time, at: now }], speed: 0 };
@@ -58,8 +59,13 @@ export function readSpeed(reading: SpeedReading | null, time: number, now: numbe
   while (samples.length > 2 && now - samples[1]!.at >= SPEED_SPAN_MS) samples.shift();
   const first = samples[0]!;
   const span = now - first.at;
-  if (span < SPEED_WINDOW_MS) return { samples, speed: reading.speed };
-  return { samples, speed: ((time - first.time) * 1000) / span };
+  const pace = ((time - first.time) * 1000) / Math.max(span, 1);
+  if (span >= SPEED_WINDOW_MS) return { samples, speed: pace };
+  // A start from standing plays at 1x until the window has its reading: held and sought frame by
+  // frame instead, the first pictures of a cut stuck and were skipped. A step faster than a video
+  // plays is a jump, not a start.
+  const starts = reading.speed === 0 && pace >= MIN_PLAY_SPEED && pace <= MAX_PLAY_SPEED;
+  return { samples, speed: starts ? 1 : reading.speed };
 }
 
 export interface FollowState {

tests/core/recording.test.ts

@@ -71,6 +71,14 @@ describe('RecordingClock', () => {
     clock.resume(8000);
     expect(clock.at(8000)).toBe(2000);
   });
+
+  it('leaves a hold before the first frame out of the time, not off it', () => {
+    const clock = new RecordingClock();
+    clock.pause(0);
+    clock.resume(2000);
+    clock.start(2000);
+    expect(clock.at(2500)).toBe(500);
+  });
 });
 
 describe('FrameGate', () => {

tests/viewport/video-follow.test.ts

@@ -23,11 +23,27 @@ describe('the speed of the asked time', () => {
       [0.083, 83],
     ] as const)
       reading = readSpeed(reading, time, now);
-    expect(reading.speed).toBe(0);
+    expect(reading.samples[0]).toEqual({ time: 0, at: 0 });
     reading = readSpeed(reading, 0.1, SPEED_WINDOW_MS);
     expect(reading.speed).toBeCloseTo(1, 5);
   });
 
+  it('counts a start from standing as 1x until the window has its reading', () => {
+    let reading = readSpeed(null, 0.8, 0);
+    reading = readSpeed(reading, 0.8, 16);
+    expect(reading.speed).toBe(0);
+    // The first step of 1/60 s after the wait: held and sought instead, the first pictures stuck.
+    reading = readSpeed(reading, 0.8 + 1 / 60, 33);
+    expect(reading.speed).toBe(1);
+    // Two steps in one frame are still a start.
+    expect(readSpeed(readSpeed(null, 0.8, 0), 0.8 + 2 / 60, 16).speed).toBe(1);
+  });
+
+  it('does not count a jump faster than a video plays, or a time running back, as a start', () => {
+    expect(readSpeed(readSpeed(null, 0.8, 0), 0.9, 16).speed).toBe(0);
+    expect(readSpeed(readSpeed(null, 0.8, 0), 0.8 - 1 / 60, 16).speed).toBe(0);
+  });
+
   it('reads steps of 1/60 s on uneven frames as 1x within 5 % once the window has grown', () => {
     let reading = readSpeed(null, 0, 0);
     const read: number[] = [];

tools/e2e/68-composition.mjs

@@ -8,7 +8,8 @@ import { MAKE_OBJECT } from './lib/scaffold.mjs';
  * it: each item becomes an object below the placement while it lasts, the first track in front, a
  * clip moves the item it overlaps, and `On composition finished` hears the end, by which time no
  * item is left. Nothing of the run stays in the scene. An item's video is waited for: the clock
- * stands until the Game tab has its first picture.
+ * stands until the Game tab has its first picture, for a clip that starts later the picture there,
+ * and then the video plays at once. A recording that runs meanwhile takes no frame.
  *
  * Everything it adds carries a `cp68` id and goes again at the end.
  */
@@ -23,7 +24,7 @@ const link = (id, from, fromPort, to, toPort) => ({
 });
 
 export async function composition(t) {
-  const { check, sleep, json, evaluate, mutate, play, stop, showTab } = t;
+  const { check, sleep, json, evaluate, mutate, press, play, stop, showTab, waitUntil } = t;
   await showTab('scene');
 
   const graph = {
@@ -192,7 +193,7 @@ export async function composition(t) {
       const loading = [];
       const step = () => {
         const time = ${clock};
-        const player = __gm.videos().find((v) => v.key.endsWith(':${id('v')}'));
+        const player = __gm.gameVideo('${id('v')}');
         if (player?.ready) return done(JSON.stringify({ loading, ready: time }));
         if (player) loading.push(time);
         if (!__gm.world() || loading.length > 300) return done(JSON.stringify({ loading, ready: null }));
@@ -219,9 +220,109 @@ export async function composition(t) {
   );
   await stop();
 
+  // 68e. The item wears a clip that starts at 0.8 s: its player loads, then seeks back to 0.8. The
+  //      clock stands until the Game tab draws that picture, and from its first step on the video
+  //      plays – it was held and sought frame by frame for 0.1 s before, its first pictures stuck.
   await mutate(`
     const p = __gm.state.project;
-    p.assets = p.assets.filter((a) => !a.id.startsWith('cp68-'));
+    p.assets.find((a) => a.id === 'cp68-video').clips = [{ id: 'cp68-late', name: 'Late', props: { start: 0.8, end: 0 } }];
+    p.assets.find((a) => a.id === 'cp68-screen').props.texture = 'cp68-late';`);
+  await play();
+  const rows = JSON.parse(
+    await evaluate(`new Promise((done) => {
+      const rows = [];
+      const step = () => {
+        const player = __gm.gameVideo('${id('v')}');
+        rows.push({ time: ${clock}, drawn: __gm.gameDrawnRect('${id('v')}') !== null,
+          at: player?.time ?? null, paused: player?.paused ?? null });
+        if (rows.length >= 240 || !__gm.world()) return done(JSON.stringify(rows));
+        requestAnimationFrame(step);
+      };
+      step();
+    })`),
+  );
+  const first = rows.findIndex((row) => row.drawn);
+  const moving = rows.filter((row) => row.drawn && row.time > 0).slice(0, 3);
+  check(
+    'a clip that starts later is waited for, then plays from its first step on',
+    first >= 0 &&
+      rows.slice(0, first).every((row) => row.time === 0) &&
+      Math.abs(rows[first].at - 0.8) < 0.02 &&
+      moving.length === 3 &&
+      moving.every((row) => row.paused === false),
+    JSON.stringify({ first, shown: rows[first], moving }),
+  );
+  await stop();
+
+  // 68f. A recording runs when Play composition starts the cut: no frame is taken while the Game
+  //      tab leaves the item's video out – it recorded the empty frames of the wait before. A clip
+  //      that starts elsewhere, so a new player loads: the one of 68e is still there, at its picture.
+  await mutate(`
+    const video = __gm.state.project.assets.find((a) => a.id === 'cp68-video');
+    video.clips.push({ id: 'cp68-later', name: 'Later', props: { start: 1.2, end: 0 } });
+    __gm.state.project.assets.find((a) => a.id === 'cp68-screen').props.texture = 'cp68-later';`);
+  const flow = structuredClone(graph);
+  flow.nodes.push(
+    node('record', 'StartRecording', 300, -150, { name: 'cp68 take', resolution: 'view', fps: '60' }),
+    node('key', 'OnKeyDown', 0, 450, { key: 'p' }),
+  );
+  flow.edges = flow.edges.filter((e) => e.id !== 'cp68-e1');
+  flow.edges.push(link('e4', 'start', 'exec', 'record', 'exec'), link('e5', 'key', 'exec', 'play', 'exec'));
+  await mutate(
+    `__gm.state.project.assets.find((a) => a.id === 'cp68-director').graph = ${JSON.stringify(flow)};`,
+  );
+  const { uploads, takes } = await json('__gm.recording()');
+  const take = `__gm.recording().takes[${takes.length}]`;
+  await play();
+  await waitUntil(`${take}?.state === 'recording' && ${take}.lastFrameAt !== null`, {
+    label: 'the recording takes frames',
+  });
+  // Sampled every frame in the page, from before the key on.
+  await evaluate(`{
+    const rows = (window.cp68Rows = []);
+    const step = () => {
+      const exists = (__gm.world()?.entities ?? []).some((e) => e.objectId === '${id('v')}');
+      rows.push({ exists, drawn: __gm.gameDrawnRect('${id('v')}') !== null, taken: ${take}?.lastFrameAt ?? null });
+      const drawn = rows.findIndex((row) => row.drawn);
+      if ((drawn < 0 || rows.length <= drawn + 20) && rows.length < 400) requestAnimationFrame(step);
+    };
+    requestAnimationFrame(step);
+  }`);
+  await press('p');
+  await waitUntil(
+    `window.cp68Rows.length >= 400 || (window.cp68Rows.some((row) => row.drawn) && window.cp68Rows.length > window.cp68Rows.findIndex((row) => row.drawn) + 20)`,
+    { label: 'the item is drawn' },
+  );
+  const takeRows = await json('window.cp68Rows');
+  await stop();
+  // The frames in which the item was there and its video still left out.
+  const missing = takeRows.filter((row) => row.exists && !row.drawn);
+  const shown = takeRows.findIndex((row) => row.drawn);
+  check(
+    'a recording takes no frame while a composition waits for its video, and goes on after',
+    missing.length >= 2 &&
+      missing.every((row) => row.taken === missing[0].taken) &&
+      shown >= 0 &&
+      takeRows.at(-1).taken > missing.at(-1).taken,
+    JSON.stringify({
+      missing: missing.length,
+      takenWhileMissing: new Set(missing.map((row) => row.taken)).size - 1,
+      shown,
+      after:
+        takeRows.at(-1).taken !== null && missing.length > 0
+          ? Math.round(takeRows.at(-1).taken - missing.at(-1).taken)
+          : null,
+    }),
+  );
+  await waitUntil(
+    `__gm.recording().uploads.length > ${uploads.length} && ['added', 'failed'].includes(__gm.recording().uploads.at(-1).state)`,
+    { timeout: 60_000, label: 'the recording is in the project' },
+  );
+  const madeId = (await json('__gm.recording()')).uploads.at(-1).assetId ?? '';
+
+  await mutate(`
+    const p = __gm.state.project;
+    p.assets = p.assets.filter((a) => !a.id.startsWith('cp68-') && a.id !== ${JSON.stringify(madeId)});
     const scene = __gm.state.activeScene;
     scene.objects = scene.objects.filter((o) => !o.id.startsWith('cp68-'));`);
 }