Load the next item's video before a composition cuts to it (#417)

Two sprites with videos back to back left a blank frame at the cut.
The video player only came into being once the item began, and a
sprite is not drawn until its video has a first frame. So the second
item was missing until its video had loaded.

- compositionAhead lists the items that begin within the next second.
  Near the end of a looping composition it also lists the items at
  its start.
- An item that is alive already is not loaded ahead. Two times would
  otherwise pull at the same player.
- The frame carries these videos hidden and paused on their first
  picture at the item's offset. Their players get the id the item
  will have, so the cut reuses them instead of opening a second one.
- This works the same in the tab's preview and in the game.
- One second of lead (AHEAD_SECONDS) is an assumption. It can go up
  if large videos still flash.

Tests:
- Unit composition: 27 green, 2 new. Unit composition-play: 14
  green, 1 new for the path in the game.
- e2e 69: ALL PASSED, 12 checks. The new 69h puts two video items
  back to back and checks the second player is open and paused at
  its offset before the cut, and drawn in the same redraw as the
  cut.
- Without loading ahead in the preview 69h fails with cut: null.
  Bending two spots in the model turned their unit tests red.
- Gate on melektaus-atx: CHECK EXIT 0, 302 files, 4665 tests.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

README.md

@@ -47,7 +47,7 @@ npm run build        # writes dist/ and regenerates index.html; the folder is wh
 The browser suite and the three example games need a machine with a GPU:
 
 ```bash
-node tools/seed-e2e.mjs && node tools/e2e.mjs      # the editor, 2032 checks (GM_STEPS=20 for one step)
+node tools/seed-e2e.mjs && node tools/e2e.mjs      # the editor, 2033 checks (GM_STEPS=20 for one step)
                                                    #   every tool signs in first – see the Accounts page
 node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs
 node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs

src/core/model/composition.ts

@@ -225,7 +225,39 @@ export function compositionAt(project: Project, comp: CompositionAsset, frame: n
  * the document.
  */
 export function compositionHolder(project: Project, comp: CompositionAsset, frame: number): CompositionAsset {
-  const live = compositionAt(project, comp, frame);
+  return holderOf(project, comp, compositionAt(project, comp, frame));
+}
+
+/** How far ahead of where a composition stands the items about to begin are written out, in seconds. */
+export const AHEAD_SECONDS = 1;
+
+/**
+ * The items that begin within `seconds` after `frame` – once it loops, those at the start too when
+ * the end is that close – each written out as at its first frame, like `compositionHolder` does. A
+ * view opens their videos ahead of the cut: opened when its item began, a video had no picture yet,
+ * and the cut showed nothing until it loaded. One alive at `frame` is left out, it wears its own time.
+ */
+export function compositionAhead(
+  project: Project,
+  comp: CompositionAsset,
+  frame: number,
+  seconds = AHEAD_SECONDS,
+): CompositionAsset {
+  const until = frame + Math.ceil(seconds * compositionFps(comp));
+  const length = compositionLength(project, comp);
+  const wrapped = compositionLoops(comp) && until >= length ? until - length : -1;
+  const coming = comp.items.filter(
+    (item) => (item.start > frame && item.start <= until) || item.start <= wrapped,
+  );
+  if (coming.length === 0) return holderOf(project, comp, []);
+  const alive = new Set(compositionAt(project, comp, frame).map((l) => l.item.id));
+  const live = coming
+    .filter((item) => !alive.has(item.id))
+    .flatMap((item) => compositionAt(project, comp, item.start).filter((l) => l.item.id === item.id));
+  return holderOf(project, comp, live);
+}
+
+function holderOf(project: Project, comp: CompositionAsset, live: LiveItem[]): CompositionAsset {
   const holder: CompositionAsset = {
     ...comp,
     props: { ...comp.props },

src/editor/composition/CompositionCanvas.ts

@@ -1,6 +1,8 @@
 import { compositionHolder } from '../../core/model/composition';
 import type { CompositionAsset } from '../../core/model/Asset';
 import { findAssetOf } from '../../core/model/Project';
+import { compositionVideosAhead } from '../../viewport/items';
+import type { Frame } from '../../viewport/WorldViewport';
 import type { EditorState } from '../state';
 import { SceneViewport } from '../viewport/SceneViewport';
 
@@ -45,7 +47,7 @@ export class CompositionCanvas extends SceneViewport {
   shownHolder(): CompositionAsset | undefined {
     const comp = this.assetId ? findAssetOf(this.state.project, this.assetId, 'composition') : undefined;
     if (!comp) return undefined;
-    const frame = Math.max(0, Math.floor(this.playhead() + 1e-6));
+    const frame = this.frameAt();
     if (this.shown?.frame !== frame || this.shown.version !== this.version)
       this.shown = {
         frame,
@@ -59,6 +61,19 @@ export class CompositionCanvas extends SceneViewport {
     return this.shownHolder();
   }
 
+  /** With the videos of the items about to begin worn ahead, so a cut to one shows its picture at once. */
+  protected override collectFrame(): Frame {
+    const frame = super.collectFrame();
+    const comp = this.assetId ? findAssetOf(this.state.project, this.assetId, 'composition') : undefined;
+    if (!comp) return frame;
+    return { ...frame, ahead: compositionVideosAhead(this.state.project, comp, this.frameAt()) };
+  }
+
+  /** The frame the playhead stands on. */
+  private frameAt(): number {
+    return Math.max(0, Math.floor(this.playhead() + 1e-6));
+  }
+
   /** Each composition keeps the view it was left with, as a container does (#171). */
   protected override framesOnOpen(): boolean {
     return false;

src/editor/debug/documentHooks.ts

@@ -92,6 +92,8 @@ export function documentHooks({ state, editor }: HookContext) {
       shown: () =>
         (editor.compositionEditor.shownHolder()?.objects ?? []).map((o) => ({ id: o.id, x: o.position.x })),
       objectScreen: (objectId: string) => editor.compositionEditor.preview.objectScreen(objectId),
+      /** Where the picture draws a sprite right now; null while it is left out, a video without its picture. */
+      drawn: (objectId: string) => editor.compositionEditor.preview.drawnRect(objectId),
     },
     tile: {
       current: () => editor.tileEditor.currentAssetId,

src/viewport/WorldViewport.ts

@@ -17,7 +17,7 @@ import type { Project } from '../core/model/Project';
 import { cancelFrameWork, onFrame, type Size, ui, Widget } from '../ui';
 import { fontLibrary } from './FontLibrary';
 import { drawSlots } from './drawOrder';
-import type { DrawItem } from './items';
+import type { DrawItem, VideoAhead } from './items';
 import { hashId } from '../core/particles/nodes';
 import { ParticleField } from './particles';
 import { particleLibrary } from './ParticleLibrary';
@@ -57,6 +57,8 @@ export interface ViewSource {
 export interface Frame {
   items: DrawItem[];
   background: number;
+  /** Driven videos (#404) worn without being drawn: those of composition items about to begin (#417). */
+  ahead?: VideoAhead[];
 }
 
 /** A part of the view by its centre and size, in local CSS pixels. */
@@ -465,6 +467,8 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
         this.isSelected(item.id),
       );
     }
+    // Opened before their item begins, so the cut to it has its picture at once.
+    for (const ahead of frame.ahead ?? []) if (this.wearVideo(ahead, false) !== ahead) this.animated = true;
     this.afterItems(items, view);
     this.drawOnScreen(items, view);
     this.drawn = { scale: view.scale, w: rect.w, h: rect.h, grid: view.grid ?? null, culled };
@@ -903,7 +907,7 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
    * The item with the player of its own a driven video (#404) has, per view and object. One that is
    * not `seen` is never heard.
    */
-  private wearVideo(item: DrawItem, seen = true): DrawItem {
+  private wearVideo<T extends VideoAhead>(item: T, seen = true): T {
     if (!item.texture) return item;
     const texture = videoLibrary.wear(
       item.texture,
@@ -914,7 +918,7 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
     return texture === item.texture ? item : { ...item, texture };
   }
 
-  private videoWearer(item: DrawItem): string {
+  private videoWearer(item: Pick<DrawItem, 'id'>): string {
     return `${this.wearer}:${item.id}`;
   }
 

src/viewport/items.ts

@@ -1,11 +1,11 @@
 import { isHandleEnabled, worldTransformOf } from '../core/graph';
 import { type AssetType, isSymbolType } from '../core/model/AssetTypes';
 import { treeOrder } from '../core/model/hierarchy';
-import { placedPicture } from '../core/model/composition';
-import { resolvedHolder } from '../core/model/instance';
+import { compositionAhead, compositionFrameAt, placedPicture } from '../core/model/composition';
+import { contentId, placedWithContent, resolvedHolder } from '../core/model/instance';
 import type { Mat } from '../core/model/matrix';
 import type { PropertyBag } from '../core/model/Properties';
-import type { ParticlesAsset, TilemapAsset } from '../core/model/Asset';
+import type { CompositionAsset, ParticlesAsset, TilemapAsset } from '../core/model/Asset';
 import {
   readTile,
   tileFps,
@@ -25,11 +25,12 @@ import {
   triggerRects,
 } from '../core/model/tilemapCollision';
 import { findAsset, findAssetOf, type ObjectHolder, type Project } from '../core/model/Project';
-import { objectName } from '../core/model/SceneObject';
+import { objectName, type SceneObject } from '../core/model/SceneObject';
 import { fontFileOf, fontSmoothOf } from '../core/model/font';
 import { resolveTexture, type TextureRef } from '../core/model/texture';
 import type { FrameOutline } from '../core/model/frameOutline';
 import { liveTilemap, type TileEdits } from '../core/model/liveTilemap';
+import { isVideoKey } from '../core/model/video';
 import { renderLayerAlong } from '../core/model/renderLayers';
 import { type ClipRegion, clipRegions } from '../core/properties/clip';
 import { colliderFrame } from '../core/properties/collider';
@@ -84,9 +85,53 @@ export interface DrawItem {
   sealedIn?: string;
 }
 
+/** A sprite not drawn yet whose driven video a view opens ahead (#417), under the id it will be drawn by. */
+export type VideoAhead = Pick<DrawItem, 'id' | 'texture' | 'props'>;
+
 interface DrawList {
   items: DrawItem[];
   background: number;
+  /** The videos of composition items about to begin, see `compositionAhead`. */
+  ahead?: VideoAhead[];
+}
+
+/**
+ * The sprites among `objects` – already laid out, container content and all – whose video is driven:
+ * what a view wears ahead for a composition's coming items.
+ */
+function drivenVideos(project: Project, objects: readonly SceneObject[]): VideoAhead[] {
+  return objects.flatMap((object) => {
+    const asset = findAsset(project, object.assetId);
+    if (asset?.type !== 'sprite') return [];
+    const props = resolveSceneObjectProps(project, asset, object);
+    const texture = resolveTexture(project, props);
+    return texture && isVideoKey(texture.file) && videoDrivenOf(props)
+      ? [{ id: object.id, texture, props }]
+      : [];
+  });
+}
+
+/** The videos of a composition's items about to begin after `frame`, by the ids its own tab draws them by. */
+export function compositionVideosAhead(
+  project: Project,
+  comp: CompositionAsset,
+  frame: number,
+): VideoAhead[] {
+  return drivenVideos(project, resolvedHolder(project, compositionAhead(project, comp, frame)).objects);
+}
+
+/** The same for every composition the run plays, by the ids its items take below their placement. */
+function worldVideosAhead(world: World): VideoAhead[] {
+  return world.entities.flatMap((placement) => {
+    const player = placement.composition;
+    const comp = player && !player.stopped && findAssetOf(world.project, placement.assetId, 'composition');
+    if (!comp) return [];
+    const frame = compositionFrameAt(comp, player.time);
+    const objects = compositionAhead(world.project, comp, frame).objects.flatMap((o) =>
+      placedWithContent(world.project, o, contentId(placement.objectId, o.id), placement.objectId),
+    );
+    return drivenVideos(world.project, objects);
+  });
 }
 
 /** One tile a tilemap paints with, as drawing needs it: the blocks and how they run. */
@@ -340,5 +385,5 @@ export function collectWorldItems(world: World): DrawList {
         (at) => at.parent ?? null,
       ),
     }));
-  return { items, background: world.backgroundColor };
+  return { items, background: world.backgroundColor, ahead: worldVideosAhead(world) };
 }

tests/core/composition-play.test.ts

@@ -24,7 +24,7 @@ import { addGameSystems } from '../../src/runtime/systems/BehaviorSystem';
 import { Input } from '../../src/runtime/systems/Input';
 import { World } from '../../src/runtime/World';
 import { stackAt } from '../../src/viewport/pick';
-import { collectHolderItems } from '../../src/viewport/items';
+import { collectHolderItems, collectWorldItems } from '../../src/viewport/items';
 import { graphEditor, makeBehavior, makeProject, type ProjectFixture } from '../helpers';
 import { PlaceAssetCommand } from '../../src/core/commands/sceneObjectCommands';
 
@@ -224,6 +224,36 @@ describe('a composition played in a scene', () => {
     expect(run.at(item)!.props[VIDEO_TIME_KEY]).toBeCloseTo(2, 9);
   });
 
+  it('hands the view the video of the next item a second ahead, by the id the item will take', () => {
+    const stage = makeStage();
+    director(stage);
+    const video = new CreateVideoCommand('Intro', FILE, {
+      width: 320,
+      height: 180,
+      duration: 10,
+      fps: 30,
+      audio: false,
+    });
+    stage.stack.execute(video);
+    const screen = new CreateAssetCommand('sprite', 'Screen', { texture: video.asset.id });
+    stage.stack.execute(screen);
+    stage.item(first(stage), screen.asset.id, 0, 20);
+    const next = stage.item(first(stage), screen.asset.id, 20, 10, 0.5);
+    const run = play(stage);
+    const ahead = () => collectWorldItems(run.world).ahead ?? [];
+    run.frame(10);
+    expect(ahead()).toEqual([]);
+    run.frame();
+    const [coming] = ahead();
+    expect(coming?.id).toBe(contentId(stage.placement, next));
+    expect(coming?.texture?.file).toMatch(/^video:/);
+    expect(coming?.props[VIDEO_CLOCK_KEY]).toBe('driven');
+    expect(coming?.props[VIDEO_TIME_KEY]).toBe(0.5);
+    run.frame(10);
+    expect(run.at(next)).toBeDefined();
+    expect(ahead()).toEqual([]);
+  });
+
   it('loops without finishing, building an item anew each time round', () => {
     const stage = makeStage();
     director(stage);

tests/core/composition.test.ts

@@ -24,6 +24,7 @@ import type {
 } from '../../src/core/model/Asset';
 import {
   clipLength,
+  compositionAhead,
   compositionAt,
   compositionFps,
   compositionHolder,
@@ -35,6 +36,7 @@ import {
 import { instanceChildId, resolveObjects } from '../../src/core/model/instance';
 import { findAssetOf, objectHolders } from '../../src/core/model/Project';
 import type { PropertyBag } from '../../src/core/model/Properties';
+import { COMPOSITION_LOOP_KEY } from '../../src/core/properties/compositions';
 import { createSceneObject } from '../../src/core/model/SceneObject';
 import {
   VIDEO_CLOCK_KEY,
@@ -532,6 +534,38 @@ describe('items wearing a video', () => {
     ).toBe(180);
   });
 
+  it('are written out a second before they begin, at their first frame, so a view opens them ahead', () => {
+    const cut = makeCut();
+    const screen = videoSprite(cut, addVideo(cut));
+    const track = firstTrack(cut);
+    cut.item(track, screen, 0, 120);
+    const next = cut.item(track, screen, 120, 120, { offset: 2 });
+    const ahead = (frame: number) => compositionAhead(cut.project, cut.comp(), frame).objects;
+    expect(ahead(59)).toEqual([]);
+    const [object] = ahead(60);
+    expect(object!.id).toBe(next);
+    expect(object!.overrides[VIDEO_CLOCK_KEY]).toBe('driven');
+    expect(object!.overrides[VIDEO_TIME_KEY]).toBe(2);
+    // Begun, it wears its own time and is no longer ahead.
+    expect(ahead(120)).toEqual([]);
+  });
+
+  it('at the start are ahead too when it loops and its end comes close – unless still alive', () => {
+    const cut = makeCut();
+    const screen = videoSprite(cut, addVideo(cut));
+    const track = firstTrack(cut);
+    const first = cut.item(track, screen, 0, 120);
+    cut.item(track, screen, 120, 120);
+    const ids = (frame: number) => compositionAhead(cut.project, cut.comp(), frame).objects.map((o) => o.id);
+    expect(ids(200)).toEqual([]);
+    cut.comp().props[COMPOSITION_LOOP_KEY] = true;
+    expect(ids(179)).toEqual([]);
+    expect(ids(200)).toEqual([first]);
+    cut.comp().items[0]!.length = 240;
+    cut.comp().items.pop();
+    expect(ids(200)).toEqual([]);
+  });
+
   it('follow the texture an override gives the item', () => {
     const cut = makeCut();
     const item = cut.item(firstTrack(cut), cut.spriteId, 0, 1);

tools/e2e/69-composition-tab.mjs

@@ -3,13 +3,19 @@
  * and picked, so the inspector shows its TIMING; a second one dropped onto the first lands right
  * after it. A dragged end stops at the next item. An animation dragged in becomes a clip that moves
  * the item it overlaps in the picture. The hierarchy lists what is alive at the playhead, Delete
- * takes an item with its object, and Play runs the playhead to the end.
+ * takes an item with its object, and Play runs the playhead to the end. The video of an item about
+ * to begin is opened ahead, so the cut to it shows its picture at once.
  *
  * Everything it adds carries a `ct69` id and goes again at the end.
  */
+import { readFileSync } from 'node:fs';
+import { uploadVideo } from '../lib/projects.mjs';
+
 const CUT = 'ct69-cut';
 const BOX = 'ct69-box';
 const SLIDE = 'ct69-slide';
+const VIDEO = 'ct69-video';
+const SCREEN = 'ct69-screen';
 
 export async function compositionTab(t) {
   const { check, sleep, json, evaluate, mutate, mouse, drag, press, showTab } = t;
@@ -182,6 +188,48 @@ export async function compositionTab(t) {
     JSON.stringify({ running, ended }),
   );
 
+  // 69h. Two video items back to back: the second one's video is opened a second ahead, so the cut
+  // draws its picture in the very redraw it begins – loaded only then, it was left out until it came.
+  const uploaded = await uploadVideo(
+    t.baseUrl,
+    readFileSync(new URL('./fixtures/colors.webm', import.meta.url)),
+    'colors.webm',
+  );
+  await mutate(`
+    const p = __gm.state.project;
+    p.assets.push(
+      { id: '${VIDEO}', type: 'video', name: 'Colors 69', props: {}, file: ${JSON.stringify(uploaded.file)},
+        width: 320, height: 180, duration: 2, fps: 30, audio: true },
+      { id: '${SCREEN}', type: 'sprite', name: 'Screen 69', props: { width: 160, height: 90, color: 0xffffff, texture: '${VIDEO}' } },
+    );
+    const c = p.assets.find((a) => a.id === '${CUT}');
+    const object = (id) => ({ ...structuredClone(c.objects[0]), id, assetId: '${SCREEN}', overrides: {}, instanceEdits: {} });
+    c.objects = [object('ct69-v1'), object('ct69-v2')];
+    c.items = [
+      { id: 'ct69-v1', track: 'ct69-t1', start: 0, length: 20, offset: 0 },
+      { id: 'ct69-v2', track: 'ct69-t1', start: 20, length: 10, offset: 1 },
+    ];
+    c.clips = [];`);
+  const second = `__gm.videos().find((v) => v.key.endsWith(':ct69-v2')) ?? null`;
+  await evaluate(`__gm.composition.scrub(5)`);
+  await sleep(300);
+  const early = await json(second);
+  await evaluate(`__gm.composition.scrub(12)`);
+  let ahead = null;
+  // Settled: a seek still landing would hold the view a frame, which is no picture missing.
+  for (let i = 0; i < 40 && !(ahead?.ready && !ahead.seeking && !ahead.landing); i++) {
+    await sleep(150);
+    ahead = await json(second);
+  }
+  const cut = await json(
+    `(() => { __gm.composition.scrub(20); return __gm.composition.drawn('ct69-v2'); })()`,
+  );
+  check(
+    'a coming video item is opened a second ahead, at its offset, and drawn in the redraw it begins',
+    early === null && ahead?.ready && ahead.paused && Math.abs(ahead.time - 1) < 0.05 && cut !== null,
+    JSON.stringify({ early, ahead, cut }),
+  );
+
   await showTab('scene');
   await mutate(`
     const p = __gm.state.project;