Let an animation drive a sprite's video picture by picture (#404)

A video used to run on its own clock as soon as a sprite wore it. An
animation could switch the sprite on and off, but never say which
picture shows at which frame. Cutting a video in an animation needs
the opposite: the animation sets the time and the video follows.

- Sprites wearing a video or a clip get two rows, Video clock and
  Video time. They show only while the texture holds a video.
- With Video clock on Driven the video does not play by itself. It
  shows the picture at Video time, counted from the clip's start.
  Past the end it goes round with Loop and holds the last picture
  without.
- Video time is a number like any other, so keys, step, easing and
  curves give stills, slow motion and ramps.
- A driven sprite gets a player of its own, keyed by the wearer's
  name. The same clip can run twice at different times.
- Following has two ways (`videoFollow.ts`). Running forward at 0.25x
  to 4x the video plays at that rate and is nudged. A still, a scrub
  or a step backwards seeks, and only the newest seek counts.
- Sound is heard only near 1x. The animation tab now plays it too, so
  a cut can follow the sound. Scrubbing is silent.
- Scene, game view, the inspector's PREVIEW and the shader preview
  show the driven picture. In a running game Play animation sets it.

Found on the way:

- After a scrub the video in the animation tab kept running. The
  preview redrew only while the playhead moved, so nobody asked for
  the video once it stood. `AnimationEditor.tick` now redraws it every
  frame while it shows a video.
- Every sprite has two more defaults (`videoClock: 'free'`,
  `videoTime: 0`). Eight tests in five files pinned the old list and
  are updated.

Tests:
- Unit: video-clock and video-follow new, 4 files with 42 tests green.
- e2e 66 (melektaus-atx): ALL PASSED, 22 checks, 3 new. 66i: two
  sprites on one clip stand at 0.2 s red and 0.7 s green. 66j: a
  scrub to frames 2, 7, 12 and 3 shows red, green, blue, red, and
  each colour still stands 0.7 s later. 66k: playing runs 10-30 ms
  behind at rate 1.00-1.03 and is not muted.
- Counterchecks: a seek that keeps the rate turns the seek test red.
  Not seeing Driven turns 66i, 66j and 66k red.
- Gate (melektaus-atx): exit 0, 297 files, 4546 tests.
- Not tested in the browser: Play animation in a running game.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/commands.test.ts

@@ -35,6 +35,8 @@ describe('asset commands', () => {
       height: 100,
       color: 0xffffff,
       texture: '',
+      videoClock: 'free',
+      videoTime: 0,
       shader: '',
     });
   });

tests/core/resolve.test.ts

@@ -36,6 +36,8 @@ describe('PropertySchema', () => {
       height: 100,
       color: 0xffffff,
       texture: '',
+      videoClock: 'free',
+      videoTime: 0,
       shader: '',
     });
   });
@@ -69,6 +71,8 @@ describe('resolve', () => {
       height: 100,
       color: 0xffffff,
       texture: '',
+      videoClock: 'free',
+      videoTime: 0,
       shader: '',
     });
   });
@@ -90,6 +94,8 @@ describe('resolve', () => {
       height: 100,
       color: 0x00ff00,
       texture: '',
+      videoClock: 'free',
+      videoTime: 0,
       shader: '',
     });
 

tests/core/serialize.test.ts

@@ -183,6 +183,8 @@ describe('serialize', () => {
       height: 100,
       color: 0xffffff,
       texture: '',
+      videoClock: 'free',
+      videoTime: 0,
       shader: '',
     });
   });

tests/core/shaders.test.ts

@@ -727,6 +727,8 @@ describe('a sprite wearing a shader', () => {
       'height',
       'color',
       'texture',
+      'videoClock',
+      'videoTime',
       'shader',
     ]);
     expect(schemaFor('sprite').find((d) => d.key === SHADER_KEY)).toMatchObject({
@@ -748,6 +750,8 @@ describe('a sprite wearing a shader', () => {
       'height',
       'color',
       'texture',
+      'videoClock',
+      'videoTime',
       'shader',
       'brightness',
       'contrast',
@@ -756,14 +760,14 @@ describe('a sprite wearing a shader', () => {
     expect(assetDeviates(f.project, f.sprite())).toBe(true);
     // A dangling id adds no rows and draws nothing – the picker still shows it.
     f.sprite().props[SHADER_KEY] = 'gone';
-    expect(keys()).toHaveLength(13);
+    expect(keys()).toHaveLength(15);
   });
 
   it('gives a placed object the rows of the shader it overrides, filled with the node defaults', () => {
     const f = wearing();
     const id = place(f, f.spriteId, { x: 0, y: 0 });
     const object = () => f.object(id);
-    expect(schemaForObject(f.project, f.sprite(), object())).toHaveLength(13);
+    expect(schemaForObject(f.project, f.sprite(), object())).toHaveLength(15);
     f.stack.execute(new SetOverrideCommand(f.sceneId, id, SHADER_KEY, f.s.id));
     expect(schemaForObject(f.project, f.sprite(), object()).map((d) => d.key)).toContain('saturation');
     expect(resolveSceneObjectProps(f.project, f.sprite(), object())).toMatchObject({

tests/core/texture-commands.test.ts

@@ -74,6 +74,8 @@ describe('CreateAssetCommand with starting props', () => {
       height: 32,
       color: 0xffffff,
       texture: texture.asset.id,
+      videoClock: 'free',
+      videoTime: 0,
       shader: '',
     });
     stack.undo();

tests/core/video-clock.test.ts

@@ -0,0 +1,94 @@
+/**
+ * A video that follows the animation (#404): a sprite whose Video clock is driven shows the picture
+ * at its Video time, a number an animation keys like any other.
+ */
+import { describe, expect, it } from 'vitest';
+import { CreateVideoCommand } from '../../src/core/commands/assetCommands';
+import { AddClipCommand } from '../../src/core/commands/clipCommands';
+import { animatableDefs } from '../../src/core/model/animation';
+import { drivenTimeOf, drivenVideoKey, isDrivenVideoKey, parseVideoKey } from '../../src/core/model/video';
+import { isRowShown, schemaFor } from '../../src/core/properties/PropertySchema';
+import {
+  VIDEO_CLOCK_KEY,
+  VIDEO_END_KEY,
+  VIDEO_START_KEY,
+  VIDEO_TIME_KEY,
+} from '../../src/core/properties/videos';
+import { makeProject } from '../helpers';
+
+const FILE = `a1b2c3d4e5f6/${'c7'.repeat(32)}.mp4`;
+const KEY = `video:${FILE}#2~4~1~1`;
+
+describe('where a driven video stands', () => {
+  const clip = { start: 2, end: 4, loop: true };
+
+  it('counts from the start of its clip', () => {
+    expect(drivenTimeOf(clip, 0.5, 10)).toBe(2.5);
+  });
+
+  it('goes round past the end with Loop', () => {
+    expect(drivenTimeOf(clip, 5, 10)).toBe(3);
+  });
+
+  it('holds the last picture without Loop, and the first one before the start', () => {
+    expect(drivenTimeOf({ ...clip, loop: false }, 9, 10)).toBe(3.999);
+    expect(drivenTimeOf(clip, -1, 10)).toBe(2);
+  });
+
+  it('measures a clip running to the end of the video by the file', () => {
+    const tail = { start: 6, end: Infinity, loop: true };
+    expect(drivenTimeOf(tail, 5, 10)).toBe(7);
+    expect(drivenTimeOf({ ...tail, loop: false }, 5, 10)).toBe(9.999);
+    expect(drivenTimeOf(tail, 5, NaN)).toBe(11);
+  });
+});
+
+describe('the player a wearer drives', () => {
+  it('goes by the clip key and the name of its wearer', () => {
+    const key = drivenVideoKey(KEY, 'view3:obj1');
+    expect(key).toBe(`${KEY}@view3:obj1`);
+    expect(isDrivenVideoKey(key)).toBe(true);
+    expect(isDrivenVideoKey(KEY)).toBe(false);
+    expect(parseVideoKey(key)).toEqual(parseVideoKey(KEY));
+  });
+});
+
+describe('the Video clock and Video time rows', () => {
+  function setup() {
+    const f = makeProject();
+    const create = new CreateVideoCommand('Intro', FILE, {
+      width: 320,
+      height: 180,
+      duration: 2,
+      fps: 30,
+      audio: false,
+    });
+    f.stack.execute(create);
+    const clip = new AddClipCommand(create.asset.id, { [VIDEO_START_KEY]: 0.5, [VIDEO_END_KEY]: 1 });
+    f.stack.execute(clip);
+    return { project: f.project, videoId: create.asset.id, clipId: clip.clipId };
+  }
+  const row = (key: string) => schemaFor('sprite').find((d) => d.key === key)!;
+
+  it('are shown only while the texture is a video or a clip of one', () => {
+    const { project, videoId, clipId } = setup();
+    const driven = { [VIDEO_CLOCK_KEY]: 'driven' };
+    for (const texture of [videoId, clipId]) {
+      expect(isRowShown(row(VIDEO_CLOCK_KEY), { texture }, project)).toBe(true);
+      expect(isRowShown(row(VIDEO_TIME_KEY), { texture, ...driven }, project)).toBe(true);
+    }
+    expect(isRowShown(row(VIDEO_CLOCK_KEY), { texture: '' }, project)).toBe(false);
+    expect(isRowShown(row(VIDEO_TIME_KEY), { texture: '', ...driven }, project)).toBe(false);
+  });
+
+  it('show the time only while the clock is driven', () => {
+    const { project, videoId } = setup();
+    expect(isRowShown(row(VIDEO_TIME_KEY), { texture: videoId }, project)).toBe(false);
+  });
+
+  it('can be keyed by an animation', () => {
+    const keys = animatableDefs('sprite').map((d) => d.key);
+    expect(keys).toContain(VIDEO_TIME_KEY);
+    expect(keys).toContain(VIDEO_CLOCK_KEY);
+  });
+});

tests/viewport/video-follow.test.ts

@@ -0,0 +1,74 @@
+import { describe, expect, it } from 'vitest';
+import { followVideo, readSpeed, SEEK_DRIFT, SPEED_WINDOW_MS } from '../../src/viewport/videoFollow';
+
+/**
+ * How a driven video keeps up with the time its wearer asks for (#404): it plays along while that
+ * time runs forward at a playable speed, and holds and seeks when it stands, jumps or runs back.
+ */
+describe('the speed of the asked time', () => {
+  it('is measured over a window, not from frame to frame', () => {
+    let reading = readSpeed(null, 0, 0);
+    // A game steps its animation at its own rate: here a frame without a step, then two in one.
+    for (const [time, now] of [
+      [0, 16],
+      [0.033, 33],
+      [0.05, 50],
+      [0.05, 66],
+      [0.083, 83],
+    ] as const)
+      reading = readSpeed(reading, time, now);
+    expect(reading.speed).toBe(0);
+    reading = readSpeed(reading, 0.1, SPEED_WINDOW_MS);
+    expect(reading.speed).toBeCloseTo(1, 5);
+  });
+
+  it('keeps the speed through a jump and measures again from there', () => {
+    const playing = { from: 3.9, since: 900, last: 3.98, speed: 1 };
+    const turned = readSpeed(playing, 0.01, 990);
+    expect(turned).toEqual({ from: 0.01, since: 990, last: 0.01, speed: 1 });
+  });
+
+  it('reads a still time as standing', () => {
+    const playing = { from: 2, since: 0, last: 2, speed: 1 };
+    expect(readSpeed(playing, 2, SPEED_WINDOW_MS).speed).toBe(0);
+  });
+});
+
+describe('following the asked time', () => {
+  const at = { target: 1, speed: 1, current: 1, end: Infinity };
+
+  it('plays along at the speed of the asked time', () => {
+    expect(followVideo(at)).toEqual({ kind: 'play', rate: 1, seek: null, heard: true });
+    expect(followVideo({ ...at, speed: 0.5 })).toEqual({ kind: 'play', rate: 0.5, seek: null, heard: false });
+    expect(followVideo({ ...at, speed: 4 })).toMatchObject({ kind: 'play', rate: 4 });
+  });
+
+  it('nudges a small lag away by a few percent, never more than a tenth', () => {
+    const behind = followVideo({ ...at, current: 0.9 });
+    expect(behind).toMatchObject({ kind: 'play', seek: null });
+    expect(behind.kind === 'play' && behind.rate).toBeCloseTo(1.05, 5);
+    const ahead = followVideo({ ...at, current: 1 + SEEK_DRIFT });
+    expect(ahead.kind === 'play' && ahead.rate).toBeCloseTo(0.9, 5);
+  });
+
+  it('seeks once it is further off than the drift allows', () => {
+    expect(followVideo({ ...at, current: 0.7 })).toEqual({ kind: 'play', rate: 1, seek: 1, heard: true });
+  });
+
+  it('holds and seeks for a still picture, a time running back or too fast', () => {
+    expect(followVideo({ ...at, speed: 0, current: 0.2 })).toEqual({ kind: 'hold', seek: 1 });
+    expect(followVideo({ ...at, speed: -1, current: 1.2 })).toEqual({ kind: 'hold', seek: 1 });
+    expect(followVideo({ ...at, speed: 8, current: 0.2 })).toEqual({ kind: 'hold', seek: 1 });
+  });
+
+  it('does not seek a held video that shows the asked picture already', () => {
+    expect(followVideo({ ...at, speed: 0, current: 1.004 })).toEqual({ kind: 'hold', seek: null });
+  });
+
+  it('waits at the asked picture once it played past the end of its stretch', () => {
+    expect(followVideo({ target: 2.999, speed: 1, current: 3.02, end: 3 })).toEqual({
+      kind: 'hold',
+      seek: 2.999,
+    });
+  });
+});

tests/viewport/video-library.test.ts

@@ -84,3 +84,30 @@ describe('VideoLibrary sweeping', () => {
     expect(videos.debugInfo()).toEqual([]);
   });
 });
+
+describe('VideoLibrary driven players (#404)', () => {
+  beforeEach(() => {
+    vi.stubGlobal('document', { hidden: false, addEventListener: () => {}, createElement: fakeVideo });
+  });
+
+  afterEach(() => vi.unstubAllGlobals());
+
+  const REF = { file: KEY, smooth: true, rect: null, sourceWidth: 320, sourceHeight: 180, slice: null };
+
+  it('leaves a sprite with a free clock on the shared player', async () => {
+    const videos = await freshLibrary();
+    expect(videos.wear(REF, { videoTime: 1 }, 'view1:a', false)).toBe(REF);
+    expect(videos.debugInfo()).toEqual([]);
+  });
+
+  it('gives every wearer of a driven video a player of its own, opened at its time', async () => {
+    const videos = await freshLibrary();
+    const a = videos.wear(REF, { videoClock: 'driven', videoTime: 0.5 }, 'view1:a', false);
+    const b = videos.wear(REF, { videoClock: 'driven', videoTime: 1.25 }, 'view1:b', false);
+    expect([a.file, b.file]).toEqual([`${KEY}@view1:a`, `${KEY}@view1:b`]);
+    expect(videos.debugInfo().map(({ key, time }) => [key, time])).toEqual([
+      [`${KEY}@view1:a`, 0.5],
+      [`${KEY}@view1:b`, 1.25],
+    ]);
+  });
+});