Let an animation drive a sprite's video picture by picture (#404)

A video used to run on its own clock as soon as a sprite wore it. An
animation could switch the sprite on and off, but never say which
picture shows at which frame. Cutting a video in an animation needs
the opposite: the animation sets the time and the video follows.

- Sprites wearing a video or a clip get two rows, Video clock and
  Video time. They show only while the texture holds a video.
- With Video clock on Driven the video does not play by itself. It
  shows the picture at Video time, counted from the clip's start.
  Past the end it goes round with Loop and holds the last picture
  without.
- Video time is a number like any other, so keys, step, easing and
  curves give stills, slow motion and ramps.
- A driven sprite gets a player of its own, keyed by the wearer's
  name. The same clip can run twice at different times.
- Following has two ways (`videoFollow.ts`). Running forward at 0.25x
  to 4x the video plays at that rate and is nudged. A still, a scrub
  or a step backwards seeks, and only the newest seek counts.
- Sound is heard only near 1x. The animation tab now plays it too, so
  a cut can follow the sound. Scrubbing is silent.
- Scene, game view, the inspector's PREVIEW and the shader preview
  show the driven picture. In a running game Play animation sets it.

Found on the way:

- After a scrub the video in the animation tab kept running. The
  preview redrew only while the playhead moved, so nobody asked for
  the video once it stood. `AnimationEditor.tick` now redraws it every
  frame while it shows a video.
- Every sprite has two more defaults (`videoClock: 'free'`,
  `videoTime: 0`). Eight tests in five files pinned the old list and
  are updated.

Tests:
- Unit: video-clock and video-follow new, 4 files with 42 tests green.
- e2e 66 (melektaus-atx): ALL PASSED, 22 checks, 3 new. 66i: two
  sprites on one clip stand at 0.2 s red and 0.7 s green. 66j: a
  scrub to frames 2, 7, 12 and 3 shows red, green, blue, red, and
  each colour still stands 0.7 s later. 66k: playing runs 10-30 ms
  behind at rate 1.00-1.03 and is not muted.
- Counterchecks: a seek that keeps the rate turns the seek test red.
  Not seeing Driven turns 66i, 66j and 66k red.
- Gate (melektaus-atx): exit 0, 297 files, 4546 tests.
- Not tested in the browser: Play animation in a running game.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/editor/animation/AnimationEditor.ts

@@ -66,10 +66,15 @@ export class AnimationEditor extends AssetTabEditor<AnimationAsset, TimelineCanv
 
   /** Only called while the tab is visible. */
   tick(dt: number): void {
-    if (!this.canvas.isPlaying) return;
-    this.canvas.tick(dt);
-    // A finished `once` pass pauses the canvas; the button has to say so.
-    this.syncPlayButton();
+    const drawn = this.preview.redrawCount;
+    if (this.canvas.isPlaying) {
+      this.canvas.tick(dt);
+      // A finished `once` pass pauses the canvas; the button has to say so.
+      this.syncPlayButton();
+    }
+    // A video moves by itself, and a driven one (#404) has to be told every frame that the
+    // playhead stands; the playhead redraws only when it moves.
+    this.preview.animate(drawn);
   }
 
   /** Whether the preview is running – a moving picture needs a frame without an event. */

src/editor/animation/PreviewCanvas.ts

@@ -87,6 +87,11 @@ export class PreviewCanvas extends FreeViewport {
     return Theme.colors.viewport;
   }
 
+  /** Heard while it plays at about 1x, so a cut can be timed by its sound (#404). */
+  protected override hearsVideo(): boolean {
+    return true;
+  }
+
   private asset(): AnimationAsset | undefined {
     return this.assetId ? findAssetOf(this.state.project, this.assetId, 'animation') : undefined;
   }

src/editor/inspector/SpritePreview.ts

@@ -8,6 +8,7 @@ import { shaderLibrary } from '../../viewport/ShaderLibrary';
 import { wornShader } from '../../viewport/shaders';
 import { disposeSpriteMesh, newSpriteMesh, syncSpriteMesh } from '../../viewport/symbols';
 import { textureLibrary } from '../../viewport/TextureLibrary';
+import { videoLibrary } from '../../viewport/VideoLibrary';
 import { type PictureSubject, previewPicture } from '../shader/previewPicture';
 import type { EditorState } from '../state';
 import { addTitle, type InspectorHost } from './host';
@@ -73,7 +74,9 @@ export class SpritePreview extends Widget {
     if (!picture) return;
     const shaders = shaderLibrary.source(this.state.project);
     const worn = wornShader(picture.props, shaders);
-    const texture = picture.texture && textureLibrary.get(picture.texture, worn?.edge ?? SHEET_EDGE);
+    // A driven video (#404) stands at its Video time here, as in the Scene tab.
+    const ref = picture.texture && videoLibrary.wear(picture.texture, picture.props, this.frameKey, false);
+    const texture = ref && textureLibrary.get(ref, worn?.edge ?? SHEET_EDGE);
     const pad = quadPadding(worn?.padding ?? 0, texture?.width ?? picture.w, texture?.height ?? picture.h);
     const ratio = ui().app.renderer.resolution;
     const { zoom, x, y } = previewFit(picture.w, picture.h, this, {
@@ -84,13 +87,7 @@ export class SpritePreview extends Widget {
     });
     const [w, h] = [picture.w * zoom, picture.h * zoom];
     const m = { a: zoom, b: 0, c: 0, d: zoom, tx: x + w / 2, ty: y + h / 2 };
-    syncSpriteMesh(
-      this.mesh,
-      { m, props: picture.props, texture: picture.texture },
-      picture,
-      textureLibrary,
-      shaders,
-    );
+    syncSpriteMesh(this.mesh, { m, props: picture.props, texture: ref }, picture, textureLibrary, shaders);
     this.drawn = { x, y, w, h };
     // A video (#402) moves by itself, and its player is let go once nobody asks for its picture.
     const video = picture.texture !== null && isVideoKey(picture.texture.file);

src/editor/inspector/assetSection.ts

@@ -97,7 +97,7 @@ export function assetSignature(project: Project, assetId: string): string {
   const shapes = asset.type === 'vectorart' ? `|${shapeListSignature(asset)}` : '';
   const values = resolveAssetProps(project, asset);
   const rows = schemaForAsset(project, asset).filter(
-    (def) => isRowShown(def, values) && !isRowHiddenWhenEmbedded(asset, def.key),
+    (def) => isRowShown(def, values, project) && !isRowHiddenWhenEmbedded(asset, def.key),
   );
   // A Type row shows the name of its Struct asset, or "Own fields" once that asset is gone (#253).
   const types = `|types:${typeNames(project, rows)}`;
@@ -207,7 +207,9 @@ const isSwitch = (key: string): boolean =>
 function buildProps(host: InspectorHost, asset: Asset, schema: readonly PropertyDef[], title: string): void {
   const project = () => host.state.project;
   const values = resolveAssetProps(project(), asset);
-  const shown = schema.filter((d) => isRowShown(d, values) && !isRowHiddenWhenEmbedded(asset, d.key));
+  const shown = schema.filter(
+    (d) => isRowShown(d, values, project()) && !isRowHiddenWhenEmbedded(asset, d.key),
+  );
   if (shown.length > 0) addTitle(host, title);
   for (const def of shown) {
     const row = createFieldRow(

src/editor/inspector/objectSection.ts

@@ -107,7 +107,7 @@ export function objectSignature(
       ? 'instance'
       : '';
   const values = resolveSceneObjectProps(project, asset, object);
-  const rows = schemaForObject(project, asset, object).filter((def) => isRowShown(def, values));
+  const rows = schemaForObject(project, asset, object).filter((def) => isRowShown(def, values, project));
   // A tilemap warns when it is placed more than once, so the number of placements shapes it too.
   const placed = asset.type === 'tilemap' ? `|placed:${findUsages(project, asset.id).length}` : '';
   // A variable turned private leaves the rows, with nothing else about it changed (#257).
@@ -257,7 +257,7 @@ export function buildOwnRows(host: InspectorHost, holderId: string, objectId: st
   if (!shown) return;
   const { object, asset } = shown;
   const values = resolveSceneObjectProps(project, asset, object);
-  const own = objectRows(project, asset, object).filter((def) => isRowShown(def, values));
+  const own = objectRows(project, asset, object).filter((def) => isRowShown(def, values, project));
   host.objectChoices = () => pickableObjects(host.state.project, holderId, objectId);
   host.objectPicker = objectPicker(host.state, holderId, objectId, host.objectChoices);
   const behavior = asset.type === 'behavior';

src/editor/shader/ShaderPreview.ts

@@ -10,6 +10,7 @@ import { syncShader } from '../../viewport/shaders';
 import { disposeSpriteMesh, newSpriteMesh, syncQuadPadding } from '../../viewport/symbols';
 import { quadPadding } from '../../viewport/quadPadding';
 import { textureLibrary } from '../../viewport/TextureLibrary';
+import { videoLibrary } from '../../viewport/VideoLibrary';
 import type { EditorState } from '../state';
 import { type PreviewPicture, TEST_IMAGE_SIZE, testImagePixels } from './previewPicture';
 
@@ -18,6 +19,8 @@ const MARGIN = 12;
 /** A 16-px sprite fills a big panel only blurred by the zoom; past this it stops growing. */
 const MAX_ZOOM = 8;
 
+let previews = 0;
+
 /**
  * One picture wearing the shown shader, with the sliders' values instead of anyone's rows (#168).
  * The shader is the scene's own (`syncShader`), so what the panel shows is what a sprite would.
@@ -35,6 +38,7 @@ export class ShaderPreview extends Widget {
   /** Null: the clock runs. A number: the preview stands at that second. */
   private held: number | null = null;
   private readonly stopPictures: () => void;
+  private readonly wearer = `shader-preview:${(previews += 1)}`;
 
   constructor(private readonly state: EditorState) {
     super();
@@ -106,9 +110,9 @@ export class ShaderPreview extends Widget {
     const w = picture?.w ?? TEST_IMAGE_SIZE;
     const h = picture?.h ?? TEST_IMAGE_SIZE;
     // A sprite whose picture is still loading shows its plain colour, like the scene does.
-    const texture = picture
-      ? ((picture.texture && textureLibrary.get(picture.texture, edge)) ?? null)
-      : testTexture(edge);
+    // A driven video (#404) stands at its Video time here, as in the Scene tab.
+    const ref = picture?.texture && videoLibrary.wear(picture.texture, picture.props, this.wearer, false);
+    const texture = picture ? ((ref && textureLibrary.get(ref, edge)) ?? null) : testTexture(edge);
     // The shader's padding in pixels of the picture, as the scene counts it; the panel keeps room for it.
     const pad = quadPadding(worn?.padding ?? 0, texture?.width ?? w, texture?.height ?? h);
     const zoom = Math.max(

src/editor/viewport/GameViewport.ts

@@ -77,6 +77,10 @@ export class GameViewport extends WorldViewport<EditorState> {
     return this.performanceShown;
   }
 
+  protected override hearsVideo(): boolean {
+    return this.playWorld !== null;
+  }
+
   setPerformanceShown(on: boolean): void {
     this.performanceShown = on;
     this.syncPerformance();