Keep composition sound in step and give compositions a bus of their own

A composition's sound ran on the audio clock while its picture ran on
the frames. A frame counts at most 0.1 s, so after a hitch the sound
stayed ahead of the picture until the next stop. Songs in a trailer
drifted off the cut. The items of a composition also had no bus of
their own, and a video sprite could not pick where its sound goes.

- CompositionSound.follow compares the audio clock with the picture
  each frame. More than 40 ms apart, it starts the sound again where
  the picture is. This holds in the tab and in a run.
- A time scale other than 1 now restarts the sound again and again.
  It stays in time but comes in pieces.
- The sprite of a video item shows "Driven (by composition)" as a
  fixed Video clock row. Video time is hidden there, because the
  composition sets it every frame.
- Every composition has a bus in the mix and a node in the master
  routing. A new one goes to the master unless other compositions are
  linked elsewhere, apart from where the sounds go.
- Audio items pick From asset, To composition, Master or a bus. New
  items start with To composition. Items saved before keep From asset,
  so music already sent through a bus does not bypass it.
- Video sprites have a new row Video bus: From asset by default, the
  master or a bus. In a composition the default is the composition's
  bus, also for video sprites inside a container item.
- The chosen bus is part of the key of the video player, so a sprite
  with another bus gets a player of its own.

Found along the way:

- Six tests with full lists of sprite defaults expected the old rows
  and now list Video bus too.

Tests:
- tsc, eslint, Prettier clean
- Unit: 971 green in the 28 files touched
- npm run check on melektaus-atx: green, 309 files, 4804 tests
- e2e 68 (melektaus-atx): 13 checks green, 68i new. The sound item
  and the video item go into mix:cp68-cut, which goes to the master.
- e2e 69 (melektaus-atx): ALL PASSED, 23 checks, 69q and 69r new.
  After a hitch of 500 ms the sound restarted twice and then lay
  0.5 to 5 ms off the playhead.
- Counter-checks in unit tests: without the comparison the hitch test
  fails, without the bus in the preload the run test fails.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/commands.test.ts

@@ -38,6 +38,7 @@ describe('asset commands', () => {
       videoClock: 'free',
       videoTime: 0,
       videoVolume: 1,
+      videoBus: '',
       shader: '',
     });
   });

tests/core/composition-audio.test.ts

@@ -28,11 +28,12 @@ import {
 import { carriedClips, itemMedia, mediaMarks } from '../../src/core/model/compositionEdit';
 import { findAssetOf } from '../../src/core/model/Project';
 import { audioItemBus, compositionPlayings, compositionSpan } from '../../src/core/music/compositionAudio';
-import { MASTER_MIX, mixBusName } from '../../src/core/music/mixer';
+import { MASTER_MIX, mixBusName, mixOf } from '../../src/core/music/mixer';
 import { Playbacks, type SequencePlay } from '../../src/core/music/playbacks';
 import { LEAD } from '../../src/core/music/Scheduler';
 import { newItem } from '../../src/core/music/song';
-import { MASTER_BUS } from '../../src/core/properties/buses';
+import { MASTER_BUS, TO_COMPOSITION_BUS } from '../../src/core/properties/buses';
+import { SLIP } from '../../src/runtime/compositionSound';
 import { SILENT_AUDIO } from '../../src/runtime/systems/Audio';
 import { addGameSystems } from '../../src/runtime/systems/BehaviorSystem';
 import { Input } from '../../src/runtime/systems/Input';
@@ -292,6 +293,14 @@ describe('the sequences of a composition', () => {
     expect(compositionPlayings(f.project, f.cut(), false).map((p) => p.route)).toEqual([MASTER_MIX]);
   });
 
+  it('go into the composition’s bus where they pick it, along their asset’s routing where they pick none (#418)', () => {
+    const f = makeCut();
+    f.lay(f.audioItem('a', 'sound', 0, 10), f.audioItem('b', 'pattern', 0, 10, { bus: TO_COMPOSITION_BUS }));
+    const routes = compositionPlayings(f.project, f.cut()).map((p) => p.route);
+    expect(routes).toEqual([MASTER_MIX, mixBusName(f.cut().id)]);
+    expect(mixOf(f.project).find((bus) => bus.name === mixBusName(f.cut().id))?.into).toBe(MASTER_MIX);
+  });
+
   it('run as long as the composition, round it where it loops', () => {
     const f = makeCut();
     f.lay(f.audioItem('a', 'sound', 5, 20));
@@ -340,6 +349,9 @@ function recordingAudio() {
     ...SILENT_AUDIO,
     started: [] as { id: number; name: string; play: SequencePlay }[],
     stopped: [] as number[],
+    /** Its audio clock, moved on by the test: 0 is none, and nothing is held to it. */
+    now: 0,
+    clock: () => sink.now,
     playSequence: (name: string, play: SequencePlay) => {
       const id = 100 + sink.started.length;
       sink.started.push({ id, name, play });
@@ -368,6 +380,14 @@ describe('a composition played in a game', () => {
     return { audio, world, frame: (times = 1) => Array.from({ length: times }, () => world.tick(0.1)) };
   }
 
+  /** Frames of 0.1 s, the audio clock moving on by `heard` each. */
+  function along(game: ReturnType<typeof run>, times: number, heard = 0.1) {
+    for (let i = 0; i < times; i++) {
+      game.audio.now += heard;
+      game.frame();
+    }
+  }
+
   it('is heard from where its clock stands once it runs, and stops with its end', () => {
     const f = makeCut();
     f.lay(f.audioItem('a', 'sound', 0, 5));
@@ -400,4 +420,36 @@ describe('a composition played in a game', () => {
     expect(game.audio.started).toHaveLength(2);
     expect(game.audio.started[1]!.play.from).toBe(Math.round((0.3 + LEAD) * 1000));
   });
+
+  it('starts its sound again where the picture is once a hitch left the two apart', () => {
+    const f = makeCut();
+    f.lay(f.audioItem('a', 'sound', 0, 50));
+    const game = run(f);
+    along(game, 3);
+    expect(game.audio.started).toHaveLength(1);
+    // A frame of 0.6 s counts 0.1 for the picture: the sound is 0.5 s ahead.
+    along(game, 1, 0.6);
+    expect(game.audio.stopped).toEqual([100]);
+    expect(game.audio.started).toHaveLength(2);
+    expect(game.audio.started[1]!.play.from).toBe(Math.round((0.3 + LEAD) * 1000));
+    // Then the two keep together, and a few milliseconds apart are no hitch.
+    along(game, 3);
+    along(game, 1, 0.1 + SLIP / 2);
+    expect(game.audio.started).toHaveLength(2);
+  });
+
+  it('holds nothing to an audio clock that stands, nor at the turn of a loop', () => {
+    const f = makeCut();
+    f.lay(f.audioItem('a', 'sound', 0, 20));
+    f.stack.execute(new SetAssetPropertyCommand(f.cut().id, 'loop', true));
+    const game = run(f);
+    along(game, 2);
+    along(game, 3, 0);
+    expect(game.audio.started).toHaveLength(1);
+    // Round the loop of 2 s three times over, the sound with it.
+    const loop = run(f);
+    along(loop, 65);
+    expect(loop.audio.started).toHaveLength(1);
+    expect(loop.audio.stopped).toEqual([]);
+  });
 });

tests/core/composition-play.test.ts

@@ -19,7 +19,14 @@ import {
 import { contentId, instanceChildId, resolveObjects } from '../../src/core/model/instance';
 import { findAsset, findAssetOf, findScene } from '../../src/core/model/Project';
 import { createSceneObject } from '../../src/core/model/SceneObject';
-import { VIDEO_CLOCK_KEY, VIDEO_LEVEL_KEY, VIDEO_TIME_KEY } from '../../src/core/properties/videos';
+import { compositionBusPick } from '../../src/core/properties/buses';
+import { parseVideoKey } from '../../src/core/model/video';
+import {
+  VIDEO_BUS_KEY,
+  VIDEO_CLOCK_KEY,
+  VIDEO_LEVEL_KEY,
+  VIDEO_TIME_KEY,
+} from '../../src/core/properties/videos';
 import { resolveSceneObjectProps } from '../../src/core/properties/resolve';
 import { addGameSystems } from '../../src/runtime/systems/BehaviorSystem';
 import type { VideoPictures } from '../../src/runtime/systems/CompositionSystem';
@@ -294,6 +301,45 @@ describe('a composition played in a scene', () => {
     expect(heard(20)).toBe(0.8);
   });
 
+  it('hears the video of an item on the composition’s bus where it picks none, ahead too (#418)', () => {
+    const stage = makeStage();
+    director(stage);
+    const video = new CreateVideoCommand('Intro', FILE, {
+      width: 320,
+      height: 180,
+      duration: 10,
+      fps: 30,
+      audio: true,
+    });
+    const film = new CreateAssetCommand('bus', 'Film');
+    stage.stack.execute(video);
+    stage.stack.execute(film);
+    const screen = new CreateAssetCommand('sprite', 'Screen', { texture: video.asset.id });
+    const own = new CreateAssetCommand('sprite', 'Own', {
+      texture: video.asset.id,
+      [VIDEO_BUS_KEY]: film.asset.id,
+    });
+    stage.stack.execute(screen);
+    stage.stack.execute(own);
+    const item = stage.item(first(stage), screen.asset.id, 0, 20);
+    const next = stage.item(first(stage), screen.asset.id, 20, 10);
+    const picked = stage.item(first(stage), own.asset.id, 0, 30);
+    const run = play(stage);
+    run.frame(11);
+    const drawn = (id: string) =>
+      collectWorldItems(run.world).items.find((i) => i.id === contentId(stage.placement, id))?.texture?.file;
+    const comp = stage.comp().id;
+    expect(run.at(item)!.props[VIDEO_BUS_KEY]).toBe(compositionBusPick(comp));
+    const key = drawn(item);
+    expect(parseVideoKey(key!)?.bus).toBe(`mix:${comp}`);
+    // The next one is worn ahead in the player it will be drawn with.
+    expect(collectWorldItems(run.world).ahead?.[0]?.texture?.file).toBe(key);
+    expect(run.at(picked)!.props[VIDEO_BUS_KEY]).toBe(film.asset.id);
+    expect(parseVideoKey(drawn(picked)!)?.bus).toBe(`mix:${film.asset.id}`);
+    run.frame(10);
+    expect(drawn(next)).toBe(key);
+  });
+
   it('stands still while the view waits for the first picture of an item’s video, 3 s at most', () => {
     const stage = makeStage();
     director(stage);

tests/core/master-routing.test.ts

@@ -39,7 +39,10 @@ const QUIET = `${'d4'.repeat(32)}.mp4`;
 function edited() {
   const f = makeProject();
   const run = (command: Command) => f.stack.execute(new KeepRoutingCommand(command));
-  const create = (type: 'bus' | 'instrument' | 'pattern' | 'song' | 'scene', name: string) => {
+  const create = (
+    type: 'bus' | 'instrument' | 'pattern' | 'song' | 'scene' | 'composition',
+    name: string,
+  ) => {
     const command = new CreateAssetCommand(type, name);
     run(command);
     return command.asset.id;
@@ -390,3 +393,27 @@ describe('the mix of the master routing (#351)', () => {
     expect(mixOf(f.project).find((bus) => bus.name === mixBusName(kick))?.sends).toEqual([]);
   });
 });
+
+describe('a composition in the master routing (#418)', () => {
+  it('comes linked where the other compositions go, else to the master, and is a bus of the mix', () => {
+    const f = edited();
+    const music = f.create('bus', 'Music');
+    const onto = (id: string, bus: string) => {
+      f.run(new DisconnectCommand(ROUTING_ID, f.project.routing.edges.find((e) => e.from.node === id)!.id));
+      f.link(id, bus);
+    };
+    for (const name of ['Theme', 'Battle']) onto(f.create('song', name), music);
+    const trailer = f.create('composition', 'Trailer');
+    expect(f.project.routing.nodes.find((n) => n.id === trailer)?.type).toBe(ROUTED_ASSET);
+    // A mix of its own: it goes where the songs go only when linked there.
+    expect(f.linksOf(trailer)).toEqual([MASTER_BUS]);
+    expect(mixOf(f.project).find((bus) => bus.name === mixBusName(trailer))).toMatchObject({
+      volume: 1,
+      into: MASTER_MIX,
+      fx: null,
+    });
+    onto(trailer, music);
+    expect(f.linksOf(f.create('composition', 'Teaser'))).toEqual([music]);
+    expect(mixOf(f.project).find((bus) => bus.name === mixBusName(trailer))?.into).toBe(mixBusName(music));
+  });
+});

tests/core/resolve.test.ts

@@ -39,6 +39,7 @@ describe('PropertySchema', () => {
       videoClock: 'free',
       videoTime: 0,
       videoVolume: 1,
+      videoBus: '',
       shader: '',
     });
   });
@@ -75,6 +76,7 @@ describe('resolve', () => {
       videoClock: 'free',
       videoTime: 0,
       videoVolume: 1,
+      videoBus: '',
       shader: '',
     });
   });
@@ -99,6 +101,7 @@ describe('resolve', () => {
       videoClock: 'free',
       videoTime: 0,
       videoVolume: 1,
+      videoBus: '',
       shader: '',
     });
 

tests/core/serialize.test.ts

@@ -186,6 +186,7 @@ describe('serialize', () => {
       videoClock: 'free',
       videoTime: 0,
       videoVolume: 1,
+      videoBus: '',
       shader: '',
     });
   });

tests/core/shaders.test.ts

@@ -730,6 +730,7 @@ describe('a sprite wearing a shader', () => {
       'videoClock',
       'videoTime',
       'videoVolume',
+      'videoBus',
       'shader',
     ]);
     expect(schemaFor('sprite').find((d) => d.key === SHADER_KEY)).toMatchObject({
@@ -754,6 +755,7 @@ describe('a sprite wearing a shader', () => {
       'videoClock',
       'videoTime',
       'videoVolume',
+      'videoBus',
       'shader',
       'brightness',
       'contrast',
@@ -762,14 +764,14 @@ describe('a sprite wearing a shader', () => {
     expect(assetDeviates(f.project, f.sprite())).toBe(true);
     // A dangling id adds no rows and draws nothing – the picker still shows it.
     f.sprite().props[SHADER_KEY] = 'gone';
-    expect(keys()).toHaveLength(16);
+    expect(keys()).toHaveLength(17);
   });
 
   it('gives a placed object the rows of the shader it overrides, filled with the node defaults', () => {
     const f = wearing();
     const id = place(f, f.spriteId, { x: 0, y: 0 });
     const object = () => f.object(id);
-    expect(schemaForObject(f.project, f.sprite(), object())).toHaveLength(16);
+    expect(schemaForObject(f.project, f.sprite(), object())).toHaveLength(17);
     f.stack.execute(new SetOverrideCommand(f.sceneId, id, SHADER_KEY, f.s.id));
     expect(schemaForObject(f.project, f.sprite(), object()).map((d) => d.key)).toContain('saturation');
     expect(resolveSceneObjectProps(f.project, f.sprite(), object())).toMatchObject({

tests/core/texture-commands.test.ts

@@ -77,6 +77,7 @@ describe('CreateAssetCommand with starting props', () => {
       videoClock: 'free',
       videoTime: 0,
       videoVolume: 1,
+      videoBus: '',
       shader: '',
     });
     stack.undo();

tests/core/videos.test.ts

@@ -3,17 +3,23 @@
  * under a key the texture library hands to its player.
  */
 import { describe, expect, it } from 'vitest';
-import { CreateVideoCommand, DuplicateAssetCommand } from '../../src/core/commands/assetCommands';
+import {
+  CreateAssetCommand,
+  CreateVideoCommand,
+  DuplicateAssetCommand,
+} from '../../src/core/commands/assetCommands';
 import { AddClipCommand, RemoveClipCommand, SetClipPropsCommand } from '../../src/core/commands/clipCommands';
 import { CommandError } from '../../src/core/commands/Command';
 import type { VideoAsset, VideoClip } from '../../src/core/model/Asset';
 import { findAssetOf } from '../../src/core/model/Project';
 import { clipsOf } from '../../src/core/model/sound';
-import { resolvePicture } from '../../src/core/model/texture';
+import { resolvePicture, resolveTexture } from '../../src/core/model/texture';
 import {
+  drivenVideoKey,
   findVideo,
   isVideoFile,
   parseVideoKey,
+  routedVideoKey,
   posterFileOf,
   videoKey,
   videoPlayOf,
@@ -21,7 +27,9 @@ import {
   videoTimeAt,
 } from '../../src/core/model/video';
 import { pictureChoices, textureRefLabel, videoChoices } from '../../src/core/properties/textureRef';
+import { compositionBusPick, FROM_ASSET_BUS, MASTER_BUS } from '../../src/core/properties/buses';
 import {
+  VIDEO_BUS_KEY,
   VIDEO_END_KEY,
   VIDEO_LOOP_KEY,
   VIDEO_START_KEY,
@@ -159,6 +167,29 @@ describe('the key a playing video goes by', () => {
     expect(videoPlayOf({ asset: video(), clip: tail })).toMatchObject({ start: 10, end: Infinity });
   });
 
+  it('names the bus a sprite picked, read back, so each bus plays in a player of its own (#418)', () => {
+    const { f, video } = setup();
+    const whole = videoKey(videoPlayOf({ asset: video(), clip: null }));
+    const film = new CreateAssetCommand('bus', 'Film');
+    const trailer = new CreateAssetCommand('composition', 'Trailer');
+    f.stack.execute(film);
+    f.stack.execute(trailer);
+    const worn = (pick: string) =>
+      resolveTexture(f.project, { texture: video().id, [VIDEO_BUS_KEY]: pick })!.file;
+    expect(worn('')).toBe(whole);
+    expect(worn(FROM_ASSET_BUS)).toBe(whole);
+    expect(worn(MASTER_BUS)).toBe(`${whole}!mix`);
+    expect(worn(film.asset.id)).toBe(`${whole}!mix:${film.asset.id}`);
+    expect(worn('gone')).toBe(`${whole}!mix`);
+    expect(worn(compositionBusPick(trailer.asset.id))).toBe(`${whole}!mix:${trailer.asset.id}`);
+    expect(parseVideoKey(worn(film.asset.id))).toEqual({
+      ...videoPlayOf({ asset: video(), clip: null }),
+      bus: `mix:${film.asset.id}`,
+    });
+    expect(parseVideoKey(drivenVideoKey(worn(MASTER_BUS), 'game:p~i'))?.bus).toBe('mix');
+    expect(routedVideoKey(worn(MASTER_BUS), null)).toBe(whole);
+  });
+
   it('is no key for anything else', () => {
     expect(parseVideoKey(FILE)).toBeNull();
     expect(parseVideoKey('gen:abc@1')).toBeNull();

tests/viewport/sound-library.test.ts

@@ -2595,6 +2595,21 @@ describe('the master and the buses of the project (#325)', () => {
     expect(library.mediaRoutes()).toEqual([]);
   });
 
+  it('hears a video on the bus its sprite picked, and on the master once the mix has that bus no more (#418)', async () => {
+    const library = new SoundLibrary(false);
+    const video = {} as HTMLMediaElement;
+    await library.setMix(mix, new Map([['a.mp4', 'mix:b']]));
+    const picked = library.mediaInput(video, 'a.mp4', true, 'mix:a')!;
+    const preview = library.mediaInput(video, 'a.mp4', false, 'mix:a')!;
+    const buses = () => library.mediaRoutes().map((route) => route.bus);
+    expect(buses()).toEqual(['mix:a', 'mix']);
+    await library.setMix([mix[0]!, mix[2]!]);
+    expect(buses()).toEqual(['mix', 'mix']);
+    await library.setMix(mix);
+    expect(buses()).toEqual(['mix:a', 'mix']);
+    for (const gain of [picked, preview]) library.releaseMedia(gain);
+  });
+
   it('sends a bus on into another one when the mix says so, and lets one go whose asset is gone', async () => {
     const library = new SoundLibrary(false);
     await library.setMix(mix);