Route a video's sound through its own Master routing node

An editor recording had no sound while the video tab played a video.
The tab played its <video> straight to the speakers, past the Web
Audio graph, so the recording tap behind the limiter never heard it.
Fixing only that would have left videos as the one sound source the
Master routing cannot reach, so they get their own node, like sounds.

- A video asset with sound has a routing node labelled "Video". One
  without sound has none, it would have nothing to send.
- Its links take the sound after volume and FX. Several links or
  another level give the video its own bus, as with a sound.
- A new video follows the bus most other videos use. With none linked
  yet it goes to the Master, not to where the sounds go. Songs and
  sounds no longer follow videos either.
- Players only know their file, so videoBusesOf maps each file to the
  bus of its first asset. setMix takes that map and moves the gains of
  running videos at once when the routing changes.
- In the game, played in the editor or released, a video's sound
  follows its links. The tabs (video, composition, animation preview)
  play straight into the Master, as the sound tabs have since #368.
- The video tab's sound now goes through the Master, so its volume, FX
  and limiter apply there and the editor recording hears it.
- The recording badge beside the menu is an HTML element over the
  canvas now. The second render that drew it on the desynchronized
  canvas let the screen show a frame without it, so it flickered. The
  element is never in the copy, and clicks go through it.

Tests:
- Unit master-routing and sound-library: 3 new, green. Red without
  video in the routing kinds, without moving the gains, and with the
  old rule for the links of a new video.
- e2e 66e-iii: the tab plays a video linked to a bus, the level is
  0.09 on the Master and 0 on the bus. 66g: in the game the bus has
  0.09. Both red with the old ways.
- e2e 67e reads the badge from a real screenshot, still green.
- The README names 2043 e2e checks.
- Gate on melektaus-atx: 304 files, 4694 tests green.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
adminandClaude Opus 5.5 committed 2026-10-12 16:15:59 +02:00
1 parent 846a8b70b3
commit 93190e185c
25 files changed
+376 -99

No files matched your search

+84 -2
View File
@@ -1,11 +1,12 @@
/**
* The master routing (#351): a node per sound, instrument, pattern, song and bus and the master,
* The master routing (#351): a node per sound, instrument, pattern, song, video and bus and the master,
* following the assets through every edit and its undo, and the mix the links make.
*/
import { describe, expect, it } from 'vitest';
import {
CreateAssetCommand,
CreateSoundCommand,
CreateVideoCommand,
DeleteAssetCommand,
DuplicateAssetCommand,
RenameAssetCommand,
@@ -16,13 +17,23 @@ import { KeepRoutingCommand } from '../../src/core/commands/routingCommands';
import { SetSendCommand } from '../../src/core/commands/songCommands';
import type { Graph } from '../../src/core/graph/Graph';
import { ROUTING_ID } from '../../src/core/model/Project';
import { MASTER_MIX, mixBusName, mixOf, playedRoute, routeOf, sameMix } from '../../src/core/music/mixer';
import {
MASTER_MIX,
mixBusName,
mixOf,
playedRoute,
routeOf,
sameMix,
videoBusesOf,
} from '../../src/core/music/mixer';
import { heldRouting, masterRoutingProblems, settleRouting } from '../../src/core/music/projectRouting';
import { ROUTED_ASSET, ROUTED_MASTER } from '../../src/core/music/routingNodes';
import { MASTER_BUS } from '../../src/core/properties/buses';
import { makeProject, roundTrip } from '../helpers';
const FILE = `${'b2'.repeat(32)}.wav`;
const VIDEO = `${'c3'.repeat(32)}.mp4`;
const QUIET = `${'d4'.repeat(32)}.mp4`;
/** A project edited as the editor edits it: every command keeps the routing in step (#351). */
function edited() {
@@ -300,6 +311,77 @@ describe('the mix of the master routing (#351)', () => {
expect(playedRoute(f.project, MASTER_BUS, song)).toBe(MASTER_MIX);
});
it('hears the sound of a video where its node goes, by its file, and gives one without sound no node', () => {
const f = edited();
const reverb = f.create('bus', 'Reverb');
const video = (name: string, file: string, audio: boolean) => {
const command = new CreateVideoCommand(name, file, {
width: 64,
height: 36,
duration: 2,
fps: 30,
audio,
});
f.run(command);
return command.asset.id;
};
const clip = video('Intro', VIDEO, true);
const quiet = video('Loop', QUIET, false);
const node = f.project.routing.nodes.find((n) => n.id === clip);
expect(node?.params).toEqual({ name: 'Intro', kind: 'video' });
expect(f.project.routing.nodes.some((n) => n.id === quiet)).toBe(false);
expect(videoBusesOf(f.project)).toEqual(
new Map([
[VIDEO, MASTER_MIX],
[QUIET, MASTER_MIX],
]),
);
// Into one bus alone, then into two: the second time on a bus of its own with both sends.
const toMaster = f.project.routing.edges.find((e) => e.from.node === clip)!.id;
f.run(new DisconnectCommand(ROUTING_ID, toMaster));
f.link(clip, reverb);
expect(videoBusesOf(f.project).get(VIDEO)).toBe(mixBusName(reverb));
f.run(new SetSendCommand(ROUTING_ID, f.link(clip, MASTER_BUS), { volume: -6, tap: 'post' }));
expect(videoBusesOf(f.project).get(VIDEO)).toBe(mixBusName(clip));
expect(
mixOf(f.project)
.find((bus) => bus.name === mixBusName(clip))
?.sends?.map((s) => s.to),
).toEqual([mixBusName(reverb), MASTER_MIX]);
// A second video of the same file plays where the first one goes.
video('Intro again', VIDEO, true);
expect(videoBusesOf(f.project).get(VIDEO)).toBe(mixBusName(clip));
});
it('link a new video where the other videos go, else to the master, never where the sounds go', () => {
const f = edited();
const sfx = f.create('bus', 'SFX');
const film = f.create('bus', 'Film');
const video = (name: string) => {
const command = new CreateVideoCommand(name, VIDEO, {
width: 64,
height: 36,
duration: 2,
fps: 30,
audio: true,
});
f.run(command);
return command.asset.id;
};
const onto = (id: string, bus: string) => {
f.run(new DisconnectCommand(ROUTING_ID, f.project.routing.edges.find((e) => e.from.node === id)!.id));
f.link(id, bus);
};
for (const name of ['Kick', 'Snare']) onto(f.sound(name), sfx);
const intro = video('Intro');
expect(f.linksOf(intro)).toEqual([MASTER_BUS]);
onto(intro, film);
expect(f.linksOf(video('Outro'))).toEqual([film]);
// Nor does the first song go by the videos, though as many of them go to Film as sounds to SFX.
expect(f.linksOf(f.create('song', 'Theme'))).toEqual([sfx]);
});
it('plays an asset with no link at all on nothing', () => {
const f = edited();
const kick = f.sound('Kick');
+21
View File
@@ -278,6 +278,7 @@ function fakeContext() {
return panner;
},
createChannelSplitter: () => wired('splitter', {}),
createMediaElementSource: () => wired('media', {}),
createAnalyser: () => {
const analyser = wired('analyser', {
fftSize: 2048,
@@ -2564,6 +2565,26 @@ describe('the master and the buses of the project (#325)', () => {
expect(library.busLevels().find((level) => level.name === `play:${id}`)?.into).toBe('mix:a');
});
it('hears a video on the bus the routing has for its file, and moves it along when that changes', async () => {
const library = new SoundLibrary(false);
const video = {} as HTMLMediaElement;
await library.setMix(mix, new Map([['a.mp4', 'mix:b']]));
const first = library.mediaInput(video, 'a.mp4', true)!;
const second = library.mediaInput(video, 'other.mp4', true)!;
// A tab's preview of the same video leaves the routing out (#368).
const preview = library.mediaInput(video, 'a.mp4', false)!;
const buses = () => library.mediaRoutes().map((route) => route.bus);
expect(buses()).toEqual(['mix:b', 'mix', 'mix']);
// The same mix with the video sent elsewhere: only the video moves, not its preview.
await library.setMix(mix, new Map([['a.mp4', 'mix:a']]));
expect(buses()).toEqual(['mix:a', 'mix', 'mix']);
// A mix set without videos keeps where they go.
await library.setMix([mix[0]!, mix[1]!, { ...mix[2]!, volume: 0.5 }]);
expect(buses()).toEqual(['mix:a', 'mix', 'mix']);
for (const gain of [second, preview, first]) library.releaseMedia(gain);
expect(library.mediaRoutes()).toEqual([]);
});
it('sends a bus on into another one when the mix says so, and lets one go whose asset is gone', async () => {
const library = new SoundLibrary(false);
await library.setMix(mix);
+8 -8
View File
@@ -105,7 +105,7 @@ describe('VideoLibrary sweeping', () => {
const props = { videoClock: 'driven', videoTime: 1.2 };
videosMade = 0;
for (let ms = 0; ms < 6000; ms += 100) {
videos.wear(ref, props, 'view2:a', false);
videos.wear(ref, props, 'view2:a', 'none');
vi.advanceTimersByTime(100);
}
// Let go and opened again would be a second video, loading while the sprite shows white.
@@ -128,14 +128,14 @@ describe('VideoLibrary driven players (#404)', () => {
it('leaves a sprite with a free clock on the shared player', async () => {
const videos = await freshLibrary();
expect(videos.wear(REF, { videoTime: 1 }, 'view1:a', false)).toBe(REF);
expect(videos.wear(REF, { videoTime: 1 }, 'view1:a', 'none')).toBe(REF);
expect(videos.debugInfo()).toEqual([]);
});
it('gives every wearer of a driven video a player of its own, opened at its time', async () => {
const videos = await freshLibrary();
const a = videos.wear(REF, { videoClock: 'driven', videoTime: 0.5 }, 'view1:a', false);
const b = videos.wear(REF, { videoClock: 'driven', videoTime: 1.25 }, 'view1:b', false);
const a = videos.wear(REF, { videoClock: 'driven', videoTime: 0.5 }, 'view1:a', 'none');
const b = videos.wear(REF, { videoClock: 'driven', videoTime: 1.25 }, 'view1:b', 'none');
expect([a.file, b.file]).toEqual([`${KEY}@view1:a`, `${KEY}@view1:b`]);
expect(videos.debugInfo().map(({ key, time }) => [key, time])).toEqual([
[`${KEY}@view1:a`, 0.5],
@@ -162,7 +162,7 @@ describe('VideoLibrary driven players (#404)', () => {
for (let i = 0; i <= 36; i++) {
frame.at = (i * 1000) / 60;
if (i === 7) late = 0;
videos.wear(REF, { videoClock: 'driven', videoTime: i / 60 }, 'view1:a', false);
videos.wear(REF, { videoClock: 'driven', videoTime: i / 60 }, 'view1:a', 'none');
}
expect(videos.debugInfo()[0]!.speed).toBeCloseTo(1, 6);
});
@@ -199,13 +199,13 @@ describe('VideoLibrary driven players (#404)', () => {
};
vi.stubGlobal('document', { hidden: false, addEventListener: () => {}, createElement: () => element });
const videos = await freshLibrary();
const key = videos.wear(REF, { videoClock: 'driven', videoTime: 0.5 }, 'view1:a', false).file;
const key = videos.wear(REF, { videoClock: 'driven', videoTime: 0.5 }, 'view1:a', 'none').file;
fire('loadeddata');
seekDone();
[frames] = [[], ...frames.map((frame) => frame())];
expect(videos.showsAsked(key)).toBe(true);
// Switched on at 1.2 s: the seek is done, and no frame follows it.
videos.wear(REF, { videoClock: 'driven', videoTime: 1.2 }, 'view1:a', false);
videos.wear(REF, { videoClock: 'driven', videoTime: 1.2 }, 'view1:a', 'none');
expect(videos.showsAsked(key)).toBe(false);
seekDone();
vi.advanceTimersByTime(200);
@@ -217,7 +217,7 @@ describe('VideoLibrary driven players (#404)', () => {
const videos = await freshLibrary();
videos.get(KEY);
expect(videos.loading('view12')).toBe(false);
const worn = videos.wear(REF, { videoClock: 'driven', videoTime: 0 }, 'view12:a', false);
const worn = videos.wear(REF, { videoClock: 'driven', videoTime: 0 }, 'view12:a', 'none');
expect(videos.hasPicture(worn.file)).toBe(false);
// A run waits only for its own view: view12 is neither view1 nor view2.
expect([videos.loading('view12'), videos.loading('view1'), videos.loading('view2')]).toEqual([