Route a video's sound through its own Master routing node

An editor recording had no sound while the video tab played a video.
The tab played its <video> straight to the speakers, past the Web
Audio graph, so the recording tap behind the limiter never heard it.
Fixing only that would have left videos as the one sound source the
Master routing cannot reach, so they get their own node, like sounds.

- A video asset with sound has a routing node labelled "Video". One
  without sound has none, it would have nothing to send.
- Its links take the sound after volume and FX. Several links or
  another level give the video its own bus, as with a sound.
- A new video follows the bus most other videos use. With none linked
  yet it goes to the Master, not to where the sounds go. Songs and
  sounds no longer follow videos either.
- Players only know their file, so videoBusesOf maps each file to the
  bus of its first asset. setMix takes that map and moves the gains of
  running videos at once when the routing changes.
- In the game, played in the editor or released, a video's sound
  follows its links. The tabs (video, composition, animation preview)
  play straight into the Master, as the sound tabs have since #368.
- The video tab's sound now goes through the Master, so its volume, FX
  and limiter apply there and the editor recording hears it.
- The recording badge beside the menu is an HTML element over the
  canvas now. The second render that drew it on the desynchronized
  canvas let the screen show a frame without it, so it flickered. The
  element is never in the copy, and clicks go through it.

Tests:
- Unit master-routing and sound-library: 3 new, green. Red without
  video in the routing kinds, without moving the gains, and with the
  old rule for the links of a new video.
- e2e 66e-iii: the tab plays a video linked to a bus, the level is
  0.09 on the Master and 0 on the bus. 66g: in the game the bus has
  0.09. Both red with the old ways.
- e2e 67e reads the badge from a real screenshot, still green.
- The README names 2043 e2e checks.
- Gate on melektaus-atx: 304 files, 4694 tests green.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
adminandClaude Opus 5.5 committed 2026-10-12 16:15:59 +02:00
1 parent 846a8b70b3
commit 93190e185c
25 files changed
+376 -99

No files matched your search

+1 -1
View File
@@ -47,7 +47,7 @@ npm run build # writes dist/ and regenerates index.html; the folder is wh
The browser suite and the three example games need a machine with a GPU: The browser suite and the three example games need a machine with a GPU:
```bash ```bash
node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2040 checks (GM_STEPS=20 for one step) node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2043 checks (GM_STEPS=20 for one step)
# every tool signs in first – see the Accounts page # every tool signs in first – see the Accounts page
node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs
node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs
+17 -3
View File
@@ -59,10 +59,12 @@ export function routeOf(project: Project, assetId: string): MixRoute {
return only && only.gain === 1 && only.tap === 'post' ? only.to : sends; return only && only.gain === 1 && only.tap === 'post' ? only.to : sends;
} }
const OWN_BUS_TYPES: readonly string[] = ['sound', 'instrument', 'video'];
/** /**
* The master and every bus of the project (#325), each where the routing sends it (#351); and a bus * The master and every bus of the project (#325), each where the routing sends it (#351); and a bus
* of its own for each sound and instrument the routing sends anywhere but straight into one bus, for * of its own for each sound, instrument and video the routing sends anywhere but straight into one
* all its voices to go through. A sound FX that does not compile is none. * bus, for all its voices to go through. A sound FX that does not compile is none.
*/ */
export function mixOf(project: Project): MixBus[] { export function mixOf(project: Project): MixBus[] {
const fxOf = (id: string) => fxVoiceOf(project, id, DEFAULT_TEMPO); const fxOf = (id: string) => fxVoiceOf(project, id, DEFAULT_TEMPO);
@@ -78,7 +80,7 @@ export function mixOf(project: Project): MixBus[] {
fx: fxOf(coerceAs(asset.props[BUS_FX_KEY], 'asset', '')), fx: fxOf(coerceAs(asset.props[BUS_FX_KEY], 'asset', '')),
}, },
]; ];
if ((asset.type !== 'sound' && asset.type !== 'instrument') || !isRouted(asset)) return []; if (!OWN_BUS_TYPES.includes(asset.type) || !isRouted(asset)) return [];
const route = routeOf(project, asset.id); const route = routeOf(project, asset.id);
return typeof route === 'string' return typeof route === 'string'
? [] ? []
@@ -125,6 +127,18 @@ export function playedBus(project: Project, picked: string, assetId: string): st
return typeof route === 'string' ? route : mixBusName(assetId); return typeof route === 'string' ? route : mixBusName(assetId);
} }
/**
* The bus the sound of each video file is heard on, by the routing of its asset: a player knows
* the file it plays, not the asset. Where two videos share a file, the first one says.
*/
export function videoBusesOf(project: Project): Map<string, string> {
const buses = new Map<string, string>();
for (const asset of project.assets)
if (asset.type === 'video' && asset.file !== '' && !buses.has(asset.file))
buses.set(asset.file, playedBus(project, '', asset.id));
return buses;
}
/** Where a playing of a pattern or a song goes: the bus a node picked, else along its routing (#351). */ /** Where a playing of a pattern or a song goes: the bus a node picked, else along its routing (#351). */
export function playedRoute(project: Project, picked: string, assetId: string): MixRoute { export function playedRoute(project: Project, picked: string, assetId: string): MixRoute {
if (picked === MASTER_BUS) return MASTER_MIX; if (picked === MASTER_BUS) return MASTER_MIX;
+10 -7
View File
@@ -1,6 +1,6 @@
/** /**
* The master routing of a project (#351): a graph with a node per sound, instrument, pattern, song and * The master routing of a project (#351): a graph with a node per sound, instrument, pattern, song,
* bus, and the master, its links the sends from one to the next. Its nodes follow the assets: one * video with sound and bus, and the master, its links the sends from one to the next. Its nodes follow the assets: one
* that comes gets a node of its own and the links most others of its type have, one that goes takes * that comes gets a node of its own and the links most others of its type have, one that goes takes
* its node along. * its node along.
*/ */
@@ -16,7 +16,7 @@ import { isEmbedded } from './embedded';
import { addLink, heldSend } from './routing'; import { addLink, heldSend } from './routing';
import { ROUTED_ASSET, ROUTED_KIND, ROUTED_MASTER, ROUTED_NAME } from './routingNodes'; import { ROUTED_ASSET, ROUTED_KIND, ROUTED_MASTER, ROUTED_NAME } from './routingNodes';
const ROUTED_TYPES: readonly AssetType[] = ['sound', 'instrument', 'pattern', 'song', 'bus']; const ROUTED_TYPES: readonly AssetType[] = ['sound', 'instrument', 'pattern', 'song', 'video', 'bus'];
/** What has its own sound FX and level on a bus of its own, and so may send from before them. */ /** What has its own sound FX and level on a bus of its own, and so may send from before them. */
const TAPPED_TYPES: readonly AssetType[] = ['bus', 'song']; const TAPPED_TYPES: readonly AssetType[] = ['bus', 'song'];
@@ -29,8 +29,9 @@ const ROW_STEP = 80;
const COLUMN_OF = (type: string) => (type === 'bus' ? 1 : 0); const COLUMN_OF = (type: string) => (type === 'bus' ? 1 : 0);
const MASTER_COLUMN = 2; const MASTER_COLUMN = 2;
/** A video only where it has sound: one without has nothing to send. */
export function isRouted(asset: Asset): boolean { export function isRouted(asset: Asset): boolean {
return ROUTED_TYPES.includes(asset.type) && !isEmbedded(asset); return ROUTED_TYPES.includes(asset.type) && !isEmbedded(asset) && (asset.type !== 'video' || asset.audio);
} }
/** Whether links out of an asset of this type may take the sound before its sound FX or its level. */ /** Whether links out of an asset of this type may take the sound before its sound FX or its level. */
@@ -132,9 +133,11 @@ export function heldRouting(graph: Graph | undefined, assets: readonly Asset[],
} }
/** /**
* The links a new sound, instrument, pattern or song takes, so it is heard where the others are: those * The links a new sound, instrument, pattern, song or video takes, so it is heard where the others are: those
* of the one whose targets most others of its type share too – the latest of them on a tie –, or of * of the one whose targets most others of its type share too – the latest of them on a tie –, or of
* all that plays where nothing of its type is linked yet. Empty where nothing is, and for a bus. * all that plays where nothing of its type is linked yet. Videos and the music go by each other
* apart: a video where no other is linked goes to the master, not where the sounds go. Empty where
* nothing is, and for a bus.
*/ */
function linksLikeTheirs( function linksLikeTheirs(
kind: string, kind: string,
@@ -159,7 +162,7 @@ function linksLikeTheirs(
}; };
const playing = nodes.filter((node) => { const playing = nodes.filter((node) => {
const of = routedKind(node); const of = routedKind(node);
return of !== '' && of !== 'bus' && out.has(node.id); return of !== '' && of !== 'bus' && (of === 'video') === (kind === 'video') && out.has(node.id);
}); });
const own = playing.filter((node) => routedKind(node) === kind); const own = playing.filter((node) => routedKind(node) === kind);
return pick(own.length > 0 ? own : playing); return pick(own.length > 0 ? own : playing);
+5 -3
View File
@@ -41,8 +41,9 @@ export function isSongRoutingDef(def: NodeDef): boolean {
} }
/** /**
* The nodes of the master routing (#351): one per sound, instrument, pattern, song and bus of the * The nodes of the master routing (#351): one per sound, instrument, pattern, song, video with sound
* project, named and typed after its asset, and the master. Only a bus and the master take links in. * and bus of the project, named and typed after its asset, and the master. Only a bus and the master
* take links in.
*/ */
export const ROUTED_ASSET = 'RoutedAsset'; export const ROUTED_ASSET = 'RoutedAsset';
export const ROUTED_MASTER = 'RoutedMaster'; export const ROUTED_MASTER = 'RoutedMaster';
@@ -54,6 +55,7 @@ const KIND_WORDS: Record<string, string> = {
instrument: 'Instrument', instrument: 'Instrument',
pattern: 'Pattern', pattern: 'Pattern',
song: 'Song', song: 'Song',
video: 'Video',
bus: 'Bus', bus: 'Bus',
}; };
@@ -62,7 +64,7 @@ registerNode({
label: 'Asset', label: 'Asset',
category: 'sound', category: 'sound',
description: description:
'A sound, an instrument, a pattern, a song or a bus of the project: all it plays, sent on to every link.', 'A sound, an instrument, a pattern, a song, a video or a bus of the project: all it plays, sent on to every link.',
inputs: [], inputs: [],
outputs: [], outputs: [],
params: [], params: [],
+2 -2
View File
@@ -2,7 +2,7 @@ import type { Application, Ticker } from 'pixi.js';
import type { Asset } from '../core/model/Asset'; import type { Asset } from '../core/model/Asset';
import { findAsset, findObjectHolder, ROUTING_ID } from '../core/model/Project'; import { findAsset, findObjectHolder, ROUTING_ID } from '../core/model/Project';
import { assetTitle } from '../core/music/embedded'; import { assetTitle } from '../core/music/embedded';
import { mixOf } from '../core/music/mixer'; import { mixOf, videoBusesOf } from '../core/music/mixer';
import type { Role } from '../core/store/AuthStore'; import type { Role } from '../core/store/AuthStore';
import type { ProjectStore } from '../core/store/ProjectStore'; import type { ProjectStore } from '../core/store/ProjectStore';
import { pluginsOf } from '../core/store/projectPlugins'; import { pluginsOf } from '../core/store/projectPlugins';
@@ -672,7 +672,7 @@ export class Editor implements PlayHost {
this.openDocument(doc); this.openDocument(doc);
}); });
// The tabs play through the buses and the master as the game would (#325). // The tabs play through the buses and the master as the game would (#325).
const syncMix = () => void soundLibrary.setMix(mixOf(state.project)); const syncMix = () => void soundLibrary.setMix(mixOf(state.project), videoBusesOf(state.project));
syncMix(); syncMix();
state.on('project', () => { state.on('project', () => {
syncMix(); syncMix();
+69 -31
View File
@@ -1,4 +1,4 @@
import { Container, Graphics, UPDATE_PRIORITY } from 'pixi.js'; import { UPDATE_PRIORITY } from 'pixi.js';
import { cursorShape, type CursorShape } from '../core/recording/cursors'; import { cursorShape, type CursorShape } from '../core/recording/cursors';
import { import {
EDITOR_RECORDING_FPS, EDITOR_RECORDING_FPS,
@@ -7,7 +7,7 @@ import {
fitPicture, fitPicture,
recordingName, recordingName,
} from '../core/recording/recording'; } from '../core/recording/recording';
import { crispText, renderCount, requestRender, textStyle, Theme, ui } from '../ui'; import { renderCount, requestRender, Theme, toHex, ui } from '../ui';
import { type FrameSource, Take, type TakeHost, type TakeInfo } from '../viewport/RecordingTake'; import { type FrameSource, Take, type TakeHost, type TakeInfo } from '../viewport/RecordingTake';
/** What the editor recorder needs of the editor. */ /** What the editor recorder needs of the editor. */
@@ -22,8 +22,9 @@ export interface EditorRecorderHost extends TakeHost {
* Records the whole editor (main menu → Start recording): the canvas as it is on screen, at its size * Records the whole editor (main menu → Start recording): the canvas as it is on screen, at its size
* in device pixels and 60 fps, with the mouse pointer drawn in, since the system draws that over the * in device pixels and 60 fps, with the mouse pointer drawn in, since the system draws that over the
* canvas. What the editor draws is copied from the canvas right after each render, so recording costs * canvas. What the editor draws is copied from the canvas right after each render, so recording costs
* no second drawing. The badge beside the menu is drawn onto the screen after that copy, so it is * no second drawing. The badge beside the menu is an HTML element over the canvas, the one exception
* never in the recording. * to the single canvas: it is never in the recording, and the canvas is drawn once per frame, which a
* `desynchronized` canvas needs to show no half-drawn picture.
*/ */
export class EditorRecorder { export class EditorRecorder {
private take: Take | null = null; private take: Take | null = null;
@@ -72,6 +73,7 @@ export class EditorRecorder {
this.source?.detach(); this.source?.detach();
this.source = null; this.source = null;
ui().app.ticker.remove(this.tick, this); ui().app.ticker.remove(this.tick, this);
this.badge.hide();
this.host.showBadge(false); this.host.showBadge(false);
requestRender(); requestRender();
} }
@@ -81,23 +83,19 @@ export class EditorRecorder {
const source = this.source; const source = this.source;
if (!take || !source) return; if (!take || !source) return;
const drawn = renderCount(); const drawn = renderCount();
const text = elapsedText(performance.now() - this.startedAt);
if (drawn !== this.seen) { if (drawn !== this.seen) {
this.seen = drawn; this.seen = drawn;
source.copy(); source.copy();
this.drawBadge(); // The top bar may have moved with this render: the badge follows it.
this.shownText = text;
this.badge.show(ui().app.canvas, this.host.badgeRoom(), text);
} else if (text !== this.shownText) {
// An idle editor draws nothing; the time moves on without it.
this.shownText = text;
this.badge.setText(text);
} }
take.capture(); take.capture();
// An idle editor draws nothing, and the time beside the menu has to move on.
if (elapsedText(performance.now() - this.startedAt) !== this.shownText) requestRender();
}
private drawBadge(): void {
this.shownText = elapsedText(performance.now() - this.startedAt);
this.badge.show(this.host.badgeRoom(), this.shownText);
const app = ui().app;
app.renderer.render({ container: this.badge, clear: false });
// Pixi hit-tests the pointer against what it drew on screen last: that has to be the stage.
(app.renderer as unknown as { _lastObjectRendered: Container })._lastObjectRendered = app.stage;
} }
info(): { info(): {
@@ -113,25 +111,65 @@ export class EditorRecorder {
} }
} }
/** The red dot and the time beside the menu while the editor records. */ /**
class RecordingBadge extends Container { * The red dot and the time beside the menu while the editor records: an HTML element laid over the
private readonly dot = new Graphics(); * room the top bar keeps free, scaled with the canvas (`?w=&h=`). Clicks go through it to the canvas.
private readonly time = crispText({ text: '', style: textStyle('body') }); */
class RecordingBadge {
private element: HTMLDivElement | null = null;
private time: HTMLSpanElement | null = null;
constructor() { show(canvas: HTMLCanvasElement, room: { x: number; y: number; w: number; h: number }, text: string): void {
super(); const element = this.element ?? this.build();
this.addChild(this.dot, this.time); const rect = canvas.getBoundingClientRect();
const shown = rect.width / ui().app.screen.width;
element.style.left = `${rect.left + room.x * shown}px`;
element.style.top = `${rect.top + room.y * shown}px`;
element.style.width = `${room.w}px`;
element.style.height = `${room.h}px`;
element.style.transform = shown === 1 ? '' : `scale(${shown})`;
this.setText(text);
} }
show(room: { x: number; y: number; w: number; h: number }, text: string): void { setText(text: string): void {
if (this.time) this.time.textContent = text;
}
hide(): void {
this.element?.remove();
this.element = null;
this.time = null;
}
private build(): HTMLDivElement {
const radius = 5; const radius = 5;
this.position.set(room.x, room.y); const element = document.createElement('div');
this.dot element.style.cssText = [
.clear() 'position: fixed',
.circle(radius + 2, room.h / 2, radius) 'display: flex',
.fill(Theme.colors.danger); 'align-items: center',
this.time.text = text; 'transform-origin: 0 0',
this.time.position.set(radius * 2 + 7, Math.round((room.h - this.time.height) / 2)); 'pointer-events: none',
'user-select: none',
`font: ${Theme.fontSize}px ${Theme.font}`,
`color: ${toHex(Theme.colors.text)}`,
'white-space: nowrap',
].join(';');
const dot = document.createElement('span');
dot.style.cssText = [
`width: ${radius * 2}px`,
`height: ${radius * 2}px`,
'border-radius: 50%',
`margin: 0 5px 0 2px`,
`background: ${toHex(Theme.colors.danger)}`,
'flex: none',
].join(';');
const time = document.createElement('span');
element.append(dot, time);
document.body.appendChild(element);
this.element = element;
this.time = time;
return element;
} }
} }
+3 -2
View File
@@ -6,6 +6,7 @@ import type { EditorState } from '../state';
import { Theme } from '../../ui'; import { Theme } from '../../ui';
import { collectHolderItems } from '../../viewport/items'; import { collectHolderItems } from '../../viewport/items';
import { FreeViewport } from '../viewport/FreeViewport'; import { FreeViewport } from '../viewport/FreeViewport';
import type { VideoHearing } from '../../viewport/VideoLibrary';
import type { Frame } from '../../viewport/WorldViewport'; import type { Frame } from '../../viewport/WorldViewport';
/** /**
@@ -88,8 +89,8 @@ export class PreviewCanvas extends FreeViewport {
} }
/** Heard while it plays at about 1x, so a cut can be timed by its sound (#404). */ /** Heard while it plays at about 1x, so a cut can be timed by its sound (#404). */
protected override hearsVideo(): boolean { protected override hearsVideo(): VideoHearing {
return true; return 'preview';
} }
private asset(): AnimationAsset | undefined { private asset(): AnimationAsset | undefined {
+3 -2
View File
@@ -2,6 +2,7 @@ import { compositionHolder } from '../../core/model/composition';
import type { CompositionAsset } from '../../core/model/Asset'; import type { CompositionAsset } from '../../core/model/Asset';
import { findAssetOf } from '../../core/model/Project'; import { findAssetOf } from '../../core/model/Project';
import { compositionVideosAhead } from '../../viewport/items'; import { compositionVideosAhead } from '../../viewport/items';
import type { VideoHearing } from '../../viewport/VideoLibrary';
import type { Frame } from '../../viewport/WorldViewport'; import type { Frame } from '../../viewport/WorldViewport';
import type { EditorState } from '../state'; import type { EditorState } from '../state';
import { SceneViewport } from '../viewport/SceneViewport'; import { SceneViewport } from '../viewport/SceneViewport';
@@ -89,7 +90,7 @@ export class CompositionCanvas extends SceneViewport {
} }
/** Heard while it plays at about 1x, so a cut can be timed by its sound (#404). */ /** Heard while it plays at about 1x, so a cut can be timed by its sound (#404). */
protected override hearsVideo(): boolean { protected override hearsVideo(): VideoHearing {
return true; return 'preview';
} }
} }
+2
View File
@@ -19,6 +19,8 @@ export function routingHooks({ state, editor }: HookContext) {
/** The loudest sample on each side of these buses of the mix, tapped on first asking (#359). */ /** The loudest sample on each side of these buses of the mix, tapped on first asking (#359). */
peaks: (names: string[]) => Object.fromEntries(soundLibrary.busPeaks(names)), peaks: (names: string[]) => Object.fromEntries(soundLibrary.busPeaks(names)),
buses: () => soundLibrary.busLevels(), buses: () => soundLibrary.busLevels(),
/** Where the sound of each video heard goes: along the routing in a game, else the master. */
media: () => soundLibrary.mediaRoutes(),
}, },
}; };
} }
+1 -1
View File
@@ -75,7 +75,7 @@ export class SpritePreview extends Widget {
const shaders = shaderLibrary.source(this.state.project); const shaders = shaderLibrary.source(this.state.project);
const worn = wornShader(picture.props, shaders); const worn = wornShader(picture.props, shaders);
// A driven video (#404) stands at its Video time here, as in the Scene tab. // A driven video (#404) stands at its Video time here, as in the Scene tab.
const ref = picture.texture && videoLibrary.wear(picture.texture, picture.props, this.frameKey, false); const ref = picture.texture && videoLibrary.wear(picture.texture, picture.props, this.frameKey, 'none');
const texture = ref && textureLibrary.get(ref, worn?.edge ?? SHEET_EDGE); const texture = ref && textureLibrary.get(ref, worn?.edge ?? SHEET_EDGE);
const pad = quadPadding(worn?.padding ?? 0, texture?.width ?? picture.w, texture?.height ?? picture.h); const pad = quadPadding(worn?.padding ?? 0, texture?.width ?? picture.w, texture?.height ?? picture.h);
const ratio = ui().app.renderer.resolution; const ratio = ui().app.renderer.resolution;
+2 -2
View File
@@ -9,8 +9,8 @@ import { TabHeader } from '../TabHeader';
const INFO_WIDTH = 220; const INFO_WIDTH = 220;
/** /**
* The Master routing tab (#351), one per project: a node per sound, instrument, pattern, song and bus * The Master routing tab (#351), one per project: a node per sound, instrument, pattern, song, video
* and the master, the links between them where each one goes. Edited as the routing of a song is * with sound and bus and the master, the links between them where each one goes. Edited as the routing of a song is
* (#350); a click on a node shows its asset in the inspector, a double click opens it. * (#350); a click on a node shows its asset in the inspector, a double click opens it.
*/ */
export class RoutingEditor extends VStack { export class RoutingEditor extends VStack {
+1 -1
View File
@@ -111,7 +111,7 @@ export class ShaderPreview extends Widget {
const h = picture?.h ?? TEST_IMAGE_SIZE; const h = picture?.h ?? TEST_IMAGE_SIZE;
// A sprite whose picture is still loading shows its plain colour, like the scene does. // A sprite whose picture is still loading shows its plain colour, like the scene does.
// A driven video (#404) stands at its Video time here, as in the Scene tab. // A driven video (#404) stands at its Video time here, as in the Scene tab.
const ref = picture?.texture && videoLibrary.wear(picture.texture, picture.props, this.wearer, false); const ref = picture?.texture && videoLibrary.wear(picture.texture, picture.props, this.wearer, 'none');
const texture = picture ? ((ref && textureLibrary.get(ref, edge)) ?? null) : testTexture(edge); const texture = picture ? ((ref && textureLibrary.get(ref, edge)) ?? null) : testTexture(edge);
// The shader's padding in pixels of the picture, as the scene counts it; the panel keeps room for it. // The shader's padding in pixels of the picture, as the scene counts it; the panel keeps room for it.
const pad = quadPadding(worn?.padding ?? 0, texture?.width ?? w, texture?.height ?? h); const pad = quadPadding(worn?.padding ?? 0, texture?.width ?? w, texture?.height ?? h);
+1 -1
View File
@@ -324,7 +324,7 @@ export class EditorState {
this.paused = false; this.paused = false;
this.project = project; this.project = project;
this.revision = revision; this.revision = revision;
// Every sound, instrument, pattern, song and bus has its node in the Master routing (#351). // Every sound, instrument, pattern, song, video with sound and bus has its node in the Master routing (#351).
settleRouting(project); settleRouting(project);
this.commands = new CommandStack(project); this.commands = new CommandStack(project);
this.selection = null; this.selection = null;
+9 -1
View File
@@ -39,6 +39,7 @@ import {
xToSeconds, xToSeconds,
zoomWave, zoomWave,
} from '../sound/waveView'; } from '../sound/waveView';
import { soundLibrary } from '../../viewport/SoundLibrary';
import { openVideo } from '../../viewport/VideoLibrary'; import { openVideo } from '../../viewport/VideoLibrary';
import { chaseSeeks, seekChase, type SeekChase, seekVideo } from '../../viewport/videoSeek'; import { chaseSeeks, seekChase, type SeekChase, seekVideo } from '../../viewport/videoSeek';
import type { EditorState } from '../state'; import type { EditorState } from '../state';
@@ -86,6 +87,8 @@ interface Preview {
chase: SeekChase; chase: SeekChase;
/** The moment last asked for, where it stands once it has its first frame. */ /** The moment last asked for, where it stands once it has its first frame. */
asked: number; asked: number;
/** Its sound on the way into the master, made at the first Play. */
gain: GainNode | null;
} }
/** /**
@@ -214,7 +217,10 @@ export class VideoCanvas extends Widget {
if (!asset || !span || !element || !preview) return; if (!asset || !span || !element || !preview) return;
this.playing = span; this.playing = span;
element.muted = false; element.muted = false;
element.volume = videoVolumeOf(asset.props); // Through the master like any tab's preview (#368), past the routing: so a recording hears it.
preview.gain ??= soundLibrary.mediaInput(element, preview.file, false);
if (preview.gain) preview.gain.gain.value = videoVolumeOf(asset.props);
else element.volume = videoVolumeOf(asset.props);
// A moment still wanted from scrubbing would send it back after the start. // A moment still wanted from scrubbing would send it back after the start.
preview.chase.wanted = null; preview.chase.wanted = null;
const head = preview.asked; const head = preview.asked;
@@ -369,6 +375,7 @@ export class VideoCanvas extends Widget {
texture: null, texture: null,
chase: seekChase(), chase: seekChase(),
asked: held, asked: held,
gain: null,
element: openVideo(`/game-maker/data/videos/${asset.file}`, held, (texture) => { element: openVideo(`/game-maker/data/videos/${asset.file}`, held, (texture) => {
if (this.preview !== preview) return texture.destroy(true); if (this.preview !== preview) return texture.destroy(true);
preview.texture = texture; preview.texture = texture;
@@ -402,6 +409,7 @@ export class VideoCanvas extends Widget {
this.playing = null; this.playing = null;
this.picture.texture = Texture.EMPTY; this.picture.texture = Texture.EMPTY;
preview.element.pause(); preview.element.pause();
if (preview.gain) soundLibrary.releaseMedia(preview.gain);
preview.element.removeAttribute('src'); preview.element.removeAttribute('src');
preview.element.load(); preview.element.load();
preview.texture?.destroy(true); preview.texture?.destroy(true);
+3 -2
View File
@@ -18,6 +18,7 @@ import { FullscreenCorner } from '../../viewport/FullscreenCorner';
import { collectHolderItems, collectWorldItems } from '../../viewport/items'; import { collectHolderItems, collectWorldItems } from '../../viewport/items';
import { routePointer } from '../../viewport/gamePointer'; import { routePointer } from '../../viewport/gamePointer';
import { cameraFactor, cameraView, type GameScale, type View } from '../../viewport/view'; import { cameraFactor, cameraView, type GameScale, type View } from '../../viewport/view';
import type { VideoHearing } from '../../viewport/VideoLibrary';
import { type Frame, WorldViewport } from '../../viewport/WorldViewport'; import { type Frame, WorldViewport } from '../../viewport/WorldViewport';
import { PerformanceOverlay } from './PerformanceOverlay'; import { PerformanceOverlay } from './PerformanceOverlay';
@@ -80,8 +81,8 @@ export class GameViewport extends WorldViewport<EditorState> {
return this.performanceShown; return this.performanceShown;
} }
protected override hearsVideo(): boolean { protected override hearsVideo(): VideoHearing {
return this.playWorld !== null; return this.playWorld !== null ? 'game' : 'none';
} }
setPerformanceShown(on: boolean): void { setPerformanceShown(on: boolean): void {
+3 -2
View File
@@ -6,6 +6,7 @@ import { FullscreenCorner } from '../viewport/FullscreenCorner';
import { routePointer } from '../viewport/gamePointer'; import { routePointer } from '../viewport/gamePointer';
import { collectWorldItems } from '../viewport/items'; import { collectWorldItems } from '../viewport/items';
import { cameraView, type View } from '../viewport/view'; import { cameraView, type View } from '../viewport/view';
import type { VideoHearing } from '../viewport/VideoLibrary';
import { type Frame, type ViewSource, WorldViewport } from '../viewport/WorldViewport'; import { type Frame, type ViewSource, WorldViewport } from '../viewport/WorldViewport';
/** /**
@@ -61,8 +62,8 @@ export class GameScreen extends WorldViewport {
return this.startable && !this.running; return this.startable && !this.running;
} }
protected override hearsVideo(): boolean { protected override hearsVideo(): VideoHearing {
return this.running !== null; return this.running !== null ? 'game' : 'none';
} }
/** null = not running. */ /** null = not running. */
+2 -2
View File
@@ -5,7 +5,7 @@
import type { SaveArea } from '../core/graph/savedValues'; import type { SaveArea } from '../core/graph/savedValues';
import type { SceneAsset } from '../core/model/Asset'; import type { SceneAsset } from '../core/model/Asset';
import { findScene, type Project } from '../core/model/Project'; import { findScene, type Project } from '../core/model/Project';
import { mixOf } from '../core/music/mixer'; import { mixOf, videoBusesOf } from '../core/music/mixer';
import type { RecordingSink } from '../core/recording/recording'; import type { RecordingSink } from '../core/recording/recording';
import type { AudioSink } from './systems/Audio'; import type { AudioSink } from './systems/Audio';
import type { VideoPictures } from './systems/CompositionSystem'; import type { VideoPictures } from './systems/CompositionSystem';
@@ -49,7 +49,7 @@ export class GameRun {
const before = scene && structuredClone<StoredObjects>({ objects: scene.objects, groups: scene.groups }); const before = scene && structuredClone<StoredObjects>({ objects: scene.objects, groups: scene.groups });
// Every run starts with the buses as their assets have them (#325). // Every run starts with the buses as their assets have them (#325).
services.audio.resetMix(); services.audio.resetMix();
services.audio.setMix(mixOf(project)); services.audio.setMix(mixOf(project), videoBusesOf(project));
const world = World.instantiate(project, sceneId); const world = World.instantiate(project, sceneId);
const run = new GameRun(project, input, services, world, addGameSystems(world, project, input, services)); const run = new GameRun(project, input, services, world, addGameSystems(world, project, input, services));
if (before) run.stored.set(sceneId, before); if (before) run.stored.set(sceneId, before);
+2 -2
View File
@@ -92,9 +92,9 @@ export interface AudioSink {
stopOwner(owner: string): void; stopOwner(owner: string): void;
/** /**
* The master and the buses of the project (#325), set as a run starts. What a game set a level to * The master and the buses of the project (#325), set as a run starts. What a game set a level to
* holds over it until `resetMix`. * holds over it until `resetMix`. `videos` names the bus of each video file (`videoBusesOf`).
*/ */
setMix(buses: readonly MixBus[]): void; setMix(buses: readonly MixBus[], videos?: ReadonlyMap<string, string>): void;
/** The level a game sets a bus of the mix to, over `time` seconds along `ease`. */ /** The level a game sets a bus of the mix to, over `time` seconds along `ease`. */
setMixVolume(bus: string, volume: number, time?: number, ease?: EaseShare): void; setMixVolume(bus: string, volume: number, time?: number, ease?: EaseShare): void;
/** The level a bus of the mix is at: what a game set, else that of its asset. */ /** The level a bus of the mix is at: what a game set, else that of its asset. */
+50 -7
View File
@@ -376,6 +376,10 @@ export class SoundLibrary implements AudioSink {
private mix: readonly MixBus[] = []; private mix: readonly MixBus[] = [];
/** The levels a running game set its buses to, over those of their assets; until `resetMix`. */ /** The levels a running game set its buses to, over those of their assets; until `resetMix`. */
private readonly mixVolumes = new Map<string, number>(); private readonly mixVolumes = new Map<string, number>();
/** The bus of each video file, by the routing of its asset; the master for one not named. */
private videoBuses: ReadonlyMap<string, string> = new Map();
/** The sound of every video heard (#402), by its gain: the file it plays, and where it goes. */
private readonly media = new Map<GainNode, { file: string; routed: boolean; bus: string }>();
/** The names streamed (#349) and the copies they stream from, by name and by copy. */ /** The names streamed (#349) and the copies they stream from, by name and by copy. */
private streams: ReadonlyMap<string, StreamedFile> = new Map(); private streams: ReadonlyMap<string, StreamedFile> = new Map();
/** The cut of each clip, with the copy it is decoded from rather than its whole file (#349). */ /** The cut of each clip, with the copy it is decoded from rather than its whole file (#349). */
@@ -826,14 +830,35 @@ export class SoundLibrary implements AudioSink {
* The master and the buses of the project (#325): each into the bus it names, at its level unless * The master and the buses of the project (#325): each into the bus it names, at its level unless
* a running game set another, through its sound FX. A bus whose asset is gone is let go. Applied * a running game set another, through its sound FX. A bus whose asset is gone is let go. Applied
* once there is a context; done once the chains are there. The same mix again changes nothing, so * once there is a context; done once the chains are there. The same mix again changes nothing, so
* the editor can hand it over on every change of the project. * the editor can hand it over on every change of the project. The sound of a video goes into the
* bus `videos` names for its file, at once also for one heard already; left out, they stay.
*/ */
async setMix(buses: readonly MixBus[]): Promise<void> { async setMix(buses: readonly MixBus[], videos = this.videoBuses): Promise<void> {
if (sameMix(this.mix, buses)) return; const rerouted = !sameEntries(this.videoBuses, videos);
this.videoBuses = videos;
if (sameMix(this.mix, buses)) {
if (rerouted) this.rerouteMedia();
return;
}
this.mix = buses; this.mix = buses;
if (rerouted) this.rerouteMedia();
if (this.context) await this.applyMix(); if (this.context) await this.applyMix();
} }
/** Every video heard along the routing into the bus of its file. */
private rerouteMedia(): void {
for (const [gain, heard] of this.media) {
if (!heard.routed) continue;
gain.disconnect();
heard.bus = this.videoBus(heard.file);
gain.connect(this.busInput(heard.bus));
}
}
private videoBus(file: string): string {
return this.videoBuses.get(file) ?? MASTER_MIX;
}
private async applyMix(): Promise<void> { private async applyMix(): Promise<void> {
const names = new Set(this.mix.map((bus) => bus.name)); const names = new Set(this.mix.map((bus) => bus.name));
const waits: Promise<void>[] = []; const waits: Promise<void>[] = [];
@@ -1678,19 +1703,33 @@ export class SoundLibrary implements AudioSink {
} }
/** /**
* The sound of a video (#402) into the master, as a voice goes: from now on `element` is heard * The sound of a video (#402) into the mix, as a voice goes: from now on `element` is heard
* through the mix and only through it. The gain is its volume. Null while a bake has the context. * through it and only through it. `routed`, into the bus the routing has for its `file`, else
* straight into the master as a tab's preview (#368). The gain is its volume; hand it to
* `releaseMedia` once the element is let go. Null while a bake has the context.
*/ */
mediaInput(element: HTMLMediaElement): GainNode | null { mediaInput(element: HTMLMediaElement, file: string, routed: boolean): GainNode | null {
this.audio(); this.audio();
const context = this.live(); const context = this.live();
if (!context) return null; if (!context) return null;
if (context.state === 'suspended' && !this.held) void context.resume(); if (context.state === 'suspended' && !this.held) void context.resume();
const gain = context.createGain(); const gain = context.createGain();
context.createMediaElementSource(element).connect(gain).connect(this.busInput('')); const bus = routed ? this.videoBus(file) : MASTER_MIX;
context.createMediaElementSource(element).connect(gain).connect(this.busInput(bus));
this.media.set(gain, { file, routed, bus });
return gain; return gain;
} }
releaseMedia(gain: GainNode): void {
this.media.delete(gain);
gain.disconnect();
}
/** The bus the sound of each video heard goes into (test hook). */
mediaRoutes(): { file: string; routed: boolean; bus: string }[] {
return [...this.media.values()].map(({ file, routed, bus }) => ({ file, routed, bus }));
}
/** The context it plays to the speakers on; null before there is one, and for a bake (#340). */ /** The context it plays to the speakers on; null before there is one, and for a bake (#340). */
private live(): AudioContext | null { private live(): AudioContext | null {
return this.render ? null : (this.context as AudioContext | null); return this.render ? null : (this.context as AudioContext | null);
@@ -2639,6 +2678,10 @@ export interface OfflineRender {
/** Seconds a bake fades out over where its notes would have rung on (#340). */ /** Seconds a bake fades out over where its notes would have rung on (#340). */
const BAKE_LET_RING = 0.3; const BAKE_LET_RING = 0.3;
function sameEntries(a: ReadonlyMap<string, string>, b: ReadonlyMap<string, string>): boolean {
return a === b || (a.size === b.size && [...a].every(([key, value]) => b.get(key) === value));
}
/** The bus of a game's playing (#336): its master for `''`, else the track of that name inside it. */ /** The bus of a game's playing (#336): its master for `''`, else the track of that name inside it. */
function groupBus(id: number, track: string): string { function groupBus(id: number, track: string): string {
return track === '' ? `play:${id}` : `play:${id}:${track}`; return track === '' ? `play:${id}` : `play:${id}:${track}`;
+18 -11
View File
@@ -4,8 +4,8 @@
* same clip share a player and show the same picture. * same clip share a player and show the same picture.
* *
* In the editor a video plays without sound, as a preview. A run starts every player over at its * In the editor a video plays without sound, as a preview. A run starts every player over at its
* clip's start and lets it be heard through the master bus; a pause holds them, and the end of the * clip's start and lets it be heard along the master routing of its video; a pause holds them, and
* run makes them quiet previews again. * the end of the run makes them quiet previews again.
* *
* A sprite whose Video clock is driven (#404) has a player of its own instead, which shows the * A sprite whose Video clock is driven (#404) has a player of its own instead, which shows the
* picture at its Video time and leaves runs and pauses to it (see `wear` and `videoFollow.ts`). * picture at its Video time and leaves runs and pauses to it (see `wear` and `videoFollow.ts`).
@@ -50,12 +50,18 @@ function frameTime(): number {
return typeof now === 'number' ? now : performance.now(); return typeof now === 'number' ? now : performance.now();
} }
/**
* Whether a wearer lets its video be heard: in a game along the master routing of the video, in a
* preview of a tab straight into the master, as every tab's preview is (#368).
*/
export type VideoHearing = 'none' | 'preview' | 'game';
/** What the wearer of a driven player (#404) asks of it, see `wear`; its seeks, see `videoSeek.ts`. */ /** What the wearer of a driven player (#404) asks of it, see `wear`; its seeks, see `videoSeek.ts`. */
interface Drive extends SeekChase { interface Drive extends SeekChase {
/** Seconds into the clip. */ /** Seconds into the clip. */
time: number; time: number;
/** Whether the wearer lets it be heard, when it plays at about 1x. */ /** Whether and how the wearer lets it be heard, when it plays at about 1x. */
heard: boolean; heard: VideoHearing;
reading: SpeedReading | null; reading: SpeedReading | null;
} }
@@ -64,7 +70,7 @@ interface Player {
element: HTMLVideoElement; element: HTMLVideoElement;
/** Made once the video has shown its first frame, see `openVideo`. */ /** Made once the video has shown its first frame, see `openVideo`. */
texture: Texture | null; texture: Texture | null;
/** The sound on its way into the master, made the first time a run lets it be heard. */ /** The sound on its way into its bus, made the first time a run lets it be heard. */
gain: GainNode | null; gain: GainNode | null;
gone: boolean; gone: boolean;
/** When a sprite last asked for its picture, by `performance.now()`. */ /** When a sprite last asked for its picture, by `performance.now()`. */
@@ -110,7 +116,7 @@ export class VideoLibrary {
* as an ask: a sprite switched off wears its player without drawing it, and one let go for that * as an ask: a sprite switched off wears its player without drawing it, and one let go for that
* loaded again when it was switched back on, white until then. * loaded again when it was switched back on, white until then.
*/ */
wear(ref: TextureRef, props: PropertyBag, wearer: string, heard: boolean): TextureRef { wear(ref: TextureRef, props: PropertyBag, wearer: string, heard: VideoHearing): TextureRef {
if (!videoDrivenOf(props) || !isVideoKey(ref.file)) return ref; if (!videoDrivenOf(props) || !isVideoKey(ref.file)) return ref;
const key = drivenVideoKey(ref.file, wearer); const key = drivenVideoKey(ref.file, wearer);
const time = videoTimeOf(props); const time = videoTimeOf(props);
@@ -196,7 +202,7 @@ export class VideoLibrary {
return; return;
} }
if (Math.abs(element.playbackRate - step.rate) > 0.001) element.playbackRate = step.rate; if (Math.abs(element.playbackRate - step.rate) > 0.001) element.playbackRate = step.rate;
if (step.heard && drive.heard) this.hear(player); if (step.heard && drive.heard !== 'none') this.hear(player, drive.heard === 'game');
else element.muted = true; else element.muted = true;
if (element.paused) void element.play().catch(() => {}); if (element.paused) void element.play().catch(() => {});
} }
@@ -261,7 +267,7 @@ export class VideoLibrary {
const play = parseVideoKey(key); const play = parseVideoKey(key);
if (!play) return null; if (!play) return null;
const drive: Drive | null = isDrivenVideoKey(key) const drive: Drive | null = isDrivenVideoKey(key)
? { time, heard: false, reading: null, wanted: null, landing: false } ? { time, heard: 'none', reading: null, wanted: null, landing: false }
: null; : null;
const at = drive ? drivenTimeOf(play, time, NaN) : play.start; const at = drive ? drivenTimeOf(play, time, NaN) : play.start;
const player: Player = { const player: Player = {
@@ -321,8 +327,9 @@ export class VideoLibrary {
element.requestVideoFrameCallback(check); element.requestVideoFrameCallback(check);
} }
private hear(player: Player): void { /** Along the master routing of its video where `routed`, else straight into the master. */
player.gain ??= soundLibrary.mediaInput(player.element); private hear(player: Player, routed = true): void {
player.gain ??= soundLibrary.mediaInput(player.element, player.play.file, routed);
if (player.gain) player.gain.gain.value = player.play.volume; if (player.gain) player.gain.gain.value = player.play.volume;
player.element.muted = false; player.element.muted = false;
} }
@@ -335,7 +342,7 @@ export class VideoLibrary {
private drop(player: Player): void { private drop(player: Player): void {
player.gone = true; player.gone = true;
player.element.pause(); player.element.pause();
player.gain?.disconnect(); if (player.gain) soundLibrary.releaseMedia(player.gain);
player.element.removeAttribute('src'); player.element.removeAttribute('src');
player.element.load(); player.element.load();
// The texture itself stays whole: a wearer that has not synced since still draws it, and a // The texture itself stays whole: a wearer that has not synced since still draws it, and a
+4 -4
View File
@@ -36,7 +36,7 @@ import { snapPlacement } from './pixelGrid';
import { cameraSees } from '../core/model/renderLayers'; import { cameraSees } from '../core/model/renderLayers';
import { drivenVideoKey, isVideoKey } from '../core/model/video'; import { drivenVideoKey, isVideoKey } from '../core/model/video';
import { videoDrivenOf } from '../core/properties/videos'; import { videoDrivenOf } from '../core/properties/videos';
import { videoLibrary } from './VideoLibrary'; import { type VideoHearing, videoLibrary } from './VideoLibrary';
/** How strongly the cut-off part of an object shows while the scene view opens its clip area (#260). */ /** How strongly the cut-off part of an object shows while the scene view opens its clip area (#260). */
const CLIP_GHOST_ALPHA = 0.3; const CLIP_GHOST_ALPHA = 0.3;
@@ -920,7 +920,7 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
item.texture, item.texture,
item.props, item.props,
this.videoWearer(item), this.videoWearer(item),
seen && this.hearsVideo(), seen ? this.hearsVideo() : 'none',
); );
return texture === item.texture ? item : { ...item, texture }; return texture === item.texture ? item : { ...item, texture };
} }
@@ -960,8 +960,8 @@ export abstract class WorldViewport<S extends ViewSource = ViewSource> extends W
} }
/** Whether a driven video (#404) drawn here may be heard: in a run, and in the animation preview. */ /** Whether a driven video (#404) drawn here may be heard: in a run, and in the animation preview. */
protected hearsVideo(): boolean { protected hearsVideo(): VideoHearing {
return false; return 'none';
} }
/** /**
+84 -2
View File
@@ -1,11 +1,12 @@
/** /**
* The master routing (#351): a node per sound, instrument, pattern, song and bus and the master, * The master routing (#351): a node per sound, instrument, pattern, song, video and bus and the master,
* following the assets through every edit and its undo, and the mix the links make. * following the assets through every edit and its undo, and the mix the links make.
*/ */
import { describe, expect, it } from 'vitest'; import { describe, expect, it } from 'vitest';
import { import {
CreateAssetCommand, CreateAssetCommand,
CreateSoundCommand, CreateSoundCommand,
CreateVideoCommand,
DeleteAssetCommand, DeleteAssetCommand,
DuplicateAssetCommand, DuplicateAssetCommand,
RenameAssetCommand, RenameAssetCommand,
@@ -16,13 +17,23 @@ import { KeepRoutingCommand } from '../../src/core/commands/routingCommands';
import { SetSendCommand } from '../../src/core/commands/songCommands'; import { SetSendCommand } from '../../src/core/commands/songCommands';
import type { Graph } from '../../src/core/graph/Graph'; import type { Graph } from '../../src/core/graph/Graph';
import { ROUTING_ID } from '../../src/core/model/Project'; import { ROUTING_ID } from '../../src/core/model/Project';
import { MASTER_MIX, mixBusName, mixOf, playedRoute, routeOf, sameMix } from '../../src/core/music/mixer'; import {
MASTER_MIX,
mixBusName,
mixOf,
playedRoute,
routeOf,
sameMix,
videoBusesOf,
} from '../../src/core/music/mixer';
import { heldRouting, masterRoutingProblems, settleRouting } from '../../src/core/music/projectRouting'; import { heldRouting, masterRoutingProblems, settleRouting } from '../../src/core/music/projectRouting';
import { ROUTED_ASSET, ROUTED_MASTER } from '../../src/core/music/routingNodes'; import { ROUTED_ASSET, ROUTED_MASTER } from '../../src/core/music/routingNodes';
import { MASTER_BUS } from '../../src/core/properties/buses'; import { MASTER_BUS } from '../../src/core/properties/buses';
import { makeProject, roundTrip } from '../helpers'; import { makeProject, roundTrip } from '../helpers';
const FILE = `${'b2'.repeat(32)}.wav`; const FILE = `${'b2'.repeat(32)}.wav`;
const VIDEO = `${'c3'.repeat(32)}.mp4`;
const QUIET = `${'d4'.repeat(32)}.mp4`;
/** A project edited as the editor edits it: every command keeps the routing in step (#351). */ /** A project edited as the editor edits it: every command keeps the routing in step (#351). */
function edited() { function edited() {
@@ -300,6 +311,77 @@ describe('the mix of the master routing (#351)', () => {
expect(playedRoute(f.project, MASTER_BUS, song)).toBe(MASTER_MIX); expect(playedRoute(f.project, MASTER_BUS, song)).toBe(MASTER_MIX);
}); });
it('hears the sound of a video where its node goes, by its file, and gives one without sound no node', () => {
const f = edited();
const reverb = f.create('bus', 'Reverb');
const video = (name: string, file: string, audio: boolean) => {
const command = new CreateVideoCommand(name, file, {
width: 64,
height: 36,
duration: 2,
fps: 30,
audio,
});
f.run(command);
return command.asset.id;
};
const clip = video('Intro', VIDEO, true);
const quiet = video('Loop', QUIET, false);
const node = f.project.routing.nodes.find((n) => n.id === clip);
expect(node?.params).toEqual({ name: 'Intro', kind: 'video' });
expect(f.project.routing.nodes.some((n) => n.id === quiet)).toBe(false);
expect(videoBusesOf(f.project)).toEqual(
new Map([
[VIDEO, MASTER_MIX],
[QUIET, MASTER_MIX],
]),
);
// Into one bus alone, then into two: the second time on a bus of its own with both sends.
const toMaster = f.project.routing.edges.find((e) => e.from.node === clip)!.id;
f.run(new DisconnectCommand(ROUTING_ID, toMaster));
f.link(clip, reverb);
expect(videoBusesOf(f.project).get(VIDEO)).toBe(mixBusName(reverb));
f.run(new SetSendCommand(ROUTING_ID, f.link(clip, MASTER_BUS), { volume: -6, tap: 'post' }));
expect(videoBusesOf(f.project).get(VIDEO)).toBe(mixBusName(clip));
expect(
mixOf(f.project)
.find((bus) => bus.name === mixBusName(clip))
?.sends?.map((s) => s.to),
).toEqual([mixBusName(reverb), MASTER_MIX]);
// A second video of the same file plays where the first one goes.
video('Intro again', VIDEO, true);
expect(videoBusesOf(f.project).get(VIDEO)).toBe(mixBusName(clip));
});
it('link a new video where the other videos go, else to the master, never where the sounds go', () => {
const f = edited();
const sfx = f.create('bus', 'SFX');
const film = f.create('bus', 'Film');
const video = (name: string) => {
const command = new CreateVideoCommand(name, VIDEO, {
width: 64,
height: 36,
duration: 2,
fps: 30,
audio: true,
});
f.run(command);
return command.asset.id;
};
const onto = (id: string, bus: string) => {
f.run(new DisconnectCommand(ROUTING_ID, f.project.routing.edges.find((e) => e.from.node === id)!.id));
f.link(id, bus);
};
for (const name of ['Kick', 'Snare']) onto(f.sound(name), sfx);
const intro = video('Intro');
expect(f.linksOf(intro)).toEqual([MASTER_BUS]);
onto(intro, film);
expect(f.linksOf(video('Outro'))).toEqual([film]);
// Nor does the first song go by the videos, though as many of them go to Film as sounds to SFX.
expect(f.linksOf(f.create('song', 'Theme'))).toEqual([sfx]);
});
it('plays an asset with no link at all on nothing', () => { it('plays an asset with no link at all on nothing', () => {
const f = edited(); const f = edited();
const kick = f.sound('Kick'); const kick = f.sound('Kick');
+21
View File
@@ -278,6 +278,7 @@ function fakeContext() {
return panner; return panner;
}, },
createChannelSplitter: () => wired('splitter', {}), createChannelSplitter: () => wired('splitter', {}),
createMediaElementSource: () => wired('media', {}),
createAnalyser: () => { createAnalyser: () => {
const analyser = wired('analyser', { const analyser = wired('analyser', {
fftSize: 2048, fftSize: 2048,
@@ -2564,6 +2565,26 @@ describe('the master and the buses of the project (#325)', () => {
expect(library.busLevels().find((level) => level.name === `play:${id}`)?.into).toBe('mix:a'); expect(library.busLevels().find((level) => level.name === `play:${id}`)?.into).toBe('mix:a');
}); });
it('hears a video on the bus the routing has for its file, and moves it along when that changes', async () => {
const library = new SoundLibrary(false);
const video = {} as HTMLMediaElement;
await library.setMix(mix, new Map([['a.mp4', 'mix:b']]));
const first = library.mediaInput(video, 'a.mp4', true)!;
const second = library.mediaInput(video, 'other.mp4', true)!;
// A tab's preview of the same video leaves the routing out (#368).
const preview = library.mediaInput(video, 'a.mp4', false)!;
const buses = () => library.mediaRoutes().map((route) => route.bus);
expect(buses()).toEqual(['mix:b', 'mix', 'mix']);
// The same mix with the video sent elsewhere: only the video moves, not its preview.
await library.setMix(mix, new Map([['a.mp4', 'mix:a']]));
expect(buses()).toEqual(['mix:a', 'mix', 'mix']);
// A mix set without videos keeps where they go.
await library.setMix([mix[0]!, mix[1]!, { ...mix[2]!, volume: 0.5 }]);
expect(buses()).toEqual(['mix:a', 'mix', 'mix']);
for (const gain of [second, preview, first]) library.releaseMedia(gain);
expect(library.mediaRoutes()).toEqual([]);
});
it('sends a bus on into another one when the mix says so, and lets one go whose asset is gone', async () => { it('sends a bus on into another one when the mix says so, and lets one go whose asset is gone', async () => {
const library = new SoundLibrary(false); const library = new SoundLibrary(false);
await library.setMix(mix); await library.setMix(mix);
+8 -8
View File
@@ -105,7 +105,7 @@ describe('VideoLibrary sweeping', () => {
const props = { videoClock: 'driven', videoTime: 1.2 }; const props = { videoClock: 'driven', videoTime: 1.2 };
videosMade = 0; videosMade = 0;
for (let ms = 0; ms < 6000; ms += 100) { for (let ms = 0; ms < 6000; ms += 100) {
videos.wear(ref, props, 'view2:a', false); videos.wear(ref, props, 'view2:a', 'none');
vi.advanceTimersByTime(100); vi.advanceTimersByTime(100);
} }
// Let go and opened again would be a second video, loading while the sprite shows white. // Let go and opened again would be a second video, loading while the sprite shows white.
@@ -128,14 +128,14 @@ describe('VideoLibrary driven players (#404)', () => {
it('leaves a sprite with a free clock on the shared player', async () => { it('leaves a sprite with a free clock on the shared player', async () => {
const videos = await freshLibrary(); const videos = await freshLibrary();
expect(videos.wear(REF, { videoTime: 1 }, 'view1:a', false)).toBe(REF); expect(videos.wear(REF, { videoTime: 1 }, 'view1:a', 'none')).toBe(REF);
expect(videos.debugInfo()).toEqual([]); expect(videos.debugInfo()).toEqual([]);
}); });
it('gives every wearer of a driven video a player of its own, opened at its time', async () => { it('gives every wearer of a driven video a player of its own, opened at its time', async () => {
const videos = await freshLibrary(); const videos = await freshLibrary();
const a = videos.wear(REF, { videoClock: 'driven', videoTime: 0.5 }, 'view1:a', false); const a = videos.wear(REF, { videoClock: 'driven', videoTime: 0.5 }, 'view1:a', 'none');
const b = videos.wear(REF, { videoClock: 'driven', videoTime: 1.25 }, 'view1:b', false); const b = videos.wear(REF, { videoClock: 'driven', videoTime: 1.25 }, 'view1:b', 'none');
expect([a.file, b.file]).toEqual([`${KEY}@view1:a`, `${KEY}@view1:b`]); expect([a.file, b.file]).toEqual([`${KEY}@view1:a`, `${KEY}@view1:b`]);
expect(videos.debugInfo().map(({ key, time }) => [key, time])).toEqual([ expect(videos.debugInfo().map(({ key, time }) => [key, time])).toEqual([
[`${KEY}@view1:a`, 0.5], [`${KEY}@view1:a`, 0.5],
@@ -162,7 +162,7 @@ describe('VideoLibrary driven players (#404)', () => {
for (let i = 0; i <= 36; i++) { for (let i = 0; i <= 36; i++) {
frame.at = (i * 1000) / 60; frame.at = (i * 1000) / 60;
if (i === 7) late = 0; if (i === 7) late = 0;
videos.wear(REF, { videoClock: 'driven', videoTime: i / 60 }, 'view1:a', false); videos.wear(REF, { videoClock: 'driven', videoTime: i / 60 }, 'view1:a', 'none');
} }
expect(videos.debugInfo()[0]!.speed).toBeCloseTo(1, 6); expect(videos.debugInfo()[0]!.speed).toBeCloseTo(1, 6);
}); });
@@ -199,13 +199,13 @@ describe('VideoLibrary driven players (#404)', () => {
}; };
vi.stubGlobal('document', { hidden: false, addEventListener: () => {}, createElement: () => element }); vi.stubGlobal('document', { hidden: false, addEventListener: () => {}, createElement: () => element });
const videos = await freshLibrary(); const videos = await freshLibrary();
const key = videos.wear(REF, { videoClock: 'driven', videoTime: 0.5 }, 'view1:a', false).file; const key = videos.wear(REF, { videoClock: 'driven', videoTime: 0.5 }, 'view1:a', 'none').file;
fire('loadeddata'); fire('loadeddata');
seekDone(); seekDone();
[frames] = [[], ...frames.map((frame) => frame())]; [frames] = [[], ...frames.map((frame) => frame())];
expect(videos.showsAsked(key)).toBe(true); expect(videos.showsAsked(key)).toBe(true);
// Switched on at 1.2 s: the seek is done, and no frame follows it. // Switched on at 1.2 s: the seek is done, and no frame follows it.
videos.wear(REF, { videoClock: 'driven', videoTime: 1.2 }, 'view1:a', false); videos.wear(REF, { videoClock: 'driven', videoTime: 1.2 }, 'view1:a', 'none');
expect(videos.showsAsked(key)).toBe(false); expect(videos.showsAsked(key)).toBe(false);
seekDone(); seekDone();
vi.advanceTimersByTime(200); vi.advanceTimersByTime(200);
@@ -217,7 +217,7 @@ describe('VideoLibrary driven players (#404)', () => {
const videos = await freshLibrary(); const videos = await freshLibrary();
videos.get(KEY); videos.get(KEY);
expect(videos.loading('view12')).toBe(false); expect(videos.loading('view12')).toBe(false);
const worn = videos.wear(REF, { videoClock: 'driven', videoTime: 0 }, 'view12:a', false); const worn = videos.wear(REF, { videoClock: 'driven', videoTime: 0 }, 'view12:a', 'none');
expect(videos.hasPicture(worn.file)).toBe(false); expect(videos.hasPicture(worn.file)).toBe(false);
// A run waits only for its own view: view12 is neither view1 nor view2. // A run waits only for its own view: view12 is neither view1 nor view2.
expect([videos.loading('view12'), videos.loading('view1'), videos.loading('view2')]).toEqual([ expect([videos.loading('view12'), videos.loading('view1'), videos.loading('view2')]).toEqual([
+55 -2
View File
@@ -21,6 +21,7 @@ const DRIVEN = 'vd66-driven';
const EARLY = 'vd66-early'; const EARLY = 'vd66-early';
const LATE = 'vd66-late'; const LATE = 'vd66-late';
const CUT = 'vd66-cut'; const CUT = 'vd66-cut';
const BUS = 'vd66-bus';
const LANE = LANES.video; const LANE = LANES.video;
const COLORS = { red: '#fe0000', green: '#00ff01', blue: '#0000ff', white: '#ffffff' }; const COLORS = { red: '#fe0000', green: '#00ff01', blue: '#0000ff', white: '#ffffff' };
@@ -184,6 +185,45 @@ export async function video(t) {
JSON.stringify(resumed), JSON.stringify(resumed),
); );
// 66e-iii. With sound the video has a node in the Master routing (#351), linked to the master.
// Sent into a bus there, the tab still plays it straight into the master, as every
// tab's preview does (#368): heard at the output, where a recording of the editor
// listens, and nothing on the bus. It used to play past the mix altogether.
const graphOf = (expr) => json(`((g) => ${expr})(__gm.routing.graph())`);
const routed = await graphOf(
`({ node: g.nodes.find((n) => n.id === ${JSON.stringify(assetId)}) ?? null,
links: g.edges.filter((e) => e.from.node === ${JSON.stringify(assetId)}).map((e) => e.to.node) })`,
);
check(
'a video with sound has a node of its own in the Master routing, linked to the master',
routed.node?.params.kind === 'video' &&
routed.node.params.name === 'colors' &&
JSON.stringify(routed.links) === '["#master"]',
JSON.stringify(routed),
);
await mutate(`
const p = __gm.state.project;
p.assets.push({ id: '${BUS}', type: 'bus', name: 'vd66 bus', props: {} });
p.routing.nodes.push({ id: '${BUS}', type: 'RoutedAsset', x: 320, y: 0, params: { name: 'vd66 bus', kind: 'bus' } });
p.routing.edges.push({ id: '${BUS}-out', from: { node: '${BUS}', port: 'out' }, to: { node: '#master', port: 'in' } });
for (const e of p.routing.edges) if (e.from.node === ${JSON.stringify(assetId)}) e.to = { node: '${BUS}', port: 'in' };`);
const metered = JSON.stringify(['mix', `mix:${BUS}`]);
await json(`__gm.routing.peaks(${metered})`);
await click(playButton.x, playButton.y);
const tabPeaks = await waitFor(
() => json(`__gm.routing.peaks(${metered})`),
(peaks) => Math.max(...(peaks.mix ?? [0])) > 0.01,
);
const tabRoutes = await json('__gm.routing.media()');
await click(playButton.x, playButton.y);
check(
'the tab plays the video straight into the master, past its routing',
Math.max(...(tabPeaks.mix ?? [0])) > 0.01 &&
Math.max(...(tabPeaks[`mix:${BUS}`] ?? [0])) === 0 &&
tabRoutes.some((r) => r.file === uploaded.file && !r.routed && r.bus === 'mix'),
JSON.stringify({ tabPeaks, tabRoutes }),
);
// 66e-ii. With the clip picked (#403), a drag in the track off its handles or in the strip beside // 66e-ii. With the clip picked (#403), a drag in the track off its handles or in the strip beside
// its band changes nothing; one on its band moves it whole, one undo step. A click in the // its band changes nothing; one on its band moves it whole, one undo step. A click in the
// track beside it picks the video, on it the clip again. // track beside it picks the video, on it the clip again.
@@ -261,7 +301,8 @@ export async function video(t) {
); );
await shot('66-video-sprite'); await shot('66-video-sprite');
// 66g. Play starts it over from the clip's start and lets it be heard; Esc makes it quiet again. // 66g. Play starts it over from the clip's start and lets it be heard, along the routing of its
// video into the bus from 66e-iii; Esc makes it quiet again.
await play(); await play();
await sleep(200); await sleep(200);
const heard = (await json('__gm.videos()'))[0]; const heard = (await json('__gm.videos()'))[0];
@@ -270,6 +311,18 @@ export async function video(t) {
heard && !heard.muted && !heard.paused && heard.time >= 0.6 && heard.time < 1.1, heard && !heard.muted && !heard.paused && heard.time >= 0.6 && heard.time < 1.1,
JSON.stringify(heard), JSON.stringify(heard),
); );
const runPeaks = await waitFor(
() => json(`__gm.routing.peaks(${metered})`),
(peaks) => Math.max(...(peaks[`mix:${BUS}`] ?? [0])) > 0.01,
20,
);
const runRoutes = await json('__gm.routing.media()');
check(
'in the game the video is heard on the bus its routing sends it to',
Math.max(...(runPeaks[`mix:${BUS}`] ?? [0])) > 0.01 &&
runRoutes.some((r) => r.file === uploaded.file && r.routed && r.bus === `mix:${BUS}`),
JSON.stringify({ runPeaks, runRoutes }),
);
await stop(); await stop();
await sleep(200); await sleep(200);
const quiet = (await json('__gm.videos()'))[0]; const quiet = (await json('__gm.videos()'))[0];
@@ -589,7 +642,7 @@ export async function video(t) {
const p = __gm.state.project; const p = __gm.state.project;
const gone = ['${OBJECT}', '${EARLY}', '${LATE}']; const gone = ['${OBJECT}', '${EARLY}', '${LATE}'];
for (const holder of p.assets) if (holder.objects) holder.objects = holder.objects.filter((o) => !gone.includes(o.id)); for (const holder of p.assets) if (holder.objects) holder.objects = holder.objects.filter((o) => !gone.includes(o.id));
p.assets = p.assets.filter((a) => ![ '${SPRITE}', '${DRIVEN}', '${CUT}', ${JSON.stringify(assetId)} ].includes(a.id)); p.assets = p.assets.filter((a) => ![ '${SPRITE}', '${DRIVEN}', '${CUT}', '${BUS}', ${JSON.stringify(assetId)} ].includes(a.id));
__gm.state.select(null); __gm.state.select(null);
__gm.lookAt(0, 0);`); __gm.lookAt(0, 0);`);
} }