Let a behavior follow the beat, bar and markers of the music (#338)
feuerwerk / gate (push) Failing after 15m41s
feuerwerk / suite (push) Canceled after 18m24s

Stage 2 of #338. A game that syncs to its music has to know where the
music is heard. This stage reads that on the audio clock and hands it
to behaviors as events and as a position.

- New event nodes On beat (playback, bar, beat, late), On bar
  (playback, bar, late) and On song marker (playback, marker, name,
  late).
- New data node Song position with bar, beat, fraction, seconds and
  playing.
- The nodes have the fields Song and Pattern. With neither picked,
  every playback counts. A linked playback input picks that one.
  Compositions do not count.
- The Marker field offers the markers of all songs as Song / Marker,
  as a sound field offers its clips.
- Everything follows what is heard: the audio clock minus the output
  latency. That holds for baked songs and for Play pattern too.
- Only the last beat since the frame before fires, and only when it is
  at most half a beat late, 0.15 s at most. Bars and markers alike.
- A copied song gets markers with ids of its own, so a Marker field
  never names two songs.
- The audio clock is only asked while something plays. Asked in every
  frame, it made the AudioContext without a user gesture (#436).

Tests:
- beats 14 and music-events 6 new, 4 in song-play, 2 in song-markers,
  2 catalog cases, the touched groups 756/756
- 16 counter-probes, all compile, all red
- e2e 65 on atx: ALL PASSED, 71 checks, 65k2 new: 11 beats 1.1 to 2.4
  and 2.1 again, none missing or twice, across the pause too. late
  0.015 to 0.027 s. Counter-probe without the last position: red
- gate on atx: CHECK EXIT 0, 334 files, 5121 tests

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
adminandClaude Opus 5.5 committed 2026-10-29 19:48:51 +01:00
1 parent 5cc1cc9ef6
commit a6a41b6b69
25 files changed
+1171 -12

No files matched your search

+135
View File
@@ -0,0 +1,135 @@
/**
* Where a playing is heard, and the beats, bars and markers it passed (#338): a frame looks now and
* then, so what passed since is told late – and too late, or more than the last, not at all.
*/
import { describe, expect, it } from 'vitest';
import {
heardPosition,
lateLimit,
newestPassed,
passedBar,
passedBeat,
passedMarkers,
type Timeline,
} from '../../src/core/music/beats';
import { COMMON_TIME } from '../../src/core/music/time';
const Q = 480;
const BAR = 4 * Q;
/** 120 BPM in 4/4: a beat is half a second, a bar two. Intro bar 1, then round bar 2. */
const LOOPED: Timeline = {
tempo: 120,
meter: COMMON_TIME,
from: 0,
loop: { start: BAR, end: 2 * BAR },
length: 2 * BAR,
};
const ONCE: Timeline = { ...LOOPED, loop: null };
const beatAt = (t: Timeline, before: number | null, now: number) => {
const beat = passedBeat(t, before, now);
return beat && [beat.bar, beat.beat, +beat.late.toFixed(6)];
};
describe('where a playing is heard', () => {
it('counts bar and beat from 1, the fraction into the beat, the seconds into the song', () => {
expect(heardPosition(LOOPED, 0)).toMatchObject({
bar: 1,
beat: 1,
fraction: 0,
seconds: 0,
playing: true,
});
expect(heardPosition(LOOPED, 1.25)).toMatchObject({ bar: 1, beat: 3, fraction: 0.5, seconds: 1.25 });
});
it('goes back round the loop with the playhead', () => {
// 5.25 s in: the intro took 2, the loop round 2 more, so a quarter second into the second round.
expect(heardPosition(LOOPED, 5.25)).toMatchObject({ bar: 2, beat: 3, fraction: 0.5, seconds: 3.25 });
});
it('stands where it starts before its first note, at its end after it, playing neither time', () => {
expect(heardPosition(LOOPED, null)).toMatchObject({ bar: 1, beat: 1, playing: false });
expect(heardPosition(LOOPED, -0.03)).toMatchObject({ bar: 1, beat: 1, playing: false });
expect(heardPosition(ONCE, 4.5)).toMatchObject({ bar: 3, beat: 1, seconds: 4, playing: false });
});
it('takes the beat from the line of the meter: an eighth in 6/8', () => {
const six: Timeline = { ...ONCE, meter: { beats: 6, unit: 8 } };
// An eighth is a quarter second at 120 quarters a minute; a bar of six is 1.5 s.
expect(heardPosition(six, 1.625)).toMatchObject({ bar: 2, beat: 1, fraction: 0.5 });
expect(lateLimit(six)).toBe(0.125);
expect(lateLimit(LOOPED)).toBe(0.15);
});
});
describe('the beats it passed', () => {
it('tells a beat passed since the last frame, with how late', () => {
expect(beatAt(LOOPED, 0.49, 0.51)).toEqual([1, 2, 0.01]);
expect(beatAt(LOOPED, 0.51, 0.53)).toBeNull();
});
it('tells the very first beat to a playing seen for the first time', () => {
expect(beatAt(LOOPED, null, 0.02)).toEqual([1, 1, 0.02]);
expect(beatAt(LOOPED, -0.01, 0.02)).toEqual([1, 1, 0.02]);
});
it('still tells a beat a slipped frame passed, but not one more than the limit ago', () => {
expect(beatAt(LOOPED, 0.4, 0.6)).toEqual([1, 2, 0.1]);
expect(beatAt(LOOPED, 0.4, 0.7)).toBeNull();
});
it('tells only the last of several passed in one frame, and nothing after a long stall', () => {
expect(beatAt(LOOPED, 0, 1.6)).toEqual([1, 4, 0.1]);
expect(beatAt(LOOPED, 0, 10.3)).toBeNull();
});
it('tells the loop’s start each time round', () => {
// 4 s in, the intro and one round are over: the head is back at bar 2.
expect(beatAt(LOOPED, 3.99, 4.01)).toEqual([2, 1, 0.01]);
expect(passedBar(LOOPED, 5.99, 6.02)).toMatchObject({ bar: 2, beat: 1 });
});
it('counts beats on the ruler’s grid where the loop starts between them', () => {
const offBeat: Timeline = { ...LOOPED, loop: { start: BAR + Q / 2, end: 2 * BAR } };
// The second round starts 4 s in at a quarter past bar 2's first beat: its next beat is 0.25 s later.
expect(beatAt(offBeat, 3.99, 4.01)).toBeNull();
expect(beatAt(offBeat, 4.2, 4.3)).toEqual([2, 2, 0.05]);
});
it('tells nothing at the end of one that plays once, nor after it', () => {
expect(beatAt(ONCE, 3.95, 4.05)).toBeNull();
expect(passedBar(ONCE, 3.95, 4.05)).toBeNull();
expect(beatAt(ONCE, 3.45, 3.55)).toEqual([2, 4, 0.05]);
});
it('tells a bar only on its first beat', () => {
expect(passedBar(LOOPED, 0.49, 0.51)).toBeNull();
expect(passedBar(LOOPED, 1.99, 2.01)).toMatchObject({ bar: 2, beat: 1 });
});
});
describe('the markers it passed', () => {
const markers = [
{ id: 'a', name: 'Verse', tick: BAR },
{ id: 'b', name: 'Also verse', tick: BAR },
{ id: 'c', name: 'Turn', tick: 2 * BAR - 10 },
];
it('tells every marker on the tick passed, each time round the loop', () => {
const passed = passedMarkers(LOOPED, 1.99, 2.01, markers);
expect(passed.map((m) => [m.id, m.name, +m.late.toFixed(6)])).toEqual([
['a', 'Verse', 0.01],
['b', 'Also verse', 0.01],
]);
expect(passedMarkers(LOOPED, 3.99, 4.01, markers).map((m) => m.id)).toEqual(['a', 'b']);
});
it('of markers passed across the loop’s end, keeps the last spot only', () => {
// "Turn" is 10 ticks before the end, heard 3.9896 s in; the loop's start follows at 4 s.
const passed = passedMarkers(LOOPED, 3.98, 4.02, markers);
expect(passed.map((m) => m.id)).toEqual(['a', 'b', 'c']);
expect(newestPassed(passed).map((m) => m.id)).toEqual(['a', 'b']);
expect(newestPassed(passed.filter((m) => m.id === 'c')).map((m) => m.id)).toEqual(['c']);
});
});
+1 -1
View File
@@ -104,7 +104,7 @@ describe('the order rule', () => {
const events = BUILT_IN_NODES.filter(
(def) => def.category === 'event' && def.type !== BEHAVIOR_START && def.type !== ON_BEHAVIOR,
);
expect(events).toHaveLength(19);
expect(events).toHaveLength(22);
for (const def of events) {
expect(
def.params.map((p) => p.key),
+200
View File
@@ -0,0 +1,200 @@
/**
* A game follows its music (#338): On beat, On bar and On song marker run as the songs and patterns
* it plays pass them, as heard, and Song position says where one stands.
*/
import { describe, expect, it } from 'vitest';
import { CreateAssetCommand, SetAssetPropertyCommand } from '../../src/core/commands/assetCommands';
import { SetSongMarkersCommand } from '../../src/core/commands/songCommands';
import type { HeardTime } from '../../src/core/music/playbacks';
import { SILENT_AUDIO } from '../../src/runtime/systems/Audio';
import { heardPlayings, positionOf } from '../../src/runtime/systems/musicPulses';
import { graphEditor, makeBehavior, makeSpriteScene, place, startWorld } from '../helpers';
const Q = 480;
const BAR = 4 * Q;
/** A song with two markers on bar 2 and one in its middle, a pattern, and a behavior to listen. */
function listening() {
const f = makeSpriteScene('Box');
const create = (type: 'song' | 'pattern', name: string) => {
const command = new CreateAssetCommand(type, name);
f.stack.execute(command);
return command.asset.id;
};
const song = create('song', 'Theme');
const pattern = create('pattern', 'Riff');
f.stack.execute(new SetAssetPropertyCommand(pattern, 'sign', '3/4'));
f.stack.execute(
new SetSongMarkersCommand(song, [
{ id: 'verse', name: 'Verse', tick: BAR },
{ id: 'also', name: 'Also', tick: BAR },
{ id: 'half', name: 'Half', tick: BAR / 2 },
]),
);
const behavior = makeBehavior(f, 'Logic');
place(f, behavior.id, { x: 0, y: 0 });
const edit = graphEditor(f, behavior);
/** An event that logs `prefix` and two of its outputs, run together. */
const logs = (type: string, params: Record<string, string>, prefix: string, a: string, b: string) => {
const event = edit.addNode(type, params);
const join = edit.addNode('Join');
const first = edit.addNode('Join', { a: prefix });
edit.connect(event, a, first, 'b');
edit.connect(first, 'text', join, 'a');
edit.connect(event, b, join, 'b');
const log = edit.addNode('Log');
edit.connect(event, 'exec', log, 'exec');
edit.connect(join, 'text', log, 'message');
return event;
};
return { ...f, song, pattern, edit, logs };
}
/** A sink whose playings are heard where a test puts them. */
function heardAudio(song: string, pattern: string) {
const times: HeardTime[] = [
{
id: 7,
name: `song:${song}`,
heard: null,
tempo: 120,
from: 0,
loop: { start: BAR, end: 2 * BAR },
length: 2 * BAR,
},
{
id: 8,
name: `pattern:${pattern}`,
heard: null,
tempo: 120,
from: 0,
loop: { start: 0, end: 3 * Q },
length: 3 * Q,
},
{ id: 9, name: 'composition:c', heard: null, tempo: 120, from: 0, loop: null, length: 2 * BAR },
];
return {
audio: { ...SILENT_AUDIO, playingTimes: () => times },
at(heard: number | null, ...ids: number[]) {
for (const time of times) if (ids.length === 0 || ids.includes(time.id)) time.heard = heard;
},
};
}
describe('On beat and On bar', () => {
it('run on every beat and bar a song plays, as heard, counting from 1', () => {
const f = listening();
f.logs('OnBeat', { song: f.song }, 'beat ', 'bar', 'beat');
f.logs('OnBar', { song: f.song }, 'bar ', 'bar', 'late');
const { audio, at } = heardAudio(f.song, f.pattern);
const { world, frame } = startWorld(f, { behaviors: true, audio });
frame();
expect(world.log).toEqual([]);
at(0.01, 7);
frame();
expect(world.log).toEqual(['beat 11', 'bar 10.01']);
// A frame the audio clock did not move in: nothing new was heard.
frame();
expect(world.log).toHaveLength(2);
at(0.3, 7);
frame();
at(0.52, 7);
frame();
expect(world.log.slice(2)).toEqual(['beat 12']);
// Round the loop: the intro and one round are over 4 s in, the head is back on bar 2.
at(3.98, 7);
frame();
at(4, 7);
frame();
expect(world.log.slice(3)).toEqual(['beat 21', 'bar 20']);
});
it('tell nothing a stalled frame passed too long ago, and of several only the last', () => {
const f = listening();
f.logs('OnBeat', {}, 'beat ', 'bar', 'beat');
const { audio, at } = heardAudio(f.song, f.pattern);
const { world, frame } = startWorld(f, { behaviors: true, audio });
at(0.6, 7);
frame();
expect(world.log).toEqual(['beat 12']);
at(1.55, 7);
frame();
at(3.3, 7);
frame();
expect(world.log.slice(1)).toEqual(['beat 14']);
});
it('hear the song or pattern picked, every one where none is, a composition never', () => {
const f = listening();
f.logs('OnBeat', {}, 'any ', 'playback', 'beat');
f.logs('OnBeat', { pattern: f.pattern }, 'pattern ', 'playback', 'beat');
f.logs('OnBeat', { song: f.song }, 'song ', 'playback', 'beat');
const { audio, at } = heardAudio(f.song, f.pattern);
const { world, frame } = startWorld(f, { behaviors: true, audio });
at(0.01);
frame();
expect(world.log).toEqual(['any 71', 'any 81', 'pattern 81', 'song 71']);
});
it('hear only the playing "playback" names where it is connected', () => {
const f = listening();
const beat = f.logs('OnBeat', {}, 'beat ', 'playback', 'beat');
const number = f.edit.addNode('Number', { value: 8 });
f.edit.connect(number, 'value', beat, 'playback');
const { audio, at } = heardAudio(f.song, f.pattern);
const { world, frame } = startWorld(f, { behaviors: true, audio });
at(0.01);
frame();
expect(world.log).toEqual(['beat 81']);
});
});
describe('On song marker', () => {
it('runs for every marker on the tick passed, or for the one picked', () => {
const f = listening();
f.logs('OnSongMarker', { song: f.song }, 'any ', 'name', 'marker');
f.logs('OnSongMarker', { marker: 'also' }, 'picked ', 'name', 'playback');
const { audio, at } = heardAudio(f.song, f.pattern);
const { world, frame } = startWorld(f, { behaviors: true, audio });
at(0.98, 7);
frame();
at(1.01, 7);
frame();
expect(world.log).toEqual(['any Halfhalf']);
at(2.01, 7);
frame();
expect(world.log.slice(1)).toEqual(['any Verseverse', 'any Alsoalso', 'picked Also7']);
});
});
describe('Song position', () => {
it('answers where the playing picked stands, the one started last where several are', () => {
const f = listening();
const { audio, at } = heardAudio(f.song, f.pattern);
at(1.25, 7);
at(0.625, 8);
const playings = heardPlayings(f.project, audio);
// The pattern is in 3/4 and runs round its one bar of 1.5 s.
expect(playings.map((p) => [p.kind, p.timeline.meter.beats])).toEqual([
['song', 4],
['pattern', 3],
]);
expect(positionOf(playings, null, f.song, '')).toMatchObject({
bar: 1,
beat: 3,
fraction: 0.5,
seconds: 1.25,
playing: true,
});
// None picked: the pattern, started after the song.
expect(positionOf(playings, null, '', '')).toMatchObject({ beat: 2, fraction: 0.25, seconds: 0.625 });
expect(positionOf(playings, 7, '', f.pattern)).toMatchObject({ seconds: 1.25 });
expect(positionOf(playings, 3, '', '')).toBeNull();
at(1.75, 8);
expect(positionOf(heardPlayings(f.project, audio), null, '', f.pattern)).toMatchObject({
bar: 1,
beat: 1,
fraction: 0.5,
});
});
});
+25
View File
@@ -2526,6 +2526,28 @@ const CASES: Record<string, Case[]> = {
outputs: { volume: 0.4 },
},
],
SongPosition: [
{
name: 'answers where the playing stands, by its number where "playback" is connected (#338)',
context: {
songPosition: (playback: number | null, song: string, pattern: string) =>
playback === 4 && song === 's' && pattern === ''
? { bar: 3, beat: 2, fraction: 0.25, seconds: 5.125, tick: 0, playing: true }
: null,
},
inputs: { playback: 4, song: 's', pattern: '' },
outputs: { bar: 3, beat: 2, fraction: 0.25, seconds: 5.125, playing: true },
},
{
name: 'asks by the song and pattern alone where "playback" is not connected, all 0 for none',
context: {
songPosition: (playback: number | null) =>
playback === null ? null : { bar: 1, beat: 1, fraction: 0, seconds: 0, tick: 0, playing: true },
},
inputs: { song: 's' },
outputs: { bar: 0, beat: 0, fraction: 0, seconds: 0, playing: false },
},
],
};
/** Two distinct handles that share a name – `Equals` must still tell them apart. */
@@ -2557,6 +2579,9 @@ const EVENT_NODES: Record<string, string> = {
OnMouseWheel: 'clip-areas.test.ts',
OnAnimationFinished: 'animation.test.ts',
OnCompositionFinished: 'composition-play.test.ts',
OnBeat: 'music-events.test.ts',
OnBar: 'music-events.test.ts',
OnSongMarker: 'music-events.test.ts',
OnStateEntered: 'state-machine.test.ts',
OnStateExited: 'state-machine.test.ts',
};
+30 -1
View File
@@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest';
import { CreateAssetCommand } from '../../src/core/commands/assetCommands';
import { CreateAssetCommand, DuplicateAssetCommand } from '../../src/core/commands/assetCommands';
import { SetSongMarkersCommand } from '../../src/core/commands/songCommands';
import type { SongAsset } from '../../src/core/model/Asset';
import {
@@ -9,6 +9,7 @@ import {
nextMarkerName,
withoutMarker,
} from '../../src/core/music/markers';
import { assetRefLabel, findSongMarker, markerChoices } from '../../src/core/properties/assetRef';
import { makeProject, roundTrip } from '../helpers';
function makeSong() {
@@ -100,4 +101,32 @@ describe('song markers (#338)', () => {
const plain = roundTrip(none.project).assets.find((a) => a.id === none.song().id) as SongAsset;
expect('markers' in plain).toBe(false);
});
it('offers the markers of every song to a marker field, each read "Song / Marker"', () => {
const f = makeSong();
f.stack.execute(
new SetSongMarkersCommand(f.song().id, [
{ id: 'm2', name: 'Boss', tick: 1920 },
{ id: 'm1', name: 'Intro', tick: 0 },
]),
);
expect(markerChoices(f.project)).toEqual([
{ id: 'm1', label: 'Tune / Intro' },
{ id: 'm2', label: 'Tune / Boss' },
]);
expect(assetRefLabel(f.project, 'm2')).toBe('Tune / Boss');
expect(findSongMarker(f.project, 'm1')?.song.id).toBe(f.song().id);
expect(assetRefLabel(f.project, 'gone')).toBe('gone?');
});
it('gives the markers of a copy of a song ids of their own, so a node names one of them', () => {
const f = makeSong();
f.stack.execute(new SetSongMarkersCommand(f.song().id, [{ id: 'm1', name: 'Intro', tick: 0 }]));
const duplicate = new DuplicateAssetCommand(f.song().id);
f.stack.execute(duplicate);
const copy = f.project.assets.find((a) => a.id === duplicate.copyId) as SongAsset;
expect(copy.markers?.map((m) => m.name)).toEqual(['Intro']);
expect(copy.markers?.[0]?.id).not.toBe('m1');
expect(findSongMarker(f.project, 'm1')?.song.id).toBe(f.song().id);
});
});
+61 -2
View File
@@ -23,9 +23,10 @@ const Q = 480;
const BAR = 4 * Q;
/** An audio clock moved by hand, its timers run as it passes them. */
function fakeHost() {
function fakeHost(baked = false) {
const host = {
time: 0,
latency: 0,
sounding: new Set<number>(),
pumps: new Set<() => void>(),
loading: [] as (() => void)[],
@@ -35,7 +36,7 @@ function fakeHost() {
};
const playbacks = new Playbacks({
now: () => host.time,
latency: () => 0,
latency: () => host.latency,
playing: (voice) => host.sounding.has(voice),
stop: (voices) => {
for (const voice of voices) host.sounding.delete(voice);
@@ -48,6 +49,7 @@ function fakeHost() {
level: (id, gain, at, seconds) => host.levels.push({ id, gain, at, seconds }),
after: (at, then) => host.timers.push({ at, then }),
closed: (id) => host.closed.push(id),
...(baked && { playBaked: () => 77 }),
});
const run = (seconds: number) => {
const end = host.time + seconds;
@@ -113,6 +115,63 @@ describe('the songs a game plays', () => {
expect(playbacks.current()[0]?.clips[0]).toMatchObject({ sound: 'drum', bus: 'track' });
});
it('tells how far into it is heard: from its first note, the output latency taken off (#338)', async () => {
const { host, playbacks, loaded } = fakeHost();
host.latency = 0.02;
playbacks.start(5, 'song:a', song([]), '');
expect(playbacks.times()).toEqual([
{
id: 5,
name: 'song:a',
heard: null,
tempo: 120,
from: 0,
loop: { start: BAR, end: 2 * BAR },
length: 2 * BAR,
},
]);
host.time = 1;
await loaded();
// Its first note is heard LEAD after it starts, and the speakers are 0.02 s behind the clock.
host.time = 3;
expect(playbacks.times()[0]!.heard).toBeCloseTo(3 - 0.02 - (1 + LEAD), 9);
});
it('asks no clock while nothing plays: that would make the audio context before a gesture (#338)', () => {
let asked = 0;
const playbacks = new Playbacks({
now: () => (asked++, 0),
latency: () => 0,
playing: () => false,
stop: () => {},
wake: () => () => {},
prepare: () => Promise.resolve(),
level: () => {},
after: () => {},
});
expect(playbacks.times()).toEqual([]);
expect(asked).toBe(0);
});
it('tells how far a baked one is heard the same way, from where its bake is heard (#338)', async () => {
const { host, playbacks, loaded } = fakeHost(true);
const bake = { file: 'b.ogg', gain: 1, playback: { end: null, until: 0, round: null } };
playbacks.start(5, 'song:a', song([], { bake }), '');
host.time = 1;
await loaded();
host.time = 2.5;
expect(playbacks.current()[0]?.baked).toBe(true);
expect(playbacks.times()[0]!.heard).toBeCloseTo(2.5 - (1 + LEAD), 9);
});
it('starts where it was asked to, as far further in as it waited on its files (#418)', async () => {
const { host, playbacks, loaded } = fakeHost();
playbacks.start(5, 'composition:c', song([], { from: 960 }), '');
host.time = 0.5;
await loaded();
expect(playbacks.times()[0]!.from).toBe(960 + 480);
});
it('rises from silence to its volume over its fade in, from its first note on', async () => {
const { host, playbacks, loaded } = fakeHost();
playbacks.start(5, 'song:a', song([], { volume: 0.5, fadeIn: 3 }), '');