Files
feuerwerk/tests/core/pattern-play.test.ts
adminandClaude Opus 5.5 b5e071ca68 Let the agent read what is heard in the running game (#436)
The agent has no ears. It could start music and sounds, but not tell
whether anything reached the speakers: a song without an instrument, a
bus at volume 0 or a browser that holds the audio clock all looked the
same from outside.

- get_runtime has a section audio. Under levels it gives the peak and
  RMS in dB of the master, each bus of the mix, each pattern, song or
  composition playing and the tracks of a song. Below -90 dB it says
  silent.
- The levels come from a tap of their own after the level and pan of
  each bus, an AnalyserNode per side as for the meters of the Song tab.
  It looks back about 0.7 s, so the gap between two notes is not
  silence. A pause holds the values, which fits play with until.
- Under voices it names what sounds now: a sound by its asset, a note
  of an instrument with its note name, each with its bus and object.
  Next to them planned, playing, waiting (bytes not here yet) and
  failed.
- Under clock it gives the state of the audio clock, with a hint when
  it stands still while the game runs. The state is told as it was
  before a pause, else the pause would hide that case.
- The buses are tapped only while the agent is switched on.
- The note now travels with a voice from the Start of a pattern or song
  to the SoundLibrary, so a voice can be named by it.
- tools.json and the relay's instructions tell the agent to check under
  audio that music and sounds are really heard.

Found on the user's project: started right after F5, before a click in
the page, the browser kept the audio clock suspended for the whole run.
Nothing was heard and the notes stayed planned. That is what clock now
shows.

Tests:
- Unit: sound-library (taps, levels, voices, clock), agent-play
  (audioView). Pattern and song tests expect the note in their timing.
- 9 counter-checks, 8 red. Not dropping a tap with its bus stays green,
  as the bus is cut off and gone anyway.
- e2e 71 (melektaus-atx): 28/28. New 71h4: a sound on the master and
  one on a bus at volume 0, both voices named, the master loud, the bus
  silent, the values held in the pause.
- Gate (melektaus-atx): CHECK EXIT 0, 327 files, 5037 tests.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-18 22:14:07 +02:00

275 lines
11 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* A game plays a pattern through Play pattern and ends it through Stop pattern: each playing on a
* scheduler of its own, numbered, owned and stopped like a voice, and gone by itself once played
* to its end.
*/
import { describe, expect, it } from 'vitest';
import {
CreateAssetCommand,
CreateSoundCommand,
SetAssetPropertyCommand,
} from '../../src/core/commands/assetCommands';
import { SetPatternNotesCommand } from '../../src/core/commands/patternCommands';
import type { InstrumentAsset } from '../../src/core/model/Asset';
import { Playbacks, type SequencePlay } from '../../src/core/music/playbacks';
import { INTERVAL, LEAD, type ScheduledNote } from '../../src/core/music/Scheduler';
import { type AudioSink, SILENT_AUDIO } from '../../src/runtime/systems/Audio';
import { graphEditor, makeBehavior, makeSpriteScene, place, startWorld } from '../helpers';
const FILE = `${'a1'.repeat(32)}.wav`;
const Q = 480;
/** An audio clock moved by hand, every voice sounding until it is stopped or let go. */
function fakeHost() {
const host = {
time: 0,
sounding: new Set<number>(),
stopped: [] as { voices: number[]; letRing: boolean }[],
pumps: new Set<() => void>(),
loading: [] as (() => void)[],
levels: [] as { id: number; gain: number; at: number; seconds: number }[],
timers: [] as { at: number; then: () => void }[],
closed: [] as number[],
};
const playbacks = new Playbacks({
now: () => host.time,
latency: () => 0,
playing: (voice) => host.sounding.has(voice),
stop: (voices, letRing = false) => {
host.stopped.push({ voices: [...voices], letRing });
for (const voice of voices) host.sounding.delete(voice);
},
wake: (pump) => {
host.pumps.add(pump);
return () => host.pumps.delete(pump);
},
prepare: () => new Promise<void>((resolve) => host.loading.push(resolve)),
level: (id, gain, at, seconds) => host.levels.push({ id, gain, at, seconds }),
after: (at, then) => host.timers.push({ at, then }),
closed: (id) => host.closed.push(id),
});
const run = (seconds: number) => {
const end = host.time + seconds;
while (host.time + INTERVAL <= end + 1e-9) {
host.time += INTERVAL;
for (const pump of [...host.pumps]) pump();
for (const timer of host.timers.filter((t) => t.at <= host.time + 1e-9)) {
host.timers.splice(host.timers.indexOf(timer), 1);
timer.then();
}
}
};
const loaded = async () => {
for (const resolve of host.loading.splice(0)) resolve();
await Promise.resolve();
};
return { host, playbacks, run, loaded };
}
/** A bar of four quarters at 120 BPM: two seconds long. Its voices sound in `sounding` until stopped. */
function bar(struck: ScheduledNote[], loop: boolean, sounding = new Set<number>()): SequencePlay {
let next = 100;
return {
sequence: {
notes: [0, 1, 2, 3].map((i) => ({
tick: i * Q,
length: Q / 2,
note: 60 + i,
velocity: 1,
instrument: 'piano',
})),
},
tempo: 120,
length: 4 * Q,
loop: loop ? { start: 0, end: 4 * Q } : null,
needs: { layers: [], files: [] },
strike: (note) => {
struck.push(note);
sounding.add(next);
return next++;
},
};
}
describe('the patterns a game plays', () => {
it('waits for its files, then plays the whole pattern once and is gone when its last note is', async () => {
const { host, playbacks, run, loaded } = fakeHost();
const struck: ScheduledNote[] = [];
playbacks.start(7, 'pattern:a', bar(struck, false, host.sounding), 'logic');
run(0.5);
expect(struck).toEqual([]);
await loaded();
run(3);
expect(struck.map((n) => n.note)).toEqual([60, 61, 62, 63]);
struck.forEach((n, i) => expect(n.at - struck[0]!.at).toBeCloseTo(i * 0.5, 9));
expect(host.pumps.size).toBe(0);
expect(playbacks.has(7)).toBe(true);
host.sounding.clear();
expect(playbacks.has(7)).toBe(false);
});
it('with loop runs round and round, 2 s to the round, until it is stopped', async () => {
const { host, playbacks, run, loaded } = fakeHost();
const struck: ScheduledNote[] = [];
playbacks.start(7, 'pattern:a', bar(struck, true, host.sounding), '');
await loaded();
run(5);
expect(struck.map((n) => n.note).slice(0, 10)).toEqual([60, 61, 62, 63, 60, 61, 62, 63, 60, 61]);
expect(struck[4]!.at - struck[0]!.at).toBeCloseTo(2, 9);
expect(struck[0]!.at).toBeCloseTo(LEAD, 9);
playbacks.stopId(7, true);
expect(host.stopped.at(-1)?.letRing).toBe(true);
expect([playbacks.has(7), host.pumps.size]).toEqual([false, 0]);
});
it('never starts when it was stopped while its files were loading', async () => {
const { host, playbacks, loaded, run } = fakeHost();
const struck: ScheduledNote[] = [];
playbacks.start(7, 'pattern:a', bar(struck, true), 'logic');
playbacks.stopOwner('logic');
await loaded();
run(1);
expect([struck.length, host.pumps.size, playbacks.has(7)]).toEqual([0, 0, false]);
});
it('stops by name, every one for no name, by owner – and an owner of nobody stops nothing', async () => {
const { playbacks, loaded } = fakeHost();
playbacks.start(1, 'pattern:a', bar([], true), 'x');
playbacks.start(2, 'pattern:a', bar([], true), '');
playbacks.start(3, 'pattern:b', bar([], true), 'y');
playbacks.start(4, 'pattern:c', bar([], true), '');
await loaded();
playbacks.stopOwner('');
expect(playbacks.current().map((p) => p.id)).toEqual([1, 2, 3, 4]);
playbacks.stopNamed('pattern:a');
expect(playbacks.current().map((p) => p.id)).toEqual([3, 4]);
playbacks.stopOwner('y');
expect(playbacks.current().map((p) => p.id)).toEqual([4]);
playbacks.stopNamed('');
expect(playbacks.current()).toEqual([]);
});
});
/** A scene whose behavior plays a pattern of C4 and E4 on start, through an instrument of one sample. */
function playingPattern(params: Record<string, string | number | boolean> = {}, instrument = true) {
const f = makeSpriteScene('Box');
const sound = new CreateSoundCommand('Blip', FILE);
f.stack.execute(sound);
const ins = new CreateAssetCommand('instrument', 'Lead');
f.stack.execute(ins);
const lead = ins.asset as InstrumentAsset;
lead.graph.nodes.find((n) => n.type === 'InstrumentSample')!.params.sound = sound.asset.id;
const pattern = new CreateAssetCommand('pattern', 'Tune');
f.stack.execute(pattern);
const id = pattern.asset.id;
if (instrument) f.stack.execute(new SetAssetPropertyCommand(id, 'instrument', lead.id));
f.stack.execute(new SetAssetPropertyCommand(id, 'tempo', 90));
f.stack.execute(new SetAssetPropertyCommand(id, 'bars', 2));
f.stack.execute(
new SetPatternNotesCommand(id, [
{ id: 'n1', tick: 0, length: Q, note: 60, velocity: 0.5 },
{ id: 'n2', tick: Q, length: Q, note: 64, velocity: 1 },
]),
);
const behavior = makeBehavior(f, 'Logic');
const box = place(f, f.spriteId, { x: 0, y: 0 });
place(f, behavior.id, { x: 0, y: 0 }, box);
const edit = graphEditor(f, behavior);
const start = edit.addNode('OnStart');
const play = edit.addNode('PlayPattern', { pattern: id, volume: 0.5, ...params });
edit.connect(start, 'exec', play, 'exec');
return { ...f, patternId: id, lead, behavior, edit, play };
}
function recordingAudio() {
const sink = {
...SILENT_AUDIO,
sequences: [] as { name: string; play: SequencePlay; owner: string }[],
notes: [] as {
name: string;
volume: number;
owner: string;
duration: number;
at?: number;
tempo?: number;
}[],
stoppedSequences: [] as string[],
stoppedVoices: [] as number[],
playSequence: (name: string, play: SequencePlay, owner: string) => {
sink.sequences.push({ name, play, owner });
return 1000 + sink.sequences.length;
},
playNote: ((name, _layers, _params, volume, owner, duration, timing = {}) => {
sink.notes.push({ name, volume, owner, duration, ...timing });
return 1;
}) satisfies AudioSink['playNote'],
stopSequence: (name: string, letRing = false) =>
sink.stoppedSequences.push(`${name}${letRing ? ' (let ring)' : ''}`),
stopVoice: (voice: number) => sink.stoppedVoices.push(voice),
};
return sink;
}
describe('Play pattern', () => {
it('hands the whole pattern to the player at its tempo, owned by the behavior, looping by default', () => {
const f = playingPattern();
const audio = recordingAudio();
const { world, frame } = startWorld(f, { behaviors: true, audio });
frame();
const logic = world.entities.find((e) => e.name === 'Logic')!;
const [asked] = audio.sequences;
expect(asked?.name).toBe(`pattern:${f.patternId}`);
expect(asked?.owner).toBe(logic.id);
expect(asked?.play).toMatchObject({ tempo: 90, length: 2 * 4 * Q, loop: { start: 0, end: 2 * 4 * Q } });
expect(asked?.play.sequence.notes.map((n) => [n.tick, n.note, n.instrument])).toEqual([
[0, 60, f.lead.id],
[Q, 64, f.lead.id],
]);
expect(asked?.play.needs.layers.length).toBe(2);
});
it('strikes each note through the instrument at the node’s volume, owned like the playing', () => {
const f = playingPattern({ detached: true, loop: false });
const audio = recordingAudio();
startWorld(f, { behaviors: true, audio }).frame();
const [asked] = audio.sequences;
expect([asked?.owner, asked?.play.loop]).toEqual(['', null]);
asked!.play.strike({ at: 3, duration: 0.25, note: 60, velocity: 0.5, instrument: f.lead.id, tempo: 90 });
expect(audio.notes).toEqual([
{
name: `instrument:${f.lead.id}`,
volume: 0.25,
owner: '',
duration: 0.25,
at: 3,
tempo: 90,
note: 60,
},
]);
});
it('plays nothing and hands out 0 for a pattern without an instrument', () => {
const f = playingPattern({}, false);
const audio = recordingAudio();
startWorld(f, { behaviors: true, audio }).frame();
expect(audio.sequences).toEqual([]);
});
});
describe('Stop pattern', () => {
it('stops one pattern by its name, every pattern for none, and one playing by its number', () => {
const f = playingPattern();
const one = f.edit.addNode('StopPattern', { pattern: f.patternId, letRing: true });
const all = f.edit.addNode('StopPattern', { pattern: '' });
const byNumber = f.edit.addNode('StopPattern', { pattern: f.patternId });
f.edit.connect(f.play, 'exec', one, 'exec');
f.edit.connect(one, 'exec', all, 'exec');
f.edit.connect(all, 'exec', byNumber, 'exec');
f.edit.connect(f.play, 'playback', byNumber, 'playback');
const audio = recordingAudio();
startWorld(f, { behaviors: true, audio }).frame();
expect(audio.stoppedSequences).toEqual([`pattern:${f.patternId} (let ring)`, 'pattern:']);
expect(audio.stoppedVoices).toEqual([1001]);
});
});