Files
adminandClaude Opus 5.5 b5e071ca68 Let the agent read what is heard in the running game (#436)
The agent has no ears. It could start music and sounds, but not tell
whether anything reached the speakers: a song without an instrument, a
bus at volume 0 or a browser that holds the audio clock all looked the
same from outside.

- get_runtime has a section audio. Under levels it gives the peak and
  RMS in dB of the master, each bus of the mix, each pattern, song or
  composition playing and the tracks of a song. Below -90 dB it says
  silent.
- The levels come from a tap of their own after the level and pan of
  each bus, an AnalyserNode per side as for the meters of the Song tab.
  It looks back about 0.7 s, so the gap between two notes is not
  silence. A pause holds the values, which fits play with until.
- Under voices it names what sounds now: a sound by its asset, a note
  of an instrument with its note name, each with its bus and object.
  Next to them planned, playing, waiting (bytes not here yet) and
  failed.
- Under clock it gives the state of the audio clock, with a hint when
  it stands still while the game runs. The state is told as it was
  before a pause, else the pause would hide that case.
- The buses are tapped only while the agent is switched on.
- The note now travels with a voice from the Start of a pattern or song
  to the SoundLibrary, so a voice can be named by it.
- tools.json and the relay's instructions tell the agent to check under
  audio that music and sounds are really heard.

Found on the user's project: started right after F5, before a click in
the page, the browser kept the audio clock suspended for the whole run.
Nothing was heard and the notes stayed planned. That is what clock now
shows.

Tests:
- Unit: sound-library (taps, levels, voices, clock), agent-play
  (audioView). Pattern and song tests expect the note in their timing.
- 9 counter-checks, 8 red. Not dropping a tap with its bus stays green,
  as the bus is cut off and gone anyway.
- e2e 71 (melektaus-atx): 28/28. New 71h4: a sound on the master and
  one on a bus at volume 0, both voices named, the master loud, the bus
  silent, the values held in the pause.
- Gate (melektaus-atx): CHECK EXIT 0, 327 files, 5037 tests.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-18 22:14:07 +02:00

475 lines
20 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { describe, expect, it } from 'vitest';
import { CreateAssetCommand } from '../../src/core/commands/assetCommands';
import { emptySharedState } from '../../src/core/graph';
import { findScene } from '../../src/core/model/Project';
import { runTool, type ToolHost, type ToolResult } from '../../src/editor/agent/agentTools';
import type { AgentView } from '../../src/editor/agent/agentView';
import { frameWithin } from '../../src/core/time';
import { type PlayDriver, pointOf, sendInput } from '../../src/editor/agent/playInput';
import { cameraHint, NO_CAMERA_HINT } from '../../src/editor/agent/runtimeView';
import type { HeardSound } from '../../src/viewport/SoundLibrary';
import { STALLED_HINT } from '../../src/editor/agent/audioView';
import {
graphEditor,
makeBehavior,
makeSpriteScene,
place,
type SceneFixture,
startWorld,
type StartedWorld,
} from '../helpers';
/** The agent plays its own game (#429, stage 4): keys and clicks in, positions and console out. */
/** A box at (200, 100) that walks left while ← is held, logs "jump" on Space and "clicked" when clicked. */
function game(withCamera = false) {
const f = makeSpriteScene('Box');
const behavior = makeBehavior(f, 'Logic', ['score']);
const box = place(f, f.spriteId, { x: 200, y: 100 });
place(f, behavior.id, { x: 200, y: 100 }, box);
if (withCamera) {
const camera = new CreateAssetCommand('camera', 'Cam');
f.stack.execute(camera);
place(f, camera.asset.id, { x: 0, y: 0 });
}
const edit = graphEditor(f, behavior);
const held = edit.addNode('OnKeyHeld', { key: 'ArrowLeft' });
const move = edit.addNode('Move', { dx: -100 });
const owner = edit.addNode('Owner');
edit.connect(held, 'exec', move, 'exec');
edit.connect(held, 'dt', move, 'dt');
edit.connect(owner, 'object', move, 'object');
const jump = edit.addNode('OnKeyDown', { key: ' ' });
edit.connect(jump, 'exec', edit.addNode('Log', { message: 'jump' }), 'exec');
const click = edit.addNode('OnClick');
edit.connect(click, 'exec', edit.addNode('Log', { message: 'clicked' }), 'exec');
const started = startWorld(f, { behaviors: true });
return { f, box, started, ...driverFor(started) };
}
/**
* Each frame is a 60th of a second of game time and 16 ms of wall time; `frozen` holds the game.
* A pause asked for at a time cuts the frame that reaches it, as `PlaySession` does.
*/
function driverFor(started: StartedWorld) {
const state = {
running: true,
paused: false,
hidden: false,
frozen: false,
wall: 0,
pausing: null as number | null,
};
const driver: PlayDriver = {
run: () => (state.running ? { world: started.world, input: started.input } : null),
paused: () => state.paused,
hidden: () => state.hidden,
frame: () => {
state.wall += 16;
if (state.frozen || state.paused) return Promise.resolve();
const left = state.pausing === null ? Infinity : state.pausing - started.world.time;
started.frame(frameWithin(1 / 60, 1, left));
if (state.pausing !== null && started.world.time >= state.pausing - 1e-9) {
state.paused = true;
state.pausing = null;
}
return Promise.resolve();
},
now: () => state.wall,
pause: () => {
state.paused = true;
},
pauseAt: (time) => {
state.pausing = time;
},
};
return { driver, state };
}
function hostFor(f: SceneFixture, started: StartedWorld, driver: PlayDriver, running = true): ToolHost {
return {
view: () => ({ play: 'playing' }) as AgentView,
screenshot: () => Promise.resolve({ data: '', width: 0, height: 0 }),
project: () => f.project,
execute: () => null,
confirmDeletes: () => Promise.resolve(null),
open: () => null,
select: () => null,
play: () => null,
undo: () => null,
redo: () => null,
game: driver,
runtime: () =>
running
? {
world: started.world,
project: f.project,
shared: emptySharedState(),
heldKeys: started.input.heldKeys,
paused: false,
}
: null,
};
}
const text = (result: ToolResult) =>
result.content.map((item) => (item.type === 'text' ? item.text : '')).join('\n');
const boxX = (started: StartedWorld, id: string) => started.world.findByObjectId(id)!.x;
describe('play_input', () => {
it('holds a key for game seconds, and the object walks that far', async () => {
const { box, started, driver } = game();
const outcome = await sendInput([{ key: 'ArrowLeft', hold: 0.5 }], driver, 25_000);
expect(outcome.seconds).toBeGreaterThanOrEqual(0.5);
expect(outcome.seconds).toBeLessThan(0.5 + 2 / 60);
// −100 per second for the frames the key was down; the frame that lets go moves no more.
expect(boxX(started, box)).toBeCloseTo(200 - 100 * outcome.seconds + 100 / 60, 0);
expect(outcome.held).toEqual([]);
});
it('taps a key for one frame: a key-down event runs once, and "Space" is the space bar', async () => {
const { started, driver } = game();
const outcome = await sendInput([{ key: 'Space' }], driver, 25_000);
expect(started.world.log).toEqual(['jump']);
expect(outcome.log).toEqual(['jump']);
expect(outcome.frames).toBe(2);
expect(outcome.held).toEqual([]);
});
it('keeps a key down across calls until it is let go', async () => {
const { box, started, driver } = game();
const first = await sendInput([{ key: 'ArrowLeft', keep: 'down' }, { wait: 0.25 }], driver, 25_000);
expect(first.held).toEqual(['ArrowLeft']);
const walked = boxX(started, box);
expect(walked).toBeLessThan(180);
const second = await sendInput(
[{ wait: 0.25 }, { key: 'ArrowLeft', keep: 'up' }, { wait: 0.25 }],
driver,
25_000,
);
expect(second.held).toEqual([]);
expect(boxX(started, box)).toBeLessThan(walked - 20);
expect(boxX(started, box)).toBeGreaterThan(walked - 30);
});
it('clicks an object by its id and tells whether the click landed on it', async () => {
const { box, started, driver } = game();
const outcome = await sendInput([{ mouse: 'click', at: { object: box } }], driver, 25_000);
expect(outcome.hits).toEqual([{ step: 1, object: box, hit: true }]);
expect(started.world.log).toEqual(['clicked']);
expect(outcome.mouse).toMatchObject({ x: 200, y: 100, down: false });
});
it('clicks an object by its name as well', async () => {
const { started, driver } = game();
const outcome = await sendInput([{ mouse: 'click', at: { object: 'Box' } }], driver, 25_000);
expect(outcome.hits).toEqual([{ step: 1, object: 'Box', hit: true }]);
expect(started.world.log).toEqual(['clicked']);
});
it('a click beside the object does not reach it', async () => {
const { started, driver } = game();
await sendInput([{ mouse: 'click', at: { x: 0, y: 0 } }], driver, 25_000);
expect(started.world.log).toEqual([]);
});
it('gives up when the game time stands still, and lets go of the key it held', async () => {
const { started, driver, state } = game();
state.frozen = true;
await expect(sendInput([{ key: 'ArrowLeft', hold: 1 }], driver, 1000)).rejects.toThrow(
/Step 1: the game time stood still/,
);
expect([...started.input.heldKeys]).toEqual([]);
});
it('turns the steps down before anything runs: not playing, paused, a tab in the background, a step doing two things', async () => {
const { f, started, driver, state } = game();
const host = hostFor(f, started, driver);
const run = (steps: object[]) => runTool('play_input', { steps }, host);
const result = await run([{ key: 'a', mouse: 'click' }]);
expect(result.isError).toBe(true);
expect(text(result)).toBe('Step 1 has to do one thing: key, mouse, wheel, wait or pause');
expect(text(await run([{ pause: true }, { wait: 1 }]))).toBe(
'Step 1: {"pause": true} ends the steps, it comes last',
);
expect(text(await run([{ wait: 10 }, { wait: 10 }, { wait: 10 }, { wait: 1 }]))).toMatch(/at most 30 s/);
state.hidden = true;
expect(text(await run([{ key: 'a' }]))).toMatch(/in the background/);
state.hidden = false;
state.paused = true;
expect(text(await run([{ key: 'a' }]))).toMatch(/paused/);
state.running = false;
expect(text(await run([{ key: 'a' }]))).toMatch(/not running/);
expect(started.world.time).toBe(0);
});
it('tells the relay when a call runs long', async () => {
const { f, started, driver } = game();
let told = 0;
const host = { ...hostFor(f, started, driver), takesLong: () => told++ };
await runTool('play_input', { steps: [{ wait: 1 }] }, host);
expect(told).toBe(0);
await runTool('play_input', { steps: [{ wait: 6 }] }, host);
expect(told).toBe(1);
});
});
describe('pausing at a game time (#435)', () => {
it('play with until holds the game at that time exactly, not at the frame after', async () => {
const { f, started, driver, state } = game();
const host = hostFor(f, started, driver);
const result = await runTool('play', { action: 'play', until: 0.505 }, host);
expect(result.isError).toBeUndefined();
expect(result.structuredContent).toEqual({ time: 0.505, paused: true });
expect(text(result)).toBe(`Paused at 0.505 s of game time. ${NO_CAMERA_HINT}`);
expect(started.world.time).toBeCloseTo(0.505, 9);
expect(state).toMatchObject({ paused: true, pausing: null });
// Paused, it holds: a frame later the time is the same.
await driver.frame();
expect(started.world.time).toBeCloseTo(0.505, 9);
});
it('resume and pause go on from where the game is', async () => {
const { f, started, driver, state } = game();
const host = hostFor(f, started, driver);
await runTool('play', { action: 'pause', until: 0.25 }, host);
expect(started.world.time).toBeCloseTo(0.25, 9);
// The host's play is a stand-in here: resuming is the test's.
const resuming = { ...host, play: () => ((state.paused = false), null) };
const resumed = await runTool('play', { action: 'resume', until: 1 }, resuming);
expect(resumed.structuredContent).toEqual({ time: 1, paused: true });
expect(started.world.time).toBeCloseTo(1, 9);
});
it('turns down what cannot be: stop, a time gone by or too far ahead, paused or running already', async () => {
const { f, started, driver, state } = game();
const host = hostFor(f, started, driver);
const play = async (args: Record<string, unknown>) => text(await runTool('play', args, host));
expect(await play({ action: 'stop', until: 1 })).toBe('until goes with play, resume or pause');
expect(await play({ action: 'pause', until: 31 })).toMatch(/31 s ahead; at most 30 s/);
expect(await play({ action: 'resume', until: 1 })).toMatch(/running already: pause it with until/);
await runTool('play', { action: 'pause', until: 0.5 }, host);
expect(await play({ action: 'pause', until: 1 })).toMatch(/paused already: resume it with until/);
expect(await play({ action: 'resume', until: 0.5 })).toBe(
'The game is at 0.5 s already: until has to lie ahead',
);
state.paused = false;
state.hidden = true;
expect(await play({ action: 'pause', until: 1 })).toMatch(/in the background/);
expect(started.world.time).toBeCloseTo(0.5, 9);
});
it('gives up when the time stands still, and calls the pause off', async () => {
const { f, started, driver, state } = game();
state.frozen = true;
const result = await runTool('play', { action: 'pause', until: 1 }, hostFor(f, started, driver));
expect(result.isError).toBe(true);
expect(text(result)).toMatch(/^The game time stood still at 0 s/);
expect(state.pausing).toBeNull();
});
it('says so when something paused the game before the time', async () => {
const { f, started, driver, state } = game();
const host = hostFor(f, started, driver);
const frame = driver.frame;
driver.frame = async () => {
await frame();
if (started.world.time > 0.2) state.paused = true;
};
const result = await runTool('play', { action: 'pause', until: 1 }, host);
expect(text(result)).toMatch(/^Paused at 0\.21\d* s of game time, before 1 s: the user or a Break node/);
expect(state.pausing).toBeNull();
});
it('a last pause step holds the game after a wait to its time exactly, and after a tap at once', async () => {
const { started, driver, state } = game();
const waited = await sendInput([{ wait: 0.31 }, { pause: true }], driver, 25_000);
expect(waited).toMatchObject({ time: 0.31, paused: true });
expect(started.world.time).toBeCloseTo(0.31, 9);
expect(state.pausing).toBeNull();
state.paused = false;
const tapped = await sendInput([{ key: 'Space' }, { pause: true }], driver, 25_000);
expect(tapped).toMatchObject({ paused: true, frames: 2 });
expect(started.world.time).toBeCloseTo(0.31 + 2 / 60, 9);
state.paused = false;
const plain = await sendInput([{ wait: 0.1 }], driver, 25_000);
expect(plain.paused).toBe(false);
});
});
describe('get_runtime', () => {
it('lists every object with its position, a behavior with its variables, and the console', async () => {
const { f, box, started, driver } = game();
const host = hostFor(f, started, driver);
await runTool('play_input', { steps: [{ key: 'ArrowLeft', hold: 0.2 }, { key: 'Space' }] }, host);
const result = await runTool('get_runtime', {}, host);
const view = result.structuredContent as {
objects: { id: string; name: string; type: string; x: number; variables?: object; parent?: string }[];
console: string[];
hint?: string;
};
const row = view.objects.find((o) => o.id === box)!;
expect(row).toMatchObject({ name: 'Box', type: 'sprite' });
expect(row.x).toBeLessThan(185);
expect(row).not.toHaveProperty('props');
const behavior = view.objects.find((o) => o.type === 'behavior');
expect(behavior).toMatchObject({ parent: box });
// Its own variables only, not the switches every behavior has (enabled, runInEditor, ...).
expect(behavior?.variables).toEqual({ score: 0 });
expect(view.console).toEqual(['jump']);
expect(view.hint).toBe(NO_CAMERA_HINT);
});
it('gives all properties of the objects named, and says which name is none', async () => {
const { f, started, driver } = game();
const host = hostFor(f, started, driver);
const result = await runTool('get_runtime', { objects: ['Box'] }, host);
const view = result.structuredContent as { objects: { props: Record<string, unknown> }[] };
expect(view.objects).toHaveLength(1);
expect(view.objects[0]!.props).toHaveProperty('width');
expect(text(await runTool('get_runtime', { objects: ['Nobody'] }, host))).toBe(
'There is no object Nobody in the running game',
);
expect(text(await runTool('get_runtime', {}, hostFor(f, started, driver, false)))).toMatch(/not running/);
});
it('says what is heard: levels in dB by bus name, voices and playings by asset, the rest left out (#436)', async () => {
const { f, started, driver } = game();
const made = (type: 'sound' | 'instrument' | 'song' | 'bus', name: string) => {
const command = new CreateAssetCommand(type, name);
f.stack.execute(command);
return command.asset;
};
const sound = made('sound', 'Jump');
if (sound.type === 'sound') sound.file = 'jump.wav';
const instrument = made('instrument', 'Lead');
const song = made('song', 'Theme');
const track = song.type === 'song' ? song.tracks[0]! : null;
const bus = made('bus', 'Music');
const owner = started.world.entities.find((entity) => entity.type === 'behavior')!.id;
const trackBus = `play:7:song:${song.id}:${track!.id}`;
const quiet = { released: false, ringing: false, startsIn: 0 };
const level = (peak: number, rms: number): Pick<HeardSound['buses'][number], 'peak' | 'rms'> => ({
peak: [peak, peak / 2],
rms: [rms, rms],
});
const heard: HeardSound = {
window: 32768 / 48000,
clock: { state: 'running', time: 12.34567, held: false },
voices: [
{ id: 1, file: 'jump.wav', owner, bus: '', ...quiet },
{
id: 2,
file: `instrument:${instrument.id}`,
owner: '',
bus: trackBus,
note: 64,
...quiet,
released: true,
},
{
id: 3,
file: `instrument:${instrument.id}`,
owner: '',
bus: trackBus,
note: 65,
...quiet,
startsIn: 0.2,
},
],
waiting: [{ id: 4, file: 'boom.wav', owner: '', bus: `mix:${bus.id}` }],
failed: ['boom.wav'],
playings: [{ id: 7, name: `song:${song.id}`, owner, playing: true, fading: null, baked: false }],
buses: [
{ name: trackBus, ...level(0.5, 0.1) },
{ name: `${trackBus}:#items`, ...level(0.5, 0.1) },
{ name: 'mix', ...level(1, 0.01) },
{ name: `mix:${bus.id}`, ...level(0, 0) },
{ name: 'play:7', ...level(0.5, 0.1) },
],
};
const host = hostFor(f, started, driver);
const facts = host.runtime!()!;
host.runtime = () => ({ ...facts, audio: heard });
const view = (await runTool('get_runtime', {}, host)).structuredContent as { audio: object };
expect(view.audio).toEqual({
clock: { state: 'running', time: 12.346 },
window: 0.68,
levels: [
{ bus: 'master', peak: 0, rms: -40 },
{ bus: `song Theme, track ${track!.name}`, peak: -6, rms: -20 },
{ bus: 'Music', silent: true },
{ bus: 'song Theme', peak: -6, rms: -20 },
],
voices: [
{ sound: 'Jump', bus: 'master', by: 'Box' },
{ instrument: 'Lead', note: 'E4', bus: `song Theme, track ${track!.name}`, released: true },
],
planned: 1,
playing: [{ song: 'Theme', by: 'Box' }],
waiting: [{ file: 'boom.wav', bus: 'Music' }],
failed: [{ file: 'boom.wav' }],
});
// A clock the browser holds while the game runs means nothing is heard, paused since or not.
const audioWith = async (clock: HeardSound['clock']) => {
host.runtime = () => ({ ...facts, audio: { ...heard, clock } });
return ((await runTool('get_runtime', {}, host)).structuredContent as { audio: { hint?: string } })
.audio;
};
expect(await audioWith({ state: 'running', time: 3, held: true })).not.toHaveProperty('hint');
expect((await audioWith({ state: 'suspended', time: 0, held: false })).hint).toBe(STALLED_HINT);
expect((await audioWith({ state: 'suspended', time: 0, held: true })).hint).toBe(STALLED_HINT);
});
});
describe('the camera', () => {
it('puts the mouse on its picture: inside its frame, outside beyond it', () => {
const { started } = game(true);
expect(pointOf(started.world, { x: 100, y: -50 })).toMatchObject({ inside: true });
expect(pointOf(started.world, { x: 5000, y: 0 })).toMatchObject({ inside: false });
expect(pointOf(game(false).started.world, { x: 0, y: 0 })).toMatchObject({ inside: false });
});
it('is missed before Play and while playing, and play says so', async () => {
const without = game(false);
const scene = findScene(without.f.project, without.f.sceneId);
expect(cameraHint(without.f.project, scene, null)).toMatch(/\("Level"\) has no camera/);
expect(cameraHint(without.f.project, scene, without.started.world)).toBe(NO_CAMERA_HINT);
const played = await runTool(
'play',
{ action: 'play' },
hostFor(without.f, without.started, without.driver),
);
expect(text(played)).toBe(`play: done. ${NO_CAMERA_HINT}`);
const withCamera = game(true);
const sceneWith = findScene(withCamera.f.project, withCamera.f.sceneId);
expect(cameraHint(withCamera.f.project, sceneWith, null)).toBeNull();
expect(cameraHint(withCamera.f.project, sceneWith, withCamera.started.world)).toBeNull();
});
});