From 209951169509d1515376ed20e730029a94e8bf90 Mon Sep 17 00:00:00 2001 From: MelekTaus Date: Mon, 19 Oct 2026 10:54:51 +0200 Subject: [PATCH] Let the agent set automation lanes of a song track (#337) Stage 4 of #337. Lanes could only be drawn in the song tab. Now the agent can set them in one step and sees which parameters a lane of a track can move. - New command set_song_lane. The target is named as it is stored: volume, pan or param:. All points come at once, each with tick, value and ease, from the same eases as animation keys, curve assets included. - Points on one tick keep the order they come in, which makes a jump. points [] removes a volume or pan lane, remove a parameter lane. - It refuses with a clear message: a value outside the range, an unknown ease, a missing curve asset, a range for volume or pan, and a new lane for a parameter nothing on the track has. The message then names the parameters there are. - Without min and max a new parameter lane gets the range the dialog in the song tab offers. - get_asset names under params of each song track the parameters a lane can move, with their default. The lanes were already in it. Live in the user's project: get_asset named both parameters of their two lanes (frequency, env) with defaults 20000 and 0.5. Playing to 3 s and 4.2 s, both tracks were heard, the audio clock running. Tests: - agent-music 13/13 (2 new), agent and MCP tests 129/129 - 6 counter-probes, all compile, all red - gate on atx: CHECK EXIT 0, 330 files, 5082 tests Co-Authored-By: Claude Opus 5.5 --- src/editor/agent/catalog/animation.ts | 2 +- src/editor/agent/catalog/music.ts | 147 ++++++++++++++++++++++++++ src/editor/agent/readTools.ts | 11 +- src/editor/agent/tools.json | 2 +- tests/editor/agent-music.test.ts | 135 ++++++++++++++++++++++- 5 files changed, 292 insertions(+), 5 deletions(-) diff --git a/src/editor/agent/catalog/animation.ts b/src/editor/agent/catalog/animation.ts index c9f73f4c..04e8f935 100644 --- a/src/editor/agent/catalog/animation.ts +++ b/src/editor/agent/catalog/animation.ts @@ -65,7 +65,7 @@ const keyOf = (args: Args, name: string): AnimationKey => { }; /** Every easing a key can name, but the curve ones: those take the id of a Curve asset. */ -const EASINGS = EASING_SHAPES.flatMap(({ value }) => +export const EASINGS = EASING_SHAPES.flatMap(({ value }) => value === 'linear' || value === 'step' ? [value] : value === 'curve' diff --git a/src/editor/agent/catalog/music.ts b/src/editor/agent/catalog/music.ts index da3859a8..b7eddd02 100644 --- a/src/editor/agent/catalog/music.ts +++ b/src/editor/agent/catalog/music.ts @@ -20,6 +20,7 @@ import { TONE_PRESETS, } from '../../../core/instrument/chipPresets'; import { HIGHEST_NOTE, LOWEST_NOTE, noteName } from '../../../core/instrument/notes'; +import { easingOf, isKeyInterpolation } from '../../../core/model/animation'; import type { PatternAsset, SongAsset } from '../../../core/model/Asset'; import { audioSeconds } from '../../../core/model/composition'; import type { PropertyBag } from '../../../core/model/Properties'; @@ -31,6 +32,15 @@ import { ROUTING_ID, } from '../../../core/model/Project'; import { clipsOf, findSound, hasClips } from '../../../core/model/sound'; +import { + type AutomationLane, + type AutomationPoint, + heldRange, + isAutomationTarget, + type LaneKind, + laneRange, + paramOf, +} from '../../../core/music/automation'; import { DEFAULT_VELOCITY, patternLength, type PatternNote } from '../../../core/music/pattern'; import { SEND_TAPS } from '../../../core/music/routing'; import { @@ -47,8 +57,11 @@ import { TRACK_MIN_DB, } from '../../../core/music/song'; import { TICKS_PER_QUARTER, ticksOfSeconds, ticksPerBar } from '../../../core/music/time'; +import { defaultParamRange, trackParams } from '../../../core/music/trackParams'; import { NO_FX } from '../../../core/properties/assetFx'; +import { withLanePoints, withoutLane, withParamLane } from '../../song/laneEdit'; import type { ArgSchema } from '../argCheck'; +import { EASINGS } from './animation'; import { type Args, type CatalogCommand, @@ -280,6 +293,81 @@ export const MUSIC_COMMANDS: readonly CatalogCommand[] = [ }, shows: theTab, }, + { + name: 'set_song_lane', + description: + 'Sets an automation lane of a song track: its volume (in dB), its pan (-1 left to 1 right) or a parameter ' + + 'changing over the song along points; the points given replace those the lane had. From each point the ' + + 'value runs to the next along its ease; before the first point it holds the first value, after the last ' + + 'the last one; two points on one tick jump there. While the lane of the volume or the pan has a point, it ' + + "rules that value in place of the track's own; with no point that lane goes and the track's value rules " + + 'again. A parameter lane moves every parameter of that name on the track: of its sound FX, of the sound FX ' + + 'of its items, of the instruments its patterns play and of their sound FX (get_asset lists them under ' + + "params of the track). It keeps a range its points lie in, round the parameter's default where none is " + + 'given, and stays without a point until remove. A note or clip starts at the value there, so what a ' + + 'voice reads only when it starts changes from the next one.', + args: object( + { + asset: SONG, + track: TRACK, + target: id( + '"volume", "pan", or "param:" for a parameter, as get_asset lists lanes under automation.', + ), + points: { + type: 'array', + items: object( + { + tick: TICK, + value: number( + 'In the units of the lane: dB, -1 to 1 for the pan, within the range of a parameter.', + ), + ease: id( + `How it runs to the next point: ${EASINGS.join(', ')}, or "curve:"; ` + + 'linear where left out.', + ), + }, + ['tick', 'value'], + ), + description: 'All points of the lane, in any order; those on one tick in the order given.', + }, + min: number('A parameter lane: the lowest value it holds.'), + max: number('A parameter lane: the highest value it holds.'), + remove: flag("Take the lane away, a parameter's range with it."), + }, + ['asset', 'track', 'target'], + ), + build: (args, { project }) => { + const song = requireSong(project, text(args, 'asset')); + const track = requireTrack(song, text(args, 'track')); + const target = args.target; + if (!isAutomationTarget(target)) + throw new CommandError( + `${JSON.stringify(target)} is no lane – give "volume", "pan" or "param:"`, + ); + const had = track.automation?.find((lane) => lane.target === target); + const given = [args.points, args.min, args.max].some((value) => value !== undefined); + let changed: SongTrack; + let label: string; + if (args.remove === true) { + if (given) throw new CommandError('remove takes no points and no range'); + if (!had) throw new CommandError(`${track.name} has no ${target} lane`); + changed = withoutLane(track, target); + label = 'Remove automation lane'; + } else { + const name = paramOf(target); + const range = name === null ? noRange(args) : paramRange(project, song, track, name, had, args); + const kind: LaneKind = { target, ...range }; + const points = args.points === undefined ? null : lanePointsOf(project, kind, args.points as Args[]); + if (name === null && !points) throw new CommandError('Give the points, [] to take the lane away'); + changed = range ? withParamLane(track, name!, range) : track; + if (points) changed = withLanePoints(changed, target, points); + label = had ? 'Set automation lane' : 'Add automation lane'; + } + const tracks = song.tracks.map((t) => (t.id === track.id ? changed : t)); + return { command: new SetSongCommand(song.id, { tracks }, label) }; + }, + shows: theTab, + }, { name: 'move_song_track', description: 'Moves a track of a song to another place in its list.', @@ -537,6 +625,65 @@ function itemLength(project: Project, song: SongAsset, kind: SongItem['kind'], p return Math.max(1, Math.round(ticksOfSeconds(seconds, songTempo(song.props)))); } +/** Refuses a range given to the lane of the volume or the pan: theirs is fixed. */ +function noRange(args: Args): null { + if (args.min !== undefined || args.max !== undefined) + throw new CommandError('Only a parameter lane has a range – the volume and the pan have theirs'); + return null; +} + +/** + * The range of a parameter's lane: the one given, else the one it has, else one round the default of + * the parameter. A new lane is refused for a parameter nothing on the track has. + */ +function paramRange( + project: Project, + song: SongAsset, + track: SongTrack, + name: string, + had: AutomationLane | undefined, + args: Args, +): { min: number; max: number } { + const offered = trackParams(project, song, track); + const param = offered.find((p) => p.name === name); + if (!had && !param) + throw new CommandError( + offered.length === 0 + ? `Nothing on ${track.name} has a parameter to move` + : `Nothing on ${track.name} has a parameter ${name} – it has ${offered.map((p) => p.name).join(', ')}`, + ); + const given = [args.min, args.max].filter((value) => value !== undefined).length; + if (given === 1) throw new CommandError('Give min and max, or neither'); + if (given === 2) { + const range = heldRange(args.min, args.max); + if (!range) throw new CommandError('max has to be above min'); + return range; + } + return had ? laneRange(had) : defaultParamRange(param!.default); +} + +/** + * The points the agent wrote, refused where a value or an easing is wrong. The song puts them in the + * order of their ticks, those on one tick as they came. + */ +function lanePointsOf(project: Project, kind: LaneKind, raw: readonly Args[]): AutomationPoint[] { + const { min, max } = laneRange(kind); + return raw.map((point): AutomationPoint => { + const tick = point.tick as number; + const value = within(point.value as number, min, max, `value at tick ${tick}`); + const ease = point.ease; + if (ease === undefined || ease === 'linear') return { tick, value }; + if (!isKeyInterpolation(ease)) + throw new CommandError( + `${JSON.stringify(ease)} is no easing – give ${EASINGS.join(', ')} or "curve:"`, + ); + const { curve } = easingOf(ease); + if (curve !== undefined && !findAssetOf(project, curve, 'curve')) + throw new CommandError(`${curve} is no Curve asset`); + return { tick, value, ease }; + }); +} + function requireTrack(song: SongAsset, trackId: string): SongTrack { const track = song.tracks.find((t) => t.id === trackId); if (!track) throw new CommandError(`${song.name} has no track ${trackId}`); diff --git a/src/editor/agent/readTools.ts b/src/editor/agent/readTools.ts index adbc1d84..3c5bdda1 100644 --- a/src/editor/agent/readTools.ts +++ b/src/editor/agent/readTools.ts @@ -17,6 +17,7 @@ import { instrumentSourcesOf, soundingNotes } from '../../core/instrument/compil import { noteName } from '../../core/instrument/notes'; import { patternInstrument, patternLength, patternSequence } from '../../core/music/pattern'; import { itemNotes, songLength } from '../../core/music/song'; +import { trackParams } from '../../core/music/trackParams'; import { TICKS_PER_QUARTER } from '../../core/music/time'; import { CLIP_ROWS } from '../../core/properties/sounds'; import { VIDEO_CLIP_ROWS } from '../../core/properties/videos'; @@ -100,7 +101,15 @@ export function getAsset(project: Project, assetId: string, readCel?: CelLookup) plain.notes = asset.notes.map((note) => ({ ...note, name: noteName(note.note) })); plain.ticks = { perQuarter: TICKS_PER_QUARTER, length: patternLength(asset) }; } - if (asset.type === 'song') plain.ticks = { perQuarter: TICKS_PER_QUARTER, length: songLength(asset) }; + if (asset.type === 'song') { + plain.ticks = { perQuarter: TICKS_PER_QUARTER, length: songLength(asset) }; + // What a lane of each track can move (#337): set_song_lane takes them as "param:". + plain.tracks = asset.tracks.map((track, i) => { + const params = trackParams(project, asset, track); + const stored = (plain.tracks as object[])[i]; + return params.length > 0 ? { ...stored, params } : stored; + }); + } const silent = silentParts(project, asset); if (silent.length > 0) plain.silent = silent; if (asset.type === 'sound' || asset.type === 'video') { diff --git a/src/editor/agent/tools.json b/src/editor/agent/tools.json index 38ba749e..b33f77bc 100644 --- a/src/editor/agent/tools.json +++ b/src/editor/agent/tools.json @@ -59,7 +59,7 @@ { "name": "get_asset", "title": "Read an asset", - "description": "One asset as stored: its property values, the rows of its properties (key, kind, default), the objects of a scene or container with their names and positions (the content of a placed container too, marked fromContainer), what the rows of an animation may aim at (targets), the cells of a small tilemap, the pixels of small pixel art as rows of palette indices (colors as \"#rrggbb\"), the notes of a pattern with their names and how many ticks it is long, the rows a clip of a sound or video has (clipProperties), how many seconds a loaded sound lasts, and under silent what of a pattern, song or composition is not heard: all of it without an instrument, or the notes its instrument does not sound. Under usedBy: every asset naming it or one of its frames or clips, with the objects, nodes, items or keys in there that do (none for a property of the asset itself), and the project's start scene, master FX or Master routing; empty when nothing uses it. A graph is only counted here; read it with get_graph.", + "description": "One asset as stored: its property values, the rows of its properties (key, kind, default), the objects of a scene or container with their names and positions (the content of a placed container too, marked fromContainer), what the rows of an animation may aim at (targets), the cells of a small tilemap, the pixels of small pixel art as rows of palette indices (colors as \"#rrggbb\"), the notes of a pattern with their names and how many ticks it is long, on each track of a song its automation lanes and the parameters a lane can move (params), the rows a clip of a sound or video has (clipProperties), how many seconds a loaded sound lasts, and under silent what of a pattern, song or composition is not heard: all of it without an instrument, or the notes its instrument does not sound. Under usedBy: every asset naming it or one of its frames or clips, with the objects, nodes, items or keys in there that do (none for a property of the asset itself), and the project's start scene, master FX or Master routing; empty when nothing uses it. A graph is only counted here; read it with get_graph.", "inputSchema": { "type": "object", "properties": { "asset": { "type": "string", "description": "Id of the asset." } }, diff --git a/tests/editor/agent-music.test.ts b/tests/editor/agent-music.test.ts index 59cde8f1..ebff2c58 100644 --- a/tests/editor/agent-music.test.ts +++ b/tests/editor/agent-music.test.ts @@ -1,12 +1,19 @@ import { afterEach, describe, expect, it } from 'vitest'; import { CreateSoundCommand } from '../../src/core/commands/assetCommands'; -import type { CompositionAsset, PatternAsset, SongAsset, SoundAsset } from '../../src/core/model/Asset'; +import type { + BehaviorAsset, + CompositionAsset, + PatternAsset, + SongAsset, + SoundAsset, + SoundFxAsset, +} from '../../src/core/model/Asset'; import { setSoundMeasure } from '../../src/core/model/composition'; import { ROUTING_ID } from '../../src/core/model/Project'; import { settleRouting } from '../../src/core/music/projectRouting'; import { TO_COMPOSITION_BUS } from '../../src/core/properties/buses'; import { runTool } from '../../src/editor/agent/agentTools'; -import { makeSpriteScene, type SceneFixture } from '../helpers'; +import { graphEditor, makeSpriteScene, type SceneFixture } from '../helpers'; import { hostFor, text } from './agentTestHost'; /** The commands of #429 stage 5d: patterns, songs, sends, clips of sounds and compositions. */ @@ -217,6 +224,130 @@ describe('songs', () => { expect(noFx.text).toContain('is no Sound FX asset'); }); + it('sets the volume lane of a track in tick order with its easings, refuses what is out of range, and clears it with []', async () => { + const fixture = makeSpriteScene(); + const { Theme, Bend } = await made(fixture, { Theme: 'song', Bend: 'curve' }); + const song = () => byId(fixture, Theme); + const track = song().tracks[0]!.id; + const set = await run( + fixture, + [ + { + command: 'set_song_lane', + args: { + asset: Theme, + track, + target: 'volume', + points: [ + { tick: 1920, value: -24, ease: 'step' }, + { tick: 0, value: -6, ease: 'sine-in' }, + { tick: 1920, value: 0, ease: `curve:${Bend}` }, + { tick: 3840, value: -12, ease: 'linear' }, + ], + }, + }, + ], + true, + ); + expect(set.result.isError).toBeUndefined(); + expect(set.shown).toEqual([{ kind: 'asset', asset: Theme, open: true }]); + // In tick order, two on one tick as given (a jump); linear is stored as no easing. + expect(song().tracks[0]!.automation).toEqual([ + { + target: 'volume', + points: [ + { tick: 0, value: -6, ease: 'sine-in' }, + { tick: 1920, value: -24, ease: 'step' }, + { tick: 1920, value: 0, ease: `curve:${Bend}` }, + { tick: 3840, value: -12 }, + ], + }, + ]); + const lanes = (await read(fixture, Theme)).tracks as { automation?: unknown[] }[]; + expect(lanes[0]!.automation).toHaveLength(1); + + const point = (value: number, ease?: string) => [{ tick: 0, value, ...(ease && { ease }) }]; + const refused = async (target: string, points: object[], extra: object = {}) => + ( + await run(fixture, [ + { command: 'set_song_lane', args: { asset: Theme, track, target, points, ...extra } }, + ]) + ).text; + expect(await refused('volume', point(13))).toContain('value at tick 0 has to be from -60 to 12'); + expect(await refused('pan', point(-2))).toContain('from -1 to 1'); + expect(await refused('volume', point(0, 'wobbly'))).toContain('"wobbly" is no easing'); + expect(await refused('volume', point(0, 'curve:nope'))).toContain('nope is no Curve asset'); + expect(await refused('loudness', point(0))).toContain('is no lane'); + expect(await refused('volume', point(0), { min: 0, max: 1 })).toContain('Only a parameter lane'); + expect(song().tracks[0]!.automation![0]!.points).toHaveLength(4); + const nothing = await run(fixture, [ + { command: 'set_song_lane', args: { asset: Theme, track, target: 'pan' } }, + ]); + expect(nothing.text).toContain('Give the points'); + + const cleared = await run(fixture, [ + { command: 'set_song_lane', args: { asset: Theme, track, target: 'volume', points: [] } }, + ]); + expect(cleared.result.isError).toBeUndefined(); + expect(song().tracks[0]!.automation).toBeUndefined(); + }); + + it('adds a parameter lane only for a parameter on the track, round its default, and get_asset lists what a lane can move', async () => { + const fixture = makeSpriteScene(); + const { Theme, Filter } = await made(fixture, { Theme: 'song', Filter: 'soundfx' }); + const fx = byId(fixture, Filter); + const edit = graphEditor(fixture, fx as unknown as BehaviorAsset); + const input = edit.addNode('SoundFxInput'); + const gain = edit.addNode('SoundFxGain'); + const output = edit.addNode('SoundFxOutput'); + input.ports = [{ key: 'cut', kind: 'number', default: 800 }]; + edit.connect(input, 'out', gain, 'in'); + edit.connect(input, 'cut', gain, 'gain'); + edit.connect(gain, 'out', output, 'in'); + const song = () => byId(fixture, Theme); + const track = song().tracks[0]!.id; + const lane = (args: object) => ({ + command: 'set_song_lane', + args: { asset: Theme, track, target: 'param:cut', ...args }, + }); + + const none = await run(fixture, [lane({})]); + expect(none.text).toContain('Nothing on Track 1 has a parameter to move'); + expect(((await read(fixture, Theme)).tracks as object[])[0]).not.toHaveProperty('params'); + + await run(fixture, [{ command: 'change_song_track', args: { asset: Theme, track, fx: Filter } }]); + expect(((await read(fixture, Theme)).tracks as { params?: object }[])[0]!.params).toEqual([ + { name: 'cut', default: 800 }, + ]); + const other = await run(fixture, [{ ...lane({}), args: { asset: Theme, track, target: 'param:tone' } }]); + expect(other.text).toContain('Nothing on Track 1 has a parameter tone – it has cut'); + + // Without a range it gets one round the default, and keeps it without a point. + const added = await run(fixture, [lane({})]); + expect(added.result.isError).toBeUndefined(); + expect(song().tracks[0]!.automation).toEqual([{ target: 'param:cut', points: [], min: 0, max: 2000 }]); + const pointed = await run(fixture, [lane({ points: [{ tick: 0, value: 1500 }] })]); + expect(pointed.result.isError).toBeUndefined(); + const over = await run(fixture, [lane({ points: [{ tick: 0, value: 2500 }] })]); + expect(over.text).toContain('from 0 to 2000'); + // A narrower range holds the points it had. + await run(fixture, [lane({ min: 100, max: 1000 })]); + expect(song().tracks[0]!.automation).toEqual([ + { target: 'param:cut', points: [{ tick: 0, value: 1000 }], min: 100, max: 1000 }, + ]); + expect((await run(fixture, [lane({ min: 5 })])).text).toContain('Give min and max'); + expect((await run(fixture, [lane({ min: 5, max: 5 })])).text).toContain('max has to be above min'); + + // A lane there is stays settable when its parameter is gone from the track. + await run(fixture, [{ command: 'change_song_track', args: { asset: Theme, track, fx: '' } }]); + expect((await run(fixture, [lane({ min: 0, max: 10 })])).result.isError).toBeUndefined(); + expect((await run(fixture, [lane({ remove: true, min: 0 })])).text).toContain('remove takes no points'); + const removed = await run(fixture, [lane({ remove: true })]); + expect(removed.result.isError).toBeUndefined(); + expect(song().tracks[0]!.automation).toBeUndefined(); + expect((await run(fixture, [lane({ remove: true })])).text).toContain('Track 1 has no param:cut lane'); + }); + it('sets the send of a song link and of a master routing link, where a sound only sends post', async () => { const fixture = makeSpriteScene(); const { Theme } = await made(fixture, { Theme: 'song' });