diff --git a/src/editor/agent/catalog/composition.ts b/src/editor/agent/catalog/composition.ts new file mode 100644 index 00000000..7067469a --- /dev/null +++ b/src/editor/agent/catalog/composition.ts @@ -0,0 +1,387 @@ +import { CommandError } from '../../../core/commands/Command'; +import { + addCompositionItem, + removeCompositionTrack, + removeFromComposition, + SetCompositionCommand, +} from '../../../core/commands/compositionCommands'; +import { addPictureItem, droppedPicture } from '../../../core/commands/compositionSprites'; +import { requireComposition } from '../../../core/commands/require'; +import { newId } from '../../../core/ids'; +import type { + CompositionAsset, + CompositionClip, + CompositionItem, + CompositionTrack, +} from '../../../core/model/Asset'; +import { + audioSeconds, + clipLength, + compositionFps, + fittedLength, + itemSounds, + newCompositionTrack, +} from '../../../core/model/composition'; +import { + audioKindOf, + clipSpans, + droppedLength, + fadedItem, + movedWithClips, + resizedItem, +} from '../../../core/model/compositionEdit'; +import { placementRejection } from '../../../core/model/placement'; +import { findAsset, findAssetOf, type Project } from '../../../core/model/Project'; +import { FADE_CURVES, type FadeCurve } from '../../../core/model/sound'; +import { MASTER_BUS, TO_COMPOSITION_BUS } from '../../../core/properties/buses'; +import { ITEM_MIN_DB, SOUND_ITEM_MAX_DB } from '../../../core/music/song'; +import type { ArgSchema } from '../argCheck'; +import { + type Args, + type CatalogCommand, + flag, + id, + ids, + number, + numberOf, + object, + oneOf, + text, + theTab, + whole, +} from './shared'; + +const COMPOSITION = id('Id of the composition asset.'); +const TRACK = id('Id of the track, as get_asset lists it under tracks.'); +const ITEM = id('Id of the item, as get_asset lists it under items.'); +const CLIP = id('Id of the clip, as get_asset lists it under clips.'); +const INDEX = whole('Place in the list of tracks, from 0 (the top one, drawn in front).', 0); +const FRAME = whole('Frames from the start, at the fps of the composition.', 0); +const CURVE = oneOf( + FADE_CURVES.map((c) => c.value), + 'The shape of the fade.', +); +const BUS: ArgSchema = { + type: 'string', + description: + `Where an audio item is heard: "${TO_COMPOSITION_BUS}" into the bus of the composition, "" along the routing ` + + `of what it plays, "${MASTER_BUS}" the master, else the id of a bus.`, +}; + +/** The tracks, items and animation clips of Composition assets. */ +export const COMPOSITION_COMMANDS: readonly CatalogCommand[] = [ + { + name: 'add_composition_track', + description: + 'Adds a track to a composition, at the bottom or at an index. The top track is drawn in front.', + args: object({ asset: COMPOSITION, name: id('Its name.'), at: INDEX }, ['asset']), + build: (args, { project }) => { + const comp = requireComposition(project, text(args, 'asset')); + const track = newCompositionTrack(comp.tracks.length + 1); + if (typeof args.name === 'string') track.name = args.name; + const tracks = [...comp.tracks]; + tracks.splice(Math.min((args.at as number | undefined) ?? tracks.length, tracks.length), 0, track); + return { command: new SetCompositionCommand(comp.id, { tracks }, 'Add track'), made: { id: track.id } }; + }, + shows: theTab, + }, + { + name: 'change_composition_track', + description: 'Renames a track of a composition.', + args: object({ asset: COMPOSITION, track: TRACK, name: id('Its name.') }, ['asset', 'track', 'name']), + build: (args, { project }) => { + const comp = requireComposition(project, text(args, 'asset')); + const track = requireTrack(comp, text(args, 'track')); + const tracks = comp.tracks.map((t): CompositionTrack => + t === track ? { ...t, name: text(args, 'name') } : t, + ); + return { command: new SetCompositionCommand(comp.id, { tracks }, 'Rename track') }; + }, + shows: theTab, + }, + { + name: 'move_composition_track', + description: 'Moves a track of a composition to another place in its list; what lies on it goes along.', + args: object({ asset: COMPOSITION, track: TRACK, to: INDEX }, ['asset', 'track', 'to']), + build: (args, { project }) => { + const comp = requireComposition(project, text(args, 'asset')); + const track = requireTrack(comp, text(args, 'track')); + const tracks = comp.tracks.filter((t) => t !== track); + tracks.splice(Math.min(numberOf(args, 'to'), tracks.length), 0, track); + return { command: new SetCompositionCommand(comp.id, { tracks }, 'Move track') }; + }, + shows: theTab, + }, + { + name: 'remove_composition_track', + description: + 'Removes a track of a composition with its items, their objects and its clips; the last track stays.', + args: object({ asset: COMPOSITION, track: TRACK }, ['asset', 'track']), + build: (args, { project }) => { + const comp = requireComposition(project, text(args, 'asset')); + const track = requireTrack(comp, text(args, 'track')); + if (comp.tracks.length <= 1) throw new CommandError('A composition keeps at least one track'); + return { command: removeCompositionTrack(project, comp.id, track.id) }; + }, + shows: theTab, + }, + { + name: 'add_composition_item', + description: + 'Lays an asset on a track of a composition, from a frame. A placeable asset (a sprite, a container, a text, ...) ' + + 'becomes an object of the composition, living as long as the item; a texture, pixel art, vector art, a video or a ' + + 'clip of one gets a sprite of the composition to wear it; a sound, a sound clip, a pattern or a song becomes an ' + + 'audio item. Without a length it is as long as its video or what it plays, else 3 s, and keeps following it. ' + + "An item's object is moved and set with the commands of area objects, in = the composition. Items of a track may " + + 'overlap. Animations go on a track with add_composition_clip.', + args: object( + { + asset: COMPOSITION, + track: TRACK, + place: id('Id of the asset (or video or sound clip) to lay on the track.'), + start: FRAME, + length: whole('Frames it lasts.', 1), + offset: number('Seconds into its video or what it plays it begins.'), + }, + ['asset', 'track', 'place', 'start'], + ), + build: (args, { project }) => { + const comp = requireComposition(project, text(args, 'asset')); + requireTrack(comp, text(args, 'track')); + const placed = text(args, 'place'); + const fits = typeof args.length !== 'number'; + const item: Omit = { + track: text(args, 'track'), + start: numberOf(args, 'start'), + length: 1, + offset: (args.offset as number | undefined) ?? 0, + ...(fits ? { fitMedia: true } : {}), + }; + const audio = audioKindOf(project, placed); + if (audio) { + if ( + fits && + audio === 'sound' && + audioSeconds(project, { kind: audio, asset: placed, volume: 0, bus: '' }) <= 0 + ) + throw new CommandError( + `The length of ${findAsset(project, placed)?.name ?? placed} is not known yet – give a length ` + + `(${compositionFps(comp)} frames a second)`, + ); + const added: CompositionItem = { + ...item, + id: newId(), + audio: { kind: audio, asset: placed, volume: 0, bus: TO_COMPOSITION_BUS }, + }; + added.length = fits ? fittedLength(project, comp, added) : numberOf(args, 'length'); + const items = [...comp.items, added]; + return { command: new SetCompositionCommand(comp.id, { items }, 'Add item'), made: { id: added.id } }; + } + const asset = findAsset(project, placed); + if (asset?.type === 'animation') + throw new CommandError('An animation goes on a track as a clip – use add_composition_clip'); + const picture = droppedPicture(project, placed); + if (!picture) { + if (!asset) throw new CommandError(`There is no asset ${placed}`); + const rejection = placementRejection(project, comp, asset); + if (rejection) throw new CommandError(rejection); + } + item.length = fits ? droppedLength(project, comp, placed) : numberOf(args, 'length'); + const { command, objectId } = picture + ? addPictureItem(project, comp.id, picture, item) + : addCompositionItem(project, comp.id, placed, item); + return { command, made: { id: objectId } }; + }, + shows: theTab, + }, + { + name: 'change_composition_item', + description: + 'Changes an item of a composition. Moving it (start, track) takes the clips that belong to it along. A length ' + + 'given is its own from then on; fitMedia true hands it back to its video or what it plays. Fades are in frames; ' + + 'volume and bus are for audio items. Give only what changes.', + args: object( + { + asset: COMPOSITION, + item: ITEM, + track: TRACK, + start: FRAME, + length: whole('Frames it lasts.', 1), + fitMedia: flag('Its length follows its video or what it plays again.'), + offset: number('Seconds into its video or what it plays it begins; below 0 it waits that long.'), + fadeIn: whole('Frames its sound rises from silence at its start; 0 for none.', 0), + fadeOut: whole('Frames its sound falls back to silence at its end; 0 for none.', 0), + curveIn: CURVE, + curveOut: CURVE, + volume: { type: 'number', description: `In dB, from ${ITEM_MIN_DB} to ${SOUND_ITEM_MAX_DB}.` }, + bus: BUS, + }, + ['asset', 'item'], + ), + build: (args, { project }) => { + const comp = requireComposition(project, text(args, 'asset')); + const item = requireItem(comp, text(args, 'item')); + let parts: { items: CompositionItem[]; clips?: CompositionClip[] } = { items: [...comp.items] }; + if (typeof args.start === 'number' || typeof args.track === 'string') { + const trackIds = comp.tracks.map((t) => t.id); + const by = + typeof args.track === 'string' + ? trackIds.indexOf(requireTrack(comp, args.track).id) - trackIds.indexOf(item.track) + : 0; + const moved = movedWithClips( + comp.items, + comp.clips, + new Set([item.id]), + trackIds, + typeof args.start === 'number' ? args.start - item.start : 0, + by, + (clip) => Math.max(1, clipLength(project, comp, clip)), + ); + if (!moved) + throw new CommandError('A clip that goes along with the item would lie over another clip there'); + parts = moved; + } + const at = parts.items.findIndex((i) => i.id === item.id); + parts.items[at] = changedItem(project, comp, parts.items[at]!, args); + return { command: new SetCompositionCommand(comp.id, parts, 'Change item') }; + }, + shows: theTab, + }, + { + name: 'remove_composition_items', + description: + 'Removes items of a composition with their objects and the clips that belong to them, and clips.', + args: object({ asset: COMPOSITION, items: ids('Ids of the items.'), clips: ids('Ids of the clips.') }, [ + 'asset', + ]), + build: (args, { project }) => { + const comp = requireComposition(project, text(args, 'asset')); + const items = (args.items ?? []) as string[]; + const clips = (args.clips ?? []) as string[]; + if (items.length + clips.length === 0) throw new CommandError('Name items or clips to remove'); + for (const itemId of items) requireItem(comp, itemId); + for (const clipId of clips) requireClip(comp, clipId); + return { command: removeFromComposition(project, comp.id, items, clips) }; + }, + shows: theTab, + }, + { + name: 'add_composition_clip', + description: + 'Lays an animation on a track of a composition as a clip, one pass long, from a frame. It plays on every item ' + + 'of its track it overlaps; its rows find their targets in the item as Play animation finds them. Clips of a ' + + 'track never overlap.', + args: object( + { asset: COMPOSITION, track: TRACK, animation: id('Id of the animation asset.'), start: FRAME }, + ['asset', 'track', 'animation', 'start'], + ), + build: (args, { project }) => { + const comp = requireComposition(project, text(args, 'asset')); + requireTrack(comp, text(args, 'track')); + const animation = findAssetOf(project, text(args, 'animation'), 'animation'); + if (!animation) throw new CommandError(`${text(args, 'animation')} is no animation asset`); + const clip: CompositionClip = { + id: newId(), + track: text(args, 'track'), + animation: animation.id, + start: numberOf(args, 'start'), + }; + if (clipLength(project, comp, clip) === 0) + throw new CommandError(`${animation.name} has no frames to play here`); + requireRoom(project, comp, clip); + return { + command: new SetCompositionCommand(comp.id, { clips: [...comp.clips, clip] }, 'Add clip'), + made: { id: clip.id }, + }; + }, + shows: theTab, + }, + { + name: 'change_composition_clip', + description: 'Moves a clip of a composition to another frame or track, where it fits whole.', + args: object({ asset: COMPOSITION, clip: CLIP, track: TRACK, start: FRAME }, ['asset', 'clip']), + build: (args, { project }) => { + const comp = requireComposition(project, text(args, 'asset')); + const clip = requireClip(comp, text(args, 'clip')); + const moved: CompositionClip = { + ...clip, + ...(typeof args.track === 'string' ? { track: requireTrack(comp, args.track).id } : {}), + ...(typeof args.start === 'number' ? { start: args.start } : {}), + }; + requireRoom(project, comp, moved); + const clips = comp.clips.map((c) => (c === clip ? moved : c)); + return { command: new SetCompositionCommand(comp.id, { clips }, 'Move clip') }; + }, + shows: theTab, + }, +]; + +/** The item with what `args` change besides its place, refused where it asks for what the item cannot have. */ +function changedItem( + project: Project, + comp: CompositionAsset, + item: CompositionItem, + args: Args, +): CompositionItem { + let out = item; + if (typeof args.offset === 'number') out = { ...out, offset: args.offset }; + if (typeof args.length === 'number') out = resizedItem(out, args.length); + if (args.fitMedia === true) out = { ...out, fitMedia: true, length: fittedLength(project, comp, out) }; + if (args.fitMedia === false) out = resizedItem(out, out.length); + const fades = ['fadeIn', 'fadeOut', 'curveIn', 'curveOut'].some((key) => args[key] !== undefined); + if (fades && !itemSounds(project, comp, out)) + throw new CommandError( + 'Only an item with a sound fades: an audio item, or one whose object wears a video', + ); + if (typeof args.fadeIn === 'number') out = fadedItem(out, 'in', out.start + args.fadeIn); + if (typeof args.fadeOut === 'number') out = fadedItem(out, 'out', out.start + out.length - args.fadeOut); + if (typeof args.curveIn === 'string') out = { ...out, curveIn: args.curveIn as FadeCurve }; + if (typeof args.curveOut === 'string') out = { ...out, curveOut: args.curveOut as FadeCurve }; + if (args.volume !== undefined || args.bus !== undefined) { + if (!out.audio) throw new CommandError('Only an audio item has a volume and a bus'); + const audio = { ...out.audio }; + if (typeof args.volume === 'number') { + if (args.volume < ITEM_MIN_DB || args.volume > SOUND_ITEM_MAX_DB) + throw new CommandError(`The volume has to be from ${ITEM_MIN_DB} to ${SOUND_ITEM_MAX_DB}`); + audio.volume = args.volume; + } + if (typeof args.bus === 'string') audio.bus = busOf(project, args.bus); + out = { ...out, audio }; + } + return out; +} + +function busOf(project: Project, bus: string): string { + if (bus === '' || bus === TO_COMPOSITION_BUS || bus === MASTER_BUS) return bus; + if (findAsset(project, bus)?.type !== 'bus') throw new CommandError(`${bus} is no bus asset`); + return bus; +} + +/** Refuses a clip that would lie over another clip of its track. */ +function requireRoom(project: Project, comp: CompositionAsset, clip: CompositionClip): void { + const length = Math.max(1, clipLength(project, comp, clip)); + const taken = clipSpans(comp.clips, clip.track, (c) => clipLength(project, comp, c), new Set([clip.id])); + const hit = taken.find((span) => span.start < clip.start + length && clip.start < span.start + span.length); + if (hit) + throw new CommandError( + `The clip (${length} frames from ${clip.start}) would lie over the one at ${hit.start}–${hit.start + hit.length}`, + ); +} + +function requireTrack(comp: CompositionAsset, trackId: string): CompositionTrack { + const track = comp.tracks.find((t) => t.id === trackId); + if (!track) throw new CommandError(`${comp.name} has no track ${trackId}`); + return track; +} + +function requireItem(comp: CompositionAsset, itemId: string): CompositionItem { + const item = comp.items.find((i) => i.id === itemId); + if (!item) throw new CommandError(`${comp.name} has no item ${itemId}`); + return item; +} + +function requireClip(comp: CompositionAsset, clipId: string): CompositionClip { + const clip = comp.clips.find((c) => c.id === clipId); + if (!clip) throw new CommandError(`${comp.name} has no clip ${clipId}`); + return clip; +} diff --git a/src/editor/agent/catalog/music.ts b/src/editor/agent/catalog/music.ts new file mode 100644 index 00000000..5f42229c --- /dev/null +++ b/src/editor/agent/catalog/music.ts @@ -0,0 +1,548 @@ +import { + AddClipCommand, + RemoveClipCommand, + RenameClipCommand, + SetClipPropsCommand, +} from '../../../core/commands/clipCommands'; +import { type Command, CommandError } from '../../../core/commands/Command'; +import { oneStep } from '../../../core/commands/CompoundCommand'; +import { SetLoopCommand, SetPatternNotesCommand } from '../../../core/commands/patternCommands'; +import { requireAsset, requireEdge, requirePattern, requireSong } from '../../../core/commands/require'; +import { SetSendCommand, SetSongCommand } from '../../../core/commands/songCommands'; +import type { SendTap } from '../../../core/graph/Graph'; +import { newId } from '../../../core/ids'; +import { HIGHEST_NOTE, LOWEST_NOTE, noteName } from '../../../core/instrument/notes'; +import type { PatternAsset, SongAsset } from '../../../core/model/Asset'; +import { audioSeconds } from '../../../core/model/composition'; +import type { PropertyBag } from '../../../core/model/Properties'; +import { + findAsset, + findAssetOf, + findGraphAsset, + type Project, + ROUTING_ID, +} from '../../../core/model/Project'; +import { clipsOf, findSound, hasClips } from '../../../core/model/sound'; +import { DEFAULT_VELOCITY, patternLength, type PatternNote } from '../../../core/music/pattern'; +import { SEND_TAPS } from '../../../core/music/routing'; +import { + itemMaxDb, + ITEM_MIN_DB, + MAX_TRANSPOSE, + newItem, + newTrack, + type SongItem, + type SongTrack, + songMeter, + songTempo, + TRACK_MAX_DB, + TRACK_MIN_DB, +} from '../../../core/music/song'; +import { TICKS_PER_QUARTER, ticksOfSeconds, ticksPerBar } from '../../../core/music/time'; +import { NO_FX } from '../../../core/properties/assetFx'; +import type { ArgSchema } from '../argCheck'; +import { + type Args, + type CatalogCommand, + flag, + id, + ids, + listOf, + number, + numberOf, + object, + oneOf, + text, + theTab, + whole, +} from './shared'; + +const PATTERN = id('Id of the pattern asset.'); +const SONG = id('Id of the song asset.'); +const TRACK = id('Id of the track, as get_asset lists it under tracks.'); +const ITEM = id('Id of the item, as get_asset lists it under items.'); +const INDEX = whole('Place in the list of tracks, from 0 (the top).', 0); +const TICK = whole(`Ticks from the start; ${TICKS_PER_QUARTER} to a quarter note.`, 0); +const NOTE: ArgSchema = { + description: 'The pitch: a MIDI number (60 is C4) or a name with sharps only, like "C4", "F#3", "A-1".', +}; +const VELOCITY: ArgSchema = { type: 'number', minimum: 0, maximum: 1, description: 'How hard, 0 to 1.' }; +const FX: ArgSchema = { type: 'string', description: 'Id of a Sound FX asset, or "" for none.' }; +const DB = (description: string): ArgSchema => ({ type: 'number', description: `${description} In dB.` }); + +/** Notes of patterns, loops, the tracks and items of songs, sends of a routing, clips of sounds and videos. */ +export const MUSIC_COMMANDS: readonly CatalogCommand[] = [ + { + name: 'add_notes', + description: + `Adds notes to a pattern; with replace, they take the place of all its notes. Time is in ticks, ${TICKS_PER_QUARTER} ` + + 'to a quarter note; how many bars long the pattern is, its meter, grid, tempo and instrument are properties. ' + + 'A note has to start before the end of the pattern.', + args: object( + { + asset: PATTERN, + notes: { + type: 'array', + minItems: 1, + items: object( + { tick: TICK, length: whole('Ticks it is held.', 1), note: NOTE, velocity: VELOCITY }, + ['tick', 'length', 'note'], + ), + description: 'The notes.', + }, + replace: flag('Take away the notes it had.'), + }, + ['asset', 'notes'], + ), + build: (args, { project }) => { + const pattern = requirePattern(project, text(args, 'asset')); + const added = (args.notes as Args[]).map((n): PatternNote => ({ + id: newId(), + tick: n.tick as number, + length: n.length as number, + note: pitchOf(n.note), + velocity: (n.velocity as number | undefined) ?? DEFAULT_VELOCITY, + })); + for (const note of added) requireInside(pattern, note); + const notes = [...(args.replace === true ? [] : pattern.notes), ...added]; + const command = new SetPatternNotesCommand(pattern.id, notes, 'Add notes'); + return added.length === 1 ? { command, made: { id: added[0]!.id } } : { command }; + }, + shows: theTab, + }, + { + name: 'change_notes', + description: + 'Changes notes of a pattern by their ids: where they start, how long, the pitch, the velocity.', + args: object( + { + asset: PATTERN, + notes: { + type: 'array', + minItems: 1, + items: object( + { + id: id('Id of the note.'), + tick: TICK, + length: whole('Ticks it is held.', 1), + note: NOTE, + velocity: VELOCITY, + }, + ['id'], + ), + description: 'Each with only what changes.', + }, + }, + ['asset', 'notes'], + ), + build: (args, { project }) => { + const pattern = requirePattern(project, text(args, 'asset')); + const changes = new Map((args.notes as Args[]).map((n) => [n.id as string, n])); + for (const noteId of changes.keys()) + if (!pattern.notes.some((n) => n.id === noteId)) + throw new CommandError(`${pattern.name} has no note ${noteId}`); + const notes = pattern.notes.map((note) => { + const change = changes.get(note.id); + if (!change) return note; + const changed: PatternNote = { + ...note, + ...(typeof change.tick === 'number' ? { tick: change.tick } : {}), + ...(typeof change.length === 'number' ? { length: change.length } : {}), + ...(change.note === undefined ? {} : { note: pitchOf(change.note) }), + ...(typeof change.velocity === 'number' ? { velocity: change.velocity } : {}), + }; + requireInside(pattern, changed); + return changed; + }); + return { command: new SetPatternNotesCommand(pattern.id, notes, 'Change notes') }; + }, + shows: theTab, + }, + { + name: 'remove_notes', + description: 'Removes notes of a pattern by their ids.', + args: object({ asset: PATTERN, notes: ids('Ids of the notes.') }, ['asset', 'notes']), + build: (args, { project }) => { + const pattern = requirePattern(project, text(args, 'asset')); + const going = new Set(listOf(args, 'notes')); + for (const noteId of going) + if (!pattern.notes.some((n) => n.id === noteId)) + throw new CommandError(`${pattern.name} has no note ${noteId}`); + const notes = pattern.notes.filter((n) => !going.has(n.id)); + return { command: new SetPatternNotesCommand(pattern.id, notes, 'Delete notes') }; + }, + shows: theTab, + }, + { + name: 'set_loop', + description: + 'Sets the loop the playhead of a pattern or a song runs round, in ticks; leave start and end out to loop the whole.', + args: object({ asset: id('Id of the pattern or song asset.'), start: TICK, end: whole('Ticks.', 1) }, [ + 'asset', + ]), + build: (args, { project }) => { + const asset = requireAsset(project, text(args, 'asset')); + if (asset.type !== 'pattern' && asset.type !== 'song') + throw new CommandError(`${asset.name} is no pattern or song`); + const given = [args.start, args.end].filter((value) => value !== undefined).length; + if (given === 1) throw new CommandError('Give start and end, or neither for the whole'); + const loop = given === 0 ? null : { start: numberOf(args, 'start'), end: numberOf(args, 'end') }; + if (loop && loop.end <= loop.start) throw new CommandError('The loop has to end after it starts'); + return { command: new SetLoopCommand(asset.type, asset.id, loop) }; + }, + shows: theTab, + }, + { + name: 'add_song_track', + description: + 'Adds a track to a song, at the bottom or at an index. Its routing gets a node for it, going to the output.', + args: object({ asset: SONG, name: id('Its name.'), at: INDEX }, ['asset']), + build: (args, { project }) => { + const song = requireSong(project, text(args, 'asset')); + const track = newTrack(song.tracks.length + 1); + if (typeof args.name === 'string') track.name = args.name; + const tracks = [...song.tracks]; + tracks.splice(Math.min((args.at as number | undefined) ?? tracks.length, tracks.length), 0, track); + return { command: new SetSongCommand(song.id, { tracks }, 'Add track'), made: { id: track.id } }; + }, + shows: theTab, + }, + { + name: 'change_song_track', + description: 'Changes a track of a song. Give only what changes.', + args: object( + { + asset: SONG, + track: TRACK, + name: id('Its name.'), + volume: DB(`From ${TRACK_MIN_DB} to ${TRACK_MAX_DB}.`), + pan: number('-1 all left, 1 all right.'), + mute: flag('Muted.'), + solo: flag('Soloed.'), + fx: { + ...FX, + description: 'Id of the Sound FX everything on the track goes through, or "" for none.', + }, + }, + ['asset', 'track'], + ), + build: (args, { project }) => { + const song = requireSong(project, text(args, 'asset')); + const track = requireTrack(song, text(args, 'track')); + const changed: SongTrack = { ...track }; + if (typeof args.name === 'string') changed.name = args.name; + if (typeof args.volume === 'number') + changed.volume = within(args.volume, TRACK_MIN_DB, TRACK_MAX_DB, 'volume'); + if (typeof args.pan === 'number') changed.pan = within(args.pan, -1, 1, 'pan'); + if (typeof args.mute === 'boolean') changed.mute = args.mute; + if (typeof args.solo === 'boolean') changed.solo = args.solo; + if (typeof args.fx === 'string') changed.fx = soundFxOf(project, args.fx); + const tracks = song.tracks.map((t) => (t.id === track.id ? changed : t)); + return { command: new SetSongCommand(song.id, { tracks }, 'Change track') }; + }, + shows: theTab, + }, + { + name: 'move_song_track', + description: 'Moves a track of a song to another place in its list.', + args: object({ asset: SONG, track: TRACK, to: INDEX }, ['asset', 'track', 'to']), + build: (args, { project }) => { + const song = requireSong(project, text(args, 'asset')); + const track = requireTrack(song, text(args, 'track')); + const tracks = song.tracks.filter((t) => t.id !== track.id); + tracks.splice(Math.min(numberOf(args, 'to'), tracks.length), 0, track); + return { command: new SetSongCommand(song.id, { tracks }, 'Move track') }; + }, + shows: theTab, + }, + { + name: 'remove_song_track', + description: 'Removes a track of a song with its items; the last track stays.', + args: object({ asset: SONG, track: TRACK }, ['asset', 'track']), + build: (args, { project }) => { + const song = requireSong(project, text(args, 'asset')); + const track = requireTrack(song, text(args, 'track')); + if (song.tracks.length <= 1) throw new CommandError('A song keeps at least one track'); + const parts = { + tracks: song.tracks.filter((t) => t.id !== track.id), + items: song.items.filter((i) => i.track !== track.id), + }; + return { command: new SetSongCommand(song.id, parts, 'Delete track') }; + }, + shows: theTab, + }, + { + name: 'add_song_item', + description: + `Lays a pattern, a sound or a clip of a sound on a track of a song. Time is in ticks of the song, ${TICKS_PER_QUARTER} ` + + 'to a quarter note at its tempo. Without a length it is as long as its pattern, or its sound at the tempo of the ' + + 'song; a pattern item repeats its pattern for as long as it lasts.', + args: object( + { + asset: SONG, + track: TRACK, + play: id('Id of the pattern, sound or sound clip.'), + tick: TICK, + ...itemFields(), + }, + ['asset', 'track', 'play', 'tick'], + ), + build: (args, { project }) => { + const song = requireSong(project, text(args, 'asset')); + requireTrack(song, text(args, 'track')); + const play = text(args, 'play'); + const kind = findAssetOf(project, play, 'pattern') + ? 'pattern' + : findSound(project, play) + ? 'sound' + : null; + if (!kind) throw new CommandError(`${play} is no pattern, sound or sound clip`); + const length = (args.length as number | undefined) ?? itemLength(project, song, kind, play); + const item = changedItem( + project, + newItem({ track: text(args, 'track'), kind, asset: play, tick: numberOf(args, 'tick'), length }), + args, + ); + return { + command: new SetSongCommand(song.id, { items: [...song.items, item] }, 'Add item'), + made: { id: item.id }, + }; + }, + shows: theTab, + }, + { + name: 'change_song_item', + description: + 'Changes an item of a song: its track, where it starts, how long, and how it plays. Give only what changes.', + args: object({ asset: SONG, item: ITEM, track: TRACK, tick: TICK, ...itemFields() }, ['asset', 'item']), + build: (args, { project }) => { + const song = requireSong(project, text(args, 'asset')); + const item = song.items.find((i) => i.id === text(args, 'item')); + if (!item) throw new CommandError(`${song.name} has no item ${text(args, 'item')}`); + if (typeof args.track === 'string') requireTrack(song, args.track); + const changed = changedItem(project, item, args); + const items = song.items.map((i) => (i.id === item.id ? changed : i)); + return { command: new SetSongCommand(song.id, { items }, 'Change item') }; + }, + shows: theTab, + }, + { + name: 'remove_song_items', + description: 'Removes items of a song.', + args: object({ asset: SONG, items: ids('Ids of the items.') }, ['asset', 'items']), + build: (args, { project }) => { + const song = requireSong(project, text(args, 'asset')); + const going = new Set(listOf(args, 'items')); + for (const itemId of going) + if (!song.items.some((i) => i.id === itemId)) + throw new CommandError(`${song.name} has no item ${itemId}`); + const items = song.items.filter((i) => !going.has(i.id)); + return { + command: new SetSongCommand(song.id, { items }, going.size === 1 ? 'Delete item' : 'Delete items'), + }; + }, + shows: theTab, + }, + { + name: 'set_send', + description: + `How loud a link of a song's routing or of the master routing (asset "${ROUTING_ID}") sends, and where it taps: ` + + 'post (after volume and pan), postFx (after the sound FX, before volume and pan) or preFx (before the sound FX). ' + + 'In the master routing only a bus or a song taps anywhere but post. Links are drawn with connect.', + args: object( + { + asset: id(`Id of the song, or "${ROUTING_ID}".`), + edge: id('Id of the link, as get_graph lists it.'), + volume: DB(`From ${TRACK_MIN_DB} to ${TRACK_MAX_DB}.`), + tap: oneOf( + SEND_TAPS.map((t) => t.value), + 'Where it taps.', + ), + }, + ['asset', 'edge'], + ), + build: (args, { project }) => { + const holder = findGraphAsset(project, text(args, 'asset')); + if (holder?.type !== 'song' && holder?.type !== 'routing') + throw new CommandError(`${text(args, 'asset')} is no song and no master routing`); + const edge = requireEdge(project, holder.id, text(args, 'edge')); + const volume = + typeof args.volume === 'number' + ? within(args.volume, TRACK_MIN_DB, TRACK_MAX_DB, 'volume') + : (edge.send?.volume ?? 0); + const tap = (args.tap as SendTap | undefined) ?? edge.send?.tap ?? 'post'; + return { command: new SetSendCommand(holder.id, edge.id, { volume, tap }) }; + }, + shows: (args) => ({ kind: 'nodes', asset: text(args, 'asset'), ids: [] }), + }, + { + name: 'add_clip', + description: + 'Adds a clip to a sound or a video: a stretch of its file with a name, which a song, a composition, a Play ' + + 'sound node or a sprite can play instead of the whole. props are its rows by key (get_asset lists them as ' + + 'clipProperties): start and end in seconds (end 0 is the end of the file), and on a sound reverse and its fades.', + args: object( + { + asset: id('Id of the sound or video asset.'), + name: id('Its name; else Clip 1, Clip 2, ...'), + props: { type: 'object', description: 'Its rows by key.' }, + }, + ['asset'], + ), + build: (args) => { + const add = new AddClipCommand(text(args, 'asset'), (args.props ?? {}) as PropertyBag); + const steps: Command[] = [add]; + if (typeof args.name === 'string') + steps.push(new RenameClipCommand(text(args, 'asset'), add.clipId, args.name)); + return { command: steps.length === 1 ? add : oneStep('Add clip', steps), made: { id: add.clipId } }; + }, + shows: theTab, + }, + { + name: 'change_clip', + description: + 'Changes a clip of a sound or a video: its name, and its rows by key. Give only what changes.', + args: object( + { + asset: id('Id of the sound or video asset.'), + clip: id('Id of the clip.'), + name: id('Its name.'), + props: { type: 'object', description: 'Rows by key.' }, + }, + ['asset', 'clip'], + ), + build: (args) => { + const steps: Command[] = []; + if (typeof args.name === 'string') + steps.push(new RenameClipCommand(text(args, 'asset'), text(args, 'clip'), args.name)); + const props = (args.props ?? {}) as PropertyBag; + if (Object.keys(props).length > 0) + steps.push(new SetClipPropsCommand(text(args, 'asset'), text(args, 'clip'), props)); + if (steps.length === 0) throw new CommandError('Name what to change: name or props'); + return { command: steps.length === 1 ? steps[0]! : oneStep('Change clip', steps) }; + }, + shows: theTab, + }, + { + name: 'remove_clip', + description: + 'Removes a clip of a sound or a video. What plays it is left naming it and reads as missing until an undo.', + args: object({ asset: id('Id of the sound or video asset.'), clip: id('Id of the clip.') }, [ + 'asset', + 'clip', + ]), + build: (args, { project }) => { + const asset = requireAsset(project, text(args, 'asset')); + if (!hasClips(asset) || !clipsOf(asset).some((c) => c.id === text(args, 'clip'))) + throw new CommandError(`${asset.name} has no clip ${text(args, 'clip')}`); + return { command: new RemoveClipCommand(asset.id, text(args, 'clip')) }; + }, + shows: theTab, + }, +]; + +/** The fields an item of a song has besides its place. */ +function itemFields(): Record { + return { + length: whole('Ticks it lasts.', 1), + offset: number('Where in what it plays it starts: ticks into a pattern, seconds into a sound.'), + volume: DB(`From ${ITEM_MIN_DB} to 0 for a pattern, to ${itemMaxDb('sound')} for a sound.`), + transpose: whole(`Semitones a pattern's notes are moved by, -${MAX_TRANSPOSE} to ${MAX_TRANSPOSE}.`), + instrument: { + type: 'string', + description: "Id of an instrument a pattern plays with instead of its own; '' for its own.", + }, + fx: { + type: 'string', + description: `Id of a Sound FX it goes through; "" for the one its asset names, "${NO_FX}" for none.`, + }, + }; +} + +/** The item with what `args` change, refused where a value is out of range or names the wrong kind of asset. */ +function changedItem(project: Project, item: SongItem, args: Args): SongItem { + const out: SongItem = { ...item }; + if (typeof args.track === 'string') out.track = args.track; + if (typeof args.tick === 'number') out.tick = args.tick; + if (typeof args.length === 'number') out.length = args.length; + if (typeof args.offset === 'number') { + if (args.offset < 0) throw new CommandError('The offset has to be 0 or more'); + out.offset = out.kind === 'pattern' ? Math.round(args.offset) : args.offset; + } + if (typeof args.volume === 'number') + out.volume = within(args.volume, ITEM_MIN_DB, itemMaxDb(out.kind), 'volume'); + if (typeof args.transpose === 'number') { + if (out.kind !== 'pattern') throw new CommandError('Only a pattern item is transposed'); + out.transpose = within(args.transpose, -MAX_TRANSPOSE, MAX_TRANSPOSE, 'transpose'); + } + if (typeof args.instrument === 'string') { + if (out.kind !== 'pattern') throw new CommandError('Only a pattern item plays with an instrument'); + if (args.instrument !== '' && !findAssetOf(project, args.instrument, 'instrument')) + throw new CommandError(`${args.instrument} is no instrument asset`); + out.instrument = args.instrument; + } + if (typeof args.fx === 'string') + out.fx = args.fx === NO_FX || args.fx === '' ? args.fx : soundFxOf(project, args.fx); + return out; +} + +/** How long an item laid in without a length is: its pattern, or its sound at the song's tempo. */ +function itemLength(project: Project, song: SongAsset, kind: SongItem['kind'], play: string): number { + const pattern = kind === 'pattern' ? findAssetOf(project, play, 'pattern') : undefined; + if (pattern) return Math.max(1, patternLength(pattern)); + const seconds = audioSeconds(project, { kind: 'sound', asset: play, volume: 0, bus: '' }); + if (seconds <= 0) + throw new CommandError( + `The length of ${findSound(project, play)?.asset.name ?? play} is not known yet – give a length ` + + `(a bar of this song is ${ticksPerBar(songMeter(song.props))} ticks)`, + ); + return Math.max(1, Math.round(ticksOfSeconds(seconds, songTempo(song.props)))); +} + +function requireTrack(song: SongAsset, trackId: string): SongTrack { + const track = song.tracks.find((t) => t.id === trackId); + if (!track) throw new CommandError(`${song.name} has no track ${trackId}`); + return track; +} + +/** A pitch the agent wrote as a MIDI number or a name like "C#4". */ +export function pitchOf(value: unknown): number { + const number = + typeof value === 'number' + ? value + : typeof value === 'string' + ? NOTE_BY_NAME.get(value.trim().toUpperCase()) + : undefined; + if (number === undefined || !Number.isInteger(number) || number < LOWEST_NOTE || number > HIGHEST_NOTE) + throw new CommandError( + `${JSON.stringify(value)} is no note – give ${LOWEST_NOTE} to ${HIGHEST_NOTE} or a name like "C4" or "F#3"`, + ); + return number; +} + +const NOTE_BY_NAME = new Map( + Array.from({ length: HIGHEST_NOTE - LOWEST_NOTE + 1 }, (_, i) => [ + noteName(LOWEST_NOTE + i), + LOWEST_NOTE + i, + ]), +); + +function requireInside(pattern: PatternAsset, note: PatternNote): void { + const end = patternLength(pattern); + if (note.tick >= end) + throw new CommandError( + `A note at tick ${note.tick} lies past the end of ${pattern.name}, which is ${end} ticks long – ` + + 'give it more bars first', + ); +} + +function within(value: number, min: number, max: number, what: string): number { + if (value < min || value > max) throw new CommandError(`The ${what} has to be from ${min} to ${max}`); + return value; +} + +function soundFxOf(project: Project, fx: string): string { + if (fx !== '' && findAsset(project, fx)?.type !== 'soundfx') + throw new CommandError(`${fx} is no Sound FX asset`); + return fx; +} diff --git a/src/editor/agent/commandCatalog.ts b/src/editor/agent/commandCatalog.ts index b97e7874..e5ebe36c 100644 --- a/src/editor/agent/commandCatalog.ts +++ b/src/editor/agent/commandCatalog.ts @@ -1,6 +1,8 @@ import { ANIMATION_COMMANDS } from './catalog/animation'; import { ASSET_COMMANDS } from './catalog/assets'; +import { COMPOSITION_COMMANDS } from './catalog/composition'; import { GRAPH_COMMANDS } from './catalog/graphs'; +import { MUSIC_COMMANDS } from './catalog/music'; import { OBJECT_COMMANDS } from './catalog/objects'; import { PIXEL_ART_COMMANDS } from './catalog/pixelart'; import { PROJECT_COMMANDS } from './catalog/project'; @@ -63,6 +65,17 @@ export const AREAS: readonly CommandArea[] = [ about: 'Shapes of Vector art assets: their outlines, fills, gradients and strokes, groups and layers.', commands: VECTOR_ART_COMMANDS, }, + { + area: 'music', + about: + 'Notes of patterns; tracks, items and loops of songs; sends of a routing; clips of sounds and videos.', + commands: MUSIC_COMMANDS, + }, + { + area: 'composition', + about: 'Tracks, items and animation clips of Composition assets.', + commands: COMPOSITION_COMMANDS, + }, { area: 'project', about: 'Settings of the project as a whole.', commands: PROJECT_COMMANDS }, ]; diff --git a/src/editor/agent/readTools.ts b/src/editor/agent/readTools.ts index 7687c24e..8a1d836b 100644 --- a/src/editor/agent/readTools.ts +++ b/src/editor/agent/readTools.ts @@ -6,6 +6,13 @@ import { ASSET_TYPE_LABELS, ASSET_TYPES, type AssetType, isPlaceableType } from import { graphLookup } from '../../core/model/graphLookup'; import { isTemplateContent, resolveObjects } from '../../core/model/instance'; import { findAsset, findGraphAsset, type Project } from '../../core/model/Project'; +import { audioSeconds } from '../../core/model/composition'; +import { noteName } from '../../core/instrument/notes'; +import { patternLength } from '../../core/music/pattern'; +import { songLength } from '../../core/music/song'; +import { TICKS_PER_QUARTER } from '../../core/music/time'; +import { CLIP_ROWS } from '../../core/properties/sounds'; +import { VIDEO_CLIP_ROWS } from '../../core/properties/videos'; import { celOf, paletteColors } from '../../core/model/pixelart'; import type { CelPixels } from '../../core/model/pixelCel'; import { objectName } from '../../core/model/SceneObject'; @@ -81,6 +88,19 @@ export function getAsset(project: Project, assetId: string, readCel?: CelLookup) plain.colors = paletteColors(project, asset).map(toHex); plain.frames = pixelFrames(asset, readCel); } + if (asset.type === 'pattern') { + plain.notes = asset.notes.map((note) => ({ ...note, name: noteName(note.note) })); + plain.ticks = { perQuarter: TICKS_PER_QUARTER, length: patternLength(asset) }; + } + if (asset.type === 'song') plain.ticks = { perQuarter: TICKS_PER_QUARTER, length: songLength(asset) }; + if (asset.type === 'sound' || asset.type === 'video') { + plain.clipProperties = (asset.type === 'sound' ? CLIP_ROWS : VIDEO_CLIP_ROWS).map(propertyRow); + } + if (asset.type === 'sound') { + // Known once the editor has loaded it; a song or composition item takes its length from it. + const seconds = audioSeconds(project, { kind: 'sound', asset: asset.id, volume: 0, bus: '' }); + plain.seconds = seconds > 0 ? seconds : 'not loaded'; + } plain.properties = schemaForAsset(project, asset).map(propertyRow); return plain; } @@ -169,6 +189,7 @@ export function getGraph(project: Project, assetId: string): object | null { from: edge.from, to: edge.to, ...(edge.label ? { label: edge.label } : {}), + ...(edge.send ? { send: edge.send } : {}), })), groups: (graph.groups ?? []).map(({ text, ...box }) => ({ ...annotationRow(box), title: text })), notes: (graph.notes ?? []).map(({ text, ...box }) => ({ ...annotationRow(box), text })), diff --git a/src/editor/agent/tools.json b/src/editor/agent/tools.json index 644228b3..1ab73f8a 100644 --- a/src/editor/agent/tools.json +++ b/src/editor/agent/tools.json @@ -44,7 +44,7 @@ { "name": "get_asset", "title": "Read an asset", - "description": "One asset as stored: its property values, the rows of its properties (key, kind, default), the objects of a scene or container with their names and positions (the content of a placed container too, marked fromContainer), what the rows of an animation may aim at (targets), the cells of a small tilemap, the pixels of small pixel art as rows of palette indices (colors as \"#rrggbb\"). A graph is only counted here; read it with get_graph.", + "description": "One asset as stored: its property values, the rows of its properties (key, kind, default), the objects of a scene or container with their names and positions (the content of a placed container too, marked fromContainer), what the rows of an animation may aim at (targets), the cells of a small tilemap, the pixels of small pixel art as rows of palette indices (colors as \"#rrggbb\"), the notes of a pattern with their names and how many ticks it is long, the rows a clip of a sound or video has (clipProperties), how many seconds a loaded sound lasts. A graph is only counted here; read it with get_graph.", "inputSchema": { "type": "object", "properties": { "asset": { "type": "string", "description": "Id of the asset." } }, @@ -56,7 +56,7 @@ { "name": "get_graph", "title": "Read a graph", - "description": "The graph of a behavior, state machine, shader, particles, sound FX or instrument: each node with its settings and its input and output ports as they are now, the edges between ports, and the groups and comments (notes). A node that declares its own ports lists them under ports.", + "description": "The graph of a behavior, state machine, shader, particles, sound FX or instrument: each node with its settings and its input and output ports as they are now, the edges between ports (a link of a routing with its send), and the groups and comments (notes). A node that declares its own ports lists them under ports.", "inputSchema": { "type": "object", "properties": { "asset": { "type": "string", "description": "Id of the asset holding the graph." } }, diff --git a/tests/editor/agent-music.test.ts b/tests/editor/agent-music.test.ts new file mode 100644 index 00000000..50be90e7 --- /dev/null +++ b/tests/editor/agent-music.test.ts @@ -0,0 +1,425 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { CreateSoundCommand } from '../../src/core/commands/assetCommands'; +import type { CompositionAsset, PatternAsset, SongAsset, SoundAsset } from '../../src/core/model/Asset'; +import { setSoundMeasure } from '../../src/core/model/composition'; +import { ROUTING_ID } from '../../src/core/model/Project'; +import { settleRouting } from '../../src/core/music/projectRouting'; +import { TO_COMPOSITION_BUS } from '../../src/core/properties/buses'; +import { runTool } from '../../src/editor/agent/agentTools'; +import { makeSpriteScene, type SceneFixture } from '../helpers'; +import { hostFor, text } from './agentTestHost'; + +/** The commands of #429 stage 5d: patterns, songs, sends, clips of sounds and compositions. */ + +const FILE = `${'a7'.repeat(32)}.wav`; + +const run = async (fixture: SceneFixture, steps: object[], follows = false) => { + const { host, shown } = hostFor(fixture, null, follows); + const result = await runTool('run_commands', { steps }, host); + return { result, shown, text: text(result) }; +}; + +const read = async (fixture: SceneFixture, asset: string) => + (await runTool('get_asset', { asset }, hostFor(fixture).host)).structuredContent as Record; + +const graph = async (fixture: SceneFixture, asset: string) => + (await runTool('get_graph', { asset }, hostFor(fixture).host)).structuredContent as { + nodes: { id: string; type: string; params: Record }[]; + edges: { id: string; from: { node: string }; to: { node: string }; send?: object }[]; + }; + +const byId = (fixture: SceneFixture, id: string) => fixture.project.assets.find((a) => a.id === id) as T; + +/** Makes the assets by name and gives their ids. */ +async function made( + fixture: SceneFixture, + assets: Record, +): Promise> { + const steps = Object.entries(assets).map(([name, type]) => ({ + command: 'create_asset', + args: { type, name }, + as: name, + })); + const done = await run(fixture, steps); + expect(done.result.isError).toBeUndefined(); + return Object.fromEntries( + Object.keys(assets).map((name) => [name, fixture.project.assets.find((a) => a.name === name)!.id]), + ) as Record; +} + +function addSound(fixture: SceneFixture, name = 'Blip'): string { + const command = new CreateSoundCommand(name, FILE); + fixture.stack.execute(command); + return command.asset.id; +} + +afterEach(() => setSoundMeasure(() => null)); + +describe('patterns', () => { + it('adds notes by number and by name, reads them back with their names and refuses a note past the end', async () => { + const fixture = makeSpriteScene(); + const { Riff } = await made(fixture, { Riff: 'pattern' }); + const { result, shown } = await run( + fixture, + [ + { + command: 'add_notes', + args: { + asset: Riff, + notes: [ + { tick: 0, length: 480, note: 'C4' }, + { tick: 480, length: 240, note: 64, velocity: 0.5 }, + { tick: 960, length: 480, note: 'f#3' }, + ], + }, + }, + ], + true, + ); + expect(result.isError).toBeUndefined(); + expect(shown).toEqual([{ kind: 'asset', asset: Riff, open: true }]); + const asset = await read(fixture, Riff); + expect(asset.ticks).toEqual({ perQuarter: 480, length: 7680 }); + expect( + (asset.notes as Record[]).map(({ id: _, ...rest }: Record) => rest), + ).toEqual([ + { tick: 0, length: 480, note: 60, velocity: 0.8, name: 'C4' }, + { tick: 480, length: 240, note: 64, velocity: 0.5, name: 'E4' }, + { tick: 960, length: 480, note: 54, velocity: 0.8, name: 'F#3' }, + ]); + + const past = await run(fixture, [ + { command: 'add_notes', args: { asset: Riff, notes: [{ tick: 7680, length: 10, note: 60 }] } }, + ]); + expect(past.result.isError).toBe(true); + expect(past.text).toContain('7680 ticks long'); + const flat = await run(fixture, [ + { command: 'add_notes', args: { asset: Riff, notes: [{ tick: 0, length: 10, note: 'Hb4' }] } }, + ]); + expect(flat.text).toContain('"Hb4" is no note'); + expect(byId(fixture, Riff).notes).toHaveLength(3); + }); + + it('changes and removes notes by id, replaces all, and sets and clears the loop', async () => { + const fixture = makeSpriteScene(); + const { Riff } = await made(fixture, { Riff: 'pattern' }); + const added = await run(fixture, [ + { command: 'add_notes', args: { asset: Riff, notes: [{ tick: 0, length: 120, note: 60 }] }, as: 'n' }, + { command: 'change_notes', args: { asset: Riff, notes: [{ id: '$n', note: 'G4', tick: 240 }] } }, + { command: 'add_notes', args: { asset: Riff, notes: [{ tick: 0, length: 60, note: 48 }] }, as: 'm' }, + { command: 'remove_notes', args: { asset: Riff, notes: ['$m'] } }, + { command: 'set_loop', args: { asset: Riff, start: 0, end: 1920 } }, + ]); + expect(added.result.isError).toBeUndefined(); + const pattern = byId(fixture, Riff); + expect(pattern.notes.map(({ tick, note }) => [tick, note])).toEqual([[240, 67]]); + expect(pattern.loop).toEqual({ start: 0, end: 1920 }); + + const replaced = await run(fixture, [ + { + command: 'add_notes', + args: { asset: Riff, notes: [{ tick: 0, length: 1, note: 0 }], replace: true }, + }, + { command: 'set_loop', args: { asset: Riff } }, + ]); + expect(replaced.result.isError).toBeUndefined(); + expect(byId(fixture, Riff).notes.map((n) => n.note)).toEqual([0]); + expect(byId(fixture, Riff).loop).toBeUndefined(); + const half = await run(fixture, [{ command: 'set_loop', args: { asset: Riff, start: 0 } }]); + expect(half.text).toContain('Give start and end'); + }); +}); + +describe('songs', () => { + it('lays a pattern as long as it is, a sound by its loaded length, and refuses one not loaded without a length', async () => { + const fixture = makeSpriteScene(); + const { Theme, Riff } = await made(fixture, { Theme: 'song', Riff: 'pattern' }); + const blip = addSound(fixture); + const song = () => byId(fixture, Theme); + const track = song().tracks[0]!.id; + + const unknown = await run(fixture, [ + { command: 'add_song_item', args: { asset: Theme, track, play: blip, tick: 0 } }, + ]); + expect(unknown.result.isError).toBe(true); + expect(unknown.text).toContain('The length of Blip is not known yet'); + expect(unknown.text).toContain('a bar of this song is 1920 ticks'); + + setSoundMeasure(() => 1.25); + const laid = await run(fixture, [ + { command: 'add_song_item', args: { asset: Theme, track, play: Riff, tick: 0, transpose: 12 } }, + { command: 'add_song_item', args: { asset: Theme, track, play: blip, tick: 7680, volume: 6 } }, + ]); + expect(laid.result.isError).toBeUndefined(); + // 1.25 s at 120 BPM is 2.5 quarters. + expect( + song().items.map(({ kind, tick, length, transpose, volume }) => [ + kind, + tick, + length, + transpose, + volume, + ]), + ).toEqual([ + ['pattern', 0, 7680, 12, 0], + ['sound', 7680, 1200, 0, 6], + ]); + expect((await read(fixture, blip)).seconds).toBe(1.25); + + const sound = song().items[1]!.id; + const wrong = await run(fixture, [ + { command: 'change_song_item', args: { asset: Theme, item: sound, transpose: 2 } }, + ]); + expect(wrong.text).toContain('Only a pattern item is transposed'); + const loud = await run(fixture, [ + { command: 'change_song_item', args: { asset: Theme, item: song().items[0]!.id, volume: 3 } }, + ]); + expect(loud.text).toContain('from -60 to 0'); + }); + + it('adds, changes, moves and removes tracks, keeps the last one, and the routing follows', async () => { + const fixture = makeSpriteScene(); + const { Theme, Riff, Verb } = await made(fixture, { Theme: 'song', Riff: 'pattern', Verb: 'soundfx' }); + const song = () => byId(fixture, Theme); + const first = song().tracks[0]!.id; + const done = await run(fixture, [ + { command: 'add_song_track', args: { asset: Theme, name: 'Drums', at: 0 }, as: 'drums' }, + { + command: 'change_song_track', + args: { asset: Theme, track: '$drums', volume: -6, pan: -0.5, solo: true, fx: Verb }, + }, + { command: 'add_song_item', args: { asset: Theme, track: '$drums', play: Riff, tick: 0 } }, + { command: 'move_song_track', args: { asset: Theme, track: '$drums', to: 1 } }, + ]); + expect(done.result.isError).toBeUndefined(); + const drums = song().tracks[1]!; + expect([song().tracks[0]!.id, drums.name, drums.volume, drums.pan, drums.solo, drums.fx]).toEqual([ + first, + 'Drums', + -6, + -0.5, + true, + Verb, + ]); + expect((await graph(fixture, Theme)).nodes).toHaveLength(3); + + const removed = await run(fixture, [ + { command: 'remove_song_track', args: { asset: Theme, track: drums.id } }, + ]); + expect(removed.result.isError).toBeUndefined(); + expect(song().items).toHaveLength(0); + expect((await graph(fixture, Theme)).nodes).toHaveLength(2); + const last = await run(fixture, [{ command: 'remove_song_track', args: { asset: Theme, track: first } }]); + expect(last.text).toContain('keeps at least one track'); + const noFx = await run(fixture, [ + { command: 'change_song_track', args: { asset: Theme, track: first, fx: Riff } }, + ]); + expect(noFx.text).toContain('is no Sound FX asset'); + }); + + it('sets the send of a song link and of a master routing link, where a sound only sends post', async () => { + const fixture = makeSpriteScene(); + const { Theme } = await made(fixture, { Theme: 'song' }); + const link = (await graph(fixture, Theme)).edges[0]!.id; + const sent = await run(fixture, [ + { command: 'set_send', args: { asset: Theme, edge: link, volume: -12, tap: 'preFx' } }, + ]); + expect(sent.result.isError).toBeUndefined(); + expect((await graph(fixture, Theme)).edges[0]!.send).toEqual({ volume: -12, tap: 'preFx' }); + + // The editor settles the master routing after every edit; the test host does not, so it is done here. + const blip = addSound(fixture); + const { Music } = await made(fixture, { Music: 'bus' }); + settleRouting(fixture.project); + const master = fixture.project.routing; + // A routed asset's node goes by the asset's id. + const linkOf = (asset: string) => master.edges.find((e) => e.from.node === asset)!; + const routed = await run(fixture, [ + { command: 'set_send', args: { asset: ROUTING_ID, edge: linkOf(blip).id, volume: -3, tap: 'preFx' } }, + { command: 'set_send', args: { asset: ROUTING_ID, edge: linkOf(Music).id, tap: 'postFx' } }, + ]); + expect(routed.result.isError).toBeUndefined(); + expect([linkOf(blip).send, linkOf(Music).send]).toEqual([ + { volume: -3, tap: 'post' }, + { volume: 0, tap: 'postFx' }, + ]); + const blipLink = linkOf(blip).id; + const nowhere = await run(fixture, [{ command: 'set_send', args: { asset: blip, edge: blipLink } }]); + expect(nowhere.text).toContain('is no song and no master routing'); + }); +}); + +describe('clips of sounds', () => { + it('adds a clip with a name and its cut, changes it, lists its rows and removes it', async () => { + const fixture = makeSpriteScene(); + const blip = addSound(fixture); + const done = await run(fixture, [ + { + command: 'add_clip', + args: { asset: blip, name: 'Hit', props: { start: 0.25, end: 0.5 } }, + as: 'clip', + }, + { command: 'change_clip', args: { asset: blip, clip: '$clip', props: { reverse: true } } }, + ]); + expect(done.result.isError).toBeUndefined(); + const sound = byId(fixture, blip); + expect(sound.clips!.map(({ name, props }) => [name, props])).toEqual([ + ['Hit', { start: 0.25, end: 0.5, reverse: true }], + ]); + const rows = (await read(fixture, blip)).clipProperties as { key: string }[]; + expect(rows.map((r) => r.key)).toContain('reverse'); + + const bad = await run(fixture, [ + { command: 'change_clip', args: { asset: blip, clip: sound.clips![0]!.id, props: { start: -1 } } }, + ]); + expect(bad.result.isError).toBe(true); + const gone = await run(fixture, [ + { command: 'remove_clip', args: { asset: blip, clip: sound.clips![0]!.id } }, + ]); + expect(gone.result.isError).toBeUndefined(); + expect(byId(fixture, blip).clips).toEqual([]); + }); +}); + +describe('compositions', () => { + it('lays an object, a picture and a pattern on tracks, each as long as the rules say', async () => { + const fixture = makeSpriteScene(); + const { Cut, Riff, Art, Slide } = await made(fixture, { + Cut: 'composition', + Riff: 'pattern', + Art: 'pixelart', + Slide: 'animation', + }); + const comp = () => byId(fixture, Cut); + const track = comp().tracks[0]!.id; + const { result, shown } = await run( + fixture, + [ + { command: 'add_composition_track', args: { asset: Cut, name: 'Music' }, as: 'music' }, + { + command: 'add_composition_item', + args: { asset: Cut, track, place: fixture.spriteId, start: 0 }, + as: 'box', + }, + { command: 'add_composition_item', args: { asset: Cut, track, place: Art, start: 30, length: 45 } }, + { command: 'add_composition_item', args: { asset: Cut, track: '$music', place: Riff, start: 0 } }, + ], + true, + ); + expect(result.isError).toBeUndefined(); + expect(shown.at(-1)).toEqual({ kind: 'asset', asset: Cut, open: true }); + const [box, art, riff] = comp().items; + // 3 s at 60 fps; a pattern of 4 bars at 120 BPM is 8 s. + expect([box!.length, box!.fitMedia, art!.length, art!.fitMedia, riff!.length]).toEqual([ + 180, + true, + 45, + undefined, + 480, + ]); + expect(riff!.audio).toEqual({ kind: 'pattern', asset: Riff, volume: 0, bus: TO_COMPOSITION_BUS }); + expect(comp().objects.map((o) => o.id)).toEqual([box!.id, art!.id]); + const wearer = fixture.project.assets.find((a) => a.id === comp().objects[1]!.assetId)!; + expect([wearer.type, wearer.owner, wearer.props.texture]).toEqual(['sprite', Cut, Art]); + + const animation = await run(fixture, [ + { command: 'add_composition_item', args: { asset: Cut, track, place: Slide, start: 0 } }, + ]); + expect(animation.text).toContain('use add_composition_clip'); + const nested = await run(fixture, [ + { command: 'add_composition_item', args: { asset: Cut, track, place: Cut, start: 0 } }, + ]); + expect(nested.result.isError).toBe(true); + }); + + it('moves an item with its clip, refuses a clip over another, fades only what sounds, and removes with the object', async () => { + const fixture = makeSpriteScene(); + const { Cut, Riff } = await made(fixture, { Cut: 'composition', Riff: 'pattern' }); + const anim = await run(fixture, [ + { command: 'create_asset', args: { type: 'animation', name: 'Slide' }, as: 'anim' }, + { command: 'set_animation_root', args: { asset: '$anim', root: fixture.spriteId } }, + { command: 'add_animation_track', args: { asset: '$anim', target: '', property: 'x' }, as: 'row' }, + { command: 'set_animation_key', args: { asset: '$anim', track: '$row', frame: 0, value: 0 } }, + { command: 'set_animation_key', args: { asset: '$anim', track: '$row', frame: 30, value: 100 } }, + ]); + expect(anim.result.isError).toBeUndefined(); + const slide = fixture.project.assets.find((a) => a.name === 'Slide')!.id; + const comp = () => byId(fixture, Cut); + const track = comp().tracks[0]!.id; + const laid = await run(fixture, [ + { + command: 'add_composition_item', + args: { asset: Cut, track, place: fixture.spriteId, start: 0, length: 120 }, + as: 'box', + }, + { + command: 'add_composition_clip', + args: { asset: Cut, track, animation: slide, start: 10 }, + as: 'clip', + }, + { command: 'add_composition_item', args: { asset: Cut, track, place: Riff, start: 200 }, as: 'riff' }, + ]); + expect(laid.result.isError).toBeUndefined(); + const [box, riff] = comp().items; + + const over = await run(fixture, [ + { command: 'add_composition_clip', args: { asset: Cut, track, animation: slide, start: 30 } }, + ]); + expect(over.text).toContain('would lie over the one at 10'); + + const moved = await run(fixture, [ + { command: 'change_composition_item', args: { asset: Cut, item: box!.id, start: 50 } }, + { + command: 'change_composition_item', + args: { asset: Cut, item: riff!.id, fadeIn: 20, volume: -6, bus: '' }, + }, + ]); + expect(moved.result.isError).toBeUndefined(); + expect([comp().items[0]!.start, comp().clips[0]!.start]).toEqual([50, 60]); + expect(comp().items[1]).toMatchObject({ fadeIn: 20, audio: { volume: -6, bus: '' } }); + + const silent = await run(fixture, [ + { command: 'change_composition_item', args: { asset: Cut, item: box!.id, fadeOut: 5 } }, + ]); + expect(silent.text).toContain('Only an item with a sound fades'); + const noBus = await run(fixture, [ + { command: 'change_composition_item', args: { asset: Cut, item: box!.id, volume: -3 } }, + ]); + expect(noBus.text).toContain('Only an audio item'); + + const gone = await run(fixture, [ + { command: 'remove_composition_items', args: { asset: Cut, items: [box!.id] } }, + ]); + expect(gone.result.isError).toBeUndefined(); + expect([comp().items.map((i) => i.id), comp().objects.length, comp().clips.map((c) => c.id)]).toEqual([ + [riff!.id], + 0, + [], + ]); + }); + + it('adds, renames, moves and removes tracks, and keeps the last one', async () => { + const fixture = makeSpriteScene(); + const { Cut } = await made(fixture, { Cut: 'composition' }); + const comp = () => byId(fixture, Cut); + const first = comp().tracks[0]!.id; + const done = await run(fixture, [ + { command: 'add_composition_track', args: { asset: Cut, at: 0 }, as: 't' }, + { command: 'change_composition_track', args: { asset: Cut, track: '$t', name: 'Front' } }, + { command: 'move_composition_track', args: { asset: Cut, track: '$t', to: 5 } }, + ]); + expect(done.result.isError).toBeUndefined(); + expect(comp().tracks.map((t) => [t.id === first, t.name])).toEqual([ + [true, 'Track 1'], + [false, 'Front'], + ]); + const removed = await run(fixture, [ + { command: 'remove_composition_track', args: { asset: Cut, track: first } }, + ]); + expect(removed.result.isError).toBeUndefined(); + const last = await run(fixture, [ + { command: 'remove_composition_track', args: { asset: Cut, track: comp().tracks[0]!.id } }, + ]); + expect(last.text).toContain('keeps at least one track'); + }); +}); diff --git a/tools/e2e/71-mcp.mjs b/tools/e2e/71-mcp.mjs index 30fae924..901ab0ad 100644 --- a/tools/e2e/71-mcp.mjs +++ b/tools/e2e/71-mcp.mjs @@ -463,6 +463,59 @@ export async function mcp(t) { }), ); + // 71g8. Music and compositions (#429 stage 5d): notes the agent writes, and the pattern and the + // sprite laid on a composition, which comes up in its tab. + const composed = await callTool('run_commands', { + label: 'compose', + steps: [ + { command: 'create_asset', args: { type: 'pattern', name: `${TOKEN_NAME} riff` }, as: 'riff' }, + { + command: 'add_notes', + args: { + asset: '$riff', + notes: [ + { tick: 0, length: 480, note: 'C4' }, + { tick: 480, length: 480, note: 'E4' }, + ], + }, + }, + { command: 'create_asset', args: { type: 'composition', name: `${TOKEN_NAME} cut` }, as: 'cut' }, + { command: 'add_composition_track', args: { asset: '$cut', name: 'Music' }, as: 'music' }, + { + command: 'add_composition_item', + args: { asset: '$cut', track: '$music', place: '$riff', start: 0 }, + }, + { + command: 'add_composition_item', + args: { asset: '$cut', track: '$music', place: heroId, start: 30 }, + }, + ], + }); + await settle(); + const composedMade = composed?.structuredContent?.made ?? []; + const riffId = composedMade.find((entry) => entry.step === 1)?.id; + const cutId = composedMade.find((entry) => entry.step === 3)?.id; + const cutView = await viewNow(); + const riffRead = riffId ? (await callTool('get_asset', { asset: riffId }))?.structuredContent : null; + const cutRead = cutId ? (await callTool('get_asset', { asset: cutId }))?.structuredContent : null; + await callTool('undo'); + await callTool('open', { asset: walkId }); + await settle(); + check( + 'notes the agent writes read back by name, and a pattern and a sprite it lays on a composition open in its tab', + composed?.isError === false && + cutView?.document?.asset?.id === cutId && + JSON.stringify(riffRead?.notes?.map((note) => note.name)) === '["C4","E4"]' && + JSON.stringify(cutRead?.items?.map((item) => [item.start, item.length])) === '[[0,480],[30,180]]' && + cutRead?.objects?.length === 1, + JSON.stringify({ + text: composed?.content?.[0]?.text?.slice(0, 300), + document: cutView?.document, + notes: riffRead?.notes, + items: cutRead?.items, + }), + ); + // 71h. Played by the agent, the object walks left while the user holds the arrow. const started = await callTool('play', { action: 'play' }); await waitUntil('__gm.state.mode === "play"', { timeout: 10000, label: 'the game runs' });