diff --git a/src/core/model/usages.ts b/src/core/model/usages.ts index 1502ffb5..95cf5f66 100644 --- a/src/core/model/usages.ts +++ b/src/core/model/usages.ts @@ -7,7 +7,9 @@ import { isKindWith } from '../properties/kinds'; import { schemaForAsset } from '../properties/PropertySchema'; import { referencesAny, trackDef } from './animation'; import type { AnimationAsset, Asset, TileAsset } from './Asset'; +import { partsOf } from './assetParts'; import { overrideSites } from './instance'; +import { pixelArtRefIds } from './pixelart'; import { type ObjectHolder, objectHolders, type Project } from './Project'; import type { SceneObject } from './SceneObject'; @@ -84,6 +86,79 @@ export function cellsPainted(project: Project, tileId: string): { tilemaps: numb return { tilemaps, cells }; } +/** The nearest thing with an id of its own a use sits in: an object, a node, an item, a key. */ +export interface UsePlace { + id: string; + name?: string; + type?: string; + kind?: string; +} + +/** An asset naming another, and where in it; no places for a use on the asset itself, like a property. */ +export interface AssetUse { + asset: Asset; + at: UsePlace[]; +} + +export interface Uses { + assets: AssetUse[]; + /** The project's own settings naming it: its start scene, the master's FX, the Master routing. */ + project: ('startScene' | 'masterFx' | 'routing')[]; +} + +/** + * Everything naming an asset or one of its parts – a frame, a clip (#433). Read as data, not field + * by field, as `namedStrings` reads it for sounds, so no place is missed: a song item, a Sample, a + * key, a cell of a tilemap and kinds of asset still to come alike. Ids, the owner a song keeps an + * asset for and the nodes an edge joins are no uses: the Master routing has a node per asset. + */ +export function usesOf(project: Project, assetId: string): Uses { + const target = project.assets.find((asset) => asset.id === assetId); + if (!target) return { assets: [], project: [] }; + const wanted = new Set( + target.type === 'pixelart' ? pixelArtRefIds(target) : [target.id, ...partsOf(target).map((p) => p.id)], + ); + const assets: AssetUse[] = []; + for (const asset of project.assets) { + if (asset.id === target.id) continue; + const found = placesNaming(asset, wanted); + if (found) assets.push({ asset, at: found }); + } + const settings: Uses['project'] = []; + if (wanted.has(project.startSceneId)) settings.push('startScene'); + if (wanted.has(project.masterFx)) settings.push('masterFx'); + if (placesNaming(project.routing, wanted)) settings.push('routing'); + return { assets, project: settings }; +} + +const NO_USE: ReadonlySet = new Set(['id', 'owner', 'node']); + +/** Where in `value` one of the ids is named; null for nowhere, no places for only outside anything with an id. */ +function placesNaming(value: unknown, wanted: ReadonlySet): UsePlace[] | null { + const places = new Map(); + const walk = (inner: unknown, place: UsePlace | null): void => { + if (typeof inner === 'string') { + if (wanted.has(inner)) places.set(place?.id ?? '', place); + } else if (Array.isArray(inner)) { + for (const item of inner) walk(item, place); + } else if (typeof inner === 'object' && inner !== null && !ArrayBuffer.isView(inner)) { + const record = inner as Record; + const here = typeof record.id === 'string' && inner !== value ? placeOf(record) : place; + for (const [key, field] of Object.entries(record)) if (!NO_USE.has(key)) walk(field, here); + } + }; + walk(value, null); + if (places.size === 0) return null; + return [...places.values()].filter((place): place is UsePlace => place !== null); +} + +function placeOf(record: Record): UsePlace { + const place: UsePlace = { id: record.id as string }; + for (const key of ['name', 'type', 'kind'] as const) + if (typeof record[key] === 'string' && record[key] !== '') place[key] = record[key]; + return place; +} + /** The animations keying any of these ids – intrinsic data, which `findPropertyUsages` does not see. */ export function animationsUsing(project: Project, ids: readonly string[]): AnimationAsset[] { const wanted = new Set(ids); diff --git a/src/editor/Editor.ts b/src/editor/Editor.ts index 3c027744..248acb66 100644 --- a/src/editor/Editor.ts +++ b/src/editor/Editor.ts @@ -120,7 +120,7 @@ import { AgentLink } from './agent/AgentLink'; import { runTool } from './agent/agentTools'; import { AgentTokensDialog } from './agent/AgentTokensDialog'; import { type AgentView, agentView, viewSummary } from './agent/agentView'; -import { editorToolHost } from './agent/editorToolHost'; +import { type EditorParts, editorToolHost } from './agent/editorToolHost'; import { cameraHint } from './agent/runtimeView'; export type CentreTab = @@ -534,7 +534,14 @@ export class Editor implements PlayHost { }, account.guest === true, ); - this.agent = account.agents ? this.makeAgentLink(app, account.agents) : null; + this.agent = account.agents + ? this.makeAgentLink(app, account.agents, { + texture: textures, + font: fonts, + sound: sounds, + video: videos, + }) + : null; root.add(this.toolbar); this.workspace = new HStack(); @@ -956,7 +963,7 @@ export class Editor implements PlayHost { ]; } - private makeAgentLink(app: Application, agents: AgentStore): AgentLink { + private makeAgentLink(app: Application, agents: AgentStore, stores: EditorParts['stores']): AgentLink { const link = new AgentLink(socketUrl(MCP_PATH, location), { ticket: () => agents.ticket(), view: () => { @@ -976,6 +983,7 @@ export class Editor implements PlayHost { showTab: (tab) => this.showTab(tab), play: this.play, follows: followsAgent, + stores, }, wait, ), diff --git a/src/editor/agent/agentTools.ts b/src/editor/agent/agentTools.ts index 97953dab..a9d4a6b6 100644 --- a/src/editor/agent/agentTools.ts +++ b/src/editor/agent/agentTools.ts @@ -4,7 +4,9 @@ import { lintLayout } from '../graph/arrange'; import { layoutOf, type MeasureNode } from '../graph/graphLayout'; import { type ArgSchema, argProblem } from './argCheck'; import type { AgentView } from './agentView'; -import { AgentEdit, deletedAssets, type EditStep } from './editPlan'; +import { AgentEdit, deletedAssets, type EditStep, filesNeeded } from './editPlan'; +import type { StoredFile } from '../../core/store/FileStore'; +import { type FileAssetType, type FileLister, type FileLookup, fileRow, offeredFiles } from './storedFiles'; import type { StepTarget } from './commandCatalog'; import { cameraOfWorld } from '../../viewport/activeCamera'; import { pixelCels } from '../../viewport/PixelCels'; @@ -70,6 +72,8 @@ export interface ToolHost { runtime?(): RuntimeFacts | null; /** Tells the relay this call takes longer than most (see `relay.mjs`, `wait`). */ takesLong?(): void; + /** The files the server holds of a kind (#431). */ + files?: FileLister; } const DEFAULT_SCREENSHOT_WIDTH = 1280; @@ -144,6 +148,15 @@ async function run(name: string, args: Record, host: ToolHost): } case 'catalog': return answer({ [args.topic as string]: catalog(host.project(), args.topic as CatalogTopic, args) }); + case 'list_files': { + if (!host.files) return failure('This editor lists no files'); + const type = args.type as FileAssetType; + const needle = ((args.query as string | undefined) ?? '').trim().toLowerCase(); + const files = offeredFiles(type, await host.files(type)).filter( + (file) => !needle || file.name.toLowerCase().includes(needle), + ); + return answer({ files: files.map((file) => fileRow(host.project(), type, file)) }); + } case 'run_commands': return runCommands(args, host); case 'open': @@ -211,7 +224,13 @@ async function runCommands(args: Record, host: ToolHost): Promi } const label = `Agent: ${(args.label as string | undefined) ?? steps.map((step) => step.command).join(', ')}`; const follows = host.follows?.() === true; - const edit = new AgentEdit(label, steps, host.measure && ((assetId) => host.measure!(assetId)), follows); + const edit = new AgentEdit( + label, + steps, + host.measure && ((assetId) => host.measure!(assetId)), + follows, + await fileLookup(filesNeeded(steps), host), + ); const refusal = host.execute(edit); if (refusal !== null) return failure(`Nothing was changed. ${refusal}`); if (follows && edit.focus) host.show?.(edit.focus); @@ -227,6 +246,14 @@ async function runCommands(args: Record, host: ToolHost): Promi return done(lines.join('\n'), { label, made }); } +/** The lists of the kinds of file an edit makes assets of, fetched before it runs: a command is built at once. */ +async function fileLookup(types: readonly FileAssetType[], host: ToolHost): Promise { + const lists = new Map(); + if (types.length > 0 && !host.files) throw new Error('This editor lists no files'); + for (const type of types) lists.set(type, offeredFiles(type, await host.files!(type))); + return (type, file) => lists.get(type)?.find((stored) => stored.file === file); +} + /** The arguments of each tool, as `tools.json` gives them to the agent and the relay. */ const ARGS: Record = Object.fromEntries( (TOOLS as unknown as { name: string; inputSchema: ArgSchema }[]).map((tool) => [ diff --git a/src/editor/agent/catalog/assets.ts b/src/editor/agent/catalog/assets.ts index aa62b321..c1ec3afa 100644 --- a/src/editor/agent/catalog/assets.ts +++ b/src/editor/agent/catalog/assets.ts @@ -15,6 +15,8 @@ import { import { ASSET_TYPES, type AssetType } from '../../../core/model/AssetTypes'; import type { PropertyBag, PropertyValue } from '../../../core/model/Properties'; import type { Scope, SharedScope } from '../../../core/model/Scope'; +import { CommandError } from '../../../core/commands/Command'; +import { FILE_ASSET_TYPES, type FileAssetType, fileAssetCommand } from '../storedFiles'; import { ASSET, anyValue, @@ -28,20 +30,18 @@ import { theAsset, } from './shared'; -/** Assets the agent cannot make: each needs a file, and there is no upload for agents. */ -const FILE_TYPES: readonly AssetType[] = ['texture', 'font', 'sound', 'video']; - export const ASSET_COMMANDS: readonly CatalogCommand[] = [ { name: 'create_asset', description: 'Makes a new asset. A scene is where objects are placed; a sprite, text or camera is placed into one; ' + - "a behavior is a graph of nodes that runs on the object it is placed under. Textures, fonts, sounds and videos need a file and can't be made here.", + 'a behavior is a graph of nodes that runs on the object it is placed under. Textures, fonts, sounds and videos are a file: ' + + 'add_file_asset makes them of one the server holds.', args: object( { type: { type: 'string', - enum: ASSET_TYPES.filter((type) => !FILE_TYPES.includes(type)), + enum: ASSET_TYPES.filter((type) => !(FILE_ASSET_TYPES as readonly AssetType[]).includes(type)), description: 'The kind of asset.', }, name: { type: 'string', minLength: 1, description: 'Its name in the asset list.' }, @@ -62,6 +62,31 @@ export const ASSET_COMMANDS: readonly CatalogCommand[] = [ }, shows: madeAsset, }, + { + name: 'add_file_asset', + description: + 'Makes a texture, font, sound or video of a file the server holds already, as its picker does: list_files lists them. ' + + 'Named after the upload unless name is given. A file the project has an asset of already is refused, with that asset named.', + args: object( + { + type: oneOf(FILE_ASSET_TYPES, 'The kind of asset.'), + file: id('The file, as list_files gives it.'), + name: { type: 'string', minLength: 1, description: 'Its name in the asset list.' }, + }, + ['type', 'file'], + ), + build: (args, { project, storedFile }) => { + const type = text(args, 'type') as FileAssetType; + const file = storedFile(type, text(args, 'file')); + if (!file) + throw new CommandError( + `The server holds no ${type} file ${text(args, 'file')}: list_files lists them`, + ); + const command = fileAssetCommand(project, type, file, args.name as string | undefined); + return { command, made: { id: command.asset.id } }; + }, + shows: madeAsset, + }, { name: 'rename_asset', description: 'Renames an asset.', @@ -82,7 +107,7 @@ export const ASSET_COMMANDS: readonly CatalogCommand[] = [ { name: 'delete_asset', description: - 'Deletes an asset and what depends on it (placed copies of it, for one). The user is asked first and may say no.', + 'Deletes an asset and what depends on it (placed copies of it, for one); get_asset lists under usedBy what names it. The user is asked first and may say no.', args: object({ asset: ASSET }, ['asset']), confirms: true, build: (args) => ({ command: new DeleteAssetCommand(text(args, 'asset')) }), diff --git a/src/editor/agent/catalog/music.ts b/src/editor/agent/catalog/music.ts index af835210..da3859a8 100644 --- a/src/editor/agent/catalog/music.ts +++ b/src/editor/agent/catalog/music.ts @@ -6,11 +6,19 @@ import { } from '../../../core/commands/clipCommands'; import { type Command, CommandError } from '../../../core/commands/Command'; import { oneStep } from '../../../core/commands/CompoundCommand'; +import { ReplaceGraphCommand } from '../../../core/commands/graphCommands'; import { SetLoopCommand, SetPatternNotesCommand } from '../../../core/commands/patternCommands'; import { requireAsset, requireEdge, requirePattern, requireSong } from '../../../core/commands/require'; import { SetSendCommand, SetSongCommand } from '../../../core/commands/songCommands'; import type { SendTap } from '../../../core/graph/Graph'; import { newId } from '../../../core/ids'; +import { + CHIP_PRESETS, + type ChipPreset, + chipPresetGraph, + EFFECT_PRESETS, + TONE_PRESETS, +} from '../../../core/instrument/chipPresets'; import { HIGHEST_NOTE, LOWEST_NOTE, noteName } from '../../../core/instrument/notes'; import type { PatternAsset, SongAsset } from '../../../core/model/Asset'; import { audioSeconds } from '../../../core/model/composition'; @@ -195,6 +203,33 @@ export const MUSIC_COMMANDS: readonly CatalogCommand[] = [ }, shows: theTab, }, + { + name: 'apply_preset', + description: + 'Replaces the graph of an Instrument asset with a preset, as the Preset menu of its tab does: the quickest way to an instrument that sounds. ' + + `To play tunes on, every note at its own pitch from ${noteName(LOWEST_NOTE)} to ${noteName(HIGHEST_NOTE)}, the same each time: ` + + `${TONE_PRESETS.map((p) => p.value).join(', ')} (drumKit has its drums on single notes). ` + + `Sounds after sfxr, one sound on every key, drawn anew each time: ${EFFECT_PRESETS.map((p) => p.value).join(', ')}.`, + args: object( + { + asset: id('Id of the instrument asset.'), + preset: oneOf( + CHIP_PRESETS.map((p) => p.value), + 'The preset.', + ), + }, + ['asset', 'preset'], + ), + build: (args, { project }) => { + const instrument = findAssetOf(project, text(args, 'asset'), 'instrument'); + if (!instrument) + throw new CommandError(`${requireAsset(project, text(args, 'asset')).name} is no instrument`); + const preset = text(args, 'preset') as ChipPreset; + const label = CHIP_PRESETS.find((p) => p.value === preset)?.label ?? preset; + return { command: new ReplaceGraphCommand(instrument.id, chipPresetGraph(preset), `Preset ${label}`) }; + }, + shows: theTab, + }, { name: 'add_song_track', description: diff --git a/src/editor/agent/catalog/shared.ts b/src/editor/agent/catalog/shared.ts index d3899f31..51427527 100644 --- a/src/editor/agent/catalog/shared.ts +++ b/src/editor/agent/catalog/shared.ts @@ -4,6 +4,7 @@ import type { Project } from '../../../core/model/Project'; import { parseHex } from '../../../ui/color'; import type { MeasureNode } from '../../graph/graphLayout'; import type { ArgSchema } from '../argCheck'; +import type { FileLookup } from '../storedFiles'; /** What a command is built from: the project as it stands before the edit, and where new nodes go. */ export interface BuildContext { @@ -12,6 +13,8 @@ export interface BuildContext { spot(assetId: string): { x: number; y: number }; /** How the nodes of a graph are measured for arranging. */ measure(assetId: string): MeasureNode; + /** The stored files, fetched before the edit for the kinds its steps make assets of (#431). */ + storedFile: FileLookup; } /** The command, and the ids of what it makes – what later steps of the same edit can name. */ diff --git a/src/editor/agent/commandCatalog.ts b/src/editor/agent/commandCatalog.ts index e5ebe36c..9ed649d0 100644 --- a/src/editor/agent/commandCatalog.ts +++ b/src/editor/agent/commandCatalog.ts @@ -68,7 +68,7 @@ export const AREAS: readonly CommandArea[] = [ { area: 'music', about: - 'Notes of patterns; tracks, items and loops of songs; sends of a routing; clips of sounds and videos.', + 'Notes of patterns; presets of instruments; tracks, items and loops of songs; sends of a routing; clips of sounds and videos.', commands: MUSIC_COMMANDS, }, { diff --git a/src/editor/agent/editPlan.ts b/src/editor/agent/editPlan.ts index 96d7db2c..1b331694 100644 --- a/src/editor/agent/editPlan.ts +++ b/src/editor/agent/editPlan.ts @@ -8,6 +8,7 @@ import { estimatedShape } from '../graph/nodeBox'; import { argProblem } from './argCheck'; import { boxAt } from './catalog/graphs'; import { type BuildContext, catalogCommand, type Frame, type StepTarget } from './commandCatalog'; +import { FILE_ASSET_TYPES, type FileAssetType, type FileLookup } from './storedFiles'; /** One step of an edit as the agent sends it. */ export interface EditStep { @@ -45,6 +46,7 @@ export class AgentEdit implements Command { private readonly steps: readonly EditStep[], private readonly measure: (assetId: string) => MeasureNode = () => estimatedShape, private readonly showScene = false, + private readonly files: FileLookup = () => undefined, ) {} execute(project: Project): void { @@ -61,6 +63,7 @@ export class AgentEdit implements Command { project, spot: (assetId) => freeSpot(project, assetId), measure: this.measure, + storedFile: this.files, }; const run = (command: Command) => { command.execute(project); @@ -131,6 +134,12 @@ export function deletedAssets(steps: readonly EditStep[]): string[] { }); } +/** The kinds of stored file the steps make assets of: their lists are fetched before the edit runs. */ +export function filesNeeded(steps: readonly EditStep[]): FileAssetType[] { + const types = steps.flatMap((step) => (step.command === 'add_file_asset' ? [step.args?.type] : [])); + return FILE_ASSET_TYPES.filter((type) => types.includes(type)); +} + /** The catalog entry of a step and its arguments, the names of earlier steps put in and checked. */ function stepEntry(step: EditStep, names: ReadonlyMap>) { const entry = catalogCommand(step.command); diff --git a/src/editor/agent/editorToolHost.ts b/src/editor/agent/editorToolHost.ts index bd713434..5528b07d 100644 --- a/src/editor/agent/editorToolHost.ts +++ b/src/editor/agent/editorToolHost.ts @@ -11,6 +11,8 @@ import type { AgentSelection, PlayAction, ToolHost } from './agentTools'; import type { AgentView } from './agentView'; import type { StepTarget } from './commandCatalog'; import { screenshot } from './screenshot'; +import type { FileAssetType } from './storedFiles'; +import type { FileStore } from '../../core/store/FileStore'; /** What the agent's tools reach of the editor beyond its state. */ export interface EditorParts { @@ -27,6 +29,8 @@ export interface EditorParts { }; /** The user's "Follow agent" switch. */ follows(): boolean; + /** Where the files of each kind are kept (#431). */ + stores: Record; } /** How long the user has to answer an agent's question before it counts as a no. */ @@ -86,6 +90,7 @@ export function editorToolHost(parts: EditorParts, wait: () => void): ToolHost { return { world, shared, project: state.project, heldKeys: input.heldKeys, paused: state.paused }; }, takesLong: wait, + files: (type) => parts.stores[type].list(), }; } diff --git a/src/editor/agent/readTools.ts b/src/editor/agent/readTools.ts index d9300d16..adbc1d84 100644 --- a/src/editor/agent/readTools.ts +++ b/src/editor/agent/readTools.ts @@ -23,6 +23,7 @@ import { VIDEO_CLIP_ROWS } from '../../core/properties/videos'; import { celOf, paletteColors } from '../../core/model/pixelart'; import type { CelPixels } from '../../core/model/pixelCel'; import { objectName } from '../../core/model/SceneObject'; +import { usesOf } from '../../core/model/usages'; import { VARIABLE_KINDS } from '../../core/properties/kinds'; import { type PropertyDef, schemaFor, schemaForAsset } from '../../core/properties/PropertySchema'; import { type PaletteEntry, paletteEntries, rankedEntries } from '../graph/paletteEntries'; @@ -111,9 +112,24 @@ export function getAsset(project: Project, assetId: string, readCel?: CelLookup) plain.seconds = seconds > 0 ? seconds : 'not loaded'; } plain.properties = schemaForAsset(project, asset).map(propertyRow); + plain.usedBy = usedBy(project, asset.id); return plain; } +/** What names the asset or a part of it (#433), and where in there: empty when nothing does. */ +function usedBy(project: Project, assetId: string): object[] { + const uses = usesOf(project, assetId); + return [ + ...uses.assets.map(({ asset, at }) => ({ + id: asset.id, + name: asset.name, + type: asset.type, + ...(at.length > 0 ? { at } : {}), + })), + ...uses.project.map((setting) => ({ project: setting })), + ]; +} + /** * What of a pattern, song or composition is not heard – nothing else says so: all of it for want of * an instrument, or the notes its instrument does not sound, outside the range of its Notes nodes. diff --git a/src/editor/agent/storedFiles.ts b/src/editor/agent/storedFiles.ts new file mode 100644 index 00000000..041ebf6d --- /dev/null +++ b/src/editor/agent/storedFiles.ts @@ -0,0 +1,105 @@ +import { type Command, CommandError } from '../../core/commands/Command'; +import { nextAssetName } from '../../core/commands/naming'; +import type { Project } from '../../core/model/Project'; +import { isBakeUpload } from '../../core/music/bake'; +import { isStreamUpload } from '../../core/music/streams'; +import type { StoredFile } from '../../core/store/FileStore'; +import type { StoredTexture } from '../../core/store/TextureStore'; +import type { StoredVideo } from '../../core/store/VideoStore'; +import { assetNameForFile } from '../assetImport'; +import { FONT_KIND } from '../fontImport'; +import { SOUND_KIND } from '../soundImport'; +import { IMAGE_KIND } from '../textureImport'; +import { VIDEO_KIND } from '../videoImport'; + +/** The assets that are a file the server holds (#431): the agent makes them of one already there. */ +export const FILE_ASSET_TYPES = ['texture', 'font', 'sound', 'video'] as const; +export type FileAssetType = (typeof FILE_ASSET_TYPES)[number]; + +/** The files the server holds of one kind, as the editor's stores list them. */ +export type FileLister = (type: FileAssetType) => Promise; +/** A file of a kind, from the lists fetched before an edit runs; undefined for one not there. */ +export type FileLookup = (type: FileAssetType, file: string) => StoredFile | undefined; + +/** What the pickers offer: the bakes and streaming copies of a sound are the editor's own files. */ +export function offeredFiles(type: FileAssetType, files: readonly StoredFile[]): StoredFile[] { + return type === 'sound' + ? files.filter((f) => !isBakeUpload(f.name) && !isStreamUpload(f.name)) + : [...files]; +} + +/** A file as `list_files` answers it, with the asset of the project already wearing it. */ +export function fileRow(project: Project, type: FileAssetType, file: StoredFile): object { + const asset = wearer(project, type, file.file); + const texture = type === 'texture' ? (file as StoredTexture) : null; + const video = type === 'video' ? (file as StoredVideo) : null; + return { + file: file.file, + name: file.name, + bytes: file.bytes, + ...(texture ? { width: texture.width, height: texture.height } : {}), + ...(video + ? { + status: video.status, + ...(video.status === 'ready' + ? { + width: video.width, + height: video.height, + seconds: video.duration, + fps: video.fps, + audio: video.audio, + } + : {}), + } + : {}), + ...(file.demo ? { demo: true } : {}), + ...(asset ? { asset: { id: asset.id, name: asset.name } } : {}), + }; +} + +function wearer(project: Project, type: FileAssetType, file: string) { + return project.assets.find((a) => a.type === type && 'file' in a && a.file === file); +} + +/** + * The command making an asset of a stored file, as the picker makes it (`importStored`): named after + * the upload unless `name` is given. Refused for a file the project has an asset of already – two + * assets on one file would disagree about it –, a picture whose size the server could not read, and + * a video not converted yet. + */ +export function fileAssetCommand( + project: Project, + type: FileAssetType, + file: StoredFile, + name?: string, +): Command & { asset: { id: string } } { + const there = wearer(project, type, file.file); + if (there) + throw new CommandError( + `${file.file} is in the project already as "${there.name}" (${there.id}): use that one`, + ); + const kind = { texture: IMAGE_KIND, font: FONT_KIND, sound: SOUND_KIND, video: VIDEO_KIND }[type]; + const named = nextAssetName(project, name ?? assetNameForFile(file.name, kind.fallbackName)); + switch (type) { + case 'texture': { + const picture = file as StoredTexture; + if (!(picture.width > 0 && picture.height > 0)) + throw new CommandError( + `The server could not read the size of ${file.file}; it cannot become a texture`, + ); + return IMAGE_KIND.create(named, picture); + } + case 'video': { + const video = file as StoredVideo; + if (video.status !== 'ready') + throw new CommandError( + `${file.file} is ${video.status}, not converted yet: it becomes an asset once it is ready`, + ); + return VIDEO_KIND.create(named, video); + } + case 'font': + return FONT_KIND.create(named, file); + case 'sound': + return SOUND_KIND.create(named, file); + } +} diff --git a/src/editor/agent/tools.json b/src/editor/agent/tools.json index 0e2061e9..5b261982 100644 --- a/src/editor/agent/tools.json +++ b/src/editor/agent/tools.json @@ -41,10 +41,25 @@ }, "annotations": { "readOnlyHint": true, "openWorldHint": false } }, + { + "name": "list_files", + "title": "List the stored files", + "description": "The pictures, typefaces, sounds or videos the server holds for this account, the demo material among them (demo), whether a project uses them or not: each with its file, the name it was uploaded as, its size, a picture's width and height, a video's state, length, frame rate and sound, and the asset already wearing it in this project (asset). The command add_file_asset makes an asset of one; uploading is for the user.", + "inputSchema": { + "type": "object", + "properties": { + "type": { "type": "string", "enum": ["texture", "font", "sound", "video"] }, + "query": { "type": "string", "description": "Only files whose upload name contains this." } + }, + "required": ["type"], + "additionalProperties": false + }, + "annotations": { "readOnlyHint": true, "openWorldHint": false } + }, { "name": "get_asset", "title": "Read an asset", - "description": "One asset as stored: its property values, the rows of its properties (key, kind, default), the objects of a scene or container with their names and positions (the content of a placed container too, marked fromContainer), what the rows of an animation may aim at (targets), the cells of a small tilemap, the pixels of small pixel art as rows of palette indices (colors as \"#rrggbb\"), the notes of a pattern with their names and how many ticks it is long, the rows a clip of a sound or video has (clipProperties), how many seconds a loaded sound lasts, and under silent what of a pattern, song or composition is not heard: all of it without an instrument, or the notes its instrument does not sound. A graph is only counted here; read it with get_graph.", + "description": "One asset as stored: its property values, the rows of its properties (key, kind, default), the objects of a scene or container with their names and positions (the content of a placed container too, marked fromContainer), what the rows of an animation may aim at (targets), the cells of a small tilemap, the pixels of small pixel art as rows of palette indices (colors as \"#rrggbb\"), the notes of a pattern with their names and how many ticks it is long, the rows a clip of a sound or video has (clipProperties), how many seconds a loaded sound lasts, and under silent what of a pattern, song or composition is not heard: all of it without an instrument, or the notes its instrument does not sound. Under usedBy: every asset naming it or one of its frames or clips, with the objects, nodes, items or keys in there that do (none for a property of the asset itself), and the project's start scene, master FX or Master routing; empty when nothing uses it. A graph is only counted here; read it with get_graph.", "inputSchema": { "type": "object", "properties": { "asset": { "type": "string", "description": "Id of the asset." } }, diff --git a/src/editor/fontImport.ts b/src/editor/fontImport.ts index f6604957..bad54509 100644 --- a/src/editor/fontImport.ts +++ b/src/editor/fontImport.ts @@ -21,7 +21,7 @@ export interface FontImportDeps { fonts: FontStore; } -const FONT_KIND: ImportKind = { +export const FONT_KIND: ImportKind = { type: 'font', noun: 'font', fallbackName: 'Font', diff --git a/src/editor/soundImport.ts b/src/editor/soundImport.ts index 7aa54af0..d2aea58a 100644 --- a/src/editor/soundImport.ts +++ b/src/editor/soundImport.ts @@ -21,7 +21,7 @@ export interface SoundImportDeps { sounds: SoundStore; } -const SOUND_KIND: ImportKind = { +export const SOUND_KIND: ImportKind = { type: 'sound', noun: 'sound', fallbackName: 'Sound', diff --git a/src/editor/textureImport.ts b/src/editor/textureImport.ts index 8bb757f0..68b49791 100644 --- a/src/editor/textureImport.ts +++ b/src/editor/textureImport.ts @@ -90,7 +90,7 @@ export interface TextureImportDeps { textures: TextureStore; } -const IMAGE_KIND: ImportKind = { +export const IMAGE_KIND: ImportKind = { type: 'texture', noun: 'image', fallbackName: 'Texture', diff --git a/src/editor/videoImport.ts b/src/editor/videoImport.ts index dad887eb..ddd065e4 100644 --- a/src/editor/videoImport.ts +++ b/src/editor/videoImport.ts @@ -21,7 +21,7 @@ export interface VideoImportDeps { * A video becomes an asset only once the worker has made it (#402): the asset copies its size, * length and frame rate, which are not known before. */ -const VIDEO_KIND: ImportKind = { +export const VIDEO_KIND: ImportKind = { type: 'video', noun: 'video', fallbackName: 'Video', diff --git a/tests/editor/agent-edit.test.ts b/tests/editor/agent-edit.test.ts index d2f9eb04..e5f6853f 100644 --- a/tests/editor/agent-edit.test.ts +++ b/tests/editor/agent-edit.test.ts @@ -506,6 +506,7 @@ describe('the catalog', () => { 'get_view', 'screenshot', 'list_assets', + 'list_files', 'get_asset', 'get_graph', 'lint_graph', diff --git a/tests/editor/agent-files.test.ts b/tests/editor/agent-files.test.ts new file mode 100644 index 00000000..8cf30077 --- /dev/null +++ b/tests/editor/agent-files.test.ts @@ -0,0 +1,256 @@ +import { describe, expect, it } from 'vitest'; +import type { InstrumentAsset } from '../../src/core/model/Asset'; +import { chipPresetGraph } from '../../src/core/instrument/chipPresets'; +import { bakeUploadName } from '../../src/core/music/bake'; +import { settleRouting } from '../../src/core/music/projectRouting'; +import type { StoredFile } from '../../src/core/store/FileStore'; +import { runTool } from '../../src/editor/agent/agentTools'; +import type { FileAssetType } from '../../src/editor/agent/storedFiles'; +import { makeSpriteScene, type SceneFixture } from '../helpers'; +import { hostFor, text } from './agentTestHost'; + +/** Files the server holds (#431), presets of instruments (#432) and what uses an asset (#433). */ + +const stored = (file: string, name: string, extra: object = {}): StoredFile => ({ + file, + name, + bytes: 1000, + updatedAt: '', + owner: 'me', + deletable: true, + ...extra, +}); + +const PICTURE = stored(`${'b2'.repeat(32)}.png`, 'hero-idle.png', { width: 32, height: 16, demo: true }); +const UNREAD = stored(`${'b3'.repeat(32)}.png`, 'broken.png', { width: 0, height: 0 }); +const BLIP = stored(`${'c4'.repeat(32)}.wav`, 'blip.wav'); +const BAKED = stored(`${'c5'.repeat(32)}.ogg`, bakeUploadName('Theme')); +const CLIP = stored(`${'d6'.repeat(32)}.mp4`, 'intro.mp4', { + status: 'ready', + width: 640, + height: 360, + duration: 4, + fps: 30, + audio: true, +}); +const QUEUED = stored(`${'d7'.repeat(32)}.mp4`, 'later.mp4', { status: 'queued' }); + +const SERVER: Record = { + texture: [PICTURE, UNREAD], + font: [], + sound: [BLIP, BAKED], + video: [CLIP, QUEUED], +}; + +function filesHost(fixture: SceneFixture) { + const made = hostFor(fixture); + const asked: FileAssetType[] = []; + made.host.files = (type) => { + asked.push(type); + return Promise.resolve(SERVER[type]); + }; + return { ...made, asked }; +} + +const run = async (fixture: SceneFixture, steps: object[]) => { + const { host, asked } = filesHost(fixture); + const result = await runTool('run_commands', { steps }, host); + return { result, asked, text: text(result) }; +}; + +const list = async (fixture: SceneFixture, type: FileAssetType, query?: string) => + ( + (await runTool('list_files', { type, ...(query ? { query } : {}) }, filesHost(fixture).host)) + .structuredContent as { + files: Record[]; + } + ).files; + +const read = async (fixture: SceneFixture, asset: string) => + (await runTool('get_asset', { asset }, hostFor(fixture).host)).structuredContent as Record; + +const named = (fixture: SceneFixture, name: string) => fixture.project.assets.find((a) => a.name === name)!; + +describe('stored files', () => { + it('lists the files of a kind as the picker offers them, with the asset wearing one', async () => { + const fixture = makeSpriteScene(); + expect(await list(fixture, 'texture')).toEqual([ + { file: PICTURE.file, name: 'hero-idle.png', bytes: 1000, width: 32, height: 16, demo: true }, + { file: UNREAD.file, name: 'broken.png', bytes: 1000, width: 0, height: 0 }, + ]); + expect((await list(fixture, 'texture', 'HERO')).map((f) => f.file)).toEqual([PICTURE.file]); + // A bake is the editor's own copy of a song, not one of the user's sounds. + expect((await list(fixture, 'sound')).map((f) => f.file)).toEqual([BLIP.file]); + expect(await list(fixture, 'video')).toEqual([ + { + file: CLIP.file, + name: 'intro.mp4', + bytes: 1000, + status: 'ready', + width: 640, + height: 360, + seconds: 4, + fps: 30, + audio: true, + }, + { file: QUEUED.file, name: 'later.mp4', bytes: 1000, status: 'queued' }, + ]); + + const done = await run(fixture, [ + { command: 'add_file_asset', args: { type: 'texture', file: PICTURE.file } }, + ]); + expect(done.result.isError).toBeUndefined(); + const asset = named(fixture, 'hero-idle'); + expect((await list(fixture, 'texture'))[0]!.asset).toEqual({ id: asset.id, name: 'hero-idle' }); + + const without = hostFor(fixture).host; + expect(text(await runTool('list_files', { type: 'sound' }, without))).toBe('This editor lists no files'); + }); + + it('makes an asset of a stored file as the picker does, one undo step, and refuses what cannot be one', async () => { + const fixture = makeSpriteScene(); + const done = await run(fixture, [ + { command: 'add_file_asset', args: { type: 'texture', file: PICTURE.file }, as: 'hero' }, + { command: 'add_file_asset', args: { type: 'sound', file: BLIP.file, name: 'Jump' } }, + { command: 'add_file_asset', args: { type: 'video', file: CLIP.file } }, + { command: 'set_asset_property', args: { asset: fixture.spriteId, key: 'texture', value: '$hero' } }, + ]); + expect(done.result.isError).toBeUndefined(); + // Only the lists the steps need are fetched, once each. + expect(done.asked).toEqual(['texture', 'sound', 'video']); + const picture = named(fixture, 'hero-idle'); + expect(picture).toMatchObject({ type: 'texture', file: PICTURE.file, width: 32, height: 16 }); + expect(named(fixture, 'Jump')).toMatchObject({ type: 'sound', file: BLIP.file }); + expect(named(fixture, 'intro')).toMatchObject({ type: 'video', file: CLIP.file, fps: 30, audio: true }); + expect(done.text).toContain(`"id":"${picture.id}"`); + + const refused = async (args: object) => { + const before = fixture.project.assets.length; + const outcome = await run(fixture, [{ command: 'add_file_asset', args }]); + expect(outcome.result.isError).toBe(true); + expect(fixture.project.assets).toHaveLength(before); + return outcome.text; + }; + expect(await refused({ type: 'texture', file: PICTURE.file })).toContain( + `is in the project already as "hero-idle" (${picture.id}): use that one`, + ); + expect(await refused({ type: 'texture', file: UNREAD.file })).toContain('could not read the size'); + expect(await refused({ type: 'video', file: QUEUED.file })).toContain('is queued, not converted yet'); + expect(await refused({ type: 'sound', file: BAKED.file })).toContain('The server holds no sound file'); + expect(await refused({ type: 'sound', file: PICTURE.file })).toContain('The server holds no sound file'); + + fixture.stack.undo(); + expect(fixture.project.assets.some((a) => a.type === 'texture' || a.type === 'sound')).toBe(false); + }); +}); + +describe('instrument presets', () => { + it('replaces the graph of an instrument with a preset, so its notes are heard, and refuses anything else', async () => { + const fixture = makeSpriteScene(); + await run(fixture, [ + { command: 'create_asset', args: { type: 'instrument', name: 'Bass' }, as: 'bass' }, + { command: 'create_asset', args: { type: 'pattern', name: 'Line' }, as: 'line' }, + { command: 'set_asset_property', args: { asset: '$line', key: 'instrument', value: '$bass' } }, + { command: 'add_notes', args: { asset: '$line', notes: [{ tick: 0, length: 480, note: 'C2' }] } }, + ]); + const bass = named(fixture, 'Bass') as InstrumentAsset; + const line = named(fixture, 'Line'); + expect((await read(fixture, line.id)).silent).toBeDefined(); + + const done = await run(fixture, [ + { command: 'apply_preset', args: { asset: bass.id, preset: 'acidBass' } }, + ]); + expect(done.result.isError).toBeUndefined(); + const types = (graph: { nodes: { type: string }[] }) => graph.nodes.map((n) => n.type).sort(); + expect(types(bass.graph)).toEqual(types(chipPresetGraph('acidBass'))); + expect(fixture.stack.undoLabel).toContain('apply_preset'); + expect((await read(fixture, line.id)).silent).toBeUndefined(); + + fixture.stack.undo(); + expect(types(bass.graph)).not.toEqual(types(chipPresetGraph('acidBass'))); + + const wrong = await run(fixture, [ + { command: 'apply_preset', args: { asset: line.id, preset: 'organ' } }, + ]); + expect(wrong.text).toContain('Line is no instrument'); + const unknown = await run(fixture, [ + { command: 'apply_preset', args: { asset: bass.id, preset: 'kazoo' } }, + ]); + expect(unknown.result.isError).toBe(true); + }); +}); + +describe('usedBy', () => { + it('names every asset naming it or a part of it, with the objects, nodes and items in there', async () => { + const fixture = makeSpriteScene(); + const done = await run(fixture, [ + { command: 'add_file_asset', args: { type: 'texture', file: PICTURE.file }, as: 'sheet' }, + { command: 'add_frame', args: { asset: '$sheet', x: 0, y: 0, w: 16, h: 16 }, as: 'frame' }, + { command: 'set_asset_property', args: { asset: fixture.spriteId, key: 'texture', value: '$frame' } }, + { + command: 'place_asset', + args: { in: fixture.sceneId, asset: fixture.spriteId, x: 0, y: 0 }, + as: 'box', + }, + { command: 'set_start_scene', args: { scene: fixture.sceneId } }, + { command: 'add_file_asset', args: { type: 'sound', file: BLIP.file }, as: 'blip' }, + { command: 'create_asset', args: { type: 'instrument', name: 'Keys' }, as: 'keys' }, + { command: 'create_asset', args: { type: 'pattern', name: 'Riff' }, as: 'riff' }, + { command: 'create_asset', args: { type: 'song', name: 'Theme' }, as: 'theme' }, + { command: 'create_asset', args: { type: 'sprite', name: 'Unused' }, as: 'unused' }, + ]); + expect(done.result.isError).toBeUndefined(); + const id = (name: string) => named(fixture, name).id; + const keys = named(fixture, 'Keys') as InstrumentAsset; + const sample = keys.graph.nodes.find((n) => n.type === 'InstrumentSample')!; + const theme = fixture.project.assets.find((a) => a.id === id('Theme')) as { tracks: { id: string }[] }; + const placed = await run(fixture, [ + { + command: 'set_node_param', + args: { asset: keys.id, node: sample.id, key: 'sound', value: id('blip') }, + }, + { command: 'set_asset_property', args: { asset: id('Riff'), key: 'instrument', value: keys.id } }, + { + command: 'add_song_item', + args: { asset: id('Theme'), track: theme.tracks[0]!.id, play: id('Riff'), tick: 0 }, + as: 'item', + }, + ]); + expect(placed.result.isError).toBeUndefined(); + const item = (fixture.project.assets.find((a) => a.id === id('Theme')) as { items: { id: string }[] }) + .items[0]!.id; + const box = fixture.sceneOf().objects.find((o) => o.assetId === fixture.spriteId)!.id; + + // The sprite wears a frame of the sheet: a use of the sheet, on the sprite itself. + expect((await read(fixture, id('hero-idle'))).usedBy).toEqual([ + { id: fixture.spriteId, name: 'Box', type: 'sprite' }, + ]); + expect((await read(fixture, fixture.spriteId)).usedBy).toEqual([ + { id: fixture.sceneId, name: 'Level', type: 'scene', at: [{ id: box }] }, + ]); + expect((await read(fixture, fixture.sceneId)).usedBy).toEqual([{ project: 'startScene' }]); + expect((await read(fixture, id('blip'))).usedBy).toEqual([ + expect.objectContaining({ + id: keys.id, + type: 'instrument', + at: [{ id: sample.id, type: 'InstrumentSample' }], + }), + ]); + expect((await read(fixture, id('Riff'))).usedBy).toEqual([ + { + id: id('Theme'), + name: 'Theme', + type: 'song', + at: [expect.objectContaining({ id: item, kind: 'pattern' })], + }, + ]); + expect((await read(fixture, keys.id)).usedBy).toEqual([ + { id: id('Riff'), name: 'Riff', type: 'pattern' }, + ]); + // The Master routing has a node for every song, joined to the output; that alone is no use of it. + settleRouting(fixture.project); + expect(fixture.project.routing.edges.some((edge) => edge.from.node === id('Theme'))).toBe(true); + expect((await read(fixture, id('Theme'))).usedBy).toEqual([]); + expect((await read(fixture, id('Unused'))).usedBy).toEqual([]); + }); +}); diff --git a/tools/e2e/71-mcp.mjs b/tools/e2e/71-mcp.mjs index 901ab0ad..825c7858 100644 --- a/tools/e2e/71-mcp.mjs +++ b/tools/e2e/71-mcp.mjs @@ -1,4 +1,6 @@ +import { sineWav } from '../lib/audio.mjs'; import { authHeaders, SESSION_COOKIE, sessionToken } from '../lib/auth.mjs'; +import { uploadSound } from '../lib/projects.mjs'; /** The tokens this step makes; a run that died halfway leaves them for the next to revoke. */ const TOKEN_NAME = 'e2e 71'; @@ -6,6 +8,7 @@ const TOOLS = [ 'get_view', 'screenshot', 'list_assets', + 'list_files', 'get_asset', 'get_graph', 'lint_graph', @@ -516,6 +519,47 @@ export async function mcp(t) { }), ); + // 71g9. Files the server holds (#431), an instrument preset (#432) and what uses an asset (#433): + // a sound uploaded by the user is listed and made an asset nothing uses yet. + const wavName = `${TOKEN_NAME} blip.wav`; + const wav = await uploadSound(baseUrl, sineWav(0.2, 660), wavName); + const filesListed = + (await callTool('list_files', { type: 'sound', query: TOKEN_NAME }))?.structuredContent?.files ?? []; + const filed = await callTool('run_commands', { + label: 'file and preset', + steps: [ + { command: 'add_file_asset', args: { type: 'sound', file: wav.file }, as: 'blip' }, + { command: 'create_asset', args: { type: 'instrument', name: `${TOKEN_NAME} keys` }, as: 'keys' }, + { command: 'apply_preset', args: { asset: '$keys', preset: 'squareLead' } }, + ], + }); + await settle(); + const filedMade = filed?.structuredContent?.made ?? []; + const blipId = filedMade.find((entry) => entry.step === 1)?.id; + const keysId = filedMade.find((entry) => entry.step === 2)?.id; + const keysGraph = keysId ? (await callTool('get_graph', { asset: keysId }))?.structuredContent : null; + const blipRead = blipId ? (await callTool('get_asset', { asset: blipId }))?.structuredContent : null; + const listedAgain = + (await callTool('list_files', { type: 'sound', query: TOKEN_NAME }))?.structuredContent?.files ?? []; + await callTool('undo'); + await settle(); + check( + 'a sound the server holds is listed, made an asset and nothing uses it yet; a preset fills an instrument', + filesListed.some((file) => file.file === wav.file && file.name === wavName && !file.asset) && + filed?.isError === false && + blipRead?.file === wav.file && + JSON.stringify(blipRead?.usedBy) === '[]' && + listedAgain.find((file) => file.file === wav.file)?.asset?.id === blipId && + keysGraph?.nodes?.some((node) => node.type === 'InstrumentChip') === true && + !(await json('__gm.state.project.assets.map((a) => a.id)')).includes(blipId), + JSON.stringify({ + listed: filesListed, + text: filed?.content?.[0]?.text?.slice(0, 300), + usedBy: blipRead?.usedBy, + nodes: keysGraph?.nodes?.map((node) => node.type), + }), + ); + // 71h. Played by the agent, the object walks left while the user holds the arrow. const started = await callTool('play', { action: 'play' }); await waitUntil('__gm.state.mode === "play"', { timeout: 10000, label: 'the game runs' }); diff --git a/tools/mcp/relay.mjs b/tools/mcp/relay.mjs index e7387ed1..2f29b1ed 100644 --- a/tools/mcp/relay.mjs +++ b/tools/mcp/relay.mjs @@ -48,6 +48,9 @@ const INSTRUCTIONS = [ '(coveredNotes) goes back above its nodes with fit_group. Music plays nothing without an', 'instrument, and an instrument sounds only the notes its Notes nodes take in, C4 to B4 in a new', 'one: get_asset lists under silent what of a pattern, song or composition is not heard.', + 'apply_preset gives an instrument that sounds every note in one step. Pictures, fonts, sounds', + 'and videos are files: list_files lists those the server holds, add_file_asset makes an asset', + 'of one. Before deleting or reworking an asset, get_asset says under usedBy what names it.', 'The editor follows your edits: after run_commands the user sees the graph, scene or asset it worked', 'on, with what it touched selected. Use open and select to show anything else.', 'Test what you built yourself: play, then play_input sends keys and mouse clicks into the game,',