From 1943539a7a1d1274d8174f84ec27b7f110e5a3ee Mon Sep 17 00:00:00 2001 From: MelekTaus Date: Sun, 18 Oct 2026 15:01:32 +0200 Subject: [PATCH] Let the agent play the game and read its state through MCP (#429) An agent could build a game object but not see whether it works. It had to ask the user to press the keys. Now it can play the game in the editor itself, send keys and mouse input and read what happened. - play_input sends steps into the running game: hold or tap a key, keep it down or let it go, click, press, release or move the mouse, turn the wheel, wait. Holding and waiting count game seconds. - A mouse target is a point in world units or an object by id or name. A click on an object reports whether it landed on it. - The answer names the game time used, the keys still held, the mouse and the new console lines. - One tool instead of press_keys and pointer from the plan. A step list can then mix both, like holding Shift during a click. - get_runtime reads the running game: objects with id, type and position, the own variables of their behaviors, the Variables assets, the camera, keys, mouse and the console. Named objects come with all their properties. - A hidden editor tab gets no frames. play_input says so at once. If game time stands still for 25 s it stops, names the step and lets go of every key it held. - get_view and play say when the scene has no active camera. The first agent test missed that and only saw it on a screenshot. - The relay instructions tell the agent to test what it built. Found in the live test on a real project: - A click on an object by its name reported hit false. The hit test looked the target up by id only. - get_runtime listed the built-in switches of a behavior (enabled, runInEditor) as its variables. Tests: - agent-play (new), agent-edit, agent-view, relay green. - 5 counter-checks red: hold counts one frame, key never released, click ignores the object, hidden tab not refused, no variable filter. - e2e 71: 21/21. The agent holds ArrowLeft 0.5 s (x -120 to -270) and clicks the object (hit true). - Gate: CHECK EXIT 0, 319 files, 4944 tests. Co-Authored-By: Claude Opus 5.5 --- src/editor/Editor.ts | 11 ++ src/editor/PlaySession.ts | 6 + src/editor/agent/agentTools.ts | 48 +++++- src/editor/agent/agentView.ts | 4 + src/editor/agent/editorToolHost.ts | 43 ++++- src/editor/agent/playInput.ts | 257 ++++++++++++++++++++++++++++ src/editor/agent/runtimeView.ts | 120 +++++++++++++ src/editor/agent/tools.json | 62 +++++++ tests/editor/agent-edit.test.ts | 2 + tests/editor/agent-play.test.ts | 264 +++++++++++++++++++++++++++++ tools/e2e/71-mcp.mjs | 27 +++ tools/mcp/relay.mjs | 3 + 12 files changed, 843 insertions(+), 4 deletions(-) create mode 100644 src/editor/agent/playInput.ts create mode 100644 src/editor/agent/runtimeView.ts create mode 100644 tests/editor/agent-play.test.ts diff --git a/src/editor/Editor.ts b/src/editor/Editor.ts index 5f101792..ec976026 100644 --- a/src/editor/Editor.ts +++ b/src/editor/Editor.ts @@ -121,6 +121,7 @@ import { runTool } from './agent/agentTools'; import { AgentTokensDialog } from './agent/AgentTokensDialog'; import { type AgentView, agentView, viewSummary } from './agent/agentView'; import { editorToolHost } from './agent/editorToolHost'; +import { cameraHint } from './agent/runtimeView'; export type CentreTab = | 'scene' @@ -1095,11 +1096,21 @@ export class Editor implements PlayHost { ? { kind: 'menu', title: popupTitle(popup), entries: popup.entries } : { kind: 'dialog', title: popupTitle(popup) }, ), + hints: withCamera ? this.agentHints() : [], }); const camera = withCamera ? this.camera() : undefined; return camera ? { ...view, camera } : view; } + private agentHints(): string[] { + const hint = cameraHint( + this.state.project, + this.state.sceneToPlay(), + this.play.agentRun()?.world ?? null, + ); + return hint ? [hint] : []; + } + private tabLabel(tab: TabId): string { return tab.startsWith('plugin:') ? tab.slice('plugin:'.length) : TAB_LABELS[tab as CentreTab]; } diff --git a/src/editor/PlaySession.ts b/src/editor/PlaySession.ts index 4ea45351..cd3b214f 100644 --- a/src/editor/PlaySession.ts +++ b/src/editor/PlaySession.ts @@ -218,6 +218,12 @@ export class PlaySession { this.behaviors?.syncFromModel(); } + /** The run as the agent drives and reads it (#429); null when nothing is playing. */ + agentRun(): { world: World; input: Input; shared: SharedState } | null { + const world = this.world; + return world ? { world, input: this.input, shared: this.behaviors?.shared ?? emptySharedState() } : null; + } + /** The values a Variables asset shares in this run; null when nothing is playing. */ liveVariables(assetId: string): PropertyBag | null { return this.behaviors?.liveVariables(assetId) ?? null; diff --git a/src/editor/agent/agentTools.ts b/src/editor/agent/agentTools.ts index cc27d7a3..23f887ae 100644 --- a/src/editor/agent/agentTools.ts +++ b/src/editor/agent/agentTools.ts @@ -6,7 +6,20 @@ import { type ArgSchema, argProblem } from './argCheck'; import type { AgentView } from './agentView'; import { AgentEdit, deletedAssets, type EditStep } from './editPlan'; import type { StepTarget } from './commandCatalog'; +import { cameraOfWorld } from '../../viewport/activeCamera'; +import { + type InputStep, + inputProblem, + inputRefusal, + LONG_DEADLINE_MS, + LONG_INPUT_SECONDS, + type PlayDriver, + SHORT_DEADLINE_MS, + sendInput, + totalSeconds, +} from './playInput'; import { catalog, type CatalogTopic, getAsset, getGraph, listAssets } from './readTools'; +import { NO_CAMERA_HINT, type RuntimeFacts, runtimeView } from './runtimeView'; import TOOLS from './tools.json'; /** What a tool answers, as the relay hands it on to the agent (MCP's `CallToolResult`). */ @@ -50,6 +63,12 @@ export interface ToolHost { follows?(): boolean; /** Brings up what an edit worked on, with its objects or nodes selected. */ show?(focus: StepTarget): void; + /** The running game as `play_input` drives it. */ + game?: PlayDriver; + /** The running game as `get_runtime` reads it; null while nothing plays. */ + runtime?(): RuntimeFacts | null; + /** Tells the relay this call takes longer than most (see `relay.mjs`, `wait`). */ + takesLong?(): void; } const DEFAULT_SCREENSHOT_WIDTH = 1280; @@ -126,8 +145,22 @@ async function run(name: string, args: Record, host: ToolHost): return outcome(host.open(args), `Opened; the user sees it now.`); case 'select': return outcome(host.select(args as unknown as AgentSelection), 'Selected.'); - case 'play': - return outcome(host.play(args.action as PlayAction), `${args.action as string}: done.`); + case 'play': { + const refusal = host.play(args.action as PlayAction); + if (refusal !== null) return failure(refusal); + const facts = args.action === 'play' ? host.runtime?.() : null; + const blind = facts ? !cameraOfWorld(facts.world) : false; + return done(`${args.action as string}: done.${blind ? ` ${NO_CAMERA_HINT}` : ''}`); + } + case 'play_input': + return playInput(args.steps as InputStep[], host); + case 'get_runtime': { + const facts = host.runtime?.(); + if (!facts) return failure('The game is not running: start it with play first'); + return answer( + runtimeView(facts, { objects: args.objects as string[], console: args.console as number }), + ); + } case 'undo': case 'redo': { const step = name === 'undo' ? () => host.undo() : () => host.redo(); @@ -153,6 +186,17 @@ function outcome(refusal: string | null, text: string): ToolResult { return refusal === null ? done(text) : failure(refusal); } +async function playInput(steps: InputStep[], host: ToolHost): Promise { + const problem = inputProblem(steps); + if (problem) return failure(problem); + if (!host.game) return failure('This editor plays no game'); + const refusal = inputRefusal(host.game); + if (refusal) return failure(refusal); + const long = totalSeconds(steps) > LONG_INPUT_SECONDS; + if (long) host.takesLong?.(); + return answer(await sendInput(steps, host.game, long ? LONG_DEADLINE_MS : SHORT_DEADLINE_MS)); +} + async function runCommands(args: Record, host: ToolHost): Promise { const steps = args.steps as EditStep[]; const deletes = deletedAssets(steps); diff --git a/src/editor/agent/agentView.ts b/src/editor/agent/agentView.ts index a81424a2..312f4f62 100644 --- a/src/editor/agent/agentView.ts +++ b/src/editor/agent/agentView.ts @@ -22,6 +22,8 @@ export interface ViewFacts { selection: Selection; /** Menus and dialogs open over the editor, topmost last. */ popups: PopupView[]; + /** What the agent should know before it goes on, like a game without a camera. */ + hints?: string[]; } /** An open menu or dialog; a menu has no title of its own, so its entries say which one it is. */ @@ -59,6 +61,7 @@ export interface AgentView { popups: PopupView[]; /** The camera of the tab showing, where it has one. */ camera?: Record; + hints?: string[]; } export function agentView(facts: ViewFacts): AgentView { @@ -72,6 +75,7 @@ export function agentView(facts: ViewFacts): AgentView { play: facts.mode === 'edit' ? 'editing' : facts.paused ? 'paused' : 'playing', saved: facts.saveStatus, popups: facts.popups, + ...(facts.hints?.length ? { hints: facts.hints } : {}), }; } diff --git a/src/editor/agent/editorToolHost.ts b/src/editor/agent/editorToolHost.ts index 49553c79..8043640b 100644 --- a/src/editor/agent/editorToolHost.ts +++ b/src/editor/agent/editorToolHost.ts @@ -1,5 +1,8 @@ -import type { Application } from 'pixi.js'; +import { type Application, UPDATE_PRIORITY } from 'pixi.js'; +import type { SharedState } from '../../core/graph'; import { findAsset, findGraphAsset, findObjectHolder } from '../../core/model/Project'; +import type { Input } from '../../runtime/systems/Input'; +import type { World } from '../../runtime/World'; import { Dialog } from '../../ui'; import { activateAsset, deleteQuestion } from '../actions'; import { measuredNodes } from '../graph/measureNodes'; @@ -15,7 +18,13 @@ export interface EditorParts { app: Application; view(): AgentView; showTab(tab: 'scene' | 'game'): void; - play: { readonly isPlaying: boolean; play(paused: boolean): void; stop(): void; togglePause(): void }; + play: { + readonly isPlaying: boolean; + play(paused: boolean): void; + stop(): void; + togglePause(): void; + agentRun(): { world: World; input: Input; shared: SharedState } | null; + }; /** The user's "Follow agent" switch. */ follows(): boolean; } @@ -63,9 +72,39 @@ export function editorToolHost(parts: EditorParts, wait: () => void): ToolHost { // A running game keeps the user's eyes: an edit then goes to the running copy anyway. follows: () => parts.follows() && !parts.play.isPlaying, show: (focus) => show(parts, focus), + game: { + run: () => parts.play.agentRun(), + paused: () => state.paused, + hidden: () => document.visibilityState === 'hidden', + frame: () => nextFrame(parts.app), + now: () => performance.now(), + }, + runtime: () => { + const run = parts.play.agentRun(); + if (!run) return null; + const { world, input, shared } = run; + return { world, shared, project: state.project, heldKeys: input.heldKeys, paused: state.paused }; + }, + takesLong: wait, }; } +/** Longest a frame is waited for: a tab gone to the background draws none, and the call has to notice. */ +const FRAME_WAIT_MS = 250; + +/** After the editor's own tick, so the game has taken the frame's input by then. */ +function nextFrame(app: Application): Promise { + return new Promise((resolve) => { + const done = () => { + clearTimeout(timer); + app.ticker.remove(done); + resolve(); + }; + const timer = setTimeout(done, FRAME_WAIT_MS); + app.ticker.addOnce(done, undefined, UPDATE_PRIORITY.LOW); + }); +} + /** Brings up what an edit worked on: a graph's tab, the scene or container, the asset in the inspector. */ function show(parts: EditorParts, focus: StepTarget): void { const { state } = parts; diff --git a/src/editor/agent/playInput.ts b/src/editor/agent/playInput.ts new file mode 100644 index 00000000..1d3c41d1 --- /dev/null +++ b/src/editor/agent/playInput.ts @@ -0,0 +1,257 @@ +import { worldPositionOf } from '../../core/graph'; +import { cameraFrame } from '../../core/properties/camera'; +import type { Entity } from '../../runtime/Entity'; +import type { Input } from '../../runtime/systems/Input'; +import type { World } from '../../runtime/World'; +import { cameraOfWorld } from '../../viewport/activeCamera'; + +/** One thing the agent does in the running game (#429); `tools.json` describes the fields. */ +export interface InputStep { + key?: string; + hold?: number; + keep?: 'down' | 'up'; + mouse?: 'click' | 'down' | 'up' | 'move'; + at?: { x?: number; y?: number; object?: string }; + wheel?: number; + wait?: number; +} + +/** What `sendInput` needs of the editor. */ +export interface PlayDriver { + /** The running game, null while nothing plays. A restart or a `Load scene` brings another world. */ + run(): { world: World; input: Input } | null; + /** Paused from the toolbar. */ + paused(): boolean; + /** The browser runs no frames for a tab in the background. */ + hidden(): boolean; + /** Resolves once the editor has drawn its next frame, the game's tick included. */ + frame(): Promise; + /** Wall-clock milliseconds. */ + now(): number; +} + +export interface InputOutcome { + /** Game seconds the steps took. */ + seconds: number; + frames: number; + /** Keys held when the steps were done: a `keep: "down"` stays held after the call. */ + held: string[]; + /** Where the mouse is, in world units, and whether it is over the game's picture. */ + mouse: { x: number; y: number; inside: boolean; down: boolean }; + /** What the console got meanwhile. */ + log: string[]; + /** Per click on an object: whether it landed on it. */ + hits: { step: number; object: string; hit: boolean }[]; +} + +/** Longest a call may ask the game to run, in game seconds: more is a test of its own. */ +export const MAX_INPUT_SECONDS = 30; +/** Beyond this the call tells the relay it takes a while (see `relay.mjs`, `wait`). */ +export const LONG_INPUT_SECONDS = 5; +/** How long a call waits for the game time to move, in real time, before it gives up. */ +export const SHORT_DEADLINE_MS = 25_000; +export const LONG_DEADLINE_MS = 100_000; + +/** Names an agent is likely to write for a key, as `KeyboardEvent.key` has them. */ +const KEY_ALIASES: Record = { space: ' ', spacebar: ' ', esc: 'Escape', return: 'Enter' }; + +export function keyOf(name: string): string { + return KEY_ALIASES[name.toLowerCase()] ?? name; +} + +/** What is wrong with the steps before anything runs; null when nothing is. */ +export function inputProblem(steps: readonly InputStep[]): string | null { + for (const [index, step] of steps.entries()) { + const at = `Step ${index + 1}`; + const kinds = (['key', 'mouse', 'wheel', 'wait'] as const).filter((kind) => step[kind] !== undefined); + if (kinds.length !== 1) return `${at} has to do one thing: key, mouse, wheel or wait`; + if (step.key === undefined && (step.hold !== undefined || step.keep !== undefined)) + return `${at}: hold and keep go with a key`; + if (step.hold !== undefined && step.keep !== undefined) return `${at}: a key is held or kept, not both`; + if (step.at !== undefined && step.mouse === undefined && step.wheel === undefined) + return `${at}: at goes with mouse or wheel`; + const point = step.at; + if (point && point.object === undefined && (point.x === undefined || point.y === undefined)) + return `${at}: at needs x and y, or an object`; + if (step.mouse === 'move' && !point) return `${at}: a move needs at`; + } + const seconds = totalSeconds(steps); + if (seconds > MAX_INPUT_SECONDS) + return `The steps ask for ${seconds} s of game time; at most ${MAX_INPUT_SECONDS} s go in one call`; + return null; +} + +export function totalSeconds(steps: readonly InputStep[]): number { + return steps.reduce((sum, step) => sum + (step.hold ?? 0) + (step.wait ?? 0), 0); +} + +/** Why the run cannot take input right now; null when it can. */ +export function inputRefusal(driver: PlayDriver): string | null { + if (!driver.run()) return 'The game is not running: start it with play first'; + if (driver.paused()) return 'The game is paused: resume it first'; + if (driver.hidden()) + return 'The editor tab is in the background, where the browser runs no frames: ask the user to bring it to the front'; + return null; +} + +/** + * Plays the steps into the running game, each one taking at least one frame, so an edge a step + * makes is seen before the next one. A tapped or held key and a clicked button are let go again + * even when a step fails; `keep: "down"` is the way to leave a key held. + */ +export async function sendInput( + steps: readonly InputStep[], + driver: PlayDriver, + deadlineMs: number, +): Promise { + const clock = new GameClock(driver, driver.now() + deadlineMs); + const first = clock.run(); + const logStart = first.world.log.length; + const startWorld = first.world; + const hits: InputOutcome['hits'] = []; + for (const [index, step] of steps.entries()) { + try { + await playStep(step, clock, (object, hit) => hits.push({ step: index + 1, object, hit })); + } catch (error) { + const why = error instanceof Error ? error.message : String(error); + throw new Error(`Step ${index + 1}: ${why} (after ${round(clock.seconds)} s of game time)`, { + cause: error, + }); + } + } + const { world, input } = clock.run(); + const { x, y, inside, down } = world.pointer; + return { + seconds: round(clock.seconds), + frames: clock.frames, + held: [...input.heldKeys], + mouse: { x: round(x), y: round(y), inside, down }, + log: world === startWorld ? world.log.slice(logStart) : [...world.log], + hits, + }; +} + +async function playStep( + step: InputStep, + clock: GameClock, + hit: (object: string, hit: boolean) => void, +): Promise { + if (step.wait !== undefined) return clock.advance(step.wait); + if (step.key !== undefined) { + const key = keyOf(step.key); + if (step.keep === 'up') { + clock.run().input.release(key); + return clock.advance(0); + } + clock.run().input.press(key); + if (step.keep === 'down') return clock.advance(0); + try { + await clock.advance(step.hold ?? 0); + } finally { + clock.current()?.input.release(key); + } + return clock.advance(0); + } + const world = clock.run().world; + if (step.at) { + const point = pointOf(world, step.at); + world.movePointer(point.x, point.y, point.inside, point.onPicture); + if (point.entity && (step.mouse === 'click' || step.mouse === 'down')) + hit(step.at.object!, world.hitTest(point.entity, point.x, point.y)); + } + if (step.wheel !== undefined) { + world.turnWheel(step.wheel); + return clock.advance(0); + } + if (step.mouse === 'down' || step.mouse === 'click') world.setPointerDown(true); + if (step.mouse === 'up') world.setPointerDown(false); + if (step.mouse !== 'click') return clock.advance(0); + try { + await clock.advance(0); + } finally { + clock.current()?.world.setPointerDown(false); + } + return clock.advance(0); +} + +/** + * Where the mouse goes: a point of the world, or an object's position, which is the centre of what + * a sprite or a collider shows. `onPicture` is the point on the main camera's picture, from its + * centre in parts of its size, the way the game view hands it over (`gamePointer.ts`). `entity` is + * the object named, if one was. + */ +export function pointOf( + world: World, + at: { x?: number; y?: number; object?: string }, +): { x: number; y: number; inside: boolean; onPicture: { x: number; y: number }; entity?: Entity } { + let { x = 0, y = 0 } = at; + let entity: Entity | undefined; + if (at.object !== undefined) { + entity = world.findByObjectId(at.object) ?? world.findByName(at.object); + if (!entity) + throw new Error(`There is no object ${at.object} in the running game – get_runtime lists them`); + ({ x, y } = worldPositionOf(entity)); + } + const camera = cameraOfWorld(world); + if (!camera) return { x, y, inside: false, onPicture: { x: 0, y: 0 }, entity }; + const frame = cameraFrame(camera.props); + const onPicture = { x: (x - camera.x) / frame.w, y: (y - camera.y) / frame.h }; + const inside = Math.abs(onPicture.x) <= 0.5 && Math.abs(onPicture.y) <= 0.5; + return { x, y, inside, onPicture, entity }; +} + +/** Counts the game time that passes over the frames, across a restart or a `Load scene` as well. */ +class GameClock { + seconds = 0; + frames = 0; + private world: World; + private last: number; + + constructor( + private readonly driver: PlayDriver, + private readonly deadline: number, + ) { + this.world = this.run().world; + this.last = this.world.time; + } + + run(): { world: World; input: Input } { + const run = this.driver.run(); + if (!run) throw new Error('the game stopped'); + return run; + } + + /** The run if there still is one, for letting go of what a failed step held. */ + current(): { world: World; input: Input } | null { + return this.driver.run(); + } + + /** Lets the game run on by `seconds` of its time, and by one frame at least. */ + async advance(seconds: number): Promise { + const goal = this.seconds + seconds; + let moved = false; + while (!moved || this.seconds < goal - 1e-6) { + if (this.driver.now() > this.deadline) + throw new Error( + 'the game time stood still: paused by a behavior, or the editor tab went to the background', + ); + await this.driver.frame(); + this.frames++; + const { world } = this.run(); + if (world !== this.world) { + this.world = world; + this.last = world.time; + continue; + } + if (world.time > this.last) { + this.seconds += world.time - this.last; + moved = true; + } + this.last = world.time; + } + } +} + +function round(value: number): number { + return Math.round(value * 100) / 100; +} diff --git a/src/editor/agent/runtimeView.ts b/src/editor/agent/runtimeView.ts new file mode 100644 index 00000000..04e713cf --- /dev/null +++ b/src/editor/agent/runtimeView.ts @@ -0,0 +1,120 @@ +import { type SharedState, worldPositionOf } from '../../core/graph'; +import { namedValue } from '../../core/graph/values'; +import type { SceneAsset } from '../../core/model/Asset'; +import { findAsset, type Project } from '../../core/model/Project'; +import type { PropertyBag } from '../../core/model/Properties'; +import { cameraFrame } from '../../core/properties/camera'; +import type { Entity } from '../../runtime/Entity'; +import type { World } from '../../runtime/World'; +import { cameraOfItems, cameraOfWorld } from '../../viewport/activeCamera'; +import { collectHolderItems } from '../../viewport/items'; + +/** The running game as `get_runtime` reads it (#429). */ +export interface RuntimeFacts { + world: World; + /** The copy of the project the run plays: edits made while it runs land here. */ + project: Project; + shared: SharedState; + heldKeys: ReadonlySet; + /** Paused from the toolbar; the game's own pause is the world's. */ + paused: boolean; +} + +/** How many console lines an answer carries unless the agent asks for more. */ +export const CONSOLE_LINES = 30; + +export const NO_CAMERA_HINT = + 'No camera is switched on, so the game shows nothing: place a camera asset in the scene.'; + +/** + * What `get_view` warns of: a game that would show nothing. Playing, the run's camera counts; before + * that, the scene Play would start (#429, the agent's first game had none and nothing said so). + */ +export function cameraHint( + project: Project, + scene: SceneAsset | undefined, + world: World | null, +): string | null { + if (world) return cameraOfWorld(world) ? null : NO_CAMERA_HINT; + if (!scene || cameraOfItems(collectHolderItems(project, scene, 0).items)) return null; + return `The scene Play starts ("${scene.name}") has no camera switched on, so the game would show nothing: place a camera asset in it.`; +} + +/** + * Where everything is and what it holds, in world units. Every object is listed briefly; the ones + * the agent names come with all their properties. + */ +export function runtimeView( + facts: RuntimeFacts, + options: { objects?: readonly string[]; console?: number } = {}, +): object { + const { world, project } = facts; + const named = options.objects ?? []; + const picked = named.length > 0 ? pickEntities(world, named) : world.entities; + const camera = cameraOfWorld(world); + const lines = options.console ?? CONSOLE_LINES; + return { + scene: { id: world.sceneId, name: findAsset(project, world.sceneId)?.name ?? '' }, + time: round(world.time), + paused: facts.paused ? 'toolbar' : world.paused ? 'by the game' : false, + camera: camera + ? { id: camera.id, x: round(camera.x), y: round(camera.y), ...cameraFrame(camera.props) } + : null, + ...(camera ? {} : { hint: NO_CAMERA_HINT }), + keys: [...facts.heldKeys], + mouse: { ...world.pointer, x: round(world.pointer.x), y: round(world.pointer.y) }, + objects: picked.map((entity) => entityRow(entity, project, named.length > 0)), + variables: sharedRows(facts.shared, project), + console: lines > 0 ? world.log.slice(-lines) : [], + consoleLines: world.log.length, + }; +} + +/** The entities named by object id or name, in the order asked; one that is not there is an error. */ +function pickEntities(world: World, names: readonly string[]): Entity[] { + return names.map((name) => { + const entity = world.findByObjectId(name) ?? world.findByName(name); + if (!entity) throw new Error(`There is no object ${name} in the running game`); + return entity; + }); +} + +function entityRow(entity: Entity, project: Project, full: boolean): object { + const { x, y } = worldPositionOf(entity); + const rotation = Number(entity.props.rotation ?? 0); + const asset = findAsset(project, entity.assetId); + const row: Record = { + id: entity.objectId, + name: entity.name, + type: entity.type, + asset: asset?.name ?? entity.assetId, + x: round(x), + y: round(y), + }; + if (rotation !== 0) row.rotation = round(rotation); + if (entity.parent) row.parent = entity.parent.objectId; + if (entity.props.enabled === false) row.enabled = false; + // A behavior's own variables are what the agent most often wants to see move; its switches are not. + const declared = asset?.type === 'behavior' ? asset.variables.map((def) => def.key) : []; + if (declared.length > 0) row.variables = named(entity.props, declared); + if (full) row.props = named(entity.props); + return row; +} + +/** The values the Variables assets share, by asset name. */ +function sharedRows(shared: SharedState, project: Project): Record { + const rows: Record = {}; + for (const [id, bag] of Object.entries(shared.variables)) + rows[findAsset(project, id)?.name ?? id] = named(bag); + return rows; +} + +function named(bag: PropertyBag, keys: readonly string[] = Object.keys(bag)): Record { + const out: Record = {}; + for (const key of keys) out[key] = namedValue(bag[key]); + return out; +} + +function round(value: number): number { + return Math.round(value * 100) / 100; +} diff --git a/src/editor/agent/tools.json b/src/editor/agent/tools.json index 0d90af65..557a4509 100644 --- a/src/editor/agent/tools.json +++ b/src/editor/agent/tools.json @@ -181,6 +181,68 @@ }, "annotations": { "readOnlyHint": false, "destructiveHint": false, "openWorldHint": false } }, + { + "name": "play_input", + "title": "Keys and mouse in the running game", + "description": "Plays keys and the mouse into the running game like a player, step by step; each step takes at least one frame, and hold and wait count game seconds. A step does one thing: {\"key\": \"ArrowLeft\", \"hold\": 0.5} holds a key; {\"key\": \"Space\"} taps it for one frame; {\"key\": \"Shift\", \"keep\": \"down\"} presses it and leaves it held after the call, \"keep\": \"up\" lets it go; {\"mouse\": \"click\", \"at\": {\"object\": \"\"}} or {\"mouse\": \"click\", \"at\": {\"x\": 10, \"y\": -20}} clicks the left button there (world units, as get_runtime gives them); \"down\", \"up\" and \"move\" do just that; {\"wheel\": -1} turns the wheel; {\"wait\": 1} lets the game run on. Keys go by their KeyboardEvent.key name: \"a\", \"ArrowUp\", \"Enter\", \"Space\". Answers the game time it took, the keys still held, where the mouse is, whether a click on an object landed on it, and the new console lines; read positions and variables with get_runtime. Needs the game playing, not paused, and the editor tab in front.", + "inputSchema": { + "type": "object", + "properties": { + "steps": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "properties": { + "key": { "type": "string", "minLength": 1 }, + "hold": { + "type": "number", + "minimum": 0, + "maximum": 10, + "description": "Seconds the key is held." + }, + "keep": { "type": "string", "enum": ["down", "up"] }, + "mouse": { "type": "string", "enum": ["click", "down", "up", "move"] }, + "at": { + "type": "object", + "properties": { + "x": { "type": "number" }, + "y": { "type": "number" }, + "object": { "type": "string", "description": "An object id (or name) of the running game." } + }, + "additionalProperties": false + }, + "wheel": { "type": "integer", "minimum": -10, "maximum": 10 }, + "wait": { "type": "number", "minimum": 0, "maximum": 10, "description": "Game seconds." } + }, + "additionalProperties": false + } + } + }, + "required": ["steps"], + "additionalProperties": false + }, + "annotations": { "readOnlyHint": false, "destructiveHint": false, "openWorldHint": false } + }, + { + "name": "get_runtime", + "title": "Read the running game", + "description": "The game as it runs now: game time, whether it is paused, the camera, the keys held and the mouse, every object with its id, type and position in world units (behaviors with their variables), the values of the Variables assets, and the last console lines. Name objects (ids or names) to get all their properties.", + "inputSchema": { + "type": "object", + "properties": { + "objects": { "type": "array", "items": { "type": "string" } }, + "console": { + "type": "integer", + "minimum": 0, + "maximum": 500, + "description": "How many of the last console lines to give (default 30)." + } + }, + "additionalProperties": false + }, + "annotations": { "readOnlyHint": true, "openWorldHint": false } + }, { "name": "undo", "title": "Undo", diff --git a/tests/editor/agent-edit.test.ts b/tests/editor/agent-edit.test.ts index f4455f09..3ca113ed 100644 --- a/tests/editor/agent-edit.test.ts +++ b/tests/editor/agent-edit.test.ts @@ -491,6 +491,8 @@ describe('the catalog', () => { 'open', 'select', 'play', + 'play_input', + 'get_runtime', 'undo', 'redo', ]); diff --git a/tests/editor/agent-play.test.ts b/tests/editor/agent-play.test.ts new file mode 100644 index 00000000..a6415c91 --- /dev/null +++ b/tests/editor/agent-play.test.ts @@ -0,0 +1,264 @@ +import { describe, expect, it } from 'vitest'; +import { CreateAssetCommand } from '../../src/core/commands/assetCommands'; +import { emptySharedState } from '../../src/core/graph'; +import { findScene } from '../../src/core/model/Project'; +import { runTool, type ToolHost, type ToolResult } from '../../src/editor/agent/agentTools'; +import type { AgentView } from '../../src/editor/agent/agentView'; +import { type PlayDriver, pointOf, sendInput } from '../../src/editor/agent/playInput'; +import { cameraHint, NO_CAMERA_HINT } from '../../src/editor/agent/runtimeView'; +import { + graphEditor, + makeBehavior, + makeSpriteScene, + place, + type SceneFixture, + startWorld, + type StartedWorld, +} from '../helpers'; + +/** The agent plays its own game (#429, stage 4): keys and clicks in, positions and console out. */ + +/** A box at (200, 100) that walks left while ← is held, logs "jump" on Space and "clicked" when clicked. */ +function game(withCamera = false) { + const f = makeSpriteScene('Box'); + const behavior = makeBehavior(f, 'Logic', ['score']); + const box = place(f, f.spriteId, { x: 200, y: 100 }); + place(f, behavior.id, { x: 200, y: 100 }, box); + if (withCamera) { + const camera = new CreateAssetCommand('camera', 'Cam'); + f.stack.execute(camera); + place(f, camera.asset.id, { x: 0, y: 0 }); + } + const edit = graphEditor(f, behavior); + const held = edit.addNode('OnKeyHeld', { key: 'ArrowLeft' }); + const move = edit.addNode('Move', { dx: -100 }); + const owner = edit.addNode('Owner'); + edit.connect(held, 'exec', move, 'exec'); + edit.connect(held, 'dt', move, 'dt'); + edit.connect(owner, 'object', move, 'object'); + const jump = edit.addNode('OnKeyDown', { key: ' ' }); + edit.connect(jump, 'exec', edit.addNode('Log', { message: 'jump' }), 'exec'); + const click = edit.addNode('OnClick'); + edit.connect(click, 'exec', edit.addNode('Log', { message: 'clicked' }), 'exec'); + const started = startWorld(f, { behaviors: true }); + return { f, box, started, ...driverFor(started) }; +} + +/** Each frame is a 60th of a second of game time and 16 ms of wall time; `frozen` holds the game. */ +function driverFor(started: StartedWorld) { + const state = { running: true, paused: false, hidden: false, frozen: false, wall: 0 }; + const driver: PlayDriver = { + run: () => (state.running ? { world: started.world, input: started.input } : null), + paused: () => state.paused, + hidden: () => state.hidden, + frame: () => { + state.wall += 16; + if (!state.frozen) started.frame(1 / 60); + return Promise.resolve(); + }, + now: () => state.wall, + }; + return { driver, state }; +} + +function hostFor(f: SceneFixture, started: StartedWorld, driver: PlayDriver, running = true): ToolHost { + return { + view: () => ({ play: 'playing' }) as AgentView, + screenshot: () => Promise.resolve({ data: '', width: 0, height: 0 }), + project: () => f.project, + execute: () => null, + confirmDeletes: () => Promise.resolve(null), + open: () => null, + select: () => null, + play: () => null, + undo: () => null, + redo: () => null, + game: driver, + runtime: () => + running + ? { + world: started.world, + project: f.project, + shared: emptySharedState(), + heldKeys: started.input.heldKeys, + paused: false, + } + : null, + }; +} + +const text = (result: ToolResult) => + result.content.map((item) => (item.type === 'text' ? item.text : '')).join('\n'); + +const boxX = (started: StartedWorld, id: string) => started.world.findByObjectId(id)!.x; + +describe('play_input', () => { + it('holds a key for game seconds, and the object walks that far', async () => { + const { box, started, driver } = game(); + const outcome = await sendInput([{ key: 'ArrowLeft', hold: 0.5 }], driver, 25_000); + + expect(outcome.seconds).toBeGreaterThanOrEqual(0.5); + expect(outcome.seconds).toBeLessThan(0.5 + 2 / 60); + // −100 per second for the frames the key was down; the frame that lets go moves no more. + expect(boxX(started, box)).toBeCloseTo(200 - 100 * outcome.seconds + 100 / 60, 0); + expect(outcome.held).toEqual([]); + }); + + it('taps a key for one frame: a key-down event runs once, and "Space" is the space bar', async () => { + const { started, driver } = game(); + const outcome = await sendInput([{ key: 'Space' }], driver, 25_000); + + expect(started.world.log).toEqual(['jump']); + expect(outcome.log).toEqual(['jump']); + expect(outcome.frames).toBe(2); + expect(outcome.held).toEqual([]); + }); + + it('keeps a key down across calls until it is let go', async () => { + const { box, started, driver } = game(); + const first = await sendInput([{ key: 'ArrowLeft', keep: 'down' }, { wait: 0.25 }], driver, 25_000); + expect(first.held).toEqual(['ArrowLeft']); + const walked = boxX(started, box); + expect(walked).toBeLessThan(180); + + const second = await sendInput( + [{ wait: 0.25 }, { key: 'ArrowLeft', keep: 'up' }, { wait: 0.25 }], + driver, + 25_000, + ); + expect(second.held).toEqual([]); + expect(boxX(started, box)).toBeLessThan(walked - 20); + expect(boxX(started, box)).toBeGreaterThan(walked - 30); + }); + + it('clicks an object by its id and tells whether the click landed on it', async () => { + const { box, started, driver } = game(); + const outcome = await sendInput([{ mouse: 'click', at: { object: box } }], driver, 25_000); + + expect(outcome.hits).toEqual([{ step: 1, object: box, hit: true }]); + expect(started.world.log).toEqual(['clicked']); + expect(outcome.mouse).toMatchObject({ x: 200, y: 100, down: false }); + }); + + it('clicks an object by its name as well', async () => { + const { started, driver } = game(); + const outcome = await sendInput([{ mouse: 'click', at: { object: 'Box' } }], driver, 25_000); + + expect(outcome.hits).toEqual([{ step: 1, object: 'Box', hit: true }]); + expect(started.world.log).toEqual(['clicked']); + }); + + it('a click beside the object does not reach it', async () => { + const { started, driver } = game(); + await sendInput([{ mouse: 'click', at: { x: 0, y: 0 } }], driver, 25_000); + + expect(started.world.log).toEqual([]); + }); + + it('gives up when the game time stands still, and lets go of the key it held', async () => { + const { started, driver, state } = game(); + state.frozen = true; + await expect(sendInput([{ key: 'ArrowLeft', hold: 1 }], driver, 1000)).rejects.toThrow( + /Step 1: the game time stood still/, + ); + expect([...started.input.heldKeys]).toEqual([]); + }); + + it('turns the steps down before anything runs: not playing, paused, a tab in the background, a step doing two things', async () => { + const { f, started, driver, state } = game(); + const host = hostFor(f, started, driver); + const run = (steps: object[]) => runTool('play_input', { steps }, host); + + const result = await run([{ key: 'a', mouse: 'click' }]); + expect(result.isError).toBe(true); + expect(text(result)).toBe('Step 1 has to do one thing: key, mouse, wheel or wait'); + expect(text(await run([{ wait: 10 }, { wait: 10 }, { wait: 10 }, { wait: 1 }]))).toMatch(/at most 30 s/); + + state.hidden = true; + expect(text(await run([{ key: 'a' }]))).toMatch(/in the background/); + state.hidden = false; + state.paused = true; + expect(text(await run([{ key: 'a' }]))).toMatch(/paused/); + state.running = false; + expect(text(await run([{ key: 'a' }]))).toMatch(/not running/); + expect(started.world.time).toBe(0); + }); + + it('tells the relay when a call runs long', async () => { + const { f, started, driver } = game(); + let told = 0; + const host = { ...hostFor(f, started, driver), takesLong: () => told++ }; + + await runTool('play_input', { steps: [{ wait: 1 }] }, host); + expect(told).toBe(0); + await runTool('play_input', { steps: [{ wait: 6 }] }, host); + expect(told).toBe(1); + }); +}); + +describe('get_runtime', () => { + it('lists every object with its position, a behavior with its variables, and the console', async () => { + const { f, box, started, driver } = game(); + const host = hostFor(f, started, driver); + await runTool('play_input', { steps: [{ key: 'ArrowLeft', hold: 0.2 }, { key: 'Space' }] }, host); + + const result = await runTool('get_runtime', {}, host); + const view = result.structuredContent as { + objects: { id: string; name: string; type: string; x: number; variables?: object; parent?: string }[]; + console: string[]; + hint?: string; + }; + const row = view.objects.find((o) => o.id === box)!; + expect(row).toMatchObject({ name: 'Box', type: 'sprite' }); + expect(row.x).toBeLessThan(185); + expect(row).not.toHaveProperty('props'); + const behavior = view.objects.find((o) => o.type === 'behavior'); + expect(behavior).toMatchObject({ parent: box }); + // Its own variables only, not the switches every behavior has (enabled, runInEditor, ...). + expect(behavior?.variables).toEqual({ score: 0 }); + expect(view.console).toEqual(['jump']); + expect(view.hint).toBe(NO_CAMERA_HINT); + }); + + it('gives all properties of the objects named, and says which name is none', async () => { + const { f, started, driver } = game(); + const host = hostFor(f, started, driver); + + const result = await runTool('get_runtime', { objects: ['Box'] }, host); + const view = result.structuredContent as { objects: { props: Record }[] }; + expect(view.objects).toHaveLength(1); + expect(view.objects[0]!.props).toHaveProperty('width'); + + expect(text(await runTool('get_runtime', { objects: ['Nobody'] }, host))).toBe( + 'There is no object Nobody in the running game', + ); + expect(text(await runTool('get_runtime', {}, hostFor(f, started, driver, false)))).toMatch(/not running/); + }); +}); + +describe('the camera', () => { + it('puts the mouse on its picture: inside its frame, outside beyond it', () => { + const { started } = game(true); + expect(pointOf(started.world, { x: 100, y: -50 })).toMatchObject({ inside: true }); + expect(pointOf(started.world, { x: 5000, y: 0 })).toMatchObject({ inside: false }); + expect(pointOf(game(false).started.world, { x: 0, y: 0 })).toMatchObject({ inside: false }); + }); + + it('is missed before Play and while playing, and play says so', async () => { + const without = game(false); + const scene = findScene(without.f.project, without.f.sceneId); + expect(cameraHint(without.f.project, scene, null)).toMatch(/\("Level"\) has no camera/); + expect(cameraHint(without.f.project, scene, without.started.world)).toBe(NO_CAMERA_HINT); + const played = await runTool( + 'play', + { action: 'play' }, + hostFor(without.f, without.started, without.driver), + ); + expect(text(played)).toBe(`play: done. ${NO_CAMERA_HINT}`); + + const withCamera = game(true); + const sceneWith = findScene(withCamera.f.project, withCamera.f.sceneId); + expect(cameraHint(withCamera.f.project, sceneWith, null)).toBeNull(); + expect(cameraHint(withCamera.f.project, sceneWith, withCamera.started.world)).toBeNull(); + }); +}); diff --git a/tools/e2e/71-mcp.mjs b/tools/e2e/71-mcp.mjs index 47e7e309..190c92d7 100644 --- a/tools/e2e/71-mcp.mjs +++ b/tools/e2e/71-mcp.mjs @@ -14,6 +14,8 @@ const TOOLS = [ 'open', 'select', 'play', + 'play_input', + 'get_runtime', 'undo', 'redo', ]; @@ -351,6 +353,31 @@ export async function mcp(t) { const x0 = (await entity(playerId))?.x; await holdKey('ArrowLeft', 400); const x1 = (await entity(playerId))?.x; + + // 71h2. The agent plays the game itself: it holds the arrow, clicks the object and reads where it went. + const played = await callTool('play_input', { + steps: [ + { key: 'ArrowLeft', hold: 0.5 }, + { mouse: 'click', at: { object: playerId } }, + ], + }); + const runtime = (await callTool('get_runtime', { objects: [playerId] }))?.structuredContent; + const x2 = runtime?.objects?.[0]?.x; + check( + 'the agent holds a key and clicks in the running game, and get_runtime reads the object walking', + played?.isError === false && + played.structuredContent?.seconds >= 0.5 && + played.structuredContent.held?.length === 0 && + played.structuredContent.hits?.[0]?.hit === true && + x2 < x1 - 100 && + x2 > x1 - 200, + JSON.stringify({ + played: played?.structuredContent ?? played?.content, + x1, + x2, + camera: runtime?.camera, + }), + ); const stopped = await callTool('play', { action: 'stop' }); await waitUntil('__gm.state.mode === "edit"', { label: 'the game stopped' }); check( diff --git a/tools/mcp/relay.mjs b/tools/mcp/relay.mjs index 2729a74a..8e82e506 100644 --- a/tools/mcp/relay.mjs +++ b/tools/mcp/relay.mjs @@ -43,6 +43,9 @@ const INSTRUCTIONS = [ 'Get or Owner node next to its user, not over a long link; check with lint_graph.', 'The editor follows your edits: after run_commands the user sees the graph, scene or asset it worked', 'on, with what it touched selected. Use open and select to show anything else.', + 'Test what you built yourself: play, then play_input sends keys and mouse clicks into the game,', + 'and get_runtime reads where the objects are, their variables and the console. A game needs a', + 'camera in its scene to show anything; get_view says when it has none.', 'When a tool answers that no editor is connected, ask the user to open the editor and switch on', '"Agent" in its account menu.', ].join(' ');