Files
feuerwerk/src/editor/agent/runtimeView.ts
T
adminandClaude Opus 5.5 1943539a7a
feuerwerk / gate (push) Successful in 5m7s
feuerwerk / suite (push) Successful in 18m44s
Let the agent play the game and read its state through MCP (#429)
An agent could build a game object but not see whether it works. It
had to ask the user to press the keys. Now it can play the game in the
editor itself, send keys and mouse input and read what happened.

- play_input sends steps into the running game: hold or tap a key,
  keep it down or let it go, click, press, release or move the mouse,
  turn the wheel, wait. Holding and waiting count game seconds.
- A mouse target is a point in world units or an object by id or
  name. A click on an object reports whether it landed on it.
- The answer names the game time used, the keys still held, the mouse
  and the new console lines.
- One tool instead of press_keys and pointer from the plan. A step
  list can then mix both, like holding Shift during a click.
- get_runtime reads the running game: objects with id, type and
  position, the own variables of their behaviors, the Variables
  assets, the camera, keys, mouse and the console. Named objects come
  with all their properties.
- A hidden editor tab gets no frames. play_input says so at once. If
  game time stands still for 25 s it stops, names the step and lets
  go of every key it held.
- get_view and play say when the scene has no active camera. The
  first agent test missed that and only saw it on a screenshot.
- The relay instructions tell the agent to test what it built.

Found in the live test on a real project:

- A click on an object by its name reported hit false. The hit test
  looked the target up by id only.
- get_runtime listed the built-in switches of a behavior (enabled,
  runInEditor) as its variables.

Tests:
- agent-play (new), agent-edit, agent-view, relay green.
- 5 counter-checks red: hold counts one frame, key never released,
  click ignores the object, hidden tab not refused, no variable
  filter.
- e2e 71: 21/21. The agent holds ArrowLeft 0.5 s (x -120 to -270)
  and clicks the object (hit true).
- Gate: CHECK EXIT 0, 319 files, 4944 tests.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-18 15:01:32 +02:00

121 lines
4.9 KiB
TypeScript

import { type SharedState, worldPositionOf } from '../../core/graph';
import { namedValue } from '../../core/graph/values';
import type { SceneAsset } from '../../core/model/Asset';
import { findAsset, type Project } from '../../core/model/Project';
import type { PropertyBag } from '../../core/model/Properties';
import { cameraFrame } from '../../core/properties/camera';
import type { Entity } from '../../runtime/Entity';
import type { World } from '../../runtime/World';
import { cameraOfItems, cameraOfWorld } from '../../viewport/activeCamera';
import { collectHolderItems } from '../../viewport/items';
/** The running game as `get_runtime` reads it (#429). */
export interface RuntimeFacts {
world: World;
/** The copy of the project the run plays: edits made while it runs land here. */
project: Project;
shared: SharedState;
heldKeys: ReadonlySet<string>;
/** Paused from the toolbar; the game's own pause is the world's. */
paused: boolean;
}
/** How many console lines an answer carries unless the agent asks for more. */
export const CONSOLE_LINES = 30;
export const NO_CAMERA_HINT =
'No camera is switched on, so the game shows nothing: place a camera asset in the scene.';
/**
* What `get_view` warns of: a game that would show nothing. Playing, the run's camera counts; before
* that, the scene Play would start (#429, the agent's first game had none and nothing said so).
*/
export function cameraHint(
project: Project,
scene: SceneAsset | undefined,
world: World | null,
): string | null {
if (world) return cameraOfWorld(world) ? null : NO_CAMERA_HINT;
if (!scene || cameraOfItems(collectHolderItems(project, scene, 0).items)) return null;
return `The scene Play starts ("${scene.name}") has no camera switched on, so the game would show nothing: place a camera asset in it.`;
}
/**
* Where everything is and what it holds, in world units. Every object is listed briefly; the ones
* the agent names come with all their properties.
*/
export function runtimeView(
facts: RuntimeFacts,
options: { objects?: readonly string[]; console?: number } = {},
): object {
const { world, project } = facts;
const named = options.objects ?? [];
const picked = named.length > 0 ? pickEntities(world, named) : world.entities;
const camera = cameraOfWorld(world);
const lines = options.console ?? CONSOLE_LINES;
return {
scene: { id: world.sceneId, name: findAsset(project, world.sceneId)?.name ?? '' },
time: round(world.time),
paused: facts.paused ? 'toolbar' : world.paused ? 'by the game' : false,
camera: camera
? { id: camera.id, x: round(camera.x), y: round(camera.y), ...cameraFrame(camera.props) }
: null,
...(camera ? {} : { hint: NO_CAMERA_HINT }),
keys: [...facts.heldKeys],
mouse: { ...world.pointer, x: round(world.pointer.x), y: round(world.pointer.y) },
objects: picked.map((entity) => entityRow(entity, project, named.length > 0)),
variables: sharedRows(facts.shared, project),
console: lines > 0 ? world.log.slice(-lines) : [],
consoleLines: world.log.length,
};
}
/** The entities named by object id or name, in the order asked; one that is not there is an error. */
function pickEntities(world: World, names: readonly string[]): Entity[] {
return names.map((name) => {
const entity = world.findByObjectId(name) ?? world.findByName(name);
if (!entity) throw new Error(`There is no object ${name} in the running game`);
return entity;
});
}
function entityRow(entity: Entity, project: Project, full: boolean): object {
const { x, y } = worldPositionOf(entity);
const rotation = Number(entity.props.rotation ?? 0);
const asset = findAsset(project, entity.assetId);
const row: Record<string, unknown> = {
id: entity.objectId,
name: entity.name,
type: entity.type,
asset: asset?.name ?? entity.assetId,
x: round(x),
y: round(y),
};
if (rotation !== 0) row.rotation = round(rotation);
if (entity.parent) row.parent = entity.parent.objectId;
if (entity.props.enabled === false) row.enabled = false;
// A behavior's own variables are what the agent most often wants to see move; its switches are not.
const declared = asset?.type === 'behavior' ? asset.variables.map((def) => def.key) : [];
if (declared.length > 0) row.variables = named(entity.props, declared);
if (full) row.props = named(entity.props);
return row;
}
/** The values the Variables assets share, by asset name. */
function sharedRows(shared: SharedState, project: Project): Record<string, unknown> {
const rows: Record<string, unknown> = {};
for (const [id, bag] of Object.entries(shared.variables))
rows[findAsset(project, id)?.name ?? id] = named(bag);
return rows;
}
function named(bag: PropertyBag, keys: readonly string[] = Object.keys(bag)): Record<string, unknown> {
const out: Record<string, unknown> = {};
for (const key of keys) out[key] = namedValue(bag[key]);
return out;
}
function round(value: number): number {
return Math.round(value * 100) / 100;
}