adminandClaude Opus 5.5
70402aa19a
The agent could not look at a short moment of a running game. Its
calls took longer than the moment lasted, so an intro window from
0.5 to 2 s slipped past twice in the 5d test. Now the editor itself
stops the clock at the time the agent names.
- play takes until: play, resume or pause let the game run on to
that game time and hold it there. The call answers once the game
is paused, so get_runtime and screenshot show that moment.
- The frame that would step past the time is cut short
(frameWithin in src/core/time.ts, used in PlaySession). The game
stands at 0.5 s, not one frame later.
- play_input takes a last step {"pause": true}. After a wait it
holds the game at the time of the wait, after a key or click at
once. Its answer names the game time and whether it is paused.
- A pause that comes first (toolbar or a Break node), a Load scene
or a Restart game call the until off, and the answer says so.
- When the clock stands still, the call gives up after 25 s with
the time it is at. A tab in the background is refused at once.
- A recording that does not run in real time is not cut. There the
game stops in the first frame that reaches the time.
- get_runtime and play_input give times to the millisecond, not to
the hundredth, so 0.505 no longer reads as 0.51.
- The relay's instructions name both ways.
Tests:
- Unit agent-play (6 new) and time (2 new): 19/19, related editor
and relay tests 1158/1158.
- 6 counter-checks, all compile, all red. One stayed green at
first: 0.3 s is exactly 18 frames. The test now takes 0.31 s.
- e2e 71 (atx): ALL PASSED, 27 checks. New 71h3 holds at 0.505 s
and 0.8 s and after a wait of 0.31 s. Without the cut in
PlaySession it went red at 0.5166 s.
- Gate (atx): CHECK EXIT 0, 327 files, 5033 tests.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
701 lines
32 KiB
JavaScript
701 lines
32 KiB
JavaScript
import { sineWav } from '../lib/audio.mjs';
|
||
import { authHeaders, SESSION_COOKIE, sessionToken } from '../lib/auth.mjs';
|
||
import { uploadSound } from '../lib/projects.mjs';
|
||
|
||
/** The tokens this step makes; a run that died halfway leaves them for the next to revoke. */
|
||
const TOKEN_NAME = 'e2e 71';
|
||
const TOOLS = [
|
||
'get_view',
|
||
'screenshot',
|
||
'list_assets',
|
||
'list_files',
|
||
'get_asset',
|
||
'get_graph',
|
||
'lint_graph',
|
||
'catalog',
|
||
'run_commands',
|
||
'open',
|
||
'select',
|
||
'play',
|
||
'play_input',
|
||
'get_runtime',
|
||
'undo',
|
||
'redo',
|
||
];
|
||
|
||
/**
|
||
* An agent in the editor over the MCP relay (#429): a token from the account menu's dialog, the
|
||
* agent switched on there, and an MCP client that reads what the user sees – the tab, the
|
||
* selection, a screenshot – and hears when it changes. It builds a walking object in one call,
|
||
* plays it, and one Ctrl+Z of the user takes it all back; deleting waits for the user's answer.
|
||
* Switched off at the badge, the agent gets no further.
|
||
*
|
||
* In CI the relay stands in front of the PHP server on a port of its own (`FEUERWERK_MCP_BASE_URL`,
|
||
* see tools/ci/serve.sh); on stein2 nginx serves it beside the editor.
|
||
*/
|
||
export async function mcp(t) {
|
||
const { check, json, sleep, waitUntil, settle, press, locate, click, cdp, baseUrl, projectId } = t;
|
||
const { clickText, menuTicked, exceptions, navigate, shot, holdKey, entity, evaluate } = t;
|
||
const site = process.env.FEUERWERK_MCP_BASE_URL ?? baseUrl;
|
||
const endpoint = `${site}mcp`;
|
||
|
||
const api = async (method, route, body) =>
|
||
fetch(`${baseUrl}api/index.php?r=${route}`, {
|
||
method,
|
||
headers: { ...(await authHeaders(baseUrl)), 'Content-Type': 'application/json' },
|
||
body: body === undefined ? undefined : JSON.stringify(body),
|
||
signal: AbortSignal.timeout(20_000),
|
||
});
|
||
const sweep = async () => {
|
||
for (const token of await (await api('GET', 'agent-tokens')).json()) {
|
||
if (token.name.startsWith(TOKEN_NAME)) await api('DELETE', `agent-tokens/${token.id}`);
|
||
}
|
||
};
|
||
|
||
let token = '';
|
||
let session = '';
|
||
const rpc = async (method, params = {}, as = token) => {
|
||
const res = await fetch(endpoint, {
|
||
method: 'POST',
|
||
headers: {
|
||
'Content-Type': 'application/json',
|
||
Accept: 'application/json, text/event-stream',
|
||
Authorization: `Bearer ${as}`,
|
||
...(session ? { 'Mcp-Session-Id': session } : {}),
|
||
},
|
||
body: JSON.stringify({ jsonrpc: '2.0', id: 1, method, params }),
|
||
signal: AbortSignal.timeout(40_000),
|
||
});
|
||
const text = await res.text();
|
||
return {
|
||
status: res.status,
|
||
session: res.headers.get('mcp-session-id'),
|
||
body: text ? JSON.parse(text) : null,
|
||
};
|
||
};
|
||
const callTool = async (name, args = {}) =>
|
||
(await rpc('tools/call', { name, arguments: args })).body?.result;
|
||
const viewNow = async () => (await callTool('get_view'))?.structuredContent;
|
||
const openAccountMenu = async () => {
|
||
await clickText('Account');
|
||
await sleep(200);
|
||
};
|
||
|
||
await sweep();
|
||
let stream = null;
|
||
try {
|
||
if (site !== baseUrl) {
|
||
await cdp.send('Network.setCookie', {
|
||
name: SESSION_COOKIE,
|
||
value: await sessionToken(baseUrl),
|
||
url: site,
|
||
});
|
||
await cdp.send('Page.navigate', { url: `${site}?project=${projectId}` });
|
||
await waitUntil('Boolean(window.__gm?.state?.project?.id)', {
|
||
timeout: 20000,
|
||
label: 'the editor behind the relay',
|
||
});
|
||
await sleep(1000);
|
||
}
|
||
|
||
// 71a. The account menu has the switch and the tokens; the dialog makes a token and shows it once,
|
||
// with the line for Claude Code.
|
||
await openAccountMenu();
|
||
const menu = await menuTicked();
|
||
await clickText('Agent tokens…');
|
||
await waitUntil('__gm.agentTokens() !== null', { label: 'the token dialog is open' });
|
||
await t.typeText(TOKEN_NAME);
|
||
await press('Enter');
|
||
await waitUntil('__gm.agentTokens()?.made != null', { label: 'the dialog shows the new token' });
|
||
const dialog = await json('__gm.agentTokens()');
|
||
token = dialog.made;
|
||
await shot('71-agent-tokens');
|
||
await press('Escape');
|
||
await settle();
|
||
const listed = await (await api('GET', 'agent-tokens')).json();
|
||
check(
|
||
'the account menu switches the agent and opens the tokens; a token made there is shown once with the Claude Code line',
|
||
menu.some((row) => row?.label === 'Agent' && row.ticked === false) &&
|
||
menu.some((row) => row?.label === 'Agent tokens…') &&
|
||
/^fwa_[0-9a-f]{64}$/.test(token) &&
|
||
dialog.tokens.includes(TOKEN_NAME) &&
|
||
dialog.command ===
|
||
`claude mcp add --transport http feuerwerk ${endpoint} --header "Authorization: Bearer ${token}"` &&
|
||
listed.some((row) => row.name === TOKEN_NAME && row.token === undefined),
|
||
JSON.stringify({
|
||
menu,
|
||
dialog: { ...dialog, made: Boolean(dialog.made), command: dialog.command?.slice(0, 60) },
|
||
}),
|
||
);
|
||
|
||
// 71b. Before the switch, the agent hears there is no editor.
|
||
const init = await rpc('initialize', {
|
||
protocolVersion: '2025-06-18',
|
||
capabilities: {},
|
||
clientInfo: { name: 'e2e' },
|
||
});
|
||
session = init.session ?? '';
|
||
const tools = (await rpc('tools/list')).body?.result?.tools.map((tool) => tool.name);
|
||
const lonely = await callTool('get_view');
|
||
check(
|
||
'an agent with the token starts a session, lists its tools, and hears no editor is on',
|
||
init.status === 200 &&
|
||
init.body?.result?.serverInfo?.name === 'feuerwerk' &&
|
||
JSON.stringify(tools) === JSON.stringify(TOOLS) &&
|
||
lonely?.isError === true &&
|
||
/No Feuerwerk editor is connected/.test(lonely.content[0].text),
|
||
JSON.stringify({ status: init.status, tools, lonely }),
|
||
);
|
||
|
||
// 71c. Switched on in the menu, the tab gets onto the relay; the badge counts the agent.
|
||
await openAccountMenu();
|
||
await clickText('Agent');
|
||
await waitUntil('__gm.agent().status === "on"', { timeout: 15000, label: 'the agent link is on' });
|
||
await waitUntil('__gm.agent().badge === "Agent · 1"', {
|
||
timeout: 5000,
|
||
label: 'the badge counts the agent',
|
||
});
|
||
await openAccountMenu();
|
||
const ticked = (await menuTicked()).find((row) => row?.label === 'Agent')?.ticked;
|
||
await press('Escape');
|
||
const agent = await json('__gm.agent()');
|
||
check(
|
||
'switched on in the account menu, the tab is on the relay and the badge says one agent is connected',
|
||
agent.status === 'on' && agent.badge === 'Agent · 1' && ticked === true,
|
||
JSON.stringify({ agent: { ...agent, view: undefined }, ticked }),
|
||
);
|
||
|
||
// 71d. get_view is what the tab shows.
|
||
await t.evaluate('__gm.showTab("scene")');
|
||
await settle();
|
||
const scene = await viewNow();
|
||
check(
|
||
'get_view names the project and the tab the user sees',
|
||
scene?.project?.id === projectId && scene?.tab?.id === 'scene' && scene.play === 'editing',
|
||
JSON.stringify(scene),
|
||
);
|
||
|
||
// 71e. Subscribed, the agent hears when the user switches the tab, and get_view follows it.
|
||
await rpc('resources/subscribe', { uri: 'feuerwerk://view' });
|
||
const events = await fetch(endpoint, {
|
||
headers: { Authorization: `Bearer ${token}`, 'Mcp-Session-Id': session, Accept: 'text/event-stream' },
|
||
});
|
||
stream = events.body.getReader();
|
||
const heard = (async () => {
|
||
let text = '';
|
||
const until = Date.now() + 10_000;
|
||
while (!text.includes('notifications/resources/updated') && Date.now() < until) {
|
||
const { value, done } = await stream.read();
|
||
if (done) break;
|
||
text += new TextDecoder().decode(value);
|
||
}
|
||
return text;
|
||
})();
|
||
await clickText('Game');
|
||
await settle();
|
||
const note = await heard;
|
||
const game = await viewNow();
|
||
check(
|
||
'a tab switched by the user reaches the subscribed agent, and get_view says the Game tab',
|
||
note.includes('"uri":"feuerwerk://view"') &&
|
||
game?.tab?.id === 'game' &&
|
||
(await json('__gm.tab()')) === 'game',
|
||
JSON.stringify({ note: note.slice(-160), tab: game?.tab }),
|
||
);
|
||
|
||
// 71f. The screenshot is the editor as drawn, scaled to the width asked for; an agent that has
|
||
// not looked since the user moved on is told so with it.
|
||
await t.evaluate('__gm.showTab("scene")');
|
||
await settle();
|
||
await sleep(800);
|
||
const shownTab = await json('__gm.tab()');
|
||
const picture = await callTool('screenshot', { maxWidth: 640 });
|
||
const png = Buffer.from(picture?.content?.[0]?.data ?? '', 'base64');
|
||
const width = png.length > 24 ? png.readUInt32BE(16) : 0;
|
||
const height = png.length > 24 ? png.readUInt32BE(20) : 0;
|
||
const canvas = await json('({ w: __gm.app.canvas.width, h: __gm.app.canvas.height })');
|
||
const told =
|
||
picture?.content?.find((item) => item.type === 'text' && item.text.includes('has changed'))?.text ?? '';
|
||
check(
|
||
'screenshot answers a PNG of the editor at the width asked for, and tells the agent the view changed',
|
||
png.subarray(1, 4).toString() === 'PNG' &&
|
||
width === 640 &&
|
||
Math.abs(height - Math.round((canvas.h * 640) / canvas.w)) <= 1 &&
|
||
told.includes('tab Scene'),
|
||
JSON.stringify({ shownTab, width, height, canvas, bytes: png.length, told }),
|
||
);
|
||
|
||
// 71g. One call builds an object that walks left while the left arrow is held: a sprite, a behavior
|
||
// placed under it, three nodes and three edges, as one undo step.
|
||
const sceneId = await json('__gm.state.sceneToPlay()?.id');
|
||
const assetsBefore = await json('JSON.stringify(__gm.state.project.assets)');
|
||
const built = await callTool('run_commands', {
|
||
label: 'walk left',
|
||
steps: [
|
||
{ command: 'create_asset', args: { type: 'sprite', name: `${TOKEN_NAME} hero` }, as: 'hero' },
|
||
{ command: 'create_asset', args: { type: 'behavior', name: `${TOKEN_NAME} walk` }, as: 'walk' },
|
||
{ command: 'place_asset', args: { in: sceneId, asset: '$hero', x: 0, y: 0 }, as: 'player' },
|
||
{ command: 'place_asset', args: { in: sceneId, asset: '$walk', x: 0, y: 0, parent: '$player' } },
|
||
{
|
||
command: 'add_node',
|
||
args: { asset: '$walk', node: 'OnKeyHeld', params: { key: 'ArrowLeft' } },
|
||
as: 'key',
|
||
},
|
||
{ command: 'add_node', args: { asset: '$walk', node: 'Move', params: { dx: -300 } }, as: 'move' },
|
||
{ command: 'add_node', args: { asset: '$walk', node: 'Owner' }, as: 'owner' },
|
||
...[
|
||
['$key', 'exec', 'exec'],
|
||
['$key', 'dt', 'dt'],
|
||
['$owner', 'object', 'object'],
|
||
].map(([node, from, to]) => ({
|
||
command: 'connect',
|
||
args: { asset: '$walk', from: { node, port: from }, to: { node: '$move', port: to } },
|
||
})),
|
||
],
|
||
});
|
||
const made = built?.structuredContent?.made ?? [];
|
||
const walkId = made.find((entry) => entry.step === 2)?.id;
|
||
const playerId = made.find((entry) => entry.step === 3)?.id;
|
||
const graph = walkId ? (await callTool('get_graph', { asset: walkId }))?.structuredContent : null;
|
||
check(
|
||
'run_commands builds a sprite with a walking behavior in one call, one undo step named by the agent',
|
||
built?.isError === false &&
|
||
(await json('__gm.state.commands.undoLabel')) === 'Agent: walk left' &&
|
||
graph?.nodes?.map((node) => node.type).join() === 'OnKeyHeld,Move,Owner' &&
|
||
graph.edges.length === 3,
|
||
JSON.stringify({ built: built?.content?.[0]?.text?.slice(0, 300), graph: graph?.edges?.length }),
|
||
);
|
||
|
||
// 71g1. The editor follows the agent: the user sees the graph it built, its nodes selected.
|
||
await settle();
|
||
const followed = await viewNow();
|
||
check(
|
||
'the editor follows the agent: the behavior it built is the open tab, with its three nodes selected',
|
||
followed?.document?.asset?.id === walkId &&
|
||
followed.selection?.kind === 'node' &&
|
||
followed.selection.in?.id === walkId &&
|
||
followed.selection.nodes?.length === 3,
|
||
JSON.stringify({ document: followed?.document, selection: followed?.selection }),
|
||
);
|
||
|
||
// 71g2. The nodes the agent gave no place come out arranged, as the drawn nodes measure.
|
||
const lint = walkId ? (await callTool('lint_graph', { asset: walkId }))?.structuredContent : null;
|
||
check(
|
||
'the agent’s nodes are arranged: no overlaps, no link over a node, none backwards',
|
||
lint?.nodes === 3 && lint.overlaps === 0 && lint.throughNodes === 0 && lint.backwards === 0,
|
||
JSON.stringify(lint),
|
||
);
|
||
|
||
// 71g3. Arrange in the graph's header puts a tangle right again, as one step of the user's own.
|
||
await callTool('open', { asset: walkId });
|
||
const [keyNode, moveNode] = graph.nodes;
|
||
await callTool('run_commands', {
|
||
label: 'tangle',
|
||
steps: [
|
||
{
|
||
command: 'move_node',
|
||
args: { asset: walkId, node: moveNode.id, x: keyNode.x - 400, y: keyNode.y + 300 },
|
||
},
|
||
],
|
||
});
|
||
await settle();
|
||
const tangled = (await callTool('lint_graph', { asset: walkId }))?.structuredContent;
|
||
await clickText('Arrange');
|
||
await settle();
|
||
const tidied = (await callTool('lint_graph', { asset: walkId }))?.structuredContent;
|
||
const arrangedLabel = await json('__gm.state.commands.undoLabel');
|
||
await press('z', { ctrl: true });
|
||
await settle();
|
||
const backTo = (await callTool('lint_graph', { asset: walkId }))?.structuredContent;
|
||
await callTool('undo');
|
||
check(
|
||
'Arrange in the header turns the links forward again in one undo step, and Ctrl+Z brings the tangle back',
|
||
tangled?.backwards > 0 &&
|
||
tidied?.backwards === 0 &&
|
||
tidied.overlaps === 0 &&
|
||
arrangedLabel === 'Arrange nodes' &&
|
||
backTo?.backwards === tangled.backwards &&
|
||
(await json('__gm.state.commands.undoLabel')) === 'Agent: walk left',
|
||
JSON.stringify({ tangled, tidied, arrangedLabel, backTo }),
|
||
);
|
||
|
||
// 71g4. In a graph tab that is open already, the camera goes after a node added far to one side,
|
||
// keeping its zoom: the user sees it rather than a selection out of sight.
|
||
const zoomBefore = await json('__gm.graph.zoom()');
|
||
const far = await callTool('run_commands', {
|
||
label: 'far node',
|
||
steps: [
|
||
{ command: 'add_node', args: { asset: walkId, node: 'Owner', x: keyNode.x + 3000, y: keyNode.y } },
|
||
],
|
||
});
|
||
const farId = far?.structuredContent?.made?.[0]?.id;
|
||
const inSight = `(() => {
|
||
const node = __gm.graph.nodeRect(${JSON.stringify(farId)});
|
||
const view = __gm.graph.canvasRect();
|
||
return Boolean(node) && node.x >= view.x && node.x + node.w <= view.x + view.w &&
|
||
node.y >= view.y && node.y + node.h <= view.y + view.h;
|
||
})()`;
|
||
const revealed = await waitUntil(inSight, { timeout: 3000, label: 'the far node in sight' }).then(
|
||
() => true,
|
||
() => false,
|
||
);
|
||
const farView = await json(
|
||
`({ node: __gm.graph.nodeRect(${JSON.stringify(farId)}), view: __gm.graph.canvasRect(), zoom: __gm.graph.zoom() })`,
|
||
);
|
||
await callTool('undo');
|
||
check(
|
||
'the open graph tab moves its camera to a node the agent added far away, at the same zoom',
|
||
revealed && farView.zoom === zoomBefore,
|
||
JSON.stringify({ zoomBefore, ...farView }),
|
||
);
|
||
|
||
// 71g5. Past nodes and edges (#429 stage 5a): a Variables node reads a variable once add_port names it,
|
||
// and a group made in the same call frames every node as drawn, the one the call arranged among them.
|
||
const gathered = await callTool('run_commands', {
|
||
label: 'speed in a group',
|
||
steps: [
|
||
{ command: 'add_variable', args: { asset: walkId, key: 'speed', kind: 'number' } },
|
||
{ command: 'add_node', args: { asset: walkId, node: 'Variables' }, as: 'vars' },
|
||
{ command: 'add_port', args: { asset: walkId, node: '$vars', key: 'speed' } },
|
||
{
|
||
command: 'add_group',
|
||
args: { asset: walkId, nodes: [...graph.nodes.map((node) => node.id), '$vars'], title: 'Walk' },
|
||
},
|
||
],
|
||
});
|
||
await settle();
|
||
const groupId = gathered?.structuredContent?.made?.find((entry) => entry.step === 4)?.id;
|
||
const varsId = gathered?.structuredContent?.made?.find((entry) => entry.step === 2)?.id;
|
||
const framing = await json(`(() => {
|
||
const box = __gm.graph.annotationRect(${JSON.stringify(groupId)});
|
||
const ids = ${JSON.stringify([...graph.nodes.map((node) => node.id), varsId])};
|
||
const nodes = ids.map((id) => __gm.graph.nodeRect(id));
|
||
const inside = Boolean(box) && nodes.every((n) => n && n.x > box.x && n.y > box.y &&
|
||
n.x + n.w < box.x + box.w && n.y + n.h < box.y + box.h);
|
||
return { box, nodes, inside };
|
||
})()`);
|
||
const readVars = (await callTool('get_graph', { asset: walkId }))?.structuredContent?.nodes?.find(
|
||
(node) => node.id === varsId,
|
||
);
|
||
const gatheredLint = (await callTool('lint_graph', { asset: walkId }))?.structuredContent;
|
||
await callTool('undo');
|
||
check(
|
||
'add_port names a variable on a Variables node, and add_group frames every drawn node of the call',
|
||
gathered?.isError === false &&
|
||
framing.inside &&
|
||
readVars?.outputs?.some((port) => port.key === 'speed' && port.type === 'number') &&
|
||
gatheredLint?.overlaps === 0,
|
||
JSON.stringify({ text: gathered?.content?.[0]?.text?.slice(0, 300), framing, readVars, gatheredLint }),
|
||
);
|
||
|
||
// 71g6. Past graphs (#429 stage 5b): an animation keyed by the agent comes up in its own tab.
|
||
const heroId = made.find((entry) => entry.step === 1)?.id;
|
||
const animated = await callTool('run_commands', {
|
||
label: 'slide',
|
||
steps: [
|
||
{ command: 'create_asset', args: { type: 'animation', name: `${TOKEN_NAME} slide` }, as: 'anim' },
|
||
{ command: 'set_animation_root', args: { asset: '$anim', root: heroId } },
|
||
{ command: 'add_animation_track', args: { asset: '$anim', target: '', property: 'x' }, as: 'row' },
|
||
{ command: 'set_animation_key', args: { asset: '$anim', track: '$row', frame: 0, value: 0 } },
|
||
{ command: 'set_animation_key', args: { asset: '$anim', track: '$row', frame: 30, value: 120 } },
|
||
],
|
||
});
|
||
await settle();
|
||
const animId = animated?.structuredContent?.made?.find((entry) => entry.step === 1)?.id;
|
||
const animView = await viewNow();
|
||
const animRead = animId ? (await callTool('get_asset', { asset: animId }))?.structuredContent : null;
|
||
await callTool('undo');
|
||
await callTool('open', { asset: walkId });
|
||
await settle();
|
||
check(
|
||
'an animation the agent keys comes up in its tab, and get_asset names the root among its targets',
|
||
animated?.isError === false &&
|
||
animView?.document?.asset?.id === animId &&
|
||
animRead?.tracks?.[0]?.keys?.length === 2 &&
|
||
animRead?.targets?.[0]?.properties?.some((property) => property.key === 'x'),
|
||
JSON.stringify({
|
||
text: animated?.content?.[0]?.text?.slice(0, 300),
|
||
document: animView?.document,
|
||
targets: animRead?.targets?.length,
|
||
}),
|
||
);
|
||
|
||
// 71g7. Drawing (#429 stage 5c): pixel art the agent paints comes up in its tab, reads back as rows,
|
||
// and its cel reaches the server when the project is saved.
|
||
const painted = await callTool('run_commands', {
|
||
label: 'paint',
|
||
steps: [
|
||
{ command: 'create_asset', args: { type: 'pixelart', name: `${TOKEN_NAME} pixels` }, as: 'art' },
|
||
{ command: 'resize_pixel_canvas', args: { asset: '$art', width: 4, height: 3 } },
|
||
{
|
||
command: 'paint_pixels',
|
||
args: { asset: '$art', draws: [{ draw: 'rows', rows: ['8..8', '.88.', '8..8'] }] },
|
||
},
|
||
],
|
||
});
|
||
await settle();
|
||
const artId = painted?.structuredContent?.made?.find((entry) => entry.step === 1)?.id;
|
||
const artView = await viewNow();
|
||
const artRead = artId ? (await callTool('get_asset', { asset: artId }))?.structuredContent : null;
|
||
const artRows = artRead ? Object.values(artRead.frames?.[0]?.layers ?? {})[0] : null;
|
||
await waitUntil('__gm.state.saveStatus === "saved"', { timeout: 10000, label: 'saved' }).catch(() => {});
|
||
const celFiles = artId
|
||
? await json(
|
||
`Object.values(__gm.state.project.assets.find((a) => a.id === ${JSON.stringify(artId)}).frames[0].cels)`,
|
||
)
|
||
: [];
|
||
const celStatus = celFiles[0]
|
||
? Number(await evaluate(`fetch('/feuerwerk/data/pixels/${celFiles[0]}').then((r) => String(r.status))`))
|
||
: null;
|
||
await callTool('undo');
|
||
await callTool('open', { asset: walkId });
|
||
await settle();
|
||
check(
|
||
'pixel art the agent paints comes up in its tab, reads back as rows, and its cel is saved to the server',
|
||
painted?.isError === false &&
|
||
artView?.document?.asset?.id === artId &&
|
||
JSON.stringify(artRows) === JSON.stringify(['8..8', '.88.', '8..8']) &&
|
||
celFiles.length === 1 &&
|
||
celStatus === 200,
|
||
JSON.stringify({
|
||
text: painted?.content?.[0]?.text?.slice(0, 300),
|
||
document: artView?.document,
|
||
artRows,
|
||
celFiles,
|
||
celStatus,
|
||
}),
|
||
);
|
||
|
||
// 71g8. Music and compositions (#429 stage 5d): notes the agent writes, and the pattern and the
|
||
// sprite laid on a composition, which comes up in its tab.
|
||
const composed = await callTool('run_commands', {
|
||
label: 'compose',
|
||
steps: [
|
||
{ command: 'create_asset', args: { type: 'pattern', name: `${TOKEN_NAME} riff` }, as: 'riff' },
|
||
{
|
||
command: 'add_notes',
|
||
args: {
|
||
asset: '$riff',
|
||
notes: [
|
||
{ tick: 0, length: 480, note: 'C4' },
|
||
{ tick: 480, length: 480, note: 'E4' },
|
||
],
|
||
},
|
||
},
|
||
{ command: 'create_asset', args: { type: 'composition', name: `${TOKEN_NAME} cut` }, as: 'cut' },
|
||
{ command: 'add_composition_track', args: { asset: '$cut', name: 'Music' }, as: 'music' },
|
||
{
|
||
command: 'add_composition_item',
|
||
args: { asset: '$cut', track: '$music', place: '$riff', start: 0 },
|
||
},
|
||
{
|
||
command: 'add_composition_item',
|
||
args: { asset: '$cut', track: '$music', place: heroId, start: 30 },
|
||
},
|
||
],
|
||
});
|
||
await settle();
|
||
const composedMade = composed?.structuredContent?.made ?? [];
|
||
const riffId = composedMade.find((entry) => entry.step === 1)?.id;
|
||
const cutId = composedMade.find((entry) => entry.step === 3)?.id;
|
||
const cutView = await viewNow();
|
||
const riffRead = riffId ? (await callTool('get_asset', { asset: riffId }))?.structuredContent : null;
|
||
const cutRead = cutId ? (await callTool('get_asset', { asset: cutId }))?.structuredContent : null;
|
||
await callTool('undo');
|
||
await callTool('open', { asset: walkId });
|
||
await settle();
|
||
check(
|
||
'notes the agent writes read back by name, and a pattern and a sprite it lays on a composition open in its tab',
|
||
composed?.isError === false &&
|
||
cutView?.document?.asset?.id === cutId &&
|
||
JSON.stringify(riffRead?.notes?.map((note) => note.name)) === '["C4","E4"]' &&
|
||
JSON.stringify(cutRead?.items?.map((item) => [item.start, item.length])) === '[[0,480],[30,180]]' &&
|
||
cutRead?.objects?.length === 1,
|
||
JSON.stringify({
|
||
text: composed?.content?.[0]?.text?.slice(0, 300),
|
||
document: cutView?.document,
|
||
notes: riffRead?.notes,
|
||
items: cutRead?.items,
|
||
}),
|
||
);
|
||
|
||
// 71g9. Files the server holds (#431), an instrument preset (#432) and what uses an asset (#433):
|
||
// a sound uploaded by the user is listed and made an asset nothing uses yet.
|
||
const wavName = `${TOKEN_NAME} blip.wav`;
|
||
const wav = await uploadSound(baseUrl, sineWav(0.2, 660), wavName);
|
||
const filesListed =
|
||
(await callTool('list_files', { type: 'sound', query: TOKEN_NAME }))?.structuredContent?.files ?? [];
|
||
const filed = await callTool('run_commands', {
|
||
label: 'file and preset',
|
||
steps: [
|
||
{ command: 'add_file_asset', args: { type: 'sound', file: wav.file }, as: 'blip' },
|
||
{ command: 'create_asset', args: { type: 'instrument', name: `${TOKEN_NAME} keys` }, as: 'keys' },
|
||
{ command: 'apply_preset', args: { asset: '$keys', preset: 'squareLead' } },
|
||
],
|
||
});
|
||
await settle();
|
||
const filedMade = filed?.structuredContent?.made ?? [];
|
||
const blipId = filedMade.find((entry) => entry.step === 1)?.id;
|
||
const keysId = filedMade.find((entry) => entry.step === 2)?.id;
|
||
const keysGraph = keysId ? (await callTool('get_graph', { asset: keysId }))?.structuredContent : null;
|
||
const blipRead = blipId ? (await callTool('get_asset', { asset: blipId }))?.structuredContent : null;
|
||
const listedAgain =
|
||
(await callTool('list_files', { type: 'sound', query: TOKEN_NAME }))?.structuredContent?.files ?? [];
|
||
await callTool('undo');
|
||
await settle();
|
||
check(
|
||
'a sound the server holds is listed, made an asset and nothing uses it yet; a preset fills an instrument',
|
||
filesListed.some((file) => file.file === wav.file && file.name === wavName && !file.asset) &&
|
||
filed?.isError === false &&
|
||
blipRead?.file === wav.file &&
|
||
JSON.stringify(blipRead?.usedBy) === '[]' &&
|
||
listedAgain.find((file) => file.file === wav.file)?.asset?.id === blipId &&
|
||
keysGraph?.nodes?.some((node) => node.type === 'InstrumentChip') === true &&
|
||
!(await json('__gm.state.project.assets.map((a) => a.id)')).includes(blipId),
|
||
JSON.stringify({
|
||
listed: filesListed,
|
||
text: filed?.content?.[0]?.text?.slice(0, 300),
|
||
usedBy: blipRead?.usedBy,
|
||
nodes: keysGraph?.nodes?.map((node) => node.type),
|
||
}),
|
||
);
|
||
|
||
// 71h. Played by the agent, the object walks left while the user holds the arrow.
|
||
const started = await callTool('play', { action: 'play' });
|
||
await waitUntil('__gm.state.mode === "play"', { timeout: 10000, label: 'the game runs' });
|
||
await sleep(300);
|
||
const x0 = (await entity(playerId))?.x;
|
||
await holdKey('ArrowLeft', 400);
|
||
const x1 = (await entity(playerId))?.x;
|
||
|
||
// 71h2. The agent plays the game itself: it holds the arrow, clicks the object and reads where it went.
|
||
const played = await callTool('play_input', {
|
||
steps: [
|
||
{ key: 'ArrowLeft', hold: 0.5 },
|
||
{ mouse: 'click', at: { object: playerId } },
|
||
],
|
||
});
|
||
const runtime = (await callTool('get_runtime', { objects: [playerId] }))?.structuredContent;
|
||
const x2 = runtime?.objects?.[0]?.x;
|
||
check(
|
||
'the agent holds a key and clicks in the running game, and get_runtime reads the object walking',
|
||
played?.isError === false &&
|
||
played.structuredContent?.seconds >= 0.5 &&
|
||
played.structuredContent.held?.length === 0 &&
|
||
played.structuredContent.hits?.[0]?.hit === true &&
|
||
x2 < x1 - 100 &&
|
||
x2 > x1 - 200,
|
||
JSON.stringify({
|
||
played: played?.structuredContent ?? played?.content,
|
||
x1,
|
||
x2,
|
||
camera: runtime?.camera,
|
||
}),
|
||
);
|
||
const stopped = await callTool('play', { action: 'stop' });
|
||
await waitUntil('__gm.state.mode === "edit"', { label: 'the game stopped' });
|
||
check(
|
||
'play started by the agent runs its behavior: the object walks left while ArrowLeft is held',
|
||
started?.isError === false && stopped?.isError === false && x0 === 0 && x1 < -40,
|
||
JSON.stringify({ started: started?.content, x0, x1 }),
|
||
);
|
||
|
||
// 71h3. A moment of the game (#435): play with until holds it at that game time exactly, resume
|
||
// with until goes on to the next, and a last pause step holds it at the end of a wait, not a
|
||
// frame beyond (the game runs between two calls, so the wait is what is exact there).
|
||
const atHalf = await callTool('play', { action: 'play', until: 0.505 });
|
||
const halfInfo = await json('__gm.playInfo()');
|
||
const halfRead = (await callTool('get_runtime', { console: 0 }))?.structuredContent;
|
||
const onward = await callTool('play', { action: 'resume', until: 0.8 });
|
||
const onwardInfo = await json('__gm.playInfo()');
|
||
await callTool('play', { action: 'resume' });
|
||
const waited = await callTool('play_input', { steps: [{ wait: 0.31 }, { pause: true }] });
|
||
const waitedInfo = await json('__gm.playInfo()');
|
||
await callTool('play', { action: 'stop' });
|
||
await waitUntil('__gm.state.mode === "edit"', { label: 'the game stopped again' });
|
||
check(
|
||
'play with until and a last pause step hold the game at the game time asked for, not a frame beyond',
|
||
atHalf?.structuredContent?.time === 0.505 &&
|
||
halfInfo.paused &&
|
||
Math.abs(halfInfo.time - 0.505) < 1e-6 &&
|
||
halfRead?.time === 0.505 &&
|
||
onward?.structuredContent?.time === 0.8 &&
|
||
onwardInfo.paused &&
|
||
Math.abs(onwardInfo.time - 0.8) < 1e-6 &&
|
||
waited?.structuredContent?.paused === true &&
|
||
waited.structuredContent.seconds === 0.31 &&
|
||
waitedInfo.paused &&
|
||
Math.abs(waitedInfo.time - waited.structuredContent.time) < 1e-3,
|
||
JSON.stringify({
|
||
atHalf: atHalf?.structuredContent ?? atHalf?.content,
|
||
halfInfo,
|
||
halfTime: halfRead?.time,
|
||
onward: onward?.structuredContent ?? onward?.content,
|
||
onwardInfo,
|
||
waited: waited?.structuredContent ?? waited?.content,
|
||
waitedInfo,
|
||
}),
|
||
);
|
||
|
||
// 71i. One Ctrl+Z of the user takes the agent's whole step back.
|
||
await press('z', { ctrl: true });
|
||
await settle();
|
||
check(
|
||
'one Ctrl+Z takes back everything the call made',
|
||
(await json('JSON.stringify(__gm.state.project.assets)')) === assetsBefore,
|
||
JSON.stringify({ undo: await json('__gm.state.commands.undoLabel') }),
|
||
);
|
||
|
||
// 71j. Deleting asks the user: the call waits for the answer, and a no leaves the asset.
|
||
const temp = await callTool('run_commands', {
|
||
steps: [{ command: 'create_asset', args: { type: 'sprite', name: `${TOKEN_NAME} temp` } }],
|
||
});
|
||
const tempId = temp?.structuredContent?.made?.[0]?.id;
|
||
const deleting = callTool('run_commands', {
|
||
steps: [{ command: 'delete_asset', args: { asset: tempId } }],
|
||
});
|
||
const asks = `The agent asks: Delete "${TOKEN_NAME} temp"?`;
|
||
await waitUntil(`Boolean(__gm.locate(${JSON.stringify(asks)}, false))`, {
|
||
label: 'the question is open',
|
||
});
|
||
await shot('71-agent-delete');
|
||
await clickText('Cancel');
|
||
const refused = await deleting;
|
||
const kept = await json(`__gm.state.project.assets.some((a) => a.id === ${JSON.stringify(tempId)})`);
|
||
await callTool('undo');
|
||
check(
|
||
'deleting asks the user first; the call waits, and on Cancel the asset stays',
|
||
refused?.isError === true &&
|
||
/said no/.test(refused.content[0].text) &&
|
||
kept === true &&
|
||
(await json('JSON.stringify(__gm.state.project.assets)')) === assetsBefore,
|
||
JSON.stringify({ refused: refused?.content, kept }),
|
||
);
|
||
|
||
// 71k. The badge is the off switch: the agent gets no further, and a revoked token not even in.
|
||
const badge = await locate('Agent · 1');
|
||
await click(badge.x, badge.y);
|
||
await waitUntil('__gm.agent().status === "off"', { label: 'the agent is off' });
|
||
const after = await callTool('get_view');
|
||
await api('DELETE', `agent-tokens/${listed.find((row) => row.name === TOKEN_NAME).id}`);
|
||
const revoked = await rpc('tools/list');
|
||
check(
|
||
'a click on the badge switches the agent off; the agent hears no editor, and after revoking it is refused',
|
||
(await json('__gm.agent().badge')) === null && after?.isError === true && revoked.status === 401,
|
||
JSON.stringify({ after, revoked: revoked.status }),
|
||
);
|
||
check('the agent session threw nothing', exceptions().length === 0, exceptions().join(' | '));
|
||
} catch (error) {
|
||
// The clean-up below can fail on a page that hangs, and its error would hide this one.
|
||
console.error('71 stopped:', error);
|
||
throw error;
|
||
} finally {
|
||
await stream?.cancel().catch(() => {});
|
||
await t.evaluate('__gm.switchAgent?.(false)').catch(() => {});
|
||
await sweep();
|
||
// An unsaved project holds the page with "Leave site?", which no one here answers.
|
||
await waitUntil('__gm.state.saveStatus === "saved"', { timeout: 10000, label: 'saved' }).catch(() => {});
|
||
if (site !== baseUrl) await navigate();
|
||
}
|
||
}
|