Let an AI agent see the open editor through MCP (#429)
Claude Code should be able to work inside the user's open editor tab. This first stage connects the two and lets the agent look: what the user sees and a picture of it. Changing the project comes in stage 2. - An agent token per account opens only /mcp, no API route. The dialog "Agent tokens..." in the account menu creates it once, shows the ready "claude mcp add" line and revokes it. The server keeps only its sha256 (api/lib/AgentTokens.php). - Wrong tokens count into the sign-in brake from #427. Changing the password, renaming or deleting an account ends its tokens. - The relay (tools/mcp/) is a small Node service without packages, WebSocket server included. It runs in the container feuerwerk-mcp because PHP-FPM cannot hold connections. Whether a token or a tab ticket is valid is still decided by the PHP API, through two routes that answer only with the relay key. - The tab joins over WebSocket on the same origin with the normal session, an Origin check and a one-time ticket. It connects only after "Agent" is switched on in the account menu. One tab per account is active, a new one takes over. Guests get nothing. - Tools: get_view (project, tab, document, selection with names, camera, play state, open dialog) and screenshot. The resource feuerwerk://view can be subscribed. Claude Code does not act on that reliably, so a changed view is also noted on the next answer. - A badge next to the account menu shows the connected agents. It counts only agents with an open stream or a request in the last two minutes. - CI starts the relay in front of php -S (tools/ci/serve.sh), and e2e 71 runs the whole path in the browser. Found on the live test: the badge showed four agents because old sessions of revoked tokens were still counted. Tests: - Gate on atx: 316 files, 4888 tests - relay protocol test with fake API and fake tabs: 23/23 - get_view, tools, socketUrl: 16/16 - PHP agents 29, rights 103 checks - e2e 71 on atx 10/10, against the live site 10/10, no CSP violation - real Claude Code against the live site: get_view answered - ZAP against /mcp: nothing at High or Medium - every new check fails when the guard it covers is taken out (6) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
edef2b6323
commit
48ec722660
32 files changed
+3324
-12
No files matched your search
+185
-1
@@ -3,7 +3,9 @@ import type { Asset } from '../core/model/Asset';
|
||||
import { findAsset, findObjectHolder, ROUTING_ID } from '../core/model/Project';
|
||||
import { assetTitle } from '../core/music/embedded';
|
||||
import { mixOf, videoBusesOf } from '../core/music/mixer';
|
||||
import type { AgentStore, MadeAgentToken } from '../core/store/AgentStore';
|
||||
import type { Role } from '../core/store/AuthStore';
|
||||
import { MCP_PATH, socketUrl } from '../core/site';
|
||||
import type { DemoStore } from '../core/store/DemoStore';
|
||||
import type { ProjectStore } from '../core/store/ProjectStore';
|
||||
import { pluginsOf } from '../core/store/projectPlugins';
|
||||
@@ -19,10 +21,12 @@ import {
|
||||
clamp,
|
||||
HStack,
|
||||
isRendering,
|
||||
Label,
|
||||
lastRenderTime,
|
||||
Menu,
|
||||
type MenuEntry,
|
||||
Panel,
|
||||
Popup,
|
||||
renderCount,
|
||||
requestRender,
|
||||
Splitter,
|
||||
@@ -112,6 +116,11 @@ import { RoutingEditor } from './routing/RoutingEditor';
|
||||
import { CompressorEditor } from './soundfx/CompressorEditor';
|
||||
import { EqEditor } from './soundfx/EqEditor';
|
||||
import { NODE_TAB_KINDS, nodeOfTab, nodeTabLabel, type NodeTabKind, nodeTabOf } from './soundfx/nodeTabs';
|
||||
import { AgentLink } from './agent/AgentLink';
|
||||
import { runTool } from './agent/agentTools';
|
||||
import { AgentTokensDialog } from './agent/AgentTokensDialog';
|
||||
import { type AgentView, agentView, viewSummary } from './agent/agentView';
|
||||
import { screenshot } from './agent/screenshot';
|
||||
|
||||
export type CentreTab =
|
||||
| 'scene'
|
||||
@@ -164,6 +173,8 @@ export interface EditorAccount {
|
||||
users: UsersStore;
|
||||
/** An administrator's: making the demo copy of the open project (#428). */
|
||||
demos?: DemoStore;
|
||||
/** The agent tokens and the relay's tickets (#429); a guest has none. */
|
||||
agents?: AgentStore;
|
||||
logOut: () => void;
|
||||
/** Rejects with `WrongCredentialsError` when the current password does not match (#182). */
|
||||
changePassword: (current: string, next: string) => Promise<void>;
|
||||
@@ -178,6 +189,15 @@ type AssetKind = Exclude<CentreTab, FixedTab | 'variables' | 'routing' | NodeTab
|
||||
|
||||
const FIXED_TABS = ['scene', 'game'] as const;
|
||||
|
||||
/** `sessionStorage` key: the agent was on in this browser tab (#429). */
|
||||
const AGENT_KEY = 'feuerwerk.agent';
|
||||
|
||||
/** A dialog's bold first line, or a menu's first entry: what tells one popup from another. */
|
||||
function popupTitle(popup: Popup): string {
|
||||
const first = popup.content.children.find((child) => child instanceof Label);
|
||||
return first instanceof Label ? first.text : '';
|
||||
}
|
||||
|
||||
const PANE_ORDER = [
|
||||
'scene',
|
||||
'game',
|
||||
@@ -361,6 +381,9 @@ export class Editor implements PlayHost {
|
||||
private readonly passwordDialog = dialogSlot<PasswordDialog>();
|
||||
private readonly generalDialog = dialogSlot<GeneralSettingsDialog>();
|
||||
private readonly recordingDialog = dialogSlot<EditorRecordingDialog>();
|
||||
private readonly agentTokensDialog = dialogSlot<AgentTokensDialog>();
|
||||
/** The tab's end of the MCP relay (#429); null for a guest. */
|
||||
readonly agent: AgentLink | null;
|
||||
/** Drawn-frame count at the last measurement, not a counter of its own. */
|
||||
private fpsFrames = 0;
|
||||
private fpsElapsed = 0;
|
||||
@@ -495,9 +518,11 @@ export class Editor implements PlayHost {
|
||||
togglePause: () => this.play.togglePause(),
|
||||
step: () => this.play.step(),
|
||||
openAccountMenu: (anchor) => this.openAccountMenu(anchor, account),
|
||||
stopAgent: () => this.switchAgent(false),
|
||||
},
|
||||
account.guest === true,
|
||||
);
|
||||
this.agent = account.agents ? this.makeAgentLink(app, account.agents) : null;
|
||||
root.add(this.toolbar);
|
||||
|
||||
this.workspace = new HStack();
|
||||
@@ -894,16 +919,172 @@ export class Editor implements PlayHost {
|
||||
onSelect: () => {
|
||||
const dialog = this.passwordDialog.open(
|
||||
new PasswordDialog(account.changePassword, () =>
|
||||
this.state.setNotice('Password changed – your other sessions were signed out'),
|
||||
this.state.setNotice('Password changed – your other sessions and agent tokens were ended'),
|
||||
),
|
||||
);
|
||||
dialog.focusFirstField();
|
||||
},
|
||||
},
|
||||
...this.agentEntries(account),
|
||||
{ separator: true },
|
||||
{ label: 'Logout', onSelect: () => account.logOut() },
|
||||
]).openBelow(anchor);
|
||||
}
|
||||
|
||||
/** The agent's switch and its tokens, in the account menu (#429). */
|
||||
private agentEntries(account: EditorAccount): MenuEntry[] {
|
||||
const link = this.agent;
|
||||
const agents = account.agents;
|
||||
if (!link || !agents) return [];
|
||||
return [
|
||||
{ separator: true },
|
||||
{ label: 'Agent', checked: () => link.on, onSelect: () => this.switchAgent(!link.on) },
|
||||
{ label: 'Agent tokens…', onSelect: () => void this.openAgentTokens(agents) },
|
||||
];
|
||||
}
|
||||
|
||||
private makeAgentLink(app: Application, agents: AgentStore): AgentLink {
|
||||
const link = new AgentLink(socketUrl(MCP_PATH, location), {
|
||||
ticket: () => agents.ticket(),
|
||||
view: () => {
|
||||
const view = this.agentView();
|
||||
return { view, summary: viewSummary(view) };
|
||||
},
|
||||
summary: () => viewSummary(this.agentView(false)),
|
||||
run: (tool, args) =>
|
||||
runTool(tool, args, {
|
||||
view: () => this.agentView(),
|
||||
screenshot: (maxWidth) => screenshot(app, maxWidth),
|
||||
}),
|
||||
replaced: () => {
|
||||
sessionStorage.removeItem(AGENT_KEY);
|
||||
this.state.setNotice('Agent switched off: another tab of your account has it now');
|
||||
},
|
||||
});
|
||||
link.onChange = () => this.syncAgentBadge();
|
||||
return link;
|
||||
}
|
||||
|
||||
/**
|
||||
* Switches the agent on or off. Kept for this browser tab alone, so a reload – a deployment –
|
||||
* comes back the way it was, and another tab does not take over by itself.
|
||||
*/
|
||||
switchAgent(on: boolean): void {
|
||||
if (!this.agent) return;
|
||||
if (on) {
|
||||
sessionStorage.setItem(AGENT_KEY, '1');
|
||||
this.agent.start();
|
||||
} else {
|
||||
sessionStorage.removeItem(AGENT_KEY);
|
||||
this.agent.stop();
|
||||
}
|
||||
}
|
||||
|
||||
private syncAgentBadge(): void {
|
||||
const link = this.agent;
|
||||
if (!link?.on) {
|
||||
this.toolbar.showAgent(null);
|
||||
return;
|
||||
}
|
||||
const off = ' Click to switch it off.';
|
||||
if (link.status === 'on') {
|
||||
this.toolbar.showAgent(
|
||||
link.agents > 0 ? `Agent · ${link.agents}` : 'Agent on',
|
||||
`An agent may work in this tab (${link.agents} connected).${off}`,
|
||||
);
|
||||
} else if (link.status === 'retrying') {
|
||||
this.toolbar.showAgent('Agent offline', `The agent relay is out of reach; trying again.${off}`);
|
||||
} else {
|
||||
this.toolbar.showAgent('Agent…', `Connecting to the agent relay.${off}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** "Agent tokens…": the list, and the token just made when there is one (shown once). */
|
||||
private async openAgentTokens(agents: AgentStore, made: MadeAgentToken | null = null): Promise<void> {
|
||||
const reopen = (next: MadeAgentToken | null = null) => void this.openAgentTokens(agents, next);
|
||||
const attempt = async (work: () => Promise<void>) => {
|
||||
try {
|
||||
await work();
|
||||
} catch (error) {
|
||||
this.state.setNotice(
|
||||
`Agent tokens: ${error instanceof Error ? error.message : String(error)}`,
|
||||
'error',
|
||||
);
|
||||
reopen();
|
||||
}
|
||||
};
|
||||
await attempt(async () => {
|
||||
const tokens = await agents.list();
|
||||
const dialog = this.agentTokensDialog.open(
|
||||
new AgentTokensDialog(tokens, made, `${location.origin}${MCP_PATH}`, {
|
||||
onCreate: (name, days) =>
|
||||
void attempt(async () => {
|
||||
const token = await agents.create(name, days);
|
||||
this.state.setNotice(`Agent token "${token.name}" made – copy it now`);
|
||||
reopen(token);
|
||||
}),
|
||||
onRevoke: (token) =>
|
||||
void attempt(async () => {
|
||||
await agents.revoke(token.id);
|
||||
this.state.setNotice(`Agent token "${token.name}" revoked`);
|
||||
reopen();
|
||||
}),
|
||||
onCopy: (text, what) =>
|
||||
void navigator.clipboard.writeText(text).then(
|
||||
() => this.state.setNotice(`${what} copied`),
|
||||
() => this.state.setNotice(`${what} could not be copied – select it and press Ctrl+C`, 'error'),
|
||||
),
|
||||
}),
|
||||
);
|
||||
if (!made) dialog.focusName();
|
||||
});
|
||||
}
|
||||
|
||||
/** The agent's badge in the toolbar, null while hidden (test hook). */
|
||||
agentBadge(): string | null {
|
||||
return this.toolbar.agentBadge();
|
||||
}
|
||||
|
||||
/** The open "Agent tokens" dialog, for the check scripts. */
|
||||
agentTokens(): AgentTokensDialog | null {
|
||||
return this.agentTokensDialog.current();
|
||||
}
|
||||
|
||||
/**
|
||||
* What the user sees, for the agent (#429). `withCamera` is off for the line the link compares
|
||||
* twice a second: the camera is not news.
|
||||
*/
|
||||
agentView(withCamera = true): AgentView {
|
||||
const { state } = this;
|
||||
const view = agentView({
|
||||
project: state.project,
|
||||
mode: state.mode,
|
||||
paused: state.paused,
|
||||
saveStatus: state.saveStatus,
|
||||
tab: { id: this.tab, label: this.tabLabel(this.tab) },
|
||||
document: this.documents.active,
|
||||
windows: [...this.floats].map(([kind, float]) => ({ kind, document: float.doc })),
|
||||
selection: state.selection,
|
||||
popups: ui()
|
||||
.overlay.children.filter((child): child is Popup => child instanceof Popup && child.visible)
|
||||
.map((popup) => ({ kind: popup instanceof Menu ? 'menu' : 'dialog', title: popupTitle(popup) })),
|
||||
});
|
||||
const camera = withCamera ? this.camera() : undefined;
|
||||
return camera ? { ...view, camera } : view;
|
||||
}
|
||||
|
||||
private tabLabel(tab: TabId): string {
|
||||
return tab.startsWith('plugin:') ? tab.slice('plugin:'.length) : TAB_LABELS[tab as CentreTab];
|
||||
}
|
||||
|
||||
/** The camera of the tab showing, where it has one: the scene, a container, a graph. */
|
||||
private camera(): Record<string, number> | undefined {
|
||||
if (this.tab === 'scene') return this.viewport.camera();
|
||||
const editor = (this.assetEditors as Partial<Record<TabId, AssetTabEditor<Asset>>>)[this.tab];
|
||||
const canvas = editor?.canvas as { camera?: () => Record<string, number> } | undefined;
|
||||
return typeof canvas?.camera === 'function' ? canvas.camera() : undefined;
|
||||
}
|
||||
|
||||
/** The open "Change password" dialog, for the check scripts. */
|
||||
accountDialog(): PasswordDialog | null {
|
||||
return this.passwordDialog.current();
|
||||
@@ -1373,6 +1554,7 @@ export class Editor implements PlayHost {
|
||||
|
||||
async boot(): Promise<void> {
|
||||
await this.projects.boot();
|
||||
if (sessionStorage.getItem(AGENT_KEY) === '1') this.agent?.start();
|
||||
if (this.guest) {
|
||||
this.state.setNotice(
|
||||
'You are trying a demo as a guest: change anything, play it, record it – nothing is saved. Sign in from the menu at the far right to keep your work.',
|
||||
@@ -1395,6 +1577,8 @@ export class Editor implements PlayHost {
|
||||
this.autosave.stop();
|
||||
if (this.play.isPlaying) this.play.stop();
|
||||
this.editorRecorder.stop();
|
||||
// Signed out, the next account in this tab starts without the agent.
|
||||
this.switchAgent(false);
|
||||
this.editorRun.shutDown();
|
||||
await this.plugins?.stopAll();
|
||||
for (const cleanup of this.cleanups.splice(0).reverse()) cleanup();
|
||||
|
||||
Reference in new issue
Block a user