rootandClaude Opus 5
3d038e11cb
typeText fired Input.insertText per character and never looked back, so a partially typed value was indistinguishable from a wrong one: a run committed x=3 for "300" and the failing check blamed the value rather than the three lost characters. Trying to reproduce that failed over 21 rounds across four variants (no pause, 400ms between characters, with the preceding scrub, and both commit paths), and the code rules out the obvious suspects: typing lands in a hidden input, each character is only a preview, objectSignature omits position so no rebuild is triggered, and NumberField.setNumber guards write-back with isEditing. Cause unknown, so this makes the symptom name itself instead of hardening against a mechanism that may not exist. Typing stays per character - that exercises the live-preview and search-filter paths - but now asserts the text is in the focused input, reporting focus and actual contents when it is not. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
434 lines
16 KiB
JavaScript
434 lines
16 KiB
JavaScript
/**
|
||
* Headless-Chromium harness for the check scripts (e2e.mjs, smoke-jumprun.mjs).
|
||
*
|
||
* Starts chromium with remote debugging, opens a CDP session and hands a set of helpers to
|
||
* the body function. Widgets are located through `window.__gm.locate(text)` (installed by
|
||
* the editor), so checks never depend on pixel coordinates.
|
||
*
|
||
* await runChecks({ projectId, baseUrl, port, shotPrefix }, async (t) => { ... });
|
||
*
|
||
* The harness always waits for chromium to actually exit (a SwiftShader GPU process kept
|
||
* rendering at full CPU once when only SIGTERM was sent) and exits with code 1 on failures.
|
||
*
|
||
* A run that ends without printing its summary exits 1 as well. Every wait here can time out for
|
||
* that reason: chromium dies under memory pressure on this machine, and a run that stops halfway
|
||
* used to exit 0 – silently reporting success for checks it never ran.
|
||
*/
|
||
import { spawn } from 'node:child_process';
|
||
import { writeFileSync } from 'node:fs';
|
||
|
||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||
|
||
/** Generous: navigation and screenshots under swiftshader are slow, a wedged chromium is forever. */
|
||
const CDP_TIMEOUT_MS = 30_000;
|
||
const SERVER_TIMEOUT_MS = 20_000;
|
||
|
||
/** Structural equality for the small plain values the checks compare (arrays of ids, selections). */
|
||
export const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
|
||
|
||
/** Keys the checks press, with the codes Chromium needs to route them like a real keyboard. */
|
||
const KEYS = {
|
||
Enter: { key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13 },
|
||
Escape: { key: 'Escape', code: 'Escape', windowsVirtualKeyCode: 27 },
|
||
Space: { key: ' ', code: 'Space', windowsVirtualKeyCode: 32 },
|
||
ArrowLeft: { key: 'ArrowLeft', code: 'ArrowLeft', windowsVirtualKeyCode: 37 },
|
||
ArrowRight: { key: 'ArrowRight', code: 'ArrowRight', windowsVirtualKeyCode: 39 },
|
||
Delete: { key: 'Delete', code: 'Delete', windowsVirtualKeyCode: 46 },
|
||
a: { key: 'a', code: 'KeyA', windowsVirtualKeyCode: 65 },
|
||
g: { key: 'g', code: 'KeyG', windowsVirtualKeyCode: 71 },
|
||
z: { key: 'z', code: 'KeyZ', windowsVirtualKeyCode: 90 },
|
||
};
|
||
|
||
/** CDP modifier bitmask: 2 = Ctrl, 8 = Shift. */
|
||
const modifierBits = ({ ctrl = false, shift = false } = {}) => (ctrl ? 2 : 0) | (shift ? 8 : 0);
|
||
|
||
class CDP {
|
||
constructor(ws) {
|
||
this.ws = ws;
|
||
this.id = 0;
|
||
this.pending = new Map();
|
||
this.listeners = [];
|
||
ws.addEventListener('message', (ev) => {
|
||
const msg = JSON.parse(ev.data);
|
||
if (msg.id && this.pending.has(msg.id)) {
|
||
const { resolve, reject } = this.pending.get(msg.id);
|
||
this.pending.delete(msg.id);
|
||
msg.error ? reject(new Error(msg.error.message)) : resolve(msg.result);
|
||
} else if (msg.method) {
|
||
for (const l of this.listeners) l(msg);
|
||
}
|
||
});
|
||
ws.addEventListener('close', () => this.failAll('the CDP socket closed – chromium is gone'));
|
||
ws.addEventListener('error', () => this.failAll('the CDP socket errored'));
|
||
}
|
||
|
||
/**
|
||
* Rejects every waiting `send`. Without this a dead socket leaves them pending forever, and
|
||
* since an unsettled promise is not a reason for node to stay alive, the run exits mid-step
|
||
* with code 0 – a dead run that reports success.
|
||
*/
|
||
failAll(reason) {
|
||
this.dead = reason;
|
||
for (const { reject } of this.pending.values()) reject(new Error(reason));
|
||
this.pending.clear();
|
||
}
|
||
|
||
send(method, params = {}) {
|
||
if (this.dead) return Promise.reject(new Error(`${this.dead} (while sending ${method})`));
|
||
const id = ++this.id;
|
||
this.ws.send(JSON.stringify({ id, method, params }));
|
||
return new Promise((resolve, reject) => {
|
||
const timer = setTimeout(() => {
|
||
this.pending.delete(id);
|
||
reject(new Error(`CDP ${method} got no answer within ${CDP_TIMEOUT_MS / 1000}s`));
|
||
}, CDP_TIMEOUT_MS);
|
||
const settle = (fn) => (value) => {
|
||
clearTimeout(timer);
|
||
fn(value);
|
||
};
|
||
this.pending.set(id, { resolve: settle(resolve), reject: settle(reject) });
|
||
});
|
||
}
|
||
|
||
on(fn) {
|
||
this.listeners.push(fn);
|
||
}
|
||
}
|
||
|
||
async function waitForChrome(port) {
|
||
for (let i = 0; i < 50; i++) {
|
||
try {
|
||
const res = await fetch(`http://127.0.0.1:${port}/json/version`, {
|
||
signal: AbortSignal.timeout(2000),
|
||
});
|
||
if (res.ok) return res.json();
|
||
} catch {
|
||
// Chromium has not opened the debugging port yet – retry.
|
||
}
|
||
await sleep(200);
|
||
}
|
||
throw new Error('chromium did not start');
|
||
}
|
||
|
||
/** The toolbox handed to a check body – everything a step needs to drive and inspect the editor. */
|
||
function makeTools({ cdp, projectId, baseUrl, consoleLog, check, shotPrefix }) {
|
||
// A throwing expression is reported, never swallowed: a mutate that dies half-way leaves the
|
||
// scene in a state no later check describes, and every one of them then passes or fails for the
|
||
// wrong reason.
|
||
const evaluate = async (expression) => {
|
||
const { result, exceptionDetails } = await cdp.send('Runtime.evaluate', {
|
||
expression,
|
||
awaitPromise: true,
|
||
returnByValue: true,
|
||
});
|
||
if (exceptionDetails)
|
||
throw new Error(
|
||
`page threw: ${exceptionDetails.exception?.description ?? exceptionDetails.text}\n in: ${expression.slice(0, 200)}`,
|
||
);
|
||
return result.value;
|
||
};
|
||
/** Evaluates an expression in the page and returns it as plain JSON; `undefined` comes back as null. */
|
||
const json = async (expression) => JSON.parse(await evaluate(`JSON.stringify((${expression}) ?? null)`));
|
||
const locate = async (text, last = false) => {
|
||
const p = await evaluate(`window.__gm && window.__gm.locate(${JSON.stringify(text)}, ${last})`);
|
||
if (!p) throw new Error(`widget with text "${text}" not found`);
|
||
return p;
|
||
};
|
||
const shot = async (name) => {
|
||
const { data } = await cdp.send('Page.captureScreenshot', { format: 'png' });
|
||
writeFileSync(`/tmp/${shotPrefix}-${name}.png`, Buffer.from(data, 'base64'));
|
||
};
|
||
|
||
// --- mouse ---------------------------------------------------------------------------------
|
||
|
||
const mouse = (type, x, y, button = 'left', extra = {}) =>
|
||
cdp.send('Input.dispatchMouseEvent', {
|
||
type,
|
||
x,
|
||
y,
|
||
button,
|
||
clickCount: type === 'mousePressed' || type === 'mouseReleased' ? 1 : 0,
|
||
...extra,
|
||
});
|
||
const click = async (x, y, button = 'left', modifiers = {}) => {
|
||
const extra = { modifiers: modifierBits(modifiers) };
|
||
await mouse('mouseMoved', x, y);
|
||
await mouse('mousePressed', x, y, button, extra);
|
||
await mouse('mouseReleased', x, y, button, extra);
|
||
await sleep(150);
|
||
};
|
||
const clickText = async (text, button = 'left') => {
|
||
const p = await locate(text);
|
||
await click(p.x, p.y, button);
|
||
};
|
||
const drag = async (x1, y1, x2, y2, steps = 12) => {
|
||
await mouse('mouseMoved', x1, y1);
|
||
await mouse('mousePressed', x1, y1);
|
||
for (let i = 1; i <= steps; i++) {
|
||
await mouse('mouseMoved', x1 + ((x2 - x1) * i) / steps, y1 + ((y2 - y1) * i) / steps, 'left', {
|
||
buttons: 1,
|
||
});
|
||
await sleep(16);
|
||
}
|
||
await mouse('mouseReleased', x2, y2);
|
||
await sleep(150);
|
||
};
|
||
|
||
// --- keyboard ------------------------------------------------------------------------------
|
||
|
||
const keyEvent = (type, name, modifiers) => {
|
||
const key = KEYS[name];
|
||
if (!key) throw new Error(`unknown key "${name}"`);
|
||
return cdp.send('Input.dispatchKeyEvent', { type, ...key, modifiers: modifierBits(modifiers) });
|
||
};
|
||
/** Press and release a key from `KEYS`, optionally with `{ ctrl, shift }`. */
|
||
const press = async (name, modifiers = {}) => {
|
||
await keyEvent('keyDown', name, modifiers);
|
||
await keyEvent('keyUp', name, modifiers);
|
||
};
|
||
/** Hold a key for `ms` milliseconds (arrow keys while playing). */
|
||
const holdKey = async (name, ms) => {
|
||
await keyEvent('keyDown', name);
|
||
await sleep(ms);
|
||
await keyEvent('keyUp', name);
|
||
};
|
||
/**
|
||
* Typed per character to exercise the per-keystroke paths (live preview, search filters), then
|
||
* verified: a partial type is otherwise indistinguishable from a wrong value – a run once
|
||
* committed x=3 for "300" and the check blamed the value, not the lost characters.
|
||
*/
|
||
const typeText = async (text) => {
|
||
for (const ch of text) await cdp.send('Input.insertText', { text: ch });
|
||
const typed = await json(
|
||
'({ tag: document.activeElement?.tagName, value: document.activeElement?.value })',
|
||
);
|
||
if (typed.tag !== 'INPUT' || !String(typed.value ?? '').includes(text))
|
||
throw new Error(
|
||
`typing "${text}" did not arrive: focus is ${typed.tag}, it holds ${JSON.stringify(typed.value)}`,
|
||
);
|
||
};
|
||
const undo = async () => {
|
||
await press('z', { ctrl: true });
|
||
await sleep(200);
|
||
};
|
||
const redo = async () => {
|
||
await press('z', { ctrl: true, shift: true });
|
||
await sleep(200);
|
||
};
|
||
|
||
// --- editor state --------------------------------------------------------------------------
|
||
|
||
/** Selection, mode, save status and notice – the facts most checks look at. */
|
||
const state = () =>
|
||
json(
|
||
'{ selection: __gm.state.selection, mode: __gm.state.mode, save: __gm.state.saveStatus, notice: __gm.state.notice }',
|
||
);
|
||
/**
|
||
* The scene's objects the way the editor lists them: the content of a placed container is
|
||
* resolved from its template and stored nowhere (see `src/core/model/instance.ts`), so it is in
|
||
* here but not in `state.activeScene.objects`. Use `storedObjects()` for the latter.
|
||
*/
|
||
const objects = () => json('__gm.sceneObjects()');
|
||
const object = (id) => json(`__gm.sceneObjects().find((o) => o.id === ${JSON.stringify(id)})`);
|
||
/** Only what the scene really holds – a placed container counts as one object, not as its content. */
|
||
const storedObjects = () => json('__gm.state.activeScene.objects');
|
||
const asset = (id) => json(`__gm.state.project.assets.find((a) => a.id === ${JSON.stringify(id)})`);
|
||
const world = () => json('__gm.world()');
|
||
const entity = async (objectId) => (await world()).entities.find((e) => e.objectId === objectId);
|
||
const select = (selection) => evaluate(`__gm.state.select(${JSON.stringify(selection)})`);
|
||
/**
|
||
* Runs `body` (JavaScript source) against the page's model and previews the result, the way a
|
||
* scrub or a command would – for putting the scene into a known state between checks.
|
||
*/
|
||
const mutate = async (body) => {
|
||
await evaluate(`(() => { ${body}; __gm.state.preview(); })()`);
|
||
await sleep(200);
|
||
};
|
||
const play = async () => {
|
||
await clickText('▶ Play');
|
||
await sleep(300);
|
||
};
|
||
const stop = async () => {
|
||
await press('Escape');
|
||
await sleep(300);
|
||
};
|
||
|
||
// --- server ----------------------------------------------------------------------------------
|
||
|
||
// Timeout, not convenience: a stalled response keeps its socket open, so node stays alive and
|
||
// the whole run hangs on this line instead of failing.
|
||
const api = async (route) => {
|
||
const url = `${baseUrl}api/index.php?r=${route}`;
|
||
try {
|
||
return await (await fetch(url, { signal: AbortSignal.timeout(SERVER_TIMEOUT_MS) })).json();
|
||
} catch (error) {
|
||
throw new Error(`request to ${route} failed: ${error.message}`, { cause: error });
|
||
}
|
||
};
|
||
const sceneOf = (doc) => doc.project.assets.find((a) => a.type === 'scene');
|
||
const exceptions = () => consoleLog.filter((l) => l.startsWith('EXCEPTION'));
|
||
/** Opens the project and asserts the page came up cleanly with the test hook installed. */
|
||
const navigate = async () => {
|
||
await cdp.send('Page.navigate', { url: `${baseUrl}?project=${projectId}` });
|
||
await sleep(4000);
|
||
check('page loaded without exceptions', exceptions().length === 0, exceptions().join(' | '));
|
||
check('debug hook present', Boolean(await evaluate('Boolean(window.__gm)')));
|
||
};
|
||
|
||
return {
|
||
cdp,
|
||
projectId,
|
||
baseUrl,
|
||
consoleLog,
|
||
exceptions,
|
||
check,
|
||
sleep,
|
||
evaluate,
|
||
json,
|
||
locate,
|
||
shot,
|
||
mouse,
|
||
click,
|
||
clickText,
|
||
drag,
|
||
press,
|
||
holdKey,
|
||
typeText,
|
||
undo,
|
||
redo,
|
||
state,
|
||
objects,
|
||
object,
|
||
storedObjects,
|
||
asset,
|
||
world,
|
||
entity,
|
||
select,
|
||
mutate,
|
||
play,
|
||
stop,
|
||
api,
|
||
sceneOf,
|
||
navigate,
|
||
};
|
||
}
|
||
|
||
export async function runChecks(options, body) {
|
||
const { projectId, baseUrl, port = 9333, shotPrefix = 'gm-e2e', width = 1280, height = 800 } = options;
|
||
const chrome = spawn(
|
||
'chromium',
|
||
[
|
||
'--headless',
|
||
'--no-sandbox',
|
||
'--disable-gpu',
|
||
'--use-gl=swiftshader',
|
||
'--enable-unsafe-swiftshader',
|
||
`--remote-debugging-port=${port}`,
|
||
`--window-size=${width},${height}`,
|
||
'--hide-scrollbars',
|
||
// Memory on this machine is tight and a long run is where chromium dies. None of this
|
||
// serves a headless check: the stderr was full of GCM sign-in retries before.
|
||
'--disable-background-networking',
|
||
'--disable-sync',
|
||
'--disable-extensions',
|
||
'--no-first-run',
|
||
'--mute-audio',
|
||
'about:blank',
|
||
],
|
||
{ stdio: ['ignore', 'ignore', 'pipe'] },
|
||
);
|
||
|
||
const consoleLog = [];
|
||
let failures = 0;
|
||
let checkCount = 0;
|
||
function check(name, ok, detail = '') {
|
||
console.log(`${ok ? 'PASS' : 'FAIL'} ${name}${detail ? ' – ' + detail : ''}`);
|
||
checkCount++;
|
||
if (!ok) failures++;
|
||
}
|
||
|
||
// Chromium dies on this machine now and then (memory pressure). Its own last words name the
|
||
// reason, and a signal here is the difference between "a check failed" and "the browser was
|
||
// killed" – questions the check output alone cannot tell apart.
|
||
const chromeErr = [];
|
||
chrome.stderr.on('data', (b) => chromeErr.push(String(b)));
|
||
let chromeDeath = '';
|
||
chrome.once('exit', (code, signal) => {
|
||
chromeDeath = signal ? `killed by ${signal}` : `exited with code ${code}`;
|
||
});
|
||
|
||
// Last line of defence: reaching the end of the run without printing a summary means the
|
||
// process walked out early. Whatever the cause, it must never look like success.
|
||
let reportedSummary = false;
|
||
process.on('exit', (code) => {
|
||
if (reportedSummary) return;
|
||
console.log(
|
||
`\nRUN DIED after ${checkCount} checks – node exited with code ${code} before the summary` +
|
||
`${chromeDeath ? `. chromium ${chromeDeath}` : ''}.`,
|
||
);
|
||
if (chromeErr.length) console.log('--- chromium stderr ---\n' + chromeErr.join('').trim().slice(-2000));
|
||
process.exitCode = 1;
|
||
});
|
||
|
||
try {
|
||
await waitForChrome(port);
|
||
const target = await (
|
||
await fetch(`http://127.0.0.1:${port}/json/new?about:blank`, {
|
||
method: 'PUT',
|
||
signal: AbortSignal.timeout(SERVER_TIMEOUT_MS),
|
||
})
|
||
).json();
|
||
const ws = new WebSocket(target.webSocketDebuggerUrl);
|
||
await new Promise((resolve, reject) => {
|
||
const timer = setTimeout(() => reject(new Error('the CDP socket never opened')), 15_000);
|
||
ws.addEventListener('open', () => {
|
||
clearTimeout(timer);
|
||
resolve();
|
||
});
|
||
ws.addEventListener('error', () => {
|
||
clearTimeout(timer);
|
||
reject(new Error('the CDP socket failed to open'));
|
||
});
|
||
});
|
||
const cdp = new CDP(ws);
|
||
await cdp.send('Page.enable');
|
||
await cdp.send('Runtime.enable');
|
||
await cdp.send('Emulation.setDeviceMetricsOverride', {
|
||
width,
|
||
height,
|
||
deviceScaleFactor: 1,
|
||
mobile: false,
|
||
});
|
||
cdp.on((msg) => {
|
||
if (msg.method === 'Runtime.consoleAPICalled')
|
||
consoleLog.push(msg.params.args.map((a) => a.value ?? a.description).join(' '));
|
||
if (msg.method === 'Runtime.exceptionThrown')
|
||
consoleLog.push(
|
||
'EXCEPTION ' +
|
||
(msg.params.exceptionDetails.exception?.description ?? msg.params.exceptionDetails.text),
|
||
);
|
||
});
|
||
await body(makeTools({ cdp, projectId, baseUrl, consoleLog, check, shotPrefix }));
|
||
ws.close();
|
||
} catch (error) {
|
||
console.error(error);
|
||
if (chromeDeath) console.error(`chromium ${chromeDeath}`);
|
||
if (chromeErr.length) console.error('--- chromium stderr ---\n' + chromeErr.join('').trim().slice(-2000));
|
||
failures++;
|
||
} finally {
|
||
await new Promise((resolve) => {
|
||
if (chrome.exitCode !== null) return resolve();
|
||
const timer = setTimeout(() => chrome.kill('SIGKILL'), 5000);
|
||
chrome.once('exit', () => {
|
||
clearTimeout(timer);
|
||
resolve();
|
||
});
|
||
chrome.kill('SIGTERM');
|
||
});
|
||
}
|
||
if (consoleLog.length) console.log('--- console ---\n' + consoleLog.slice(0, 20).join('\n'));
|
||
reportedSummary = true;
|
||
console.log(failures ? `${failures} FAILED` : `ALL PASSED – ${checkCount} checks`);
|
||
process.exit(failures ? 1 : 0);
|
||
}
|