/** * Headless-Chromium harness for the checks; widgets are found via `__gm.locate(text)`, not coordinates. * It always waits for chromium to exit (a leftover GPU process burns CPU) and exits 1 on failures or when * the summary line is missing, since chromium can die halfway under memory pressure. */ import { writeFileSync } from 'node:fs'; import { authHeaders, forgetSession, SESSION_COOKIE, sessionToken } from './auth.mjs'; import { launchBrowser } from './browser.mjs'; /** * Every wait the checks do goes through here, and the time is booked. A run that takes ten * minutes needs to say how much of that was the editor working and how much was the harness * waiting for a deadline it made up – the two call for opposite fixes. */ export const waited = { ms: 0, calls: 0 }; const sleep = (ms) => { waited.ms += ms; waited.calls++; return new Promise((r) => setTimeout(r, ms)); }; /** Generous on purpose: a slow navigation still finishes, a wedged chromium never does. */ const CDP_TIMEOUT_MS = 30_000; const SERVER_TIMEOUT_MS = 20_000; /** Structural equality for the small plain values the checks compare (arrays of ids, selections). */ export const same = (a, b) => JSON.stringify(a) === JSON.stringify(b); /** Sum of the channel differences of two `#rrggbb` colours. */ export function colourDistance(a, b) { const channels = (hex) => [1, 3, 5].map((i) => parseInt(hex.slice(i, i + 2), 16)); const [ar, ag, ab] = channels(a); const [br, bg, bb] = channels(b); return Math.abs(ar - br) + Math.abs(ag - bg) + Math.abs(ab - bb); } /** Keys the checks press, with the codes Chromium needs to route them like a real keyboard. */ const KEYS = { Enter: { key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13 }, Escape: { key: 'Escape', code: 'Escape', windowsVirtualKeyCode: 27 }, Space: { key: ' ', code: 'Space', windowsVirtualKeyCode: 32 }, ArrowLeft: { key: 'ArrowLeft', code: 'ArrowLeft', windowsVirtualKeyCode: 37 }, ArrowUp: { key: 'ArrowUp', code: 'ArrowUp', windowsVirtualKeyCode: 38 }, ArrowRight: { key: 'ArrowRight', code: 'ArrowRight', windowsVirtualKeyCode: 39 }, ArrowDown: { key: 'ArrowDown', code: 'ArrowDown', windowsVirtualKeyCode: 40 }, Delete: { key: 'Delete', code: 'Delete', windowsVirtualKeyCode: 46 }, Alt: { key: 'Alt', code: 'AltLeft', windowsVirtualKeyCode: 18 }, a: { key: 'a', code: 'KeyA', windowsVirtualKeyCode: 65 }, c: { key: 'c', code: 'KeyC', windowsVirtualKeyCode: 67 }, d: { key: 'd', code: 'KeyD', windowsVirtualKeyCode: 68 }, g: { key: 'g', code: 'KeyG', windowsVirtualKeyCode: 71 }, p: { key: 'p', code: 'KeyP', windowsVirtualKeyCode: 80 }, s: { key: 's', code: 'KeyS', windowsVirtualKeyCode: 83 }, v: { key: 'v', code: 'KeyV', windowsVirtualKeyCode: 86 }, w: { key: 'w', code: 'KeyW', windowsVirtualKeyCode: 87 }, z: { key: 'z', code: 'KeyZ', windowsVirtualKeyCode: 90 }, }; /** CDP modifier bitmask: 1 = Alt, 2 = Ctrl, 8 = Shift. */ const modifierBits = ({ alt = false, ctrl = false, shift = false } = {}) => (alt ? 1 : 0) | (ctrl ? 2 : 0) | (shift ? 8 : 0); class CDP { constructor(ws) { this.ws = ws; this.id = 0; this.pending = new Map(); this.listeners = []; ws.addEventListener('message', (ev) => { const msg = JSON.parse(ev.data); if (msg.id && this.pending.has(msg.id)) { const { resolve, reject } = this.pending.get(msg.id); this.pending.delete(msg.id); msg.error ? reject(new Error(msg.error.message)) : resolve(msg.result); } else if (msg.method) { for (const l of this.listeners) l(msg); } }); ws.addEventListener('close', () => this.failAll('the CDP socket closed – chromium is gone')); ws.addEventListener('error', () => this.failAll('the CDP socket errored')); } /** * Rejects every waiting `send`. Without this a dead socket leaves them pending forever, and * since an unsettled promise is not a reason for node to stay alive, the run exits mid-step * with code 0 – a dead run that reports success. */ failAll(reason) { this.dead = reason; for (const { reject } of this.pending.values()) reject(new Error(reason)); this.pending.clear(); } send(method, params = {}) { if (this.dead) return Promise.reject(new Error(`${this.dead} (while sending ${method})`)); const id = ++this.id; this.ws.send(JSON.stringify({ id, method, params })); return new Promise((resolve, reject) => { const timer = setTimeout(() => { this.pending.delete(id); reject(new Error(`CDP ${method} got no answer within ${CDP_TIMEOUT_MS / 1000}s`)); }, CDP_TIMEOUT_MS); const settle = (fn) => (value) => { clearTimeout(timer); fn(value); }; this.pending.set(id, { resolve: settle(resolve), reject: settle(reject) }); }); } on(fn) { this.listeners.push(fn); } } async function waitForChrome(port) { for (let i = 0; i < 50; i++) { try { const res = await fetch(`http://127.0.0.1:${port}/json/version`, { signal: AbortSignal.timeout(2000), }); if (res.ok) return res.json(); } catch { // Chromium has not opened the debugging port yet – retry. } await sleep(200); } throw new Error('chromium did not start'); } /** The toolbox handed to a check body – everything a step needs to drive and inspect the editor. */ function makeTools({ cdp, projectId, baseUrl, consoleLog, check, shotPrefix }) { // A throwing expression is reported, never swallowed: a mutate that dies half-way leaves the // scene in a state no later check describes, and every one of them then passes or fails for the // wrong reason. const evaluate = async (expression) => { const { result, exceptionDetails } = await cdp.send('Runtime.evaluate', { expression, awaitPromise: true, returnByValue: true, }); if (exceptionDetails) throw new Error( `page threw: ${exceptionDetails.exception?.description ?? exceptionDetails.text}\n in: ${expression.slice(0, 200)}`, ); return result.value; }; /** Evaluates an expression in the page and returns it as plain JSON; `undefined` comes back as null. */ const json = async (expression) => JSON.parse(await evaluate(`JSON.stringify((${expression}) ?? null)`)); /** * Waits for the editor to have drawn, instead of for a deadline someone guessed. A frame costs * ~16 ms where the fixed waits this replaces cost 150–300, and it is the editor itself that * says when it has caught up – so the same call is both faster and less flaky than the number * it replaces. Two frames by default: one for the handler, one for the layout it invalidated. * * Falls back to a fixed wait when the page has no such hook (an old build, or before the first * navigation), because a silent no-wait would turn every later check into a coin toss. */ const settle = async (count = 2) => { const at = Date.now(); const ok = await evaluate( `window.__gm?.afterFrames ? __gm.afterFrames(${count}).then(() => true) : false`, ); if (!ok) return sleep(150); waited.ms += Date.now() - at; waited.calls++; }; /** * Waits until `expression` is true in the page – for the things a frame cannot answer, such as a * save that is still in flight. Polls, because the conditions live all over the model and none * of them has an event worth inventing one for. */ const waitUntil = async (expression, { timeout = 5000, label = expression } = {}) => { const at = Date.now(); while (Date.now() - at < timeout) { if (await json(`Boolean(${expression})`)) { waited.ms += Date.now() - at; waited.calls++; return; } await sleep(25); } throw new Error(`waited ${timeout} ms and ${label} never came true`); }; /** Like `locate`, but null instead of throwing – which a check for absence needs. */ const find = async (text, last = false) => (await evaluate(`window.__gm && window.__gm.locate(${JSON.stringify(text)}, ${last})`)) || null; const has = async (text) => Boolean(await find(text)); /** * A picture by the id of its SVG file (#214) – `{ x, y, menu }` or null. `menu` picks the form: * true only a button that opens a menu, false only one that acts, left out either. */ const findIcon = async (name, menu) => (await evaluate( `window.__gm && (__gm.locateIcons(${JSON.stringify(name)}).find((hit) => ${ menu === undefined ? 'true' : `hit.menu === ${menu}` }) ?? null)`, )) || null; const locate = async (text, last = false) => { const p = await find(text, last); if (!p) throw new Error(`widget with text "${text}" not found`); return p; }; const shot = async (name) => { // The editor only draws on change, so a screenshot asks for one frame first: without it the // shot shows whatever the compositor still holds, which is right but needlessly fragile. await evaluate('window.__gm?.wake()').catch(() => {}); await sleep(50); const { data } = await cdp.send('Page.captureScreenshot', { format: 'png' }); writeFileSync(`/tmp/${shotPrefix}-${name}.png`, Buffer.from(data, 'base64')); }; const mouse = (type, x, y, button = 'left', extra = {}) => cdp.send('Input.dispatchMouseEvent', { type, x, y, button, clickCount: type === 'mousePressed' || type === 'mouseReleased' ? 1 : 0, ...extra, }); const click = async (x, y, button = 'left', modifiers = {}) => { const extra = { modifiers: modifierBits(modifiers) }; await mouse('mouseMoved', x, y); await mouse('mousePressed', x, y, button, extra); await mouse('mouseReleased', x, y, button, extra); await settle(); }; /** * A real double-click: two press/release pairs in quick succession, the second carrying * `clickCount: 2` the way a browser reports one. `click()` twice would not do – the editor * only counts presses that arrive close enough in time. */ const doubleClick = async (x, y) => { await mouse('mouseMoved', x, y); for (const clickCount of [1, 2]) { await mouse('mousePressed', x, y, 'left', { clickCount }); await mouse('mouseReleased', x, y, 'left', { clickCount }); } await settle(); }; /** Mouse wheel over (x, y); positive `dy` scrolls down. */ const wheel = async (x, y, dy) => { await mouse('mouseMoved', x, y); await mouse('mouseWheel', x, y, 'none', { deltaX: 0, deltaY: dy }); await settle(); }; const clickText = async (text, button = 'left') => { const p = await locate(text); await click(p.x, p.y, button); }; /** The open menu's entries with their tick box, `ticked` null where an entry has none (#214). */ const menuTicked = async () => (await json( '(__gm.menuRows() ?? []).map((label, i) => label === null ? null : { label, ticked: __gm.menuTicks()[i] })', )) ?? []; const clickIcon = async (name, menu) => { const p = await findIcon(name, menu); if (!p) throw new Error(`picture "${name}" not found`); await click(p.x, p.y); }; /** CDP reports held buttons as a bitmask, which is not the button name it takes elsewhere. */ const BUTTON_MASK = { left: 1, right: 2, middle: 4 }; /** The fifth argument is a step count or `{ steps, button, modifiers }`. */ const drag = async (x1, y1, x2, y2, options = {}) => { const { steps = 12, button = 'left', modifiers = {}, } = typeof options === 'number' ? { steps: options } : options; const extra = { modifiers: modifierBits(modifiers) }; const held = { ...extra, buttons: BUTTON_MASK[button] }; await mouse('mouseMoved', x1, y1); await mouse('mousePressed', x1, y1, button, extra); for (let i = 1; i <= steps; i++) { await mouse('mouseMoved', x1 + ((x2 - x1) * i) / steps, y1 + ((y2 - y1) * i) / steps, button, held); await sleep(16); } await mouse('mouseReleased', x2, y2, button, extra); await settle(); }; /** * By id, not text: the container picker shows the same name as the row. Press and moves are separate * events because the panel starts a drag only once the pointer has travelled. */ const dragAsset = async (assetId, x, y, steps = 8) => { const from = await json(`__gm.assetRow(${JSON.stringify(assetId)})`); if (!from) throw new Error(`no asset row for "${assetId}" – is the panel scrolled past it?`); await mouse('mouseMoved', from.x, from.y); await mouse('mousePressed', from.x, from.y); for (let i = 1; i <= steps; i++) { await mouse( 'mouseMoved', from.x + ((x - from.x) * i) / steps, from.y + ((y - from.y) * i) / steps, 'left', { buttons: 1, }, ); await sleep(16); } await mouse('mouseReleased', x, y); await settle(); }; const keyEvent = (type, name, modifiers) => { const key = KEYS[name]; if (!key) throw new Error(`unknown key "${name}"`); return cdp.send('Input.dispatchKeyEvent', { type, ...key, modifiers: modifierBits(modifiers) }); }; /** Press and release a key from `KEYS`, optionally with `{ ctrl, shift }`. */ const press = async (name, modifiers = {}) => { await keyEvent('keyDown', name, modifiers); await keyEvent('keyUp', name, modifiers); await settle(); }; /** Hold a key for `ms` milliseconds (arrow keys while playing). */ const holdKey = async (name, ms) => { await keyEvent('keyDown', name); await sleep(ms); await keyEvent('keyUp', name); }; // Separate halves, because `holdKey` cannot express two keys whose presses overlap – which is // exactly what a diagonal is. const keyDown = (name) => keyEvent('keyDown', name); const keyUp = (name) => keyEvent('keyUp', name); /** * Typed per character to exercise per-keystroke paths, then verified: a partial type is otherwise * indistinguishable from a wrong value. */ const typeText = async (text) => { for (const ch of text) await cdp.send('Input.insertText', { text: ch }); const typed = await json( '({ tag: document.activeElement?.tagName, value: document.activeElement?.value })', ); if (typed.tag !== 'INPUT' || !String(typed.value ?? '').includes(text)) throw new Error( `typing "${text}" did not arrive: focus is ${typed.tag}, it holds ${JSON.stringify(typed.value)}`, ); await settle(); }; const undo = async () => { await press('z', { ctrl: true }); }; const redo = async () => { await press('z', { ctrl: true, shift: true }); }; /** Selection, mode, save status and notice – the facts most checks look at. */ const state = () => json( '{ selection: __gm.state.selection, mode: __gm.state.mode, save: __gm.state.saveStatus, notice: __gm.state.notice }', ); /** * The scene's objects the way the editor lists them: the content of a placed container is * resolved from its template and stored nowhere (see `src/core/model/instance.ts`), so it is in * here but not in `state.activeScene.objects`. Use `storedObjects()` for the latter. */ const objects = () => json('__gm.sceneObjects()'); const object = (id) => json(`__gm.sceneObjects().find((o) => o.id === ${JSON.stringify(id)})`); /** Only what the scene really holds – a placed container counts as one object, not as its content. */ const storedObjects = () => json('__gm.state.activeScene.objects'); const asset = (id) => json(`__gm.state.project.assets.find((a) => a.id === ${JSON.stringify(id)})`); const world = () => json('__gm.world()'); const entity = async (objectId) => (await world())?.entities.find((e) => e.objectId === objectId); const select = async (selection) => { await evaluate(`__gm.state.select(${JSON.stringify(selection)})`); await settle(); }; /** * Switching the centre tab rebuilds a whole panel, so it needs the same wait an action gets. * It is a helper rather than a bare `evaluate` in every step because a tab change without one * reads the old panel – which is a check that passes on stale content, the worst kind. */ const showTab = async (tab) => { await evaluate(`__gm.showTab(${JSON.stringify(tab)})`); await settle(); }; /** * Waits for the autosave to have reached the server. The status goes `dirty` → `saving` → * `saved`, and only the last one means the document a reload would fetch holds the change. */ const saved = () => waitUntil("__gm.state.saveStatus === 'saved'", { timeout: 9000, label: 'the save finished' }); /** * Runs `body` (JavaScript source) against the page's model and previews the result, the way a * scrub or a command would – for putting the scene into a known state between checks. */ const mutate = async (body) => { await evaluate(`(() => { ${body}; __gm.state.preview(); })()`); await settle(); }; /** * Clicks one of the top bar's icon buttons – menu, undo, redo, play, pause (#183). They carry no * caption any more, so `clickText` has nothing to find and the position comes from the hook. */ const clickToolbar = async (name) => { const at = await json(`__gm.toolbarAt(${JSON.stringify(name)})`); await click(at.x, at.y); }; /** * Starting and stopping a run build or tear down a whole world, which is more than the two frames * `settle()` grants – so these wait for the mode itself to have changed, then for a drawn frame so * the panels that follow the mode are on screen too. */ const play = async () => { await clickToolbar('play'); await waitUntil("__gm.state.mode === 'play'", { label: 'the run started' }); await settle(); }; const stop = async () => { await press('Escape'); await waitUntil("__gm.state.mode === 'edit'", { label: 'the run stopped' }); await settle(); }; // Timeout, not convenience: a stalled response keeps its socket open, so node stays alive and // the whole run hangs on this line instead of failing. const api = async (route) => { const url = `${baseUrl}api/index.php?r=${route}`; const request = async () => fetch(url, { headers: await authHeaders(baseUrl), signal: AbortSignal.timeout(SERVER_TIMEOUT_MS) }); try { let res = await request(); // The session can have been ended by a password change in the login step (#182). if (res.status === 401) { forgetSession(baseUrl); res = await request(); } return await res.json(); } catch (error) { throw new Error(`request to ${route} failed: ${error.message}`, { cause: error }); } }; const sceneOf = (doc) => doc.project.assets.find((a) => a.type === 'scene'); const exceptions = () => consoleLog.filter((l) => l.startsWith('EXCEPTION')); /** Opens the project and asserts the page came up cleanly with the test hook installed. */ /** * Hands the browser the session the tools log in with (#177). Without it the editor would put * up its sign-in screen instead of the project, and every step after it would find no editor. * The sign-in screen itself is driven by hand in the step that tests it. */ const signIn = async () => { await cdp.send('Network.enable'); // The browser keeps the default profile, disk cache included, and nginx sends no // Cache-Control: a run could test the index.html of a build before the last one. await cdp.send('Network.setCacheDisabled', { cacheDisabled: true }); await cdp.send('Network.setCookie', { name: SESSION_COOKIE, value: await sessionToken(baseUrl), url: baseUrl, }); }; const navigate = async () => { await signIn(); await cdp.send('Page.navigate', { url: `${baseUrl}?project=${projectId}` }); await sleep(4000); check('page loaded without exceptions', exceptions().length === 0, exceptions().join(' | ')); // `state` rather than the bare hook: the sign-in screen installs one of its own, and a run // that stopped there must not look like a loaded editor. check('debug hook present', Boolean(await evaluate('Boolean(window.__gm?.state)'))); }; return { cdp, projectId, baseUrl, consoleLog, exceptions, check, sleep, evaluate, json, settle, waitUntil, find, has, findIcon, locate, shot, mouse, click, doubleClick, clickText, clickIcon, menuTicked, clickToolbar, wheel, drag, dragAsset, press, holdKey, keyDown, keyUp, typeText, undo, redo, state, objects, object, storedObjects, asset, world, entity, select, showTab, saved, mutate, play, stop, api, signIn, sceneOf, navigate, }; } export async function runChecks(options, body) { const { projectId, baseUrl, port = 9333, shotPrefix = 'gm-e2e', width = 1280, height = 800 } = options; const { proc: chrome, stop: stopBrowser, where } = launchBrowser({ port, width, height }); console.log(`browser: ${where}`); const consoleLog = []; let failures = 0; let checkCount = 0; function check(name, ok, detail = '') { const text = typeof detail === 'string' ? detail : JSON.stringify(detail); console.log(`${ok ? 'PASS' : 'FAIL'} ${name}${text ? ' – ' + text : ''}`); checkCount++; if (!ok) failures++; } // Chromium dies on this machine now and then (memory pressure). Its own last words name the // reason, and a signal here is the difference between "a check failed" and "the browser was // killed" – questions the check output alone cannot tell apart. const chromeErr = []; chrome.stderr.on('data', (b) => chromeErr.push(String(b))); let chromeDeath = ''; chrome.once('exit', (code, signal) => { chromeDeath = signal ? `killed by ${signal}` : `exited with code ${code}`; }); // Last line of defence: reaching the end of the run without printing a summary means the // process walked out early. Whatever the cause, it must never look like success. let reportedSummary = false; process.on('exit', (code) => { if (reportedSummary) return; console.log( `\nRUN DIED after ${checkCount} checks – node exited with code ${code} before the summary` + `${chromeDeath ? `. chromium ${chromeDeath}` : ''}.`, ); if (chromeErr.length) console.log('--- chromium stderr ---\n' + chromeErr.join('').trim().slice(-2000)); process.exitCode = 1; }); try { await waitForChrome(port); const target = await ( await fetch(`http://127.0.0.1:${port}/json/new?about:blank`, { method: 'PUT', signal: AbortSignal.timeout(SERVER_TIMEOUT_MS), }) ).json(); const ws = new WebSocket(target.webSocketDebuggerUrl); await new Promise((resolve, reject) => { const timer = setTimeout(() => reject(new Error('the CDP socket never opened')), 15_000); ws.addEventListener('open', () => { clearTimeout(timer); resolve(); }); ws.addEventListener('error', () => { clearTimeout(timer); reject(new Error('the CDP socket failed to open')); }); }); const cdp = new CDP(ws); await cdp.send('Page.enable'); await cdp.send('Runtime.enable'); await cdp.send('Emulation.setDeviceMetricsOverride', { width, height, deviceScaleFactor: 1, mobile: false, }); cdp.on((msg) => { if (msg.method === 'Runtime.consoleAPICalled') consoleLog.push(msg.params.args.map((a) => a.value ?? a.description).join(' ')); if (msg.method === 'Runtime.exceptionThrown') consoleLog.push( 'EXCEPTION ' + (msg.params.exceptionDetails.exception?.description ?? msg.params.exceptionDetails.text), ); }); await body(makeTools({ cdp, projectId, baseUrl, consoleLog, check, shotPrefix })); ws.close(); } catch (error) { console.error(error); if (chromeDeath) console.error(`chromium ${chromeDeath}`); if (chromeErr.length) console.error('--- chromium stderr ---\n' + chromeErr.join('').trim().slice(-2000)); failures++; } finally { await new Promise((resolve) => { if (chrome.exitCode !== null) return resolve(); const timer = setTimeout(() => chrome.kill('SIGKILL'), 15_000); chrome.once('exit', () => { clearTimeout(timer); resolve(); }); stopBrowser(); }); } if (consoleLog.length) console.log('--- console ---\n' + consoleLog.slice(0, 20).join('\n')); reportedSummary = true; console.log(failures ? `${failures} FAILED` : `ALL PASSED – ${checkCount} checks`); process.exit(failures ? 1 : 0); }