feuerwerk/tools/lib/harness.mjs

/**
 * Headless-Chromium harness for the checks; widgets are found via `__gm.locate(text)`, not coordinates.
 * It always waits for chromium to exit (a leftover GPU process burns CPU) and exits 1 on failures or when
 * the summary line is missing, since chromium can die halfway under memory pressure.
 */
import { writeFileSync } from 'node:fs';
import { launchBrowser } from './browser.mjs';

/**
 * Every wait the checks do goes through here, and the time is booked. A run that takes ten
 * minutes needs to say how much of that was the editor working and how much was the harness
 * waiting for a deadline it made up – the two call for opposite fixes.
 */
export const waited = { ms: 0, calls: 0 };
const sleep = (ms) => {
  waited.ms += ms;
  waited.calls++;
  return new Promise((r) => setTimeout(r, ms));
};

/** Generous on purpose: a slow navigation still finishes, a wedged chromium never does. */
const CDP_TIMEOUT_MS = 30_000;
const SERVER_TIMEOUT_MS = 20_000;

/** Structural equality for the small plain values the checks compare (arrays of ids, selections). */
export const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);

/** Sum of the channel differences of two `#rrggbb` colours. */
export function colourDistance(a, b) {
  const channels = (hex) => [1, 3, 5].map((i) => parseInt(hex.slice(i, i + 2), 16));
  const [ar, ag, ab] = channels(a);
  const [br, bg, bb] = channels(b);
  return Math.abs(ar - br) + Math.abs(ag - bg) + Math.abs(ab - bb);
}

/** Keys the checks press, with the codes Chromium needs to route them like a real keyboard. */
const KEYS = {
  Enter: { key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13 },
  Escape: { key: 'Escape', code: 'Escape', windowsVirtualKeyCode: 27 },
  Space: { key: ' ', code: 'Space', windowsVirtualKeyCode: 32 },
  ArrowLeft: { key: 'ArrowLeft', code: 'ArrowLeft', windowsVirtualKeyCode: 37 },
  ArrowUp: { key: 'ArrowUp', code: 'ArrowUp', windowsVirtualKeyCode: 38 },
  ArrowRight: { key: 'ArrowRight', code: 'ArrowRight', windowsVirtualKeyCode: 39 },
  ArrowDown: { key: 'ArrowDown', code: 'ArrowDown', windowsVirtualKeyCode: 40 },
  Delete: { key: 'Delete', code: 'Delete', windowsVirtualKeyCode: 46 },
  a: { key: 'a', code: 'KeyA', windowsVirtualKeyCode: 65 },
  d: { key: 'd', code: 'KeyD', windowsVirtualKeyCode: 68 },
  g: { key: 'g', code: 'KeyG', windowsVirtualKeyCode: 71 },
  s: { key: 's', code: 'KeyS', windowsVirtualKeyCode: 83 },
  w: { key: 'w', code: 'KeyW', windowsVirtualKeyCode: 87 },
  z: { key: 'z', code: 'KeyZ', windowsVirtualKeyCode: 90 },
};

/** CDP modifier bitmask: 2 = Ctrl, 8 = Shift. */
const modifierBits = ({ ctrl = false, shift = false } = {}) => (ctrl ? 2 : 0) | (shift ? 8 : 0);

class CDP {
  constructor(ws) {
    this.ws = ws;
    this.id = 0;
    this.pending = new Map();
    this.listeners = [];
    ws.addEventListener('message', (ev) => {
      const msg = JSON.parse(ev.data);
      if (msg.id && this.pending.has(msg.id)) {
        const { resolve, reject } = this.pending.get(msg.id);
        this.pending.delete(msg.id);
        msg.error ? reject(new Error(msg.error.message)) : resolve(msg.result);
      } else if (msg.method) {
        for (const l of this.listeners) l(msg);
      }
    });
    ws.addEventListener('close', () => this.failAll('the CDP socket closed – chromium is gone'));
    ws.addEventListener('error', () => this.failAll('the CDP socket errored'));
  }

  /**
   * Rejects every waiting `send`. Without this a dead socket leaves them pending forever, and
   * since an unsettled promise is not a reason for node to stay alive, the run exits mid-step
   * with code 0 – a dead run that reports success.
   */
  failAll(reason) {
    this.dead = reason;
    for (const { reject } of this.pending.values()) reject(new Error(reason));
    this.pending.clear();
  }

  send(method, params = {}) {
    if (this.dead) return Promise.reject(new Error(`${this.dead} (while sending ${method})`));
    const id = ++this.id;
    this.ws.send(JSON.stringify({ id, method, params }));
    return new Promise((resolve, reject) => {
      const timer = setTimeout(() => {
        this.pending.delete(id);
        reject(new Error(`CDP ${method} got no answer within ${CDP_TIMEOUT_MS / 1000}s`));
      }, CDP_TIMEOUT_MS);
      const settle = (fn) => (value) => {
        clearTimeout(timer);
        fn(value);
      };
      this.pending.set(id, { resolve: settle(resolve), reject: settle(reject) });
    });
  }

  on(fn) {
    this.listeners.push(fn);
  }
}

async function waitForChrome(port) {
  for (let i = 0; i < 50; i++) {
    try {
      const res = await fetch(`http://127.0.0.1:${port}/json/version`, {
        signal: AbortSignal.timeout(2000),
      });
      if (res.ok) return res.json();
    } catch {
      // Chromium has not opened the debugging port yet – retry.
    }
    await sleep(200);
  }
  throw new Error('chromium did not start');
}

/** The toolbox handed to a check body – everything a step needs to drive and inspect the editor. */
function makeTools({ cdp, projectId, baseUrl, consoleLog, check, shotPrefix }) {
  // A throwing expression is reported, never swallowed: a mutate that dies half-way leaves the
  // scene in a state no later check describes, and every one of them then passes or fails for the
  // wrong reason.
  const evaluate = async (expression) => {
    const { result, exceptionDetails } = await cdp.send('Runtime.evaluate', {
      expression,
      awaitPromise: true,
      returnByValue: true,
    });
    if (exceptionDetails)
      throw new Error(
        `page threw: ${exceptionDetails.exception?.description ?? exceptionDetails.text}\n  in: ${expression.slice(0, 200)}`,
      );
    return result.value;
  };
  /** Evaluates an expression in the page and returns it as plain JSON; `undefined` comes back as null. */
  const json = async (expression) => JSON.parse(await evaluate(`JSON.stringify((${expression}) ?? null)`));
  /**
   * Waits for the editor to have drawn, instead of for a deadline someone guessed. A frame costs
   * ~16 ms where the fixed waits this replaces cost 150–300, and it is the editor itself that
   * says when it has caught up – so the same call is both faster and less flaky than the number
   * it replaces. Two frames by default: one for the handler, one for the layout it invalidated.
   *
   * Falls back to a fixed wait when the page has no such hook (an old build, or before the first
   * navigation), because a silent no-wait would turn every later check into a coin toss.
   */
  const settle = async (count = 2) => {
    const at = Date.now();
    const ok = await evaluate(
      `window.__gm?.afterFrames ? __gm.afterFrames(${count}).then(() => true) : false`,
    );
    if (!ok) return sleep(150);
    waited.ms += Date.now() - at;
    waited.calls++;
  };
  /**
   * Waits until `expression` is true in the page – for the things a frame cannot answer, such as a
   * save that is still in flight. Polls, because the conditions live all over the model and none
   * of them has an event worth inventing one for.
   */
  const waitUntil = async (expression, { timeout = 5000, label = expression } = {}) => {
    const at = Date.now();
    while (Date.now() - at < timeout) {
      if (await json(`Boolean(${expression})`)) {
        waited.ms += Date.now() - at;
        waited.calls++;
        return;
      }
      await sleep(25);
    }
    throw new Error(`waited ${timeout} ms and ${label} never came true`);
  };
  /** Like `locate`, but null instead of throwing – which a check for absence needs. */
  const find = async (text, last = false) =>
    (await evaluate(`window.__gm && window.__gm.locate(${JSON.stringify(text)}, ${last})`)) || null;
  const has = async (text) => Boolean(await find(text));
  const locate = async (text, last = false) => {
    const p = await find(text, last);
    if (!p) throw new Error(`widget with text "${text}" not found`);
    return p;
  };
  const shot = async (name) => {
    // The editor only draws on change, so a screenshot asks for one frame first: without it the
    // shot shows whatever the compositor still holds, which is right but needlessly fragile.
    await evaluate('window.__gm?.wake()').catch(() => {});
    await sleep(50);
    const { data } = await cdp.send('Page.captureScreenshot', { format: 'png' });
    writeFileSync(`/tmp/${shotPrefix}-${name}.png`, Buffer.from(data, 'base64'));
  };

  const mouse = (type, x, y, button = 'left', extra = {}) =>
    cdp.send('Input.dispatchMouseEvent', {
      type,
      x,
      y,
      button,
      clickCount: type === 'mousePressed' || type === 'mouseReleased' ? 1 : 0,
      ...extra,
    });
  const click = async (x, y, button = 'left', modifiers = {}) => {
    const extra = { modifiers: modifierBits(modifiers) };
    await mouse('mouseMoved', x, y);
    await mouse('mousePressed', x, y, button, extra);
    await mouse('mouseReleased', x, y, button, extra);
    await settle();
  };
  /**
   * A real double-click: two press/release pairs in quick succession, the second carrying
   * `clickCount: 2` the way a browser reports one. `click()` twice would not do – the editor
   * only counts presses that arrive close enough in time.
   */
  const doubleClick = async (x, y) => {
    await mouse('mouseMoved', x, y);
    for (const clickCount of [1, 2]) {
      await mouse('mousePressed', x, y, 'left', { clickCount });
      await mouse('mouseReleased', x, y, 'left', { clickCount });
    }
    await settle();
  };
  /** Mouse wheel over (x, y); positive `dy` scrolls down. */
  const wheel = async (x, y, dy) => {
    await mouse('mouseMoved', x, y);
    await mouse('mouseWheel', x, y, 'none', { deltaX: 0, deltaY: dy });
    await settle();
  };
  const clickText = async (text, button = 'left') => {
    const p = await locate(text);
    await click(p.x, p.y, button);
  };
  /** CDP reports held buttons as a bitmask, which is not the button name it takes elsewhere. */
  const BUTTON_MASK = { left: 1, right: 2, middle: 4 };
  /** The fifth argument is a step count or `{ steps, button, modifiers }`. */
  const drag = async (x1, y1, x2, y2, options = {}) => {
    const {
      steps = 12,
      button = 'left',
      modifiers = {},
    } = typeof options === 'number' ? { steps: options } : options;
    const extra = { modifiers: modifierBits(modifiers) };
    const held = { ...extra, buttons: BUTTON_MASK[button] };
    await mouse('mouseMoved', x1, y1);
    await mouse('mousePressed', x1, y1, button, extra);
    for (let i = 1; i <= steps; i++) {
      await mouse('mouseMoved', x1 + ((x2 - x1) * i) / steps, y1 + ((y2 - y1) * i) / steps, button, held);
      await sleep(16);
    }
    await mouse('mouseReleased', x2, y2, button, extra);
    await settle();
  };

  /**
   * By id, not text: the container picker shows the same name as the row. Press and moves are separate
   * events because the panel starts a drag only once the pointer has travelled.
   */
  const dragAsset = async (assetId, x, y, steps = 8) => {
    const from = await json(`__gm.assetRow(${JSON.stringify(assetId)})`);
    if (!from) throw new Error(`no asset row for "${assetId}" – is the panel scrolled past it?`);
    await mouse('mouseMoved', from.x, from.y);
    await mouse('mousePressed', from.x, from.y);
    for (let i = 1; i <= steps; i++) {
      await mouse(
        'mouseMoved',
        from.x + ((x - from.x) * i) / steps,
        from.y + ((y - from.y) * i) / steps,
        'left',
        {
          buttons: 1,
        },
      );
      await sleep(16);
    }
    await mouse('mouseReleased', x, y);
    await settle();
  };

  const keyEvent = (type, name, modifiers) => {
    const key = KEYS[name];
    if (!key) throw new Error(`unknown key "${name}"`);
    return cdp.send('Input.dispatchKeyEvent', { type, ...key, modifiers: modifierBits(modifiers) });
  };
  /** Press and release a key from `KEYS`, optionally with `{ ctrl, shift }`. */
  const press = async (name, modifiers = {}) => {
    await keyEvent('keyDown', name, modifiers);
    await keyEvent('keyUp', name, modifiers);
    await settle();
  };
  /** Hold a key for `ms` milliseconds (arrow keys while playing). */
  const holdKey = async (name, ms) => {
    await keyEvent('keyDown', name);
    await sleep(ms);
    await keyEvent('keyUp', name);
  };
  // Separate halves, because `holdKey` cannot express two keys whose presses overlap – which is
  // exactly what a diagonal is.
  const keyDown = (name) => keyEvent('keyDown', name);
  const keyUp = (name) => keyEvent('keyUp', name);
  /**
   * Typed per character to exercise per-keystroke paths, then verified: a partial type is otherwise
   * indistinguishable from a wrong value.
   */
  const typeText = async (text) => {
    for (const ch of text) await cdp.send('Input.insertText', { text: ch });
    const typed = await json(
      '({ tag: document.activeElement?.tagName, value: document.activeElement?.value })',
    );
    if (typed.tag !== 'INPUT' || !String(typed.value ?? '').includes(text))
      throw new Error(
        `typing "${text}" did not arrive: focus is ${typed.tag}, it holds ${JSON.stringify(typed.value)}`,
      );
    await settle();
  };
  const undo = async () => {
    await press('z', { ctrl: true });
  };
  const redo = async () => {
    await press('z', { ctrl: true, shift: true });
  };

  /** Selection, mode, save status and notice – the facts most checks look at. */
  const state = () =>
    json(
      '{ selection: __gm.state.selection, mode: __gm.state.mode, save: __gm.state.saveStatus, notice: __gm.state.notice }',
    );
  /**
   * The scene's objects the way the editor lists them: the content of a placed container is
   * resolved from its template and stored nowhere (see `src/core/model/instance.ts`), so it is in
   * here but not in `state.activeScene.objects`. Use `storedObjects()` for the latter.
   */
  const objects = () => json('__gm.sceneObjects()');
  const object = (id) => json(`__gm.sceneObjects().find((o) => o.id === ${JSON.stringify(id)})`);
  /** Only what the scene really holds – a placed container counts as one object, not as its content. */
  const storedObjects = () => json('__gm.state.activeScene.objects');
  const asset = (id) => json(`__gm.state.project.assets.find((a) => a.id === ${JSON.stringify(id)})`);
  const world = () => json('__gm.world()');
  const entity = async (objectId) => (await world())?.entities.find((e) => e.objectId === objectId);
  const select = async (selection) => {
    await evaluate(`__gm.state.select(${JSON.stringify(selection)})`);
    await settle();
  };
  /**
   * Switching the centre tab rebuilds a whole panel, so it needs the same wait an action gets.
   * It is a helper rather than a bare `evaluate` in every step because a tab change without one
   * reads the old panel – which is a check that passes on stale content, the worst kind.
   */
  const showTab = async (tab) => {
    await evaluate(`__gm.showTab(${JSON.stringify(tab)})`);
    await settle();
  };
  /**
   * Waits for the autosave to have reached the server. The status goes `dirty` → `saving` →
   * `saved`, and only the last one means the document a reload would fetch holds the change.
   */
  const saved = () =>
    waitUntil("__gm.state.saveStatus === 'saved'", { timeout: 9000, label: 'the save finished' });
  /**
   * Runs `body` (JavaScript source) against the page's model and previews the result, the way a
   * scrub or a command would – for putting the scene into a known state between checks.
   */
  const mutate = async (body) => {
    await evaluate(`(() => { ${body}; __gm.state.preview(); })()`);
    await settle();
  };
  /**
   * Starting and stopping a run build or tear down a whole world, which is more than the two frames
   * `settle()` grants – so these wait for the mode itself to have changed, then for a drawn frame so
   * the panels that follow the mode are on screen too.
   */
  const play = async () => {
    await clickText('▶  Play');
    await waitUntil("__gm.state.mode === 'play'", { label: 'the run started' });
    await settle();
  };
  const stop = async () => {
    await press('Escape');
    await waitUntil("__gm.state.mode === 'edit'", { label: 'the run stopped' });
    await settle();
  };

  // Timeout, not convenience: a stalled response keeps its socket open, so node stays alive and
  // the whole run hangs on this line instead of failing.
  const api = async (route) => {
    const url = `${baseUrl}api/index.php?r=${route}`;
    try {
      return await (await fetch(url, { signal: AbortSignal.timeout(SERVER_TIMEOUT_MS) })).json();
    } catch (error) {
      throw new Error(`request to ${route} failed: ${error.message}`, { cause: error });
    }
  };
  const sceneOf = (doc) => doc.project.assets.find((a) => a.type === 'scene');
  const exceptions = () => consoleLog.filter((l) => l.startsWith('EXCEPTION'));
  /** Opens the project and asserts the page came up cleanly with the test hook installed. */
  const navigate = async () => {
    await cdp.send('Page.navigate', { url: `${baseUrl}?project=${projectId}` });
    await sleep(4000);
    check('page loaded without exceptions', exceptions().length === 0, exceptions().join(' | '));
    check('debug hook present', Boolean(await evaluate('Boolean(window.__gm)')));
  };

  return {
    cdp,
    projectId,
    baseUrl,
    consoleLog,
    exceptions,
    check,
    sleep,
    evaluate,
    json,
    settle,
    waitUntil,
    find,
    has,
    locate,
    shot,
    mouse,
    click,
    doubleClick,
    clickText,
    wheel,
    drag,
    dragAsset,
    press,
    holdKey,
    keyDown,
    keyUp,
    typeText,
    undo,
    redo,
    state,
    objects,
    object,
    storedObjects,
    asset,
    world,
    entity,
    select,
    showTab,
    saved,
    mutate,
    play,
    stop,
    api,
    sceneOf,
    navigate,
  };
}

export async function runChecks(options, body) {
  const { projectId, baseUrl, port = 9333, shotPrefix = 'gm-e2e', width = 1280, height = 800 } = options;
  const { proc: chrome, stop: stopBrowser, where } = launchBrowser({ port, width, height });
  console.log(`browser: ${where}`);

  const consoleLog = [];
  let failures = 0;
  let checkCount = 0;
  function check(name, ok, detail = '') {
    const text = typeof detail === 'string' ? detail : JSON.stringify(detail);
    console.log(`${ok ? 'PASS' : 'FAIL'}  ${name}${text ? ' – ' + text : ''}`);
    checkCount++;
    if (!ok) failures++;
  }

  // Chromium dies on this machine now and then (memory pressure). Its own last words name the
  // reason, and a signal here is the difference between "a check failed" and "the browser was
  // killed" – questions the check output alone cannot tell apart.
  const chromeErr = [];
  chrome.stderr.on('data', (b) => chromeErr.push(String(b)));
  let chromeDeath = '';
  chrome.once('exit', (code, signal) => {
    chromeDeath = signal ? `killed by ${signal}` : `exited with code ${code}`;
  });

  // Last line of defence: reaching the end of the run without printing a summary means the
  // process walked out early. Whatever the cause, it must never look like success.
  let reportedSummary = false;
  process.on('exit', (code) => {
    if (reportedSummary) return;
    console.log(
      `\nRUN DIED after ${checkCount} checks – node exited with code ${code} before the summary` +
        `${chromeDeath ? `. chromium ${chromeDeath}` : ''}.`,
    );
    if (chromeErr.length) console.log('--- chromium stderr ---\n' + chromeErr.join('').trim().slice(-2000));
    process.exitCode = 1;
  });

  try {
    await waitForChrome(port);
    const target = await (
      await fetch(`http://127.0.0.1:${port}/json/new?about:blank`, {
        method: 'PUT',
        signal: AbortSignal.timeout(SERVER_TIMEOUT_MS),
      })
    ).json();
    const ws = new WebSocket(target.webSocketDebuggerUrl);
    await new Promise((resolve, reject) => {
      const timer = setTimeout(() => reject(new Error('the CDP socket never opened')), 15_000);
      ws.addEventListener('open', () => {
        clearTimeout(timer);
        resolve();
      });
      ws.addEventListener('error', () => {
        clearTimeout(timer);
        reject(new Error('the CDP socket failed to open'));
      });
    });
    const cdp = new CDP(ws);
    await cdp.send('Page.enable');
    await cdp.send('Runtime.enable');
    await cdp.send('Emulation.setDeviceMetricsOverride', {
      width,
      height,
      deviceScaleFactor: 1,
      mobile: false,
    });
    cdp.on((msg) => {
      if (msg.method === 'Runtime.consoleAPICalled')
        consoleLog.push(msg.params.args.map((a) => a.value ?? a.description).join(' '));
      if (msg.method === 'Runtime.exceptionThrown')
        consoleLog.push(
          'EXCEPTION ' +
            (msg.params.exceptionDetails.exception?.description ?? msg.params.exceptionDetails.text),
        );
    });
    await body(makeTools({ cdp, projectId, baseUrl, consoleLog, check, shotPrefix }));
    ws.close();
  } catch (error) {
    console.error(error);
    if (chromeDeath) console.error(`chromium ${chromeDeath}`);
    if (chromeErr.length) console.error('--- chromium stderr ---\n' + chromeErr.join('').trim().slice(-2000));
    failures++;
  } finally {
    await new Promise((resolve) => {
      if (chrome.exitCode !== null) return resolve();
      const timer = setTimeout(() => chrome.kill('SIGKILL'), 15_000);
      chrome.once('exit', () => {
        clearTimeout(timer);
        resolve();
      });
      stopBrowser();
    });
  }
  if (consoleLog.length) console.log('--- console ---\n' + consoleLog.slice(0, 20).join('\n'));
  reportedSummary = true;
  console.log(failures ? `${failures} FAILED` : `ALL PASSED – ${checkCount} checks`);
  process.exit(failures ? 1 : 0);
}