feuerwerk/tools/perf-shaders.mjs

#!/usr/bin/env node
/**
 * What a shader costs: the same scene of N sprites drawn with and without `Adjust colors`, measured
 * as frame time in the running game. Up to #67 every sprite wearing a shader got its own Pixi
 * filter, so its own render pass into an intermediate texture and back; since #77 the fragment
 * runs in the sprite's own draw, which leaves the batch – one draw call per shaded sprite, no pass.
 *
 * The values are neutral (1), so both runs put the very same pixels on screen – the only
 * difference between them is how the sprite is drawn.
 *
 * Usage: node tools/seed-e2e.mjs && node tools/perf-shaders.mjs
 *   GM_PERF_COUNTS=...   sprite counts, GM_PERF_SIZE=...  edge length of one sprite
 *
 * It is a measuring tool, not a test: frame times depend on the machine, so CI does not run it.
 *
 * Two things the numbers hang on. The rate is frames divided by sampling time, never the median
 * frame time: the distribution is bimodal (most frames arrive on a vsync tick, some miss several),
 * so at 800 sprites the median still reads 17 ms while only a third of the frames are there.
 * And the 60 Hz cap cannot be lifted here – measured, `--disable-gpu-vsync` and
 * `--disable-frame-rate-limit` change nothing in headless chromium (57.6 fps with and without).
 * So this measures where the frame rate breaks, not what a pass costs inside an unfilled frame.
 */
import { readFileSync } from 'node:fs';
import { runChecks } from './lib/harness.mjs';
import { PROJECTS, cliArgs } from './lib/projects.mjs';

const { projectId, baseUrl } = cliArgs('perf-shaders.mjs', PROJECTS.fixture);

const COUNTS = (process.env.GM_PERF_COUNTS ?? '50,100,200,400,800').split(',').map(Number);
const SIZE = Number(process.env.GM_PERF_SIZE ?? 48);
const WARMUP_MS = 1500;
const SAMPLE_MS = 4000;

const median = (xs) => [...xs].sort((a, b) => a - b)[Math.floor(xs.length / 2)];
const percentile = (xs, p) => [...xs].sort((a, b) => a - b)[Math.floor((xs.length - 1) * p)];
const round = (x, digits = 2) => Number(x.toFixed(digits));
const load = () => readFileSync('/proc/loadavg', 'utf8').split(' ').slice(0, 2).join(' ');

await runChecks({ projectId, baseUrl, port: 9347, shotPrefix: 'gm-perf' }, async (t) => {
  const { check, sleep, json, evaluate, mutate, play, stop, shot } = t;

  await t.navigate();

  // One sprite asset for all of them: flipping its `shader` prop switches every instance at once.
  await mutate(`
    const p = __gm.state.project;
    p.assets = p.assets.filter((a) => a.id !== 'perf-sprite');
    p.assets.push({ id: 'perf-sprite', type: 'sprite', name: 'Perf sprite',
      props: { width: ${SIZE}, height: ${SIZE}, color: 0x4080ff, shader: '' } });
  `);

  /** Camera-sized grid of `n` instances, all inside the view so none of them is skipped. */
  const buildScene = async (n) => {
    await mutate(`
      const scene = __gm.state.activeScene;
      const cam = scene.objects.find((o) => o.assetId === 'cam');
      const objects = [cam];
      const cols = Math.ceil(Math.sqrt(${n}));
      for (let i = 0; i < ${n}; i++) {
        const col = i % cols, row = Math.floor(i / cols);
        objects.push({ id: 'perf-' + i, assetId: 'perf-sprite', name: '',
          position: { x: cam.position.x - 300 + (col + 0.5) * (600 / cols),
                      y: cam.position.y - 160 + (row + 0.5) * (320 / cols), z: i },
          editorOffset: { x: 0, y: 0 }, overrides: {}, parentId: '', instanceEdits: {} });
      }
      scene.objects = objects;
    `);
  };

  const setShader = (on) =>
    mutate(
      `__gm.state.project.assets.find((a) => a.id === 'perf-sprite').props.shader = ${on ? "'adjust-colors'" : "''"}`,
    );

  /** Frame times as the page itself sees them, over `SAMPLE_MS` after a warm-up. */
  const sample = async () => {
    await sleep(WARMUP_MS);
    await evaluate(`(() => {
      window.__perfSamples = [];
      window.__perfOn = true;
      let last = performance.now();
      const tick = () => {
        const now = performance.now();
        window.__perfSamples.push(now - last);
        last = now;
        if (window.__perfOn) requestAnimationFrame(tick);
      };
      requestAnimationFrame(tick);
    })()`);
    await sleep(SAMPLE_MS);
    const frames = await json('(window.__perfOn = false, window.__perfSamples.slice(1))');
    const reported = await json('__gm.fps()');
    const shaders = await json('__gm.shaderCount()');
    const filters = await json('__gm.filterCount()');
    const elapsed = frames.reduce((a, b) => a + b, 0);
    return {
      frames: frames.length,
      mean: round(elapsed / frames.length),
      med: round(median(frames)),
      p95: round(percentile(frames, 0.95)),
      fps: round((frames.length * 1000) / elapsed, 1),
      reported,
      shaders,
      filters,
    };
  };

  const rows = [];
  for (const n of COUNTS) {
    await buildScene(n);
    const run = {};
    for (const on of [false, true]) {
      await setShader(on);
      const before = load();
      await play();
      run[on ? 'on' : 'off'] = { ...(await sample()), load: before };
      await stop();
    }
    check(
      `${n} sprites: a shader per sprite and no filter with the shader on, neither without`,
      run.on.shaders === n && run.on.filters === 0 && run.off.shaders === 0 && run.off.filters === 0,
      { on: run.on, off: run.off },
    );
    rows.push({ n, ...run });
    console.log(
      `  n=${String(n).padStart(4)}  off ${String(run.off.mean).padStart(6)} ms (${run.off.fps} fps)` +
        `   on ${String(run.on.mean).padStart(6)} ms (${run.on.fps} fps)` +
        `   +${round(((run.on.mean - run.off.mean) * 1000) / n, 1)} us/sprite   load ${run.off.load} / ${run.on.load}`,
    );
  }

  // The comparison is only worth reading while the scene without shaders still holds the cap:
  // once it misses ticks by itself, both columns measure something else.
  check(
    'the baseline holds the frame rate at every count',
    rows.every((r) => r.off.fps > 55),
    rows.map((r) => `${r.n}:${r.off.fps}`).join(' '),
  );

  // The screenshot is of the last, heaviest scene with the shader on – proof it really drew.
  await setShader(true);
  await play();
  await sleep(500);
  await shot('heaviest');
  await stop();

  console.log(
    `\nsprite ${SIZE}x${SIZE}, ${SAMPLE_MS} ms per point, neutral values, ` +
      `60 Hz cap (cannot be lifted in headless)`,
  );
  console.log('| sprites | no shader | with shader | per sprite | shaders | filters |');
  console.log('| --- | --- | --- | --- | --- | --- |');
  for (const r of rows)
    console.log(
      `| ${r.n} | ${r.off.mean} ms / ${r.off.fps} fps | ${r.on.mean} ms / ${r.on.fps} fps |` +
        ` ${round(((r.on.mean - r.off.mean) * 1000) / r.n, 1)} us | ${r.on.shaders} | ${r.on.filters} |`,
    );
  console.log(JSON.stringify(rows));
});