feuerwerk/tools/e2e/67-recording.mjs

import { MAKE_OBJECT } from './lib/scaffold.mjs';

/**
 * Recording the game view (#405). A behavior starts a recording at 1080p and 30 fps when the run
 * starts, and V stops it: the server converts the file like any upload, and once the run is over a
 * video asset of that name is in the project, 1920×1080 with sound, its picture the red sprite that
 * covers the camera. G starts another at the size of the game view; a second G is ignored and says
 * so in the log, and Esc ends the run, which saves that one too, named after when it started.
 *
 * Everything it adds carries an `rc67` id or is one of the two video assets, and goes again at the end.
 */
const CONTROL = 'rc67-control';
const COVER = 'rc67-cover';
const NAME = 'rc67 take';

const node = (id, type, x, y, params = {}) => ({ id: `rc67-${id}`, type, x, y, params });
const edge = (id, from, to) => ({
  id: `rc67-${id}`,
  from: { node: `rc67-${from}`, port: 'exec' },
  to: { node: `rc67-${to}`, port: 'exec' },
});

export async function recording(t) {
  const { check, sleep, json, evaluate, mutate, press, play, stop, showTab, waitUntil } = t;
  await showTab('scene');

  const graph = {
    nodes: [
      node('start', 'OnStart', 0, 0),
      node('record', 'StartRecording', 300, 0, { name: NAME, resolution: '1080p', fps: '30' }),
      node('v', 'OnKeyDown', 0, 200, { key: 'v' }),
      node('stop', 'StopRecording', 300, 200),
      node('g', 'OnKeyDown', 0, 400, { key: 'g' }),
      node('again', 'StartRecording', 300, 400),
    ],
    edges: [edge('e1', 'start', 'record'), edge('e2', 'v', 'stop'), edge('e3', 'g', 'again')],
  };
  // Red over the whole camera picture, blue over its right half and green over its lower left
  // quarter, in front of what earlier steps left there: a frame drawn at the wrong place, or read
  // back upside down, shows the wrong colour.
  await mutate(`
    const p = __gm.state.project;
    const scene = __gm.state.activeScene;
    const cam = scene.objects.find((o) => o.id === 'o3').position;
    p.assets.push({ id: '${CONTROL}', type: 'behavior', name: 'Recorder 67', props: { enabled: true, runInEditor: false }, variables: [], graph: ${JSON.stringify(graph)} });
    ${MAKE_OBJECT}
    const sprite = p.assets.find((a) => a.type === 'sprite');
    const cover = make('${COVER}', sprite.id, '', 0, 'Cover 67');
    Object.assign(cover.position, { x: cam.x, y: cam.y, z: 1000 });
    cover.overrides = { width: 900, height: 600, color: 0xff0000, texture: '' };
    const right = make('rc67-right', sprite.id, '', 0, 'Right 67');
    Object.assign(right.position, { x: cam.x + 225, y: cam.y, z: 1001 });
    right.overrides = { width: 450, height: 600, color: 0x0000ff, texture: '' };
    const low = make('rc67-low', sprite.id, '', 0, 'Low 67');
    Object.assign(low.position, { x: cam.x - 225, y: cam.y + 150, z: 1001 });
    low.overrides = { width: 450, height: 300, color: 0x00ff00, texture: '' };
    const logic = make('rc67-logic', '${CONTROL}', '', 0, 'Recorder 67');
    Object.assign(logic.position, { x: cam.x + 400, y: cam.y });
    scene.objects.push(cover, right, low, logic);
  `);
  const recordings = () => json('__gm.recording()');

  // 67a. Recorded from the start of the run, at 1080p and 30 fps, with the master's sound. The window
  //      is smaller than 1080p: the frame is drawn into a texture of its own (`PictureReader`).
  await play();
  await sleep(1200);
  await press('v');
  await waitUntil(`__gm.recording().takes[0]?.state === 'done'`, { label: 'the first take is saved' });
  const first = (await recordings()).takes[0];
  check(
    'Start recording records the game view at the size asked, with sound, and Stop recording saves it',
    first.name === NAME &&
      first.width === 1920 &&
      first.height === 1080 &&
      first.fps === 30 &&
      first.sound &&
      first.soundFrames > 0 &&
      first.frames >= 25 &&
      first.frames <= 45 &&
      first.failure === null,
    JSON.stringify(first),
  );

  // 67b. A second start while one records is ignored, and says so in the log.
  await press('g');
  await sleep(300);
  await press('g');
  await sleep(900);
  const log = (await json('__gm.world()'))?.log ?? [];
  const during = await recordings();
  check(
    'a second Start recording while one records is ignored and says so in the log',
    during.takes.length === 2 &&
      log.some((line) => line.includes('Start recording is ignored') && line.includes(during.takes[1].name)),
    JSON.stringify({ log, takes: during.takes.map((take) => take.name) }),
  );
  await stop();

  // 67c. Ending the run saved the second take; both became video assets once the server had them.
  await waitUntil(
    `__gm.recording().uploads.length === 2 && __gm.recording().uploads.every((u) => u.state === 'added' || u.state === 'failed')`,
    {
      timeout: 60_000,
      label: 'both recordings are in the project',
    },
  );
  const { takes, uploads } = await recordings();
  const second = takes[1];
  const assets = await json(`__gm.state.project.assets
    .filter((a) => a.type === 'video' && ${JSON.stringify(uploads.map((u) => u.assetId))}.includes(a.id))
    .map((a) => ({ id: a.id, name: a.name, file: a.file, width: a.width, height: a.height, duration: a.duration, fps: a.fps, audio: a.audio }))`);
  const named = assets.find((a) => a.name === NAME);
  check(
    'the recording becomes a video asset of its name: 1920×1080, its length, with sound',
    named !== undefined &&
      named.width === 1920 &&
      named.height === 1080 &&
      named.audio === true &&
      Math.abs(named.duration - first.durationMs / 1000) < 0.1 &&
      named.fps === 30,
    JSON.stringify({ assets, uploads }),
  );
  const timed = assets.find((a) => a.name !== NAME);
  check(
    'ending the run saves a recording still going, named after when it started, at the game view size',
    second.state === 'done' &&
      /^Recording \d{4}-\d\d-\d\d \d\d:\d\d:\d\d$/.test(second.name) &&
      timed?.name === second.name &&
      timed.width === second.width &&
      timed.height === second.height &&
      second.width % 2 === 0 &&
      second.fps === 60,
    JSON.stringify({ second, timed }),
  );

  // 67d. The picture is the camera's: red at the top left, green below it and blue on the right,
  //      none of what the view lays over it, the right way up, and as red as it was drawn; every
  //      4th pixel is read.
  //      From the MP4 as Chrome decodes it: the poster is no witness, ffmpeg cuts it with BT.601
  //      whatever the file says (a red of 255 comes out 229).
  const colorsOf = (file) =>
    evaluate(`new Promise((resolve) => {
    const video = document.createElement('video');
    video.muted = true;
    video.onerror = () => resolve('null');
    video.src = '/game-maker/data/videos/${file}';
    video.play().then(() => video.requestVideoFrameCallback(() => {
      video.pause();
      const canvas = document.createElement('canvas');
      canvas.width = video.videoWidth;
      canvas.height = video.videoHeight;
      const pen = canvas.getContext('2d');
      pen.drawImage(video, 0, 0);
      const { data } = pen.getImageData(0, 0, canvas.width, canvas.height);
      const off = [];
      for (let y = 2; y < canvas.height; y += 4)
        for (let x = 2; x < canvas.width; x += 4) {
          const i = (y * canvas.width + x) * 4;
          const edge = canvas.width / 2;
          const middle = canvas.height / 2;
          if (Math.abs(x - edge) < canvas.width / 100 || Math.abs(y - middle) < canvas.height / 100) continue;
          const hot = x >= edge ? 2 : y >= middle ? 1 : 0;
          const rgb = [data[i], data[i + 1], data[i + 2]];
          if (rgb.some((v, c) => (c === hot ? v < 245 : v > 12))) off.push([x, y, ...rgb]);
        }
      resolve(JSON.stringify({ w: canvas.width, h: canvas.height, off: off.length, first: off.slice(0, 5) }));
    }), () => resolve('null'));
  })`);
  const colors = `[${await colorsOf(named?.file ?? '')},${await colorsOf(timed?.file ?? '')}]`;
  const red = JSON.parse(colors);
  check(
    'the recorded picture is the camera picture, the right way up, as red as it was drawn',
    red.every((frame) => frame !== null && frame.w > 0 && frame.off === 0),
    colors,
  );

  // 67e. The editor records itself from the main menu: the whole canvas at 60 fps, the mouse pointer
  //      drawn in where the system draws it over the canvas, and the red dot beside the menu on
  //      screen only. The frame is read where the pointer rested on the empty middle of the top bar.
  const { mouse, cdp, clickToolbar, clickText } = t;
  await clickToolbar('menu');
  const idle = await json('__gm.menuRows()');
  await clickText('Start recording');
  await waitUntil(`__gm.recording().editor.takes[0]?.state === 'recording'`, { label: 'the editor records' });
  const redo = await json('__gm.toolbarAt("redo")');
  const playAt = await json('__gm.toolbarAt("play")');
  const rest = { x: Math.round((redo.x + playAt.x) / 2), y: Math.round(playAt.y) };
  await mouse('mouseMoved', rest.x, rest.y);
  await sleep(200);
  const restFrom = (await json('__gm.recording().editor.takes[0]')).durationMs;
  await sleep(1000);
  const restTo = (await json('__gm.recording().editor.takes[0]')).durationMs;
  const canvas = await json(
    "({ w: document.querySelector('canvas').width, h: document.querySelector('canvas').height })",
  );
  // Points of the canvas below the top bar, whose notice changes, while the editor rests: the frame
  // of that moment has to show the same. Only points amid one colour count: H.264 shares a colour
  // between 2×2 pixels, so a hairline over red comes out pink.
  const near = [];
  for (let gy = 1; gy < 14; gy++)
    for (let gx = 1; gx < 24; gx++) {
      const x = Math.round((canvas.w * gx) / 24);
      const y = 40 + Math.round(((canvas.h - 40) * gy) / 14);
      for (let dy = -2; dy <= 2; dy += 2)
        for (let dx = -2; dx <= 2; dx += 2) near.push({ x: x + dx, y: y + dy });
    }
  const around = await json(`__gm.pixels(${JSON.stringify(near)})`);
  const grid = [];
  const live = [];
  for (let i = 0; i < near.length; i += 9) {
    const patch = around.slice(i, i + 9);
    if (patch.every((c) => c === patch[0])) {
      grid.push(near[i + 4]);
      live.push(patch[4]);
    }
  }
  const badge = (await json('__gm.recording().editor')).badge;
  const dot = { x: Math.round(badge.x + 7), y: Math.round(badge.y + badge.h / 2) };
  await evaluate('window.__gm.wake()');
  await sleep(100);
  const { data: screen } = await cdp.send('Page.captureScreenshot', { format: 'png' });
  const onScreen = JSON.parse(
    await evaluate(`new Promise((resolve) => {
      const image = new Image();
      image.onload = () => {
        const canvas = document.createElement('canvas');
        canvas.width = image.width;
        canvas.height = image.height;
        const pen = canvas.getContext('2d');
        pen.drawImage(image, 0, 0);
        resolve(JSON.stringify([...pen.getImageData(${dot.x}, ${dot.y}, 1, 1).data].slice(0, 3)));
      };
      image.src = 'data:image/png;base64,${screen}';
    })`),
  );
  await clickToolbar('menu');
  const busy = await json('__gm.menuRows()');
  await clickText('Stop recording');
  await waitUntil(`__gm.recording().editor.takes[0]?.state === 'done'`, {
    label: 'the editor take is saved',
  });
  const take = (await json('__gm.recording().editor')).takes[0];
  await waitUntil(
    `__gm.recording().uploads.some((u) => u.name === ${JSON.stringify(take.name)} && (u.state === 'added' || u.state === 'failed'))`,
    { timeout: 60_000, label: 'the editor recording is in the project' },
  );
  const upload = (await recordings()).uploads.find((u) => u.name === take.name);
  const made = await json(
    `__gm.state.project.assets.filter((a) => a.id === ${JSON.stringify(upload.assetId)}).map((a) => ({ file: a.file, fps: a.fps }))[0] ?? null`,
  );
  const file = made?.file ?? '';
  const at = (restFrom + restTo) / 2000;
  const frame = JSON.parse(
    await evaluate(`new Promise((resolve) => {
      const video = document.createElement('video');
      video.muted = true;
      video.onerror = () => resolve('null');
      video.src = '/game-maker/data/videos/${file}';
      video.play().then(() => video.requestVideoFrameCallback(() => {
        video.pause();
        video.onseeked = () => setTimeout(() => {
          const canvas = document.createElement('canvas');
          canvas.width = video.videoWidth;
          canvas.height = video.videoHeight;
          const pen = canvas.getContext('2d');
          pen.drawImage(video, 0, 0);
          const px = (x, y) => [...pen.getImageData(x, y, 1, 1).data].slice(0, 3);
          resolve(JSON.stringify({
            w: canvas.width,
            h: canvas.height,
            inside: px(${rest.x + 2}, ${rest.y + 8}),
            beside: px(${rest.x - 4}, ${rest.y + 8}),
            dot: px(${dot.x}, ${dot.y}),
            grid: ${JSON.stringify(grid)}.map((p) => px(p.x, p.y)),
          }));
        }, 150);
        video.currentTime = ${at};
      }), () => resolve('null'));
    })`),
  );
  const hex = (rgb) => rgb.map((v) => v.toString(16).padStart(2, '0')).join('');
  const off = frame
    ? frame.grid.filter((rgb, i) => {
        const want = live[i]
          .slice(1)
          .match(/../g)
          .map((v) => parseInt(v, 16));
        return rgb.some((v, c) => Math.abs(v - want[c]) > 24);
      }).length
    : -1;
  check(
    'Start recording in the main menu records the whole editor at 60 fps with the pointer drawn in, the red dot on screen only, and Stop recording saves it',
    idle.includes('Start recording') &&
      busy.includes('Stop recording') &&
      take.width === canvas.w - (canvas.w % 2) &&
      take.height === canvas.h - (canvas.h % 2) &&
      take.fps === 60 &&
      made?.fps === 60 &&
      upload.state === 'added' &&
      frame !== null &&
      frame.w === take.width &&
      Math.max(...frame.inside) < 60 &&
      Math.min(...frame.beside) > 25 &&
      onScreen[0] > 180 &&
      onScreen[1] < 120 &&
      frame.dot[0] < 120 &&
      grid.length >= 50 &&
      off <= grid.length / 20,
    JSON.stringify({
      idle,
      busy,
      take,
      made,
      upload,
      canvas,
      at,
      onScreen,
      frame: frame && {
        ...frame,
        inside: hex(frame.inside),
        beside: hex(frame.beside),
        dot: hex(frame.dot),
        grid: undefined,
      },
      flat: grid.length,
      off,
    }),
  );

  await mutate(`
    const p = __gm.state.project;
    const made = ${JSON.stringify([...assets.map((a) => a.id), upload.assetId])};
    p.assets = p.assets.filter((a) => a.id !== '${CONTROL}' && !made.includes(a.id));
    const scene = __gm.state.activeScene;
    scene.objects = scene.objects.filter((o) => !o.id.startsWith('rc67-'));`);
}