feuerwerk/tools/e2e/67-recording.mjs

import { MAKE_OBJECT, pickAssetType } from './lib/scaffold.mjs';

/**
 * Recording the game view (#405). A behavior starts a recording at 1080p and 30 fps when the run
 * starts, and V stops it: the server converts the file like any upload, and once the run is over a
 * video asset of that name is in the project, 1920×1080 with sound, its picture the red sprite that
 * covers the camera. G starts another at the size of the game view; a second G is ignored and says
 * so in the log, and Esc ends the run, which saves that one too, named after when it started.
 *
 * Everything it adds carries an `rc67` id or is one of the two video assets, and goes again at the end.
 */
const CONTROL = 'rc67-control';
const COVER = 'rc67-cover';
const NAME = 'rc67 take';

const node = (id, type, x, y, params = {}) => ({ id: `rc67-${id}`, type, x, y, params });
const edge = (id, from, to) => ({
  id: `rc67-${id}`,
  from: { node: `rc67-${from}`, port: 'exec' },
  to: { node: `rc67-${to}`, port: 'exec' },
});

export async function recording(t) {
  const { check, sleep, json, evaluate, mutate, press, play, stop, showTab, waitUntil } = t;
  await showTab('scene');

  const graph = {
    nodes: [
      node('start', 'OnStart', 0, 0),
      node('record', 'StartRecording', 300, 0, { name: NAME, resolution: '1080p', fps: '30' }),
      node('v', 'OnKeyDown', 0, 200, { key: 'v' }),
      node('stop', 'StopRecording', 300, 200),
      node('g', 'OnKeyDown', 0, 400, { key: 'g' }),
      node('again', 'StartRecording', 300, 400),
    ],
    edges: [edge('e1', 'start', 'record'), edge('e2', 'v', 'stop'), edge('e3', 'g', 'again')],
  };
  // Red over the whole camera picture, blue over its right half and green over its lower left
  // quarter, in front of what earlier steps left there: a frame drawn at the wrong place, or read
  // back upside down, shows the wrong colour.
  await mutate(`
    const p = __gm.state.project;
    const scene = __gm.state.activeScene;
    const cam = scene.objects.find((o) => o.id === 'o3').position;
    p.assets.push({ id: '${CONTROL}', type: 'behavior', name: 'Recorder 67', props: { enabled: true, runInEditor: false }, variables: [], graph: ${JSON.stringify(graph)} });
    ${MAKE_OBJECT}
    const sprite = p.assets.find((a) => a.type === 'sprite');
    const cover = make('${COVER}', sprite.id, '', 0, 'Cover 67');
    Object.assign(cover.position, { x: cam.x, y: cam.y, z: 1000 });
    cover.overrides = { width: 900, height: 600, color: 0xff0000, texture: '' };
    const right = make('rc67-right', sprite.id, '', 0, 'Right 67');
    Object.assign(right.position, { x: cam.x + 225, y: cam.y, z: 1001 });
    right.overrides = { width: 450, height: 600, color: 0x0000ff, texture: '' };
    const low = make('rc67-low', sprite.id, '', 0, 'Low 67');
    Object.assign(low.position, { x: cam.x - 225, y: cam.y + 150, z: 1001 });
    low.overrides = { width: 450, height: 300, color: 0x00ff00, texture: '' };
    const logic = make('rc67-logic', '${CONTROL}', '', 0, 'Recorder 67');
    Object.assign(logic.position, { x: cam.x + 400, y: cam.y });
    scene.objects.push(cover, right, low, logic);
  `);
  const recordings = () => json('__gm.recording()');

  // 67a. Recorded from the start of the run, at 1080p and 30 fps, with the master's sound. The window
  //      is smaller than 1080p: the frame is drawn into a texture of its own (`PictureReader`).
  //      The game time is written down each frame until the first frame is taken.
  await evaluate(`(() => {
    window.__rc67 = { at: null };
    const look = () => {
      const take = __gm.recording().takes[0];
      if (take?.lastFrameAt != null) {
        window.__rc67.at = __gm.playInfo().time;
        return;
      }
      requestAnimationFrame(look);
    };
    requestAnimationFrame(look);
  })()`);
  await play();
  await sleep(1200);
  await press('v');
  await waitUntil(`__gm.recording().takes[0]?.state === 'done'`, { label: 'the first take is saved' });
  const first = (await recordings()).takes[0];
  check(
    'Start recording records the game view at the size asked, with sound, and Stop recording saves it',
    first.name === NAME &&
      first.width === 1920 &&
      first.height === 1080 &&
      first.fps === 30 &&
      first.sound &&
      first.soundFrames > 0 &&
      first.frames >= 25 &&
      first.frames <= 45 &&
      first.failure === null,
    JSON.stringify(first),
  );

  // 67a-ii. The run waits while the recording sets up its encoders: a recording started in
  //         `On start` has the first frames of the game, not what the game shows a moment later.
  const firstAt = await json('window.__rc67.at');
  check(
    'a recording started in On start takes the first frames of the run',
    typeof firstAt === 'number' && firstAt <= 3 / 60,
    JSON.stringify({ firstAt }),
  );

  // 67b. A second start while one records is ignored, and says so in the log.
  await press('g');
  await sleep(300);
  await press('g');
  await sleep(900);
  const log = (await json('__gm.world()'))?.log ?? [];
  const during = await recordings();
  check(
    'a second Start recording while one records is ignored and says so in the log',
    during.takes.length === 2 &&
      log.some((line) => line.includes('Start recording is ignored') && line.includes(during.takes[1].name)),
    JSON.stringify({ log, takes: during.takes.map((take) => take.name) }),
  );
  await stop();

  // 67c. Ending the run saved the second take; both became video assets once the server had them.
  await waitUntil(
    `__gm.recording().uploads.length === 2 && __gm.recording().uploads.every((u) => u.state === 'added' || u.state === 'failed')`,
    {
      timeout: 60_000,
      label: 'both recordings are in the project',
    },
  );
  const { takes, uploads } = await recordings();
  const second = takes[1];
  const assets = await json(`__gm.state.project.assets
    .filter((a) => a.type === 'video' && ${JSON.stringify(uploads.map((u) => u.assetId))}.includes(a.id))
    .map((a) => ({ id: a.id, name: a.name, file: a.file, width: a.width, height: a.height, duration: a.duration, fps: a.fps, audio: a.audio }))`);
  const named = assets.find((a) => a.name === NAME);
  check(
    'the recording becomes a video asset of its name: 1920×1080, its length, with sound',
    named !== undefined &&
      named.width === 1920 &&
      named.height === 1080 &&
      named.audio === true &&
      Math.abs(named.duration - first.durationMs / 1000) < 0.1 &&
      named.fps === 30,
    JSON.stringify({ assets, uploads }),
  );
  const timed = assets.find((a) => a.name !== NAME);
  check(
    'ending the run saves a recording still going, named after when it started, at the game view size',
    second.state === 'done' &&
      /^Recording \d{4}-\d\d-\d\d \d\d:\d\d:\d\d$/.test(second.name) &&
      timed?.name === second.name &&
      timed.width === second.width &&
      timed.height === second.height &&
      second.width % 2 === 0 &&
      second.fps === 60,
    JSON.stringify({ second, timed }),
  );

  // 67d. The picture is the camera's: red at the top left, green below it and blue on the right,
  //      none of what the view lays over it, the right way up, and as red as it was drawn; every
  //      4th pixel is read.
  //      From the MP4 as Chrome decodes it: the poster is no witness, ffmpeg cuts it with BT.601
  //      whatever the file says (a red of 255 comes out 229).
  const colorsOf = (file) =>
    evaluate(`new Promise((resolve) => {
    const video = document.createElement('video');
    video.muted = true;
    video.onerror = () => resolve('null');
    video.src = '/game-maker/data/videos/${file}';
    video.play().then(() => video.requestVideoFrameCallback(() => {
      video.pause();
      const canvas = document.createElement('canvas');
      canvas.width = video.videoWidth;
      canvas.height = video.videoHeight;
      const pen = canvas.getContext('2d');
      pen.drawImage(video, 0, 0);
      const { data } = pen.getImageData(0, 0, canvas.width, canvas.height);
      const off = [];
      for (let y = 2; y < canvas.height; y += 4)
        for (let x = 2; x < canvas.width; x += 4) {
          const i = (y * canvas.width + x) * 4;
          const edge = canvas.width / 2;
          const middle = canvas.height / 2;
          if (Math.abs(x - edge) < canvas.width / 100 || Math.abs(y - middle) < canvas.height / 100) continue;
          const hot = x >= edge ? 2 : y >= middle ? 1 : 0;
          const rgb = [data[i], data[i + 1], data[i + 2]];
          if (rgb.some((v, c) => (c === hot ? v < 245 : v > 12))) off.push([x, y, ...rgb]);
        }
      resolve(JSON.stringify({ w: canvas.width, h: canvas.height, off: off.length, first: off.slice(0, 5) }));
    }), () => resolve('null'));
  })`);
  const colors = `[${await colorsOf(named?.file ?? '')},${await colorsOf(timed?.file ?? '')}]`;
  const red = JSON.parse(colors);
  check(
    'the recorded picture is the camera picture, the right way up, as red as it was drawn',
    red.every((frame) => frame !== null && frame.w > 0 && frame.off === 0),
    colors,
  );

  // 67e. The editor records itself from the main menu: the whole canvas at 60 fps, the mouse pointer
  //      drawn in where the system draws it over the canvas, and the red dot beside the menu on
  //      screen only. The frame is read where the pointer rested on the empty middle of the top bar.
  const { mouse, cdp, click, clickToolbar, clickText } = t;
  // Downloads are caught rather than saved: what counts is which file and under which name.
  await evaluate(`window.__downloads = [];
    HTMLAnchorElement.prototype.click = function () { window.__downloads.push({ name: this.download, href: this.href }); };`);
  const startDialog = async (change) => {
    await clickToolbar('menu');
    const rows = await json('__gm.menuRows()');
    await clickText('Start recording…');
    await waitUntil('__gm.recordingDialog() !== null', { label: 'the recording dialog opens' });
    await evaluate(`__gm.recordingDialogChoose(${JSON.stringify(change)})`);
    return rows;
  };
  const DEFAULTS = {
    name: '',
    resolution: 'screen',
    fps: 60,
    pointer: true,
    sound: true,
    convert: true,
    adopt: true,
  };
  const idle = await startDialog(DEFAULTS);
  const start = (await json('__gm.recordingDialog()')).start;
  await click(start.x, start.y);
  await waitUntil(`__gm.recording().editor.takes[0]?.state === 'recording'`, { label: 'the editor records' });
  const redo = await json('__gm.toolbarAt("redo")');
  const playAt = await json('__gm.toolbarAt("play")');
  const rest = { x: Math.round((redo.x + playAt.x) / 2), y: Math.round(playAt.y) };
  await mouse('mouseMoved', rest.x, rest.y);
  await sleep(200);
  const restFrom = (await json('__gm.recording().editor.takes[0]')).durationMs;
  await sleep(1000);
  const restTo = (await json('__gm.recording().editor.takes[0]')).durationMs;
  const canvas = await json(
    "({ w: document.querySelector('canvas').width, h: document.querySelector('canvas').height })",
  );
  // Points of the canvas below the top bar, whose notice changes, while the editor rests: the frame
  // of that moment has to show the same. Only points amid one colour count: H.264 shares a colour
  // between 2×2 pixels, so a hairline over red comes out pink.
  const near = [];
  for (let gy = 1; gy < 14; gy++)
    for (let gx = 1; gx < 24; gx++) {
      const x = Math.round((canvas.w * gx) / 24);
      const y = 40 + Math.round(((canvas.h - 40) * gy) / 14);
      for (let dy = -2; dy <= 2; dy += 2)
        for (let dx = -2; dx <= 2; dx += 2) near.push({ x: x + dx, y: y + dy });
    }
  const around = await json(`__gm.pixels(${JSON.stringify(near)})`);
  const grid = [];
  const live = [];
  for (let i = 0; i < near.length; i += 9) {
    const patch = around.slice(i, i + 9);
    if (patch.every((c) => c === patch[0])) {
      grid.push(near[i + 4]);
      live.push(patch[4]);
    }
  }
  const badge = (await json('__gm.recording().editor')).badge;
  const dot = { x: Math.round(badge.x + 7), y: Math.round(badge.y + badge.h / 2) };
  await evaluate('window.__gm.wake()');
  await sleep(100);
  const { data: screen } = await cdp.send('Page.captureScreenshot', { format: 'png' });
  const onScreen = JSON.parse(
    await evaluate(`new Promise((resolve) => {
      const image = new Image();
      image.onload = () => {
        const canvas = document.createElement('canvas');
        canvas.width = image.width;
        canvas.height = image.height;
        const pen = canvas.getContext('2d');
        pen.drawImage(image, 0, 0);
        resolve(JSON.stringify([...pen.getImageData(${dot.x}, ${dot.y}, 1, 1).data].slice(0, 3)));
      };
      image.src = 'data:image/png;base64,${screen}';
    })`),
  );
  await clickToolbar('menu');
  const busy = await json('__gm.menuRows()');
  await clickText('Stop recording');
  await waitUntil(`__gm.recording().editor.takes[0]?.state === 'done'`, {
    label: 'the editor take is saved',
  });
  const take = (await json('__gm.recording().editor')).takes[0];
  await waitUntil(
    `__gm.recording().uploads.some((u) => u.name === ${JSON.stringify(take.name)} && (u.state === 'added' || u.state === 'failed'))`,
    { timeout: 60_000, label: 'the editor recording is in the project' },
  );
  const upload = (await recordings()).uploads.find((u) => u.name === take.name);
  const made = await json(
    `__gm.state.project.assets.filter((a) => a.id === ${JSON.stringify(upload.assetId)}).map((a) => ({ file: a.file, fps: a.fps }))[0] ?? null`,
  );
  const file = made?.file ?? '';
  const at = (restFrom + restTo) / 2000;
  const frame = JSON.parse(
    await evaluate(`new Promise((resolve) => {
      const video = document.createElement('video');
      video.muted = true;
      video.onerror = () => resolve('null');
      video.src = '/game-maker/data/videos/${file}';
      video.play().then(() => video.requestVideoFrameCallback(() => {
        video.pause();
        video.onseeked = () => setTimeout(() => {
          const canvas = document.createElement('canvas');
          canvas.width = video.videoWidth;
          canvas.height = video.videoHeight;
          const pen = canvas.getContext('2d');
          pen.drawImage(video, 0, 0);
          const px = (x, y) => [...pen.getImageData(x, y, 1, 1).data].slice(0, 3);
          resolve(JSON.stringify({
            w: canvas.width,
            h: canvas.height,
            inside: px(${rest.x + 2}, ${rest.y + 8}),
            beside: px(${rest.x - 4}, ${rest.y + 8}),
            dot: px(${dot.x}, ${dot.y}),
            grid: ${JSON.stringify(grid)}.map((p) => px(p.x, p.y)),
          }));
        }, 150);
        video.currentTime = ${at};
      }), () => resolve('null'));
    })`),
  );
  const hex = (rgb) => rgb.map((v) => v.toString(16).padStart(2, '0')).join('');
  const off = frame
    ? frame.grid.filter((rgb, i) => {
        const want = live[i]
          .slice(1)
          .match(/../g)
          .map((v) => parseInt(v, 16));
        return rgb.some((v, c) => Math.abs(v - want[c]) > 24);
      }).length
    : -1;
  check(
    'Start recording… in the main menu records the whole editor at 60 fps with the pointer drawn in, the red dot on screen only, and Stop recording saves it',
    idle.includes('Start recording…') &&
      busy.includes('Stop recording') &&
      take.width === canvas.w - (canvas.w % 2) &&
      take.height === canvas.h - (canvas.h % 2) &&
      take.fps === 60 &&
      made?.fps === 60 &&
      upload.state === 'added' &&
      frame !== null &&
      frame.w === take.width &&
      Math.max(...frame.inside) < 60 &&
      Math.min(...frame.beside) > 25 &&
      onScreen[0] > 180 &&
      onScreen[1] < 120 &&
      frame.dot[0] < 120 &&
      grid.length >= 50 &&
      off <= grid.length / 20,
    JSON.stringify({
      idle,
      busy,
      take,
      made,
      upload,
      canvas,
      at,
      onScreen,
      frame: frame && {
        ...frame,
        inside: hex(frame.inside),
        beside: hex(frame.beside),
        dot: hex(frame.dot),
        grid: undefined,
      },
      flat: grid.length,
      off,
    }),
  );

  // 67e-ii. The video tab's Download saves the recording as it was recorded – the Matroska file the
  //         editor uploaded, kept beside the MP4 the server made of it.
  const firstBytes = (href) =>
    evaluate(
      `fetch(${JSON.stringify(href)}).then((r) => r.arrayBuffer()).then((b) => [...new Uint8Array(b.slice(0, 4))].map((v) => v.toString(16).padStart(2, '0')).join(''))`,
    );
  await evaluate(`__gm.openDocument(${JSON.stringify(`asset:${upload.assetId}`)})`);
  await waitUntil(`__gm.videoTab.current() === ${JSON.stringify(upload.assetId)}`, {
    label: 'the recording opens in the video tab',
  });
  const downloadAt = await json('__gm.videoTab.downloadButton()');
  await click(downloadAt.x, downloadAt.y);
  await waitUntil('__gm.videoTab.lastDownload() !== null', { label: 'the video tab downloads' });
  const tabDownload = await json('__gm.videoTab.lastDownload()');
  const tabHead = await firstBytes(tabDownload.href);
  check(
    "the video tab's Download saves the recording as it was recorded, not the MP4",
    tabDownload.href === `/game-maker/data/videos/${file.replace(/\.mp4$/, '.source.mkv')}` &&
      tabDownload.name === `${take.name}.mkv` &&
      (await json('window.__downloads')).some((d) => d.name === tabDownload.name) &&
      tabHead === '1a45dfa3',
    JSON.stringify({ tabDownload, tabHead, file }),
  );

  // 67f. "Start recording…" with its options. 1440p from the 1280×800 window is drawn sharper while
  //      it records (resolution 1.8) rather than scaled up, here at 30 fps without sound. Without
  //      Convert, Import goes grey, the file is only kept on the server and nothing comes into the
  //      project. The video picker downloads it as it is, and converts it on asking at its full size,
  //      above the 1080p the server used to shrink every video to.
  const assetsBefore = await json('__gm.state.project.assets.length');
  await startDialog({ resolution: '1440p', fps: 30, pointer: false, sound: false });
  const convertAt = (await json('__gm.recordingDialog()')).switches.convert;
  await click(convertAt.x, convertAt.y);
  const asked = await json('__gm.recordingDialog()');
  await click(asked.start.x, asked.start.y);
  await waitUntil(`__gm.recording().editor.takes[1]?.state === 'recording'`, {
    label: 'the second editor take records',
  });
  const raised = (await json('__gm.recording().editor')).resolution;
  await sleep(1000);
  await clickToolbar('menu');
  await clickText('Stop recording');
  await waitUntil(`__gm.recording().editor.takes[1]?.state === 'done'`, {
    label: 'the second editor take is saved',
  });
  const sharp = (await json('__gm.recording().editor')).takes[1];
  const back = (await json('__gm.recording().editor')).resolution;
  await waitUntil(
    `__gm.recording().uploads.some((u) => u.name === ${JSON.stringify(sharp.name)} && (u.state === 'kept' || u.state === 'failed'))`,
    { timeout: 60_000, label: 'the second editor recording is uploaded' },
  );
  const kept = (await recordings()).uploads.find((u) => u.name === sharp.name);
  await showTab('scene');
  await clickText('+');
  await pickAssetType(t, 'Video');
  await waitUntil(`__gm.videoPicker()?.files.includes(${JSON.stringify(kept.file)})`, {
    label: 'the picker lists the kept recording',
  });
  const keptRow = await json(`__gm.videoPickerRow(${JSON.stringify(kept.file)})`);
  await click(keptRow.x, keptRow.y);
  const keptPick = await json('__gm.videoPicker()');
  const [pickerDownload, pickerConvert] = keptPick.extras;
  await click(pickerDownload.x, pickerDownload.y);
  const pickerSaved = (await json('__gm.videoPicker()')).lastDownload;
  const pickerHead = pickerSaved ? await firstBytes(pickerSaved.href) : '';
  await click(pickerConvert.x, pickerConvert.y);
  await waitUntil(`__gm.videoPicker()?.detail.includes('2304 × 1440')`, {
    timeout: 60_000,
    label: 'the kept recording is converted on asking',
  });
  const converted = await json('__gm.videoPicker()');
  await press('Escape');
  check(
    'Start recording… records 1440p sharper at 30 fps without sound; without Convert it is only kept, the picker downloads it as it is and converts it on asking',
    asked.adoptEnabled === false &&
      asked.values.convert === false &&
      asked.size === '2304 × 1440 at 30 fps, from this window' &&
      raised === 1.8 &&
      back === 1 &&
      sharp.width === 2304 &&
      sharp.height === 1440 &&
      sharp.fps === 30 &&
      sharp.sound === false &&
      kept.state === 'kept' &&
      keptPick.detail.includes('Not converted') &&
      pickerDownload.enabled &&
      pickerConvert.enabled &&
      pickerSaved?.name === `${sharp.name}.mkv` &&
      pickerSaved.href === `/game-maker/data/videos/${kept.file.replace(/\.mp4$/, '.source.mkv')}` &&
      pickerHead === '1a45dfa3' &&
      converted.detail.includes('2304 × 1440') &&
      converted.extras[1].enabled === false &&
      (await json('__gm.state.project.assets.length')) === assetsBefore,
    JSON.stringify({ asked, raised, back, sharp, kept, keptPick, pickerSaved, pickerHead, converted }),
  );

  await mutate(`
    const p = __gm.state.project;
    const made = ${JSON.stringify([...assets.map((a) => a.id), upload.assetId])};
    p.assets = p.assets.filter((a) => a.id !== '${CONTROL}' && !made.includes(a.id));
    const scene = __gm.state.activeScene;
    scene.objects = scene.objects.filter((o) => !o.id.startsWith('rc67-'));`);
}