feuerwerk/tools/e2e/66-video.mjs

import { readFileSync } from 'node:fs';
import { uploadVideo } from '../lib/projects.mjs';
import { LANES, MAKE_OBJECT, pickAssetType } from './lib/scaffold.mjs';

/**
 * Videos (#402). A WebM goes up and the worker turns it into an MP4 (in CI beside the PHP server,
 * on stein2 in the container game-maker-video). "+ → Video" offers it with what it is; Use makes
 * the asset and opens its tab, which shows the picture. A drag in the clip strip spans a clip, one
 * undo step. A sprite wearing the clip plays only that stretch, over and over, without sound in the
 * editor; Play starts it over from the clip's start and lets it be heard, Esc makes it quiet again.
 *
 * The test video is 2 s at 30 fps, a quarter each red, green, blue and white, with a tone. Colours
 * are read from the video the tab holds and from the screen.
 *
 * Everything it adds carries a `vd66` id or is the asset it made, and goes again at the end.
 */
const SPRITE = 'vd66-sprite';
const OBJECT = 'vd66-object';
const DRIVEN = 'vd66-driven';
const EARLY = 'vd66-early';
const LATE = 'vd66-late';
const CUT = 'vd66-cut';
const LANE = LANES.video;
const COLORS = { red: '#fe0000', green: '#00ff01', blue: '#0000ff', white: '#ffffff' };

export async function video(t) {
  const { check, sleep, json, mutate, click, clickText, drag, shot, showTab, play, stop } = t;
  const waitFor = async (read, done, tries = 40) => {
    let value = await read();
    for (let i = 0; i < tries && !done(value); i++) {
      await sleep(150);
      value = await read();
    }
    return value;
  };

  // 66a. Converted on the server: the asked size, length and frame rate, as an MP4 with a poster.
  const uploaded = await uploadVideo(
    t.baseUrl,
    readFileSync(new URL('./fixtures/colors.webm', import.meta.url)),
    'colors.webm',
  );
  check(
    'an uploaded WebM is converted to an MP4 with its size, length, frame rate and sound',
    uploaded.status === 'ready' &&
      /^[a-f0-9]{12}\/[a-f0-9]{64}\.mp4$/.test(uploaded.file) &&
      uploaded.width === 320 &&
      uploaded.height === 180 &&
      Math.abs(uploaded.duration - 2) < 0.05 &&
      uploaded.fps === 30 &&
      uploaded.audio === true &&
      uploaded.poster === uploaded.file.replace(/\.mp4$/, '.jpg'),
    JSON.stringify(uploaded),
  );

  // 66b. "+ → Video" offers it with what it is, Use makes the asset and opens the tab.
  await showTab('scene');
  await clickText('+');
  await pickAssetType(t, 'Video');
  const picker = await waitFor(
    () => json('__gm.videoPicker()'),
    (p) => p?.files.includes(uploaded.file),
  );
  check(
    'the + menu opens the video picker on what the server holds',
    Boolean(picker?.files.includes(uploaded.file)),
    JSON.stringify(picker),
  );
  const row = await json(`__gm.videoPickerRow(${JSON.stringify(uploaded.file)})`);
  await click(row.x, row.y);
  const picked = await json('__gm.videoPicker()');
  check(
    'a ready video says its size, length, frame rate and sound',
    picked.detail.includes('320 × 180 · 2 s · 30 fps · with sound'),
    picked.detail,
  );
  await shot('66-video-picker');
  await clickText('Use video');
  const assetId = await waitFor(
    () => json('__gm.videoTab.current()'),
    (id) => Boolean(id),
  );
  const asset = await json(`__gm.state.project.assets.find((a) => a.id === ${JSON.stringify(assetId)})`);
  check(
    'Use makes a video asset of it and opens its tab',
    asset?.type === 'video' &&
      asset.file === uploaded.file &&
      asset.width === 320 &&
      asset.fps === 30 &&
      asset.name === 'colors',
    JSON.stringify(asset),
  );

  // 66c. The tab shows the picture: red at the start, read from the video and from the screen.
  const preview = await waitFor(
    () => json('__gm.videoTab.preview()'),
    (p) => p?.shown && p.pixel?.[0] > 200,
  );
  const shownColor = (p) => json(`__gm.pixelAt(${Math.round(p.x + p.w / 2)}, ${Math.round(p.y + p.h / 2)})`);
  check(
    'the tab shows the video, its first frame red',
    preview?.shown &&
      preview.pixel[0] > 200 &&
      preview.pixel[1] < 50 &&
      (await shownColor(preview)) === COLORS.red,
    JSON.stringify({ preview, screen: await shownColor(preview) }),
  );

  // 66d. A drag in the strip spans a clip from 0.6 s to 1.4 s, which is then picked; one undo step.
  const from = await json('__gm.videoTab.stripAt(0.6)');
  const to = await json('__gm.videoTab.stripAt(1.4)');
  await drag(from.x, from.y, to.x, to.y);
  await sleep(300);
  const clips = await json('__gm.videoTab.clips()');
  const clip = clips[0];
  check(
    'a drag in the strip spans a clip, which is picked',
    clips.length === 1 &&
      Math.abs(clip.props.start - 0.6) < 0.02 &&
      Math.abs(clip.props.end - 1.4) < 0.02 &&
      (await json('__gm.videoTab.pickedClip()')) === clip.id,
    JSON.stringify({ clips, picked: await json('__gm.videoTab.pickedClip()') }),
  );
  const atClip = await waitFor(
    () => json('__gm.videoTab.preview()'),
    (p) => p?.pixel?.[1] > 200 && p.pixel[0] < 50,
  );
  check(
    'the tab stands at the picked clip’s start, green',
    atClip.pixel[1] > 200 && atClip.pixel[0] < 50,
    JSON.stringify(atClip),
  );
  await t.undo();
  const undone = await json('__gm.videoTab.clips()');
  await t.redo();
  check(
    'the clip is one undo step',
    undone.length === 0 && (await json('__gm.videoTab.clips()')).length === 1,
    JSON.stringify(undone),
  );

  // 66e. The undo left the video picked; a click on the band picks the clip again. Play in the tab
  //      runs the clip and moves the playhead; Stop holds it.
  const band = await json('__gm.videoTab.stripAt(1)');
  await click(band.x, band.y);
  check('a click on its band picks the clip', (await json('__gm.videoTab.pickedClip()')) === clip.id);
  const playButton = await json('__gm.videoTab.playButton()');
  await click(playButton.x, playButton.y);
  await sleep(500);
  const running = await json(
    '({ playing: __gm.videoTab.playing(), head: __gm.videoTab.playhead(), button: __gm.videoTab.playButton().text })',
  );
  check(
    'Play in the tab plays the picked clip from its start',
    running.playing && running.head > 0.6 && running.head < 1.4 && running.button === 'Stop',
    JSON.stringify(running),
  );
  await click(playButton.x, playButton.y);
  check('Stop holds it', (await json('__gm.videoTab.playing()')) === false);

  // 66e-ii. With the clip picked (#403), a drag in the track off its handles or in the strip beside
  //         its band changes nothing; one on its band moves it whole, one undo step. A click in the
  //         track beside it picks the video, on it the clip again.
  const track = (s) => json(`__gm.videoTab.trackAt(${s})`);
  const strip = (s) => json(`__gm.videoTab.stripAt(${s})`);
  const clipsNow = async () => JSON.stringify(await json('__gm.videoTab.clips()'));
  const spanBefore = await clipsNow();
  for (const [at, from, to] of [
    [track, 1, 1.8],
    [strip, 1.6, 1.9],
  ]) {
    const a = await at(from);
    const b = await at(to);
    await drag(a.x, a.y, b.x, b.y);
  }
  const untouched = await clipsNow();
  const bandFrom = await strip(1);
  const bandTo = await strip(1.2);
  await drag(bandFrom.x, bandFrom.y, bandTo.x, bandTo.y);
  await sleep(150);
  const moved = (await json('__gm.videoTab.clips()'))[0];
  await t.undo();
  const undoneMove = await clipsNow();
  const besideTrack = await track(1.8);
  await click(besideTrack.x, besideTrack.y);
  const pickedBeside = await json('__gm.videoTab.pickedClip()');
  const onTrack = await track(1);
  await click(onTrack.x, onTrack.y);
  check(
    'with the clip picked a drag off its handles changes nothing, on its band it moves the clip whole; a click in the track picks',
    untouched === spanBefore &&
      Math.abs(moved?.props.start - 0.8) < 0.02 &&
      Math.abs(moved?.props.end - 1.6) < 0.02 &&
      undoneMove === spanBefore &&
      pickedBeside === null &&
      (await json('__gm.videoTab.pickedClip()')) === clip.id,
    JSON.stringify({ spanBefore, untouched, moved, undoneMove, pickedBeside }),
  );
  await shot('66-video-tab');

  // 66f. A sprite wearing the clip shows only its stretch, over and over: green and blue, never red
  //      or white, without sound in the editor.
  await showTab('scene');
  await mutate(`
    const p = __gm.state.project;
    const scene = __gm.state.activeScene;
    ${MAKE_OBJECT}
    p.assets.unshift({ id: '${SPRITE}', type: 'sprite', name: '${SPRITE}', props: { width: 320, height: 180, color: 0xffffff, texture: ${JSON.stringify(clip.id)}, shader: '' } });
    scene.objects.push(make('${OBJECT}', '${SPRITE}', '', ${LANE}, 'video'));
    __gm.state.select(null);
    __gm.lookAt(0, ${LANE});`);
  await waitFor(
    () => json('__gm.videos()'),
    (players) => players.some((p) => p.ready),
  );
  const seen = new Set();
  for (let i = 0; i < 14; i++) {
    const r = await json(`__gm.objectRect('${OBJECT}')`);
    seen.add(await json(`__gm.pixelAt(${Math.round(r.x)}, ${Math.round(r.y)})`));
    await sleep(120);
  }
  const players = await json('__gm.videos()');
  check(
    'a sprite wearing the clip plays only its stretch, round and round',
    seen.has(COLORS.green) && seen.has(COLORS.blue) && !seen.has(COLORS.red) && !seen.has(COLORS.white),
    JSON.stringify([...seen]),
  );
  check(
    'the editor plays it without sound',
    players.length === 1 && players[0].muted && !players[0].paused && players[0].key.includes('#0.6~1.4~1~1'),
    JSON.stringify(players),
  );
  await shot('66-video-sprite');

  // 66g. Play starts it over from the clip's start and lets it be heard; Esc makes it quiet again.
  await play();
  await sleep(200);
  const heard = (await json('__gm.videos()'))[0];
  check(
    'Play starts the clip over and lets it be heard',
    heard && !heard.muted && !heard.paused && heard.time >= 0.6 && heard.time < 1.1,
    JSON.stringify(heard),
  );
  await stop();
  await sleep(200);
  const quiet = (await json('__gm.videos()'))[0];
  check('Esc makes it a quiet preview again', quiet && quiet.muted, JSON.stringify(quiet));

  // 66h. The sprite picked but not in the scene: only the inspector's preview wears the video. It
  //      asks for it every frame, so the player is not let go after 2 s; a let-go texture used to
  //      leave the preview drawing a destroyed one, and the batcher threw on every frame.
  await mutate(`
    const p = __gm.state.project;
    for (const holder of p.assets) if (holder.objects) holder.objects = holder.objects.filter((o) => o.id !== '${OBJECT}');
    __gm.state.select({ kind: 'asset', assetId: '${SPRITE}' });`);
  const logFrom = t.consoleLog.length;
  await sleep(4500);
  const inspected = await json('__gm.inspectorSpritePreview()');
  const thrown = t.consoleLog.slice(logFrom).filter((line) => /EXCEPTION|alphaMode/.test(line));
  const kept = await json('__gm.videos()');
  const inPreview = inspected && (await shownColor(inspected));
  check(
    'a sprite shown only in the inspector keeps its video playing there',
    thrown.length === 0 &&
      inspected?.animates === true &&
      kept.length === 1 &&
      kept[0].ready &&
      (inPreview === COLORS.green || inPreview === COLORS.blue),
    JSON.stringify({ thrown, inspected, kept, inPreview }),
  );

  // 66i. Driven by its Video time (#404): two sprites wearing the same video stand still in the
  //      scene, each at its own time (0.2 s red, 0.7 s green), each on a player of its own.
  await mutate(`
    const p = __gm.state.project;
    const scene = __gm.state.activeScene;
    ${MAKE_OBJECT}
    p.assets.unshift({ id: '${DRIVEN}', type: 'sprite', name: '${DRIVEN}', props: { width: 320, height: 180, color: 0xffffff, texture: ${JSON.stringify(assetId)}, shader: '', videoClock: 'driven', videoTime: 0 } });
    const early = make('${EARLY}', '${DRIVEN}', '', ${LANE}, 'at 0.2');
    const late = make('${LATE}', '${DRIVEN}', '', ${LANE}, 'at 0.7');
    early.overrides = { videoTime: 0.2 };
    late.position.x = 400;
    late.overrides = { videoTime: 0.7 };
    scene.objects.push(early, late);
    __gm.state.select(null);
    __gm.lookAt(200, ${LANE});`);
  const colorAt = (r) => r && json(`__gm.pixelAt(${Math.round(r.x)}, ${Math.round(r.y)})`);
  const pair = async () => [
    await colorAt(await json(`__gm.objectRect('${EARLY}')`)),
    await colorAt(await json(`__gm.objectRect('${LATE}')`)),
  ];
  const stills = await waitFor(pair, ([a, b]) => a === COLORS.red && b === COLORS.green);
  await sleep(700);
  const later = await pair();
  const drivenPlayers = (await json('__gm.videos()')).filter((v) => v.key.includes('@'));
  check(
    'driven sprites stand still at their Video time, two of the same video on players of their own',
    stills[0] === COLORS.red &&
      stills[1] === COLORS.green &&
      later[0] === COLORS.red &&
      later[1] === COLORS.green &&
      drivenPlayers.length === 2 &&
      drivenPlayers.every((v) => v.paused),
    JSON.stringify({ stills, later, drivenPlayers }),
  );
  await shot('66-video-driven');

  // 66j. Keyed in an animation, the preview shows the picture of the time at the playhead, back and
  //      forth: frame 2 = 0.2 s red, 7 = green, 12 = blue, and back to 3 = red.
  await mutate(`
    __gm.state.project.assets.push({ id: '${CUT}', type: 'animation', name: '${CUT}', props: { fps: 10, playback: 'loop' },
      root: '${DRIVEN}',
      tracks: [{ id: 'vd66-row', target: '', property: 'videoTime', keys: [{ frame: 0, value: 0 }, { frame: 20, value: 2 }] }] });`);
  await t.evaluate(`__gm.openDocument('asset:${CUT}')`);
  await sleep(300);
  const rootId = await json("__gm.animation.preview.objectId('')");
  // Each picture must also stay: a video playing by itself shows every colour within 2 s.
  const previewColor = async () =>
    colorAt(await json(`__gm.animation.preview.rect(${JSON.stringify(rootId)})`));
  const scrubbed = [];
  const wanted = [COLORS.red, COLORS.green, COLORS.blue, COLORS.red];
  for (const [i, frame] of [2, 7, 12, 3].entries()) {
    await t.evaluate(`__gm.animation.scrub(${frame})`);
    const first = await waitFor(previewColor, (color) => color === wanted[i], 10);
    await sleep(700);
    scrubbed.push([first, await previewColor()]);
  }
  check(
    'scrubbing the animation shows the picture of the keyed Video time and holds it, back and forth',
    scrubbed.every(([first, held], i) => first === wanted[i] && held === wanted[i]),
    JSON.stringify(scrubbed),
  );
  await shot('66-video-scrubbed');

  // 66k. Played in the tab, the video plays along at 1x, heard, at the time the playhead asks for
  //      (the clip and the animation both loop at 2 s, so the gap is measured round the loop).
  const playCut = await json('__gm.animation.playAt()');
  await click(playCut.x, playCut.y);
  await sleep(600);
  const along = [];
  for (let i = 0; i < 5; i++) {
    const asked = (await json(`__gm.animation.preview.item(${JSON.stringify(rootId)})`))?.props.videoTime;
    const player = (await json('__gm.videos()')).find(
      (v) => v.key.includes(`@`) && !v.key.includes(EARLY) && !v.key.includes(LATE),
    );
    along.push({ asked, ...player });
    await sleep(200);
  }
  await click(playCut.x, playCut.y);
  const gap = (a, b) => Math.min(Math.abs(a - b), 2 - Math.abs(a - b));
  check(
    'playing the animation plays the video along at 1x and lets it be heard',
    along.every((p) => !p.paused && !p.muted && Math.abs(p.rate - 1) <= 0.1 && gap(p.time, p.asked) < 0.15),
    JSON.stringify(along),
  );

  // 66l. Right-click → "Video at 1× to its end" (#407): from the playhead (frame 5, 0.5 s) the
  //      last whole frame before the clip ends, the key on 9 between them gone; one undo step.
  const rowKeys = () =>
    json(
      `__gm.state.project.assets.find((a) => a.id === '${CUT}').tracks[0].keys.map((k) => [k.frame, k.value])`,
    );
  await mutate(`
    __gm.state.project.assets.find((a) => a.id === '${CUT}').tracks[0].keys =
      [{ frame: 0, value: 0 }, { frame: 5, value: 0.5 }, { frame: 9, value: 1.7 }];`);
  const keyed = JSON.stringify(await rowKeys());
  await t.evaluate('__gm.animation.scrub(5)');
  const inRow = await json("__gm.animation.cellAt('vd66-row', 2)");
  await click(inRow.x, inRow.y, 'right');
  const runItem = await t.find('Video at 1× to its end from frame 5', true);
  await click(runItem.x, runItem.y);
  const steps = Math.floor((uploaded.duration - 0.5) * 10 + 1e-6);
  const ran = await rowKeys();
  await t.undo();
  check(
    'the menu keys the video at 1x from the playhead to the last frame before its end, in one undo step',
    ran.length === 3 &&
      JSON.stringify(ran.slice(0, 2)) === '[[0,0],[5,0.5]]' &&
      ran[2][0] === 5 + steps &&
      Math.abs(ran[2][1] - (0.5 + steps / 10)) < 1e-6 &&
      JSON.stringify(await rowKeys()) === keyed,
    JSON.stringify({ ran, steps, duration: uploaded.duration, undone: await rowKeys() }),
  );
  await showTab('scene');

  // 66m. Switched off, a driven sprite keeps its player ready at its Video time, longer than a player
  //      nobody asks for lives: switched on, the first frame shows 1.2 s blue, not its plain colour.
  //      4.5 s, as the sweeper looks only every 2 s and lets go of what is 2 s idle.
  const switchLate = (on) =>
    mutate(`
      const late = __gm.state.activeScene.objects.find((o) => o.id === '${LATE}');
      late.overrides = { ...late.overrides, videoTime: 1.2, enabled: ${on} };`);
  await switchLate(false);
  await sleep(4500);
  const readied = (await json('__gm.videos()')).find((v) => v.key.includes(LATE));
  await switchLate(true);
  await t.evaluate('__gm.afterFrames(1)');
  const firstShown = await colorAt(await json(`__gm.objectRect('${LATE}')`));
  check(
    'a driven sprite switched off keeps its video ready and shows its picture the frame it is switched on',
    readied?.ready === true &&
      readied.paused &&
      readied.muted &&
      Math.abs(readied.time - 1.2) < 0.05 &&
      firstShown === COLORS.blue,
    JSON.stringify({ readied, firstShown }),
  );

  // 66n. The same in a run: the world used to leave a switched-off object out of what the Game tab
  //      draws, so a cut to it loaded the video only then and showed white meanwhile. Nor did wearing
  //      the player count as asking for it: let go after 2 to 4 s, it was opened again.
  await switchLate(false);
  const sceneKeys = new Set((await json('__gm.videos()')).map((v) => v.key));
  await play();
  await sleep(4500);
  const inRun = (await json('__gm.videos()')).filter((v) => v.key.includes(LATE) && !sceneKeys.has(v.key));
  await stop();
  await switchLate(true);
  check(
    'in a run, a driven sprite switched off keeps its video ready too',
    inRun.length === 1 &&
      inRun[0].ready &&
      inRun[0].paused &&
      inRun[0].muted &&
      Math.abs(inRun[0].time - 1.2) < 0.05,
    JSON.stringify(inRun),
  );

  await mutate(`
    const p = __gm.state.project;
    const gone = ['${OBJECT}', '${EARLY}', '${LATE}'];
    for (const holder of p.assets) if (holder.objects) holder.objects = holder.objects.filter((o) => !gone.includes(o.id));
    p.assets = p.assets.filter((a) => ![ '${SPRITE}', '${DRIVEN}', '${CUT}', ${JSON.stringify(assetId)} ].includes(a.id));
    __gm.state.select(null);
    __gm.lookAt(0, 0);`);
}