adminandClaude Opus 5.5
abb8fa51d8
Chrome stops calling requestAnimationFrame for a hidden tab, and Pixi's ticker hangs on it. The run, the drawing and the recording all stood still as soon as the user switched to another tab. A plain run keeps standing still there, so a forgotten game does not play on unheard. - backgroundBeat.ts ticks the editor from a worker at the recording's fps while a recording goes and the tab is hidden, or the page has not drawn a frame of its own for 250 ms. Once the page draws again, the beat keeps out of the way. - The worker looks every 4 ms whether a tick is due. A 16 ms interval came in bursts behind a busy page, and the recording threw away the second tick of each burst. - backgroundTimers.ts and timerWorker.ts give the short waits of a recording timers in a worker. Chrome slows the page's own timers to once a second in a hidden tab. This covers the waits while saving and the wait for a video's picture after a seek. - Behind a tab Chrome does not call requestVideoFrameCallback, but the picture is there at seeked. The wait there is 10 ms instead of 150 ms while the tab is hidden. - The beat runs for Start recording, the exported game and Start recording... in the main menu. Tests: - tsc, eslint, Prettier clean - tests/viewport: 374 green, 3 new in background-beat.test.ts - e2e 68k (3 checks) on melektaus-atx with the tab really hidden: not in real time all 120 frames, colour changes every 30 frames, the blip 8 ms from green; in real time 97 of 120 frames (73 with the tab in front) - Counter-check without the beat: all three 68k checks red, 3 frames in 60 s not in real time, 7 frames in real time - Gate on melektaus-atx: green, 311 files, 4821 tests Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
752 lines
33 KiB
JavaScript
752 lines
33 KiB
JavaScript
import { execFileSync } from 'node:child_process';
|
||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||
import { tmpdir } from 'node:os';
|
||
import { join } from 'node:path';
|
||
import { uploadVideo } from '../lib/projects.mjs';
|
||
import { MAKE_OBJECT } from './lib/scaffold.mjs';
|
||
|
||
/**
|
||
* A composition placed in the scene (#417). In the editor the placement shows the composition's
|
||
* first frame, and a click on that picture picks the placement. Played, a behavior below it starts
|
||
* it: each item becomes an object below the placement while it lasts, the first track in front, a
|
||
* clip moves the item it overlaps, and `On composition finished` hears the end, by which time no
|
||
* item is left. Nothing of the run stays in the scene. An item's video is waited for: the clock
|
||
* stands until the Game tab has its first picture, for a clip that starts later the picture there,
|
||
* and then the video plays at once. A recording that runs meanwhile takes no frame. When the item
|
||
* ends, its video stops and goes quiet with it. A sound item is planned once the clock runs, timed
|
||
* to the cut. Recorded not in real time from a slowed editor, the file still has every frame, the
|
||
* video's picture at each, and its sound and the sound item's in step. Behind another tab, where the
|
||
* page draws no frames, a recording goes on all the same, not in real time and in real time.
|
||
*
|
||
* Everything it adds carries a `cp68` id and goes again at the end.
|
||
*/
|
||
const PLACE = 'cp68-place';
|
||
const DONE = 'cut 68 done';
|
||
|
||
const node = (id, type, x, y, params = {}) => ({ id: `cp68-${id}`, type, x, y, params });
|
||
const link = (id, from, fromPort, to, toPort) => ({
|
||
id: `cp68-${id}`,
|
||
from: { node: `cp68-${from}`, port: fromPort },
|
||
to: { node: `cp68-${to}`, port: toPort },
|
||
});
|
||
|
||
export async function composition(t) {
|
||
const { check, sleep, json, evaluate, mutate, press, play, stop, showTab, waitUntil } = t;
|
||
await showTab('scene');
|
||
|
||
const graph = {
|
||
nodes: [
|
||
node('start', 'OnStart', 0, 0),
|
||
node('owner', 'Owner', 0, 150),
|
||
node('play', 'PlayComposition', 300, 0),
|
||
node('finished', 'OnCompositionFinished', 0, 300, { composition: 'cp68-cut' }),
|
||
node('log', 'Log', 300, 300, { message: DONE }),
|
||
],
|
||
edges: [
|
||
link('e1', 'start', 'exec', 'play', 'exec'),
|
||
link('e2', 'owner', 'object', 'play', 'object'),
|
||
link('e3', 'finished', 'exec', 'log', 'exec'),
|
||
],
|
||
};
|
||
// Red from the start, a larger blue behind it from the next frame – built after red, it still goes
|
||
// behind – and green after them, moved by a clip; ten frames a second, so the cut lasts 1.2 s. Placed where the camera looks, in front of what is there.
|
||
await mutate(`
|
||
const p = __gm.state.project;
|
||
const scene = __gm.state.activeScene;
|
||
const cam = scene.objects.find((o) => o.id === 'o3').position;
|
||
${MAKE_OBJECT}
|
||
const sprite = p.assets.find((a) => a.type === 'sprite');
|
||
const look = (o, size, color) => { o.overrides = { width: size, height: size, color, texture: '' }; return o; };
|
||
const red = look(make('cp68-red', sprite.id, '', 0, 'Red 68'), 200, 0xff0000);
|
||
const blue = look(make('cp68-blue', sprite.id, '', 0, 'Blue 68'), 300, 0x0000ff);
|
||
const green = look(make('cp68-green', sprite.id, '', 0, 'Green 68'), 200, 0x00ff00);
|
||
for (const o of [red, blue, green]) delete o.groupId;
|
||
p.assets.push(
|
||
{
|
||
id: 'cp68-cut',
|
||
type: 'composition',
|
||
name: 'Cut 68',
|
||
props: { fps: 10, loop: false },
|
||
objects: [red, blue, green],
|
||
groups: [],
|
||
tracks: [{ id: 'cp68-front', name: 'Front' }, { id: 'cp68-back', name: 'Back' }],
|
||
items: [
|
||
{ id: 'cp68-red', track: 'cp68-front', start: 0, length: 6, offset: 0 },
|
||
{ id: 'cp68-blue', track: 'cp68-back', start: 1, length: 5, offset: 0 },
|
||
{ id: 'cp68-green', track: 'cp68-front', start: 6, length: 6, offset: 0 },
|
||
],
|
||
clips: [{ id: 'cp68-clip', track: 'cp68-front', animation: 'cp68-move', start: 6 }],
|
||
},
|
||
{
|
||
id: 'cp68-move',
|
||
type: 'animation',
|
||
name: 'Slide 68',
|
||
props: { fps: 10, playback: 'once' },
|
||
root: sprite.id,
|
||
tracks: [{ id: 'cp68-row', target: '', property: 'x', keys: [{ frame: 0, value: 0 }, { frame: 5, value: 100 }] }],
|
||
},
|
||
{ id: 'cp68-director', type: 'behavior', name: 'Director 68', props: { enabled: true, runInEditor: false }, variables: [], graph: ${JSON.stringify(graph)} },
|
||
);
|
||
const place = make('${PLACE}', 'cp68-cut', '', 0, 'Cut 68');
|
||
Object.assign(place.position, { x: cam.x, y: cam.y, z: 2000 });
|
||
scene.objects.push(place, make('cp68-logic', 'cp68-director', '${PLACE}', 0, 'Director 68'));
|
||
__gm.lookAt(cam.x, cam.y);
|
||
`);
|
||
const id = (item) => `${PLACE}~cp68-${item}`;
|
||
|
||
// 68a. In the editor the placement shows its first frame – red alone – and a click on the picture
|
||
// picks the placement, since what it shows is edited in the composition.
|
||
const order = await json('__gm.drawOrder()');
|
||
// Off the centre, where the placement's own symbol sits.
|
||
const rect = await json(`__gm.objectRect(${JSON.stringify(id('red'))})`);
|
||
const stack = rect
|
||
? await json(`__gm.sceneStack(${rect.x + rect.w * 0.35}, ${rect.y + rect.h * 0.35})`)
|
||
: null;
|
||
check(
|
||
'a placed composition shows its first frame, and a click on it picks the placement',
|
||
order.includes(id('red')) &&
|
||
!order.includes(id('blue')) &&
|
||
!order.includes(id('green')) &&
|
||
JSON.stringify(stack) === JSON.stringify([PLACE]),
|
||
JSON.stringify({ order: order.filter((o) => o.startsWith('cp68')), stack }),
|
||
);
|
||
|
||
// 68b. Played: the items come and go below the placement, sampled every frame of the editor.
|
||
await play();
|
||
const seen = await evaluate(`new Promise((resolve) => {
|
||
const frames = [];
|
||
const sample = () => {
|
||
const world = __gm.world();
|
||
if (!world) return resolve(JSON.stringify(frames));
|
||
const items = world.entities.filter((e) => e.objectId.startsWith('${PLACE}~'));
|
||
const green = items.find((e) => e.objectId === '${id('green')}');
|
||
frames.push({
|
||
ids: items.map((e) => e.objectId.slice('${PLACE}~cp68-'.length)),
|
||
parents: [...new Set(items.map((e) => e.parent))],
|
||
greenX: green ? green.x : null,
|
||
done: world.log.includes('${DONE}'),
|
||
});
|
||
if (frames.length > 400 || frames.at(-1).done) return resolve(JSON.stringify(frames));
|
||
requestAnimationFrame(sample);
|
||
};
|
||
sample();
|
||
})`);
|
||
const frames = JSON.parse(seen);
|
||
const both = frames.find((f) => f.ids.length === 2);
|
||
const greenXs = frames.map((f) => f.greenX).filter((x) => x !== null);
|
||
check(
|
||
'Play composition builds the items below the placement, the first track in front, one after the other',
|
||
both !== undefined &&
|
||
JSON.stringify(both.ids) === JSON.stringify(['blue', 'red']) &&
|
||
both.parents.length === 1 &&
|
||
both.parents[0] === PLACE &&
|
||
!frames.some((f) => f.ids.join('+') === 'red+blue') &&
|
||
frames.some((f) => JSON.stringify(f.ids) === JSON.stringify(['green'])) &&
|
||
!frames.some((f) => f.ids.includes('green') && f.ids.includes('red')),
|
||
JSON.stringify(frames.filter((_, i) => i % 5 === 0).map((f) => f.ids.join('+'))),
|
||
);
|
||
check(
|
||
'a clip moves the item it overlaps, from its value on',
|
||
greenXs.length >= 3 &&
|
||
greenXs[0] < 40 &&
|
||
greenXs.at(-1) > greenXs[0] &&
|
||
greenXs.every((x, i) => i === 0 || x >= greenXs[i - 1]),
|
||
JSON.stringify(greenXs),
|
||
);
|
||
const last = frames.at(-1);
|
||
check(
|
||
'On composition finished hears the end, by which time no item is left',
|
||
last?.done === true && last.ids.length === 0,
|
||
JSON.stringify(last),
|
||
);
|
||
await sleep(300);
|
||
await stop();
|
||
|
||
// 68c. The run's objects stay in the run: the scene holds the placement and its behavior only.
|
||
const stored = await json(
|
||
`__gm.state.activeScene.objects.filter((o) => o.id.startsWith('cp68')).map((o) => o.id).sort()`,
|
||
);
|
||
check(
|
||
'nothing of a played composition stays in the scene',
|
||
JSON.stringify(stored) === JSON.stringify(['cp68-logic', PLACE].sort()),
|
||
JSON.stringify(stored),
|
||
);
|
||
|
||
// 68d. A video item at frame 0, played from On start: the composition's clock stands while the Game
|
||
// tab loads the video, so its first frames are seen – it ran on without them before.
|
||
const uploaded = await uploadVideo(
|
||
t.baseUrl,
|
||
readFileSync(new URL('./fixtures/colors.webm', import.meta.url)),
|
||
'colors.webm',
|
||
);
|
||
await mutate(`
|
||
const p = __gm.state.project;
|
||
p.assets.push(
|
||
{ id: 'cp68-video', type: 'video', name: 'Colors 68', props: {}, file: ${JSON.stringify(uploaded.file)},
|
||
width: 320, height: 180, duration: 2, fps: 30, audio: true },
|
||
{ id: 'cp68-screen', type: 'sprite', name: 'Screen 68', props: { width: 160, height: 90, color: 0xffffff, texture: 'cp68-video' } },
|
||
);
|
||
const c = p.assets.find((a) => a.id === 'cp68-cut');
|
||
// Sixty frames a second: in steps of a tenth, a clock that ran on for a few frames still read 0.
|
||
c.props.fps = 60;
|
||
c.objects = [{ ...structuredClone(c.objects[0]), id: 'cp68-v', assetId: 'cp68-screen', name: 'Video 68', overrides: {} }];
|
||
c.items = [{ id: 'cp68-v', track: 'cp68-front', start: 0, length: 120, offset: 0 }];
|
||
c.clips = [];`);
|
||
// The video time the composition writes into its item every frame.
|
||
const clock = `__gm.world()?.entities.find((e) => e.objectId === '${id('v')}')?.props.videoTime ?? null`;
|
||
// Sampled from before the click on Play: play() settles after the start, and a video that loads
|
||
// quickly is ready by then, with no wait left to see (CI task 669).
|
||
await evaluate(`{
|
||
const loading = [];
|
||
const until = performance.now() + 20000;
|
||
window.cp68Wait = new Promise((done) => {
|
||
const step = () => {
|
||
const playing = __gm.state.mode === 'play';
|
||
const time = playing ? ${clock} : null;
|
||
const player = playing ? __gm.gameVideo('${id('v')}') : null;
|
||
if (player?.ready) return done(JSON.stringify({ loading, ready: time }));
|
||
if (player) loading.push(time);
|
||
if (performance.now() > until) return done(JSON.stringify({ loading, ready: null }));
|
||
requestAnimationFrame(step);
|
||
};
|
||
step();
|
||
});
|
||
// Not the promise as the block's value: evaluate would wait for it, and Play would come too late.
|
||
null;
|
||
}`);
|
||
await play();
|
||
const wait = JSON.parse(await evaluate('window.cp68Wait'));
|
||
await sleep(500);
|
||
const after = await json(clock);
|
||
check(
|
||
'a played composition waits for the first picture of an item’s video, then runs on',
|
||
wait.loading.length > 0 &&
|
||
wait.loading.every((time) => time === 0) &&
|
||
wait.ready !== null &&
|
||
wait.ready === 0 &&
|
||
after > 0.2,
|
||
JSON.stringify({
|
||
frames: wait.loading.length,
|
||
loading: wait.loading.slice(0, 8),
|
||
ready: wait.ready,
|
||
after,
|
||
}),
|
||
);
|
||
await stop();
|
||
|
||
// 68e. The item wears a clip that starts at 0.8 s: its player loads, then seeks back to 0.8. The
|
||
// clock stands until the Game tab draws that picture, and from its first step on the video
|
||
// plays – it was held and sought frame by frame for 0.1 s before, its first pictures stuck.
|
||
await mutate(`
|
||
const p = __gm.state.project;
|
||
p.assets.find((a) => a.id === 'cp68-video').clips = [{ id: 'cp68-late', name: 'Late', props: { start: 0.8, end: 0 } }];
|
||
p.assets.find((a) => a.id === 'cp68-screen').props.texture = 'cp68-late';`);
|
||
await play();
|
||
const rows = JSON.parse(
|
||
await evaluate(`new Promise((done) => {
|
||
const rows = [];
|
||
const step = () => {
|
||
const player = __gm.gameVideo('${id('v')}');
|
||
rows.push({ time: ${clock}, drawn: __gm.gameDrawnRect('${id('v')}') !== null,
|
||
at: player?.time ?? null, paused: player?.paused ?? null });
|
||
if (rows.length >= 240 || !__gm.world()) return done(JSON.stringify(rows));
|
||
requestAnimationFrame(step);
|
||
};
|
||
step();
|
||
})`),
|
||
);
|
||
const first = rows.findIndex((row) => row.drawn);
|
||
const moving = rows.filter((row) => row.drawn && row.time > 0).slice(0, 3);
|
||
check(
|
||
'a clip that starts later is waited for, then plays from its first step on',
|
||
first >= 0 &&
|
||
rows.slice(0, first).every((row) => row.time === 0) &&
|
||
Math.abs(rows[first].at - 0.8) < 0.02 &&
|
||
moving.length === 3 &&
|
||
moving.every((row) => row.paused === false),
|
||
JSON.stringify({ first, shown: rows[first], moving }),
|
||
);
|
||
await stop();
|
||
|
||
// 68f. A recording runs when Play composition starts the cut: no frame is taken while the Game
|
||
// tab leaves the item's video out – it recorded the empty frames of the wait before. A clip
|
||
// that starts elsewhere, so a new player loads: the one of 68e is still there, at its picture.
|
||
await mutate(`
|
||
const video = __gm.state.project.assets.find((a) => a.id === 'cp68-video');
|
||
video.clips.push({ id: 'cp68-later', name: 'Later', props: { start: 1.2, end: 0 } });
|
||
__gm.state.project.assets.find((a) => a.id === 'cp68-screen').props.texture = 'cp68-later';`);
|
||
const flow = structuredClone(graph);
|
||
flow.nodes.push(
|
||
node('record', 'StartRecording', 300, -150, { name: 'cp68 take', resolution: 'view', fps: '60' }),
|
||
node('key', 'OnKeyDown', 0, 450, { key: 'p' }),
|
||
);
|
||
flow.edges = flow.edges.filter((e) => e.id !== 'cp68-e1');
|
||
flow.edges.push(link('e4', 'start', 'exec', 'record', 'exec'), link('e5', 'key', 'exec', 'play', 'exec'));
|
||
await mutate(
|
||
`__gm.state.project.assets.find((a) => a.id === 'cp68-director').graph = ${JSON.stringify(flow)};`,
|
||
);
|
||
const { uploads, takes } = await json('__gm.recording()');
|
||
const take = `__gm.recording().takes[${takes.length}]`;
|
||
await play();
|
||
await waitUntil(`${take}?.state === 'recording' && ${take}.lastFrameAt !== null`, {
|
||
label: 'the recording takes frames',
|
||
});
|
||
// Sampled every frame in the page, from before the key on.
|
||
await evaluate(`{
|
||
const rows = (window.cp68Rows = []);
|
||
const step = () => {
|
||
const exists = (__gm.world()?.entities ?? []).some((e) => e.objectId === '${id('v')}');
|
||
rows.push({ exists, drawn: __gm.gameDrawnRect('${id('v')}') !== null, taken: ${take}?.lastFrameAt ?? null });
|
||
const drawn = rows.findIndex((row) => row.drawn);
|
||
if ((drawn < 0 || rows.length <= drawn + 20) && rows.length < 400) requestAnimationFrame(step);
|
||
};
|
||
requestAnimationFrame(step);
|
||
}`);
|
||
await press('p');
|
||
await waitUntil(
|
||
`window.cp68Rows.length >= 400 || (window.cp68Rows.some((row) => row.drawn) && window.cp68Rows.length > window.cp68Rows.findIndex((row) => row.drawn) + 20)`,
|
||
{ label: 'the item is drawn' },
|
||
);
|
||
const takeRows = await json('window.cp68Rows');
|
||
await stop();
|
||
// The frames in which the item was there and its video still left out.
|
||
const missing = takeRows.filter((row) => row.exists && !row.drawn);
|
||
const shown = takeRows.findIndex((row) => row.drawn);
|
||
check(
|
||
'a recording takes no frame while a composition waits for its video, and goes on after',
|
||
missing.length >= 2 &&
|
||
missing.every((row) => row.taken === missing[0].taken) &&
|
||
shown >= 0 &&
|
||
takeRows.at(-1).taken > missing.at(-1).taken,
|
||
JSON.stringify({
|
||
missing: missing.length,
|
||
takenWhileMissing: new Set(missing.map((row) => row.taken)).size - 1,
|
||
shown,
|
||
after:
|
||
takeRows.at(-1).taken !== null && missing.length > 0
|
||
? Math.round(takeRows.at(-1).taken - missing.at(-1).taken)
|
||
: null,
|
||
}),
|
||
);
|
||
await waitUntil(
|
||
`__gm.recording().uploads.length > ${uploads.length} && ['added', 'failed'].includes(__gm.recording().uploads.at(-1).state)`,
|
||
{ timeout: 60_000, label: 'the recording is in the project' },
|
||
);
|
||
const madeId = (await json('__gm.recording()')).uploads.at(-1).assetId ?? '';
|
||
|
||
// 68g. The item ends after half a second, its video of two seconds with sound in the middle: the
|
||
// item's player stops and goes quiet at once – it played on, heard, until it was let go 2-4 s
|
||
// later.
|
||
await mutate(`
|
||
const p = __gm.state.project;
|
||
p.assets.find((a) => a.id === 'cp68-director').graph = ${JSON.stringify(graph)};
|
||
p.assets.find((a) => a.id === 'cp68-screen').props.texture = 'cp68-video';
|
||
p.assets.find((a) => a.id === 'cp68-cut').items[0].length = 30;`);
|
||
await play();
|
||
const cut = JSON.parse(
|
||
await evaluate(`new Promise((done) => {
|
||
const rows = [];
|
||
let gone = -1;
|
||
const step = () => {
|
||
const exists = (__gm.world()?.entities ?? []).some((e) => e.objectId === '${id('v')}');
|
||
const player = __gm.gameVideo('${id('v')}');
|
||
rows.push({ exists, paused: player?.paused ?? null, muted: player?.muted ?? null });
|
||
if (gone < 0 && rows.some((row) => row.exists) && !exists) gone = rows.length - 1;
|
||
if ((gone >= 0 && rows.length > gone + 30) || rows.length >= 600 || !__gm.world())
|
||
return done(JSON.stringify({ gone, heard: rows.filter((row) => row.exists && row.paused === false && row.muted === false).length, after: rows.slice(gone + 12) }));
|
||
requestAnimationFrame(step);
|
||
};
|
||
step();
|
||
})`),
|
||
);
|
||
await stop();
|
||
check(
|
||
'the video of an item that ends stops and goes quiet with it',
|
||
cut.gone > 0 &&
|
||
cut.heard > 0 &&
|
||
cut.after.length > 0 &&
|
||
cut.after.every((row) => row.paused === true && row.muted === true),
|
||
JSON.stringify({
|
||
gone: cut.gone,
|
||
heard: cut.heard,
|
||
after: cut.after.slice(0, 3),
|
||
last: cut.after.at(-1),
|
||
}),
|
||
);
|
||
|
||
// 68h (#418). A sound item half a second in, beside the video item from 68d: the cut waits for the
|
||
// video, and its sound with it – planned once the clock runs, for the second of the audio
|
||
// clock the cut reaches the item at, and to its end.
|
||
const sounds = await evaluate(`fetch('/feuerwerk/api/index.php?r=sounds').then((r) => r.json())`);
|
||
const blip = sounds.find((item) => item.bytes === 4044);
|
||
await mutate(`
|
||
const p = __gm.state.project;
|
||
p.assets.push({ id: 'cp68-blip', type: 'sound', name: 'Blip 68', props: {}, file: ${JSON.stringify(blip?.file ?? '')} });
|
||
p.assets.find((a) => a.id === 'cp68-screen').props.texture = 'cp68-video';
|
||
const c = p.assets.find((a) => a.id === 'cp68-cut');
|
||
c.items = [
|
||
{ id: 'cp68-v', track: 'cp68-front', start: 0, length: 120, offset: 0 },
|
||
{ id: 'cp68-tone', track: 'cp68-back', start: 30, length: 30, offset: 0,
|
||
audio: { kind: 'sound', asset: 'cp68-blip', volume: 0, bus: '#composition' } },
|
||
];`);
|
||
await evaluate(`{
|
||
window.cp68Sound = new Promise((done) => {
|
||
const rows = [];
|
||
const until = performance.now() + 20000;
|
||
const step = () => {
|
||
if (__gm.state.mode === 'play') {
|
||
const time = ${clock};
|
||
const tone = __gm.sound.playbacks().find((p) => p.name === 'composition:cp68-cut');
|
||
rows.push({ time, clock: __gm.sound.clock(), planned: tone?.clips.length ?? 0, clip: tone?.clips[0] ?? null });
|
||
if (time !== null && time >= 0.6) {
|
||
const buses = __gm.routing.buses();
|
||
rows.at(-1).routes = {
|
||
tone: buses.find((b) => b.name === 'play:' + tone?.id)?.into ?? null,
|
||
cut: buses.find((b) => b.name === 'mix:cp68-cut')?.into ?? null,
|
||
videos: __gm.routing.media().filter((m) => m.routed).map((m) => m.bus),
|
||
};
|
||
return done(JSON.stringify(rows));
|
||
}
|
||
}
|
||
if (performance.now() > until) return done(JSON.stringify(rows));
|
||
requestAnimationFrame(step);
|
||
};
|
||
step();
|
||
});
|
||
null;
|
||
}`);
|
||
await play();
|
||
const toneRows = JSON.parse(await evaluate('window.cp68Sound'));
|
||
await stop();
|
||
const ran = toneRows.findIndex((row) => row.time > 0);
|
||
const reached = toneRows.find((row) => row.time >= 0.5);
|
||
const toneClip = toneRows.at(-1)?.clip;
|
||
const early = reached && toneClip ? reached.clock - toneClip.at : null;
|
||
check(
|
||
'a sound item is planned only once the cut’s clock runs, for when the cut reaches it, to its end',
|
||
ran > 0 &&
|
||
toneRows.slice(0, ran).every((row) => row.planned === 0) &&
|
||
toneClip?.sound === 'cp68-blip' &&
|
||
toneClip.offset === 0 &&
|
||
Math.abs(toneClip.duration - 0.5) < 0.002 &&
|
||
early !== null &&
|
||
early > -0.02 &&
|
||
early < 0.05,
|
||
JSON.stringify({ waited: ran, early, reached, toneClip }),
|
||
);
|
||
// 68i. The sound item picked the composition's bus, the video item picked none: both go into the
|
||
// composition's bus, and that one into the master.
|
||
const routes = toneRows.at(-1)?.routes;
|
||
check(
|
||
'a sound item and a video item go into the composition’s bus, and that one into the master',
|
||
routes?.tone === 'mix:cp68-cut' &&
|
||
routes.cut === 'mix' &&
|
||
routes.videos.length > 0 &&
|
||
routes.videos.every((bus) => bus === 'mix:cp68-cut'),
|
||
JSON.stringify(routes),
|
||
);
|
||
|
||
// 68j (#423). Recorded not in real time while the editor draws a frame every 45 ms, about 20 a
|
||
// second: the file still has every frame, 1/60 s apart. The video item covers the picture,
|
||
// its colours turn every half second of its file and so every 30 frames of the recording, and
|
||
// its sound is heard from the start. The blip item starts as green does.
|
||
await mutate(`
|
||
const p = __gm.state.project;
|
||
p.assets.find((a) => a.id === 'cp68-screen').props = { width: 4000, height: 3000, color: 0xffffff, texture: 'cp68-video' };
|
||
const c = p.assets.find((a) => a.id === 'cp68-cut');
|
||
c.items = [
|
||
{ id: 'cp68-v', track: 'cp68-front', start: 0, length: 120, offset: 0 },
|
||
{ id: 'cp68-tone', track: 'cp68-back', start: 30, length: 30, offset: 0,
|
||
audio: { kind: 'sound', asset: 'cp68-blip', volume: 0, bus: '#composition' } },
|
||
];
|
||
p.assets.find((a) => a.id === 'cp68-director').graph = ${JSON.stringify({
|
||
nodes: [
|
||
node('start', 'OnStart', 0, 0),
|
||
node('owner', 'Owner', 0, 150),
|
||
node('record', 'StartRecording', 300, 0, {
|
||
name: 'cp68 steady',
|
||
resolution: '720p',
|
||
fps: '60',
|
||
convert: false,
|
||
adopt: false,
|
||
realTime: false,
|
||
}),
|
||
node('play', 'PlayComposition', 600, 0),
|
||
node('finished', 'OnCompositionFinished', 0, 300, { composition: 'cp68-cut' }),
|
||
node('stop', 'StopRecording', 300, 300),
|
||
],
|
||
edges: [
|
||
link('e1', 'start', 'exec', 'record', 'exec'),
|
||
link('e2', 'record', 'exec', 'play', 'exec'),
|
||
link('e3', 'owner', 'object', 'play', 'object'),
|
||
link('e4', 'finished', 'exec', 'stop', 'exec'),
|
||
],
|
||
})};`);
|
||
const before = await json('__gm.recording()');
|
||
const steadyTake = `__gm.recording().takes[${before.takes.length}]`;
|
||
await evaluate(`(() => {
|
||
window.__cp68slow = true;
|
||
const slow = () => {
|
||
if (!window.__cp68slow) return;
|
||
const until = performance.now() + 45;
|
||
while (performance.now() < until);
|
||
requestAnimationFrame(slow);
|
||
};
|
||
requestAnimationFrame(slow);
|
||
})()`);
|
||
await play();
|
||
await waitUntil(`${steadyTake}?.state === 'done' || ${steadyTake}?.state === 'failed'`, {
|
||
timeout: 60_000,
|
||
label: 'the recording not in real time is saved',
|
||
});
|
||
await evaluate('window.__cp68slow = false; 0');
|
||
await stop();
|
||
await waitUntil(
|
||
`__gm.recording().uploads.length > ${before.uploads.length} && ['kept', 'failed'].includes(__gm.recording().uploads.at(-1).state)`,
|
||
{ timeout: 30_000, label: 'the recording is uploaded' },
|
||
);
|
||
const steady = await json(steadyTake);
|
||
const steadyUpload = (await json('__gm.recording()')).uploads.at(-1);
|
||
const steadyLines = (await json('__gm.console.lines()'))
|
||
.map((line) => line.text)
|
||
.filter((text) => text.includes('"cp68 steady"'));
|
||
/**
|
||
* What ffmpeg reads from the original of a recording: its frames, where they are not 1/60 s
|
||
* apart, where the video's colour turns, the video's tone and where the blip lies (from green).
|
||
*/
|
||
const readRecording = async (upload) => {
|
||
const original = await fetch(`${t.baseUrl}data/videos/${upload.file.replace(/\.mp4$/, '.source.mkv')}`);
|
||
const file = { frames: 0, gaps: [], turns: [], toneBefore: null, toneDuring: null, blipAt: null };
|
||
if (original.ok) {
|
||
const dir = mkdtempSync(join(tmpdir(), 'cp68-'));
|
||
const mkv = join(dir, 'steady.mkv');
|
||
writeFileSync(mkv, new Uint8Array(await original.arrayBuffer()));
|
||
const pts = execFileSync('ffprobe', [
|
||
'-v',
|
||
'error',
|
||
'-select_streams',
|
||
'v:0',
|
||
'-show_entries',
|
||
'packet=pts_time',
|
||
'-of',
|
||
'csv=p=0',
|
||
mkv,
|
||
])
|
||
.toString()
|
||
.trim()
|
||
.split('\n')
|
||
.map(Number)
|
||
.sort((a, b) => a - b);
|
||
file.frames = pts.length;
|
||
file.gaps = pts.slice(1).flatMap((at, i) => (Math.abs(at - pts[i] - 1 / 60) > 0.0015 ? [i + 1] : []));
|
||
// Each frame as 8x4 pixels: the colour of the video item covering it, or of what lies behind.
|
||
const pixels = execFileSync(
|
||
'ffmpeg',
|
||
[
|
||
'-v',
|
||
'error',
|
||
'-i',
|
||
mkv,
|
||
'-map',
|
||
'0:v',
|
||
'-vf',
|
||
'scale=8:4',
|
||
'-f',
|
||
'rawvideo',
|
||
'-pix_fmt',
|
||
'rgb24',
|
||
'-',
|
||
],
|
||
{ maxBuffer: 64 * 1024 * 1024 },
|
||
);
|
||
const colours = [];
|
||
for (let at = 0; at + 96 <= pixels.length; at += 96) {
|
||
const [r, g, b] = [0, 1, 2].map((c) => pixels[at + 13 * 3 + c]);
|
||
colours.push(
|
||
r > 180 && g > 180 && b > 180 ? 'W' : r > 150 ? 'R' : g > 150 ? 'G' : b > 150 ? 'B' : '-',
|
||
);
|
||
}
|
||
file.turns = colours.flatMap((colour, i) =>
|
||
i === 0 || colour !== colours[i - 1] ? [`${i}${colour}`] : [],
|
||
);
|
||
const sound = execFileSync(
|
||
'ffmpeg',
|
||
['-v', 'error', '-i', mkv, '-map', '0:a', '-ac', '1', '-ar', '48000', '-f', 'f32le', '-'],
|
||
{ maxBuffer: 64 * 1024 * 1024 },
|
||
);
|
||
const samples = new Float32Array(sound.buffer, sound.byteOffset, sound.length / 4);
|
||
const rms = (from, to) => {
|
||
let sum = 0;
|
||
const a = Math.max(0, Math.round(from * 48000));
|
||
const b = Math.min(samples.length, Math.round(to * 48000));
|
||
for (let i = a; i < b; i++) sum += samples[i] * samples[i];
|
||
return b > a ? Math.sqrt(sum / (b - a)) : 0;
|
||
};
|
||
const red = Number.parseInt(file.turns.find((turn) => turn.endsWith('R')) ?? '-1', 10);
|
||
const green = Number.parseInt(file.turns.find((turn) => turn.endsWith('G')) ?? '-1', 10);
|
||
if (red >= 0 && green > red) {
|
||
file.toneBefore = red > 6 ? rms(0, (red - 3) / 60) : 0;
|
||
file.toneDuring = rms((red + 3) / 60, (green - 3) / 60);
|
||
// The blip is loud over the tone of the video: the first 2 ms that are.
|
||
for (let at = red / 60; at < (green + 30) / 60; at += 0.002)
|
||
if (rms(at, at + 0.002) > 3 * file.toneDuring + 0.05) {
|
||
file.blipAt = Math.round((at - green / 60) * 1000);
|
||
break;
|
||
}
|
||
}
|
||
rmSync(dir, { recursive: true, force: true });
|
||
}
|
||
// The video's four colours one after the other, whatever lay behind it before and after.
|
||
const colourOrder = file.turns.map((turn) => turn.at(-1)).join('');
|
||
const turned = file.turns
|
||
.slice(colourOrder.indexOf('RGBW'), colourOrder.indexOf('RGBW') + 4)
|
||
.map((turn) => Number.parseInt(turn, 10));
|
||
return { ...file, ok: original.ok, status: original.status, colourOrder, turned };
|
||
};
|
||
const file = await readRecording(steadyUpload);
|
||
const original = { ok: file.ok, status: file.status };
|
||
const { colourOrder, turned } = file;
|
||
check(
|
||
'a recording not in real time has every frame, 1/60 s apart, though the editor drew about 20 a second',
|
||
original.ok &&
|
||
steady.realTime === false &&
|
||
steady.failure === null &&
|
||
steady.skipped === 0 &&
|
||
steady.dropped === 0 &&
|
||
file.frames === steady.frames &&
|
||
file.frames >= 120 &&
|
||
file.gaps.length === 0 &&
|
||
steady.tookMs > steady.durationMs * 1.5,
|
||
JSON.stringify({ status: original.status, steady, frames: file.frames, gaps: file.gaps.slice(0, 5) }),
|
||
);
|
||
check(
|
||
'its picture is the video at every frame: the colours turn exactly every 30 frames',
|
||
colourOrder.includes('RGBW') && turned.slice(1).every((at, i) => at - turned[i] === 30),
|
||
JSON.stringify(file.turns),
|
||
);
|
||
check(
|
||
'and its sound is in step: the video heard from its first picture, the blip as green begins',
|
||
file.toneBefore !== null &&
|
||
file.toneBefore < 0.005 &&
|
||
file.toneDuring > 0.02 &&
|
||
file.blipAt !== null &&
|
||
Math.abs(file.blipAt) <= 20 &&
|
||
steadyLines.some((text) => text.endsWith('with sound, not in real time')) &&
|
||
steadyLines.some((text) => text.includes('x real time, and has every frame')),
|
||
JSON.stringify({
|
||
toneBefore: file.toneBefore,
|
||
toneDuring: file.toneDuring,
|
||
blipAtMs: file.blipAt,
|
||
steadyLines,
|
||
}),
|
||
);
|
||
|
||
// 68k. Behind another tab Chrome draws no frames, and the run stood still and the recording with
|
||
// it. While it records, a worker's beat ticks the page instead: not in real time the file has
|
||
// every frame, the video's picture at each and the blip in step; in real time the run goes on
|
||
// at its pace and the recording with it.
|
||
const tabs = 'http://127.0.0.1:9333/json';
|
||
const ownTab = (await t.cdp.send('Target.getTargetInfo')).targetInfo.targetId;
|
||
const recordBehind = async (name, realTime) => {
|
||
await mutate(`
|
||
const g = __gm.state.project.assets.find((a) => a.id === 'cp68-director').graph;
|
||
Object.assign(g.nodes.find((n) => n.id === 'cp68-record').params, { name: ${JSON.stringify(name)}, realTime: ${realTime} });`);
|
||
const before = await json('__gm.recording()');
|
||
const take = `__gm.recording().takes[${before.takes.length}]`;
|
||
await evaluate(`(() => {
|
||
window.__cp68frames = 0;
|
||
window.__cp68count = true;
|
||
const count = () => {
|
||
if (!window.__cp68count) return;
|
||
window.__cp68frames++;
|
||
requestAnimationFrame(count);
|
||
};
|
||
requestAnimationFrame(count);
|
||
})()`);
|
||
await play();
|
||
const other = await (await fetch(`${tabs}/new?about:blank`, { method: 'PUT' })).json();
|
||
const seen = { hidden: false, stillHidden: null, framesBehind: null, error: '' };
|
||
try {
|
||
await fetch(`${tabs}/activate/${other.id}`);
|
||
await waitUntil('document.hidden', { label: 'the tab went behind another' });
|
||
seen.hidden = true;
|
||
const framesAt = await json('window.__cp68frames');
|
||
await waitUntil(`${take}?.state === 'done' || ${take}?.state === 'failed'`, {
|
||
timeout: 60_000,
|
||
label: `"${name}" is saved behind another tab`,
|
||
});
|
||
seen.framesBehind = (await json('window.__cp68frames')) - framesAt;
|
||
seen.stillHidden = await json('document.hidden');
|
||
} catch (error) {
|
||
seen.error = String(error?.message ?? error);
|
||
} finally {
|
||
await fetch(`${tabs}/activate/${ownTab}`);
|
||
await fetch(`${tabs}/close/${other.id}`);
|
||
}
|
||
await waitUntil('!document.hidden', { label: 'the tab is in front again' });
|
||
await evaluate('window.__cp68count = false; 0');
|
||
await stop();
|
||
const made = await json(take);
|
||
if (made?.state !== 'done') return { made, seen, beats: 0, file: null };
|
||
await waitUntil(
|
||
`__gm.recording().uploads.length > ${before.uploads.length} && ['kept', 'failed'].includes(__gm.recording().uploads.at(-1).state)`,
|
||
{ timeout: 30_000, label: 'the recording is uploaded' },
|
||
);
|
||
const after = await json('__gm.recording()');
|
||
return { made, seen, beats: after.beats - before.beats, file: await readRecording(after.uploads.at(-1)) };
|
||
};
|
||
const behind = await recordBehind('cp68 behind', false);
|
||
const behindFile = behind.file;
|
||
check(
|
||
'behind another tab, where the page draws no frame, a recording not in real time still has every frame',
|
||
behind.seen.hidden &&
|
||
behind.seen.stillHidden === true &&
|
||
behind.seen.framesBehind <= 2 &&
|
||
behind.beats > 100 &&
|
||
behind.made?.failure === null &&
|
||
behindFile?.ok &&
|
||
behindFile.frames >= 120 &&
|
||
behindFile.gaps.length === 0,
|
||
JSON.stringify({
|
||
seen: behind.seen,
|
||
beats: behind.beats,
|
||
made: behind.made,
|
||
frames: behindFile?.frames,
|
||
gaps: behindFile?.gaps.slice(0, 5),
|
||
}),
|
||
);
|
||
check(
|
||
'and its picture and sound are those in front: colours every 30 frames, the blip as green begins',
|
||
Boolean(behindFile) &&
|
||
behindFile.colourOrder.includes('RGBW') &&
|
||
behindFile.turned.slice(1).every((at, i) => at - behindFile.turned[i] === 30) &&
|
||
behindFile.blipAt !== null &&
|
||
Math.abs(behindFile.blipAt) <= 20,
|
||
JSON.stringify({
|
||
turns: behindFile?.turns,
|
||
blipAtMs: behindFile?.blipAt,
|
||
toneDuring: behindFile?.toneDuring,
|
||
}),
|
||
);
|
||
const live = await recordBehind('cp68 behind live', true);
|
||
check(
|
||
'and in real time the run goes on behind another tab, and the recording with it',
|
||
live.seen.hidden &&
|
||
live.seen.stillHidden === true &&
|
||
live.seen.framesBehind <= 2 &&
|
||
live.beats > 60 &&
|
||
live.made?.failure === null &&
|
||
live.file?.ok &&
|
||
// Half its frames at least: in front of the others it has about 60 % here, the encoder in WSL
|
||
// being slow; standing still behind them it had none.
|
||
live.file.frames >= live.made.durationMs * 0.03,
|
||
JSON.stringify({ seen: live.seen, beats: live.beats, made: live.made, frames: live.file?.frames }),
|
||
);
|
||
|
||
await mutate(`
|
||
const p = __gm.state.project;
|
||
p.assets = p.assets.filter((a) => !a.id.startsWith('cp68-') && a.id !== ${JSON.stringify(madeId)});
|
||
const scene = __gm.state.activeScene;
|
||
scene.objects = scene.objects.filter((o) => !o.id.startsWith('cp68-'));`);
|
||
}
|