meleki/feuerwerk
1
0 Fork 0
Code Issues 38 Actions Projects 1 Wiki Activity

Keep a recording going behind another browser tab

Browse Source 
Chrome stops calling requestAnimationFrame for a hidden tab, and Pixi's
ticker hangs on it. The run, the drawing and the recording all stood
still as soon as the user switched to another tab. A plain run keeps
standing still there, so a forgotten game does not play on unheard.

- backgroundBeat.ts ticks the editor from a worker at the recording's
  fps while a recording goes and the tab is hidden, or the page has
  not drawn a frame of its own for 250 ms. Once the page draws again,
  the beat keeps out of the way.
- The worker looks every 4 ms whether a tick is due. A 16 ms interval
  came in bursts behind a busy page, and the recording threw away the
  second tick of each burst.
- backgroundTimers.ts and timerWorker.ts give the short waits of a
  recording timers in a worker. Chrome slows the page's own timers to
  once a second in a hidden tab. This covers the waits while saving
  and the wait for a video's picture after a seek.
- Behind a tab Chrome does not call requestVideoFrameCallback, but the
  picture is there at seeked. The wait there is 10 ms instead of
  150 ms while the tab is hidden.
- The beat runs for Start recording, the exported game and Start
  recording... in the main menu.

Tests:
- tsc, eslint, Prettier clean
- tests/viewport: 374 green, 3 new in background-beat.test.ts
- e2e 68k (3 checks) on melektaus-atx with the tab really hidden:
  not in real time all 120 frames, colour changes every 30 frames,
  the blip 8 ms from green; in real time 97 of 120 frames (73 with
  the tab in front)
- Counter-check without the beat: all three 68k checks red, 3 frames
  in 60 s not in real time, 7 frames in real time
- Gate on melektaus-atx: green, 311 files, 4821 tests

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
adminandClaude Opus 5.5 committed 2026-10-14 13:56:56 +02:00
1 parent 87b2ee9402
commit abb8fa51d8
10 files changed
+474 -94
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/e2e/68-composition.mjs
+196 -83
View File
@@ -15,7 +15,8 @@ import { MAKE_OBJECT } from './lib/scaffold.mjs';
* and then the video plays at once. A recording that runs meanwhile takes no frame. When the item
* ends, its video stops and goes quiet with it. A sound item is planned once the clock runs, timed
* to the cut. Recorded not in real time from a slowed editor, the file still has every frame, the
* video's picture at each, and its sound and the sound item's in step.
* video's picture at each, and its sound and the sound item's in step. Behind another tab, where the
* page draws no frames, a recording goes on all the same, not in real time and in real time.
*
* Everything it adds carries a `cp68` id and goes again at the end.
*/
@@ -509,92 +510,102 @@ export async function composition(t) {
const steadyLines = (await json('__gm.console.lines()'))
.map((line) => line.text)
.filter((text) => text.includes('"cp68 steady"'));
const original = await fetch(
`${t.baseUrl}data/videos/${steadyUpload.file.replace(/\.mp4$/, '.source.mkv')}`,
);
const file = { frames: 0, gaps: [], turns: [], toneBefore: null, toneDuring: null, blipAt: null };
if (original.ok) {
const dir = mkdtempSync(join(tmpdir(), 'cp68-'));
const mkv = join(dir, 'steady.mkv');
writeFileSync(mkv, new Uint8Array(await original.arrayBuffer()));
const pts = execFileSync('ffprobe', [
'-v',
'error',
'-select_streams',
'v:0',
'-show_entries',
'packet=pts_time',
'-of',
'csv=p=0',
mkv,
])
.toString()
.trim()
.split('\n')
.map(Number)
.sort((a, b) => a - b);
file.frames = pts.length;
file.gaps = pts.slice(1).flatMap((at, i) => (Math.abs(at - pts[i] - 1 / 60) > 0.0015 ? [i + 1] : []));
// Each frame as 8x4 pixels: the colour of the video item covering it, or of what lies behind.
const pixels = execFileSync(
'ffmpeg',
[
/**
* What ffmpeg reads from the original of a recording: its frames, where they are not 1/60 s
* apart, where the video's colour turns, the video's tone and where the blip lies (from green).
*/
const readRecording = async (upload) => {
const original = await fetch(`${t.baseUrl}data/videos/${upload.file.replace(/\.mp4$/, '.source.mkv')}`);
const file = { frames: 0, gaps: [], turns: [], toneBefore: null, toneDuring: null, blipAt: null };
if (original.ok) {
const dir = mkdtempSync(join(tmpdir(), 'cp68-'));
const mkv = join(dir, 'steady.mkv');
writeFileSync(mkv, new Uint8Array(await original.arrayBuffer()));
const pts = execFileSync('ffprobe', [
'-v',
'error',
'-i',
'-select_streams',
'v:0',
'-show_entries',
'packet=pts_time',
'-of',
'csv=p=0',
mkv,
'-map',
'0:v',
'-vf',
'scale=8:4',
'-f',
'rawvideo',
'-pix_fmt',
'rgb24',
'-',
],
{ maxBuffer: 64 * 1024 * 1024 },
);
const colours = [];
for (let at = 0; at + 96 <= pixels.length; at += 96) {
const [r, g, b] = [0, 1, 2].map((c) => pixels[at + 13 * 3 + c]);
colours.push(r > 180 && g > 180 && b > 180 ? 'W' : r > 150 ? 'R' : g > 150 ? 'G' : b > 150 ? 'B' : '-');
])
.toString()
.trim()
.split('\n')
.map(Number)
.sort((a, b) => a - b);
file.frames = pts.length;
file.gaps = pts.slice(1).flatMap((at, i) => (Math.abs(at - pts[i] - 1 / 60) > 0.0015 ? [i + 1] : []));
// Each frame as 8x4 pixels: the colour of the video item covering it, or of what lies behind.
const pixels = execFileSync(
'ffmpeg',
[
'-v',
'error',
'-i',
mkv,
'-map',
'0:v',
'-vf',
'scale=8:4',
'-f',
'rawvideo',
'-pix_fmt',
'rgb24',
'-',
],
{ maxBuffer: 64 * 1024 * 1024 },
);
const colours = [];
for (let at = 0; at + 96 <= pixels.length; at += 96) {
const [r, g, b] = [0, 1, 2].map((c) => pixels[at + 13 * 3 + c]);
colours.push(
r > 180 && g > 180 && b > 180 ? 'W' : r > 150 ? 'R' : g > 150 ? 'G' : b > 150 ? 'B' : '-',
);
}
file.turns = colours.flatMap((colour, i) =>
i === 0 || colour !== colours[i - 1] ? [`${i}${colour}`] : [],
);
const sound = execFileSync(
'ffmpeg',
['-v', 'error', '-i', mkv, '-map', '0:a', '-ac', '1', '-ar', '48000', '-f', 'f32le', '-'],
{ maxBuffer: 64 * 1024 * 1024 },
);
const samples = new Float32Array(sound.buffer, sound.byteOffset, sound.length / 4);
const rms = (from, to) => {
let sum = 0;
const a = Math.max(0, Math.round(from * 48000));
const b = Math.min(samples.length, Math.round(to * 48000));
for (let i = a; i < b; i++) sum += samples[i] * samples[i];
return b > a ? Math.sqrt(sum / (b - a)) : 0;
};
const red = Number.parseInt(file.turns.find((turn) => turn.endsWith('R')) ?? '-1', 10);
const green = Number.parseInt(file.turns.find((turn) => turn.endsWith('G')) ?? '-1', 10);
if (red >= 0 && green > red) {
file.toneBefore = red > 6 ? rms(0, (red - 3) / 60) : 0;
file.toneDuring = rms((red + 3) / 60, (green - 3) / 60);
// The blip is loud over the tone of the video: the first 2 ms that are.
for (let at = red / 60; at < (green + 30) / 60; at += 0.002)
if (rms(at, at + 0.002) > 3 * file.toneDuring + 0.05) {
file.blipAt = Math.round((at - green / 60) * 1000);
break;
}
}
rmSync(dir, { recursive: true, force: true });
}
file.turns = colours.flatMap((colour, i) =>
i === 0 || colour !== colours[i - 1] ? [`${i}${colour}`] : [],
);
const sound = execFileSync(
'ffmpeg',
['-v', 'error', '-i', mkv, '-map', '0:a', '-ac', '1', '-ar', '48000', '-f', 'f32le', '-'],
{ maxBuffer: 64 * 1024 * 1024 },
);
const samples = new Float32Array(sound.buffer, sound.byteOffset, sound.length / 4);
const rms = (from, to) => {
let sum = 0;
const a = Math.max(0, Math.round(from * 48000));
const b = Math.min(samples.length, Math.round(to * 48000));
for (let i = a; i < b; i++) sum += samples[i] * samples[i];
return b > a ? Math.sqrt(sum / (b - a)) : 0;
};
const red = Number.parseInt(file.turns.find((turn) => turn.endsWith('R')) ?? '-1', 10);
const green = Number.parseInt(file.turns.find((turn) => turn.endsWith('G')) ?? '-1', 10);
if (red >= 0 && green > red) {
file.toneBefore = red > 6 ? rms(0, (red - 3) / 60) : 0;
file.toneDuring = rms((red + 3) / 60, (green - 3) / 60);
// The blip is loud over the tone of the video: the first 2 ms that are.
for (let at = red / 60; at < (green + 30) / 60; at += 0.002)
if (rms(at, at + 0.002) > 3 * file.toneDuring + 0.05) {
file.blipAt = Math.round((at - green / 60) * 1000);
break;
}
}
rmSync(dir, { recursive: true, force: true });
}
// The video's four colours one after the other, whatever lay behind it before and after.
const colourOrder = file.turns.map((turn) => turn.at(-1)).join('');
const turned = file.turns
.slice(colourOrder.indexOf('RGBW'), colourOrder.indexOf('RGBW') + 4)
.map((turn) => Number.parseInt(turn, 10));
// The video's four colours one after the other, whatever lay behind it before and after.
const colourOrder = file.turns.map((turn) => turn.at(-1)).join('');
const turned = file.turns
.slice(colourOrder.indexOf('RGBW'), colourOrder.indexOf('RGBW') + 4)
.map((turn) => Number.parseInt(turn, 10));
return { ...file, ok: original.ok, status: original.status, colourOrder, turned };
};
const file = await readRecording(steadyUpload);
const original = { ok: file.ok, status: file.status };
const { colourOrder, turned } = file;
check(
'a recording not in real time has every frame, 1/60 s apart, though the editor drew about 20 a second',
original.ok &&
@@ -630,6 +641,108 @@ export async function composition(t) {
}),
);
// 68k. Behind another tab Chrome draws no frames, and the run stood still and the recording with
// it. While it records, a worker's beat ticks the page instead: not in real time the file has
// every frame, the video's picture at each and the blip in step; in real time the run goes on
// at its pace and the recording with it.
const tabs = 'http://127.0.0.1:9333/json';
const ownTab = (await t.cdp.send('Target.getTargetInfo')).targetInfo.targetId;
const recordBehind = async (name, realTime) => {
await mutate(`
const g = __gm.state.project.assets.find((a) => a.id === 'cp68-director').graph;
Object.assign(g.nodes.find((n) => n.id === 'cp68-record').params, { name: ${JSON.stringify(name)}, realTime: ${realTime} });`);
const before = await json('__gm.recording()');
const take = `__gm.recording().takes[${before.takes.length}]`;
await evaluate(`(() => {
window.__cp68frames = 0;
window.__cp68count = true;
const count = () => {
if (!window.__cp68count) return;
window.__cp68frames++;
requestAnimationFrame(count);
};
requestAnimationFrame(count);
})()`);
await play();
const other = await (await fetch(`${tabs}/new?about:blank`, { method: 'PUT' })).json();
const seen = { hidden: false, stillHidden: null, framesBehind: null, error: '' };
try {
await fetch(`${tabs}/activate/${other.id}`);
await waitUntil('document.hidden', { label: 'the tab went behind another' });
seen.hidden = true;
const framesAt = await json('window.__cp68frames');
await waitUntil(`${take}?.state === 'done' || ${take}?.state === 'failed'`, {
timeout: 60_000,
label: `"${name}" is saved behind another tab`,
});
seen.framesBehind = (await json('window.__cp68frames')) - framesAt;
seen.stillHidden = await json('document.hidden');
} catch (error) {
seen.error = String(error?.message ?? error);
} finally {
await fetch(`${tabs}/activate/${ownTab}`);
await fetch(`${tabs}/close/${other.id}`);
}
await waitUntil('!document.hidden', { label: 'the tab is in front again' });
await evaluate('window.__cp68count = false; 0');
await stop();
const made = await json(take);
if (made?.state !== 'done') return { made, seen, beats: 0, file: null };
await waitUntil(
`__gm.recording().uploads.length > ${before.uploads.length} && ['kept', 'failed'].includes(__gm.recording().uploads.at(-1).state)`,
{ timeout: 30_000, label: 'the recording is uploaded' },
);
const after = await json('__gm.recording()');
return { made, seen, beats: after.beats - before.beats, file: await readRecording(after.uploads.at(-1)) };
};
const behind = await recordBehind('cp68 behind', false);
const behindFile = behind.file;
check(
'behind another tab, where the page draws no frame, a recording not in real time still has every frame',
behind.seen.hidden &&
behind.seen.stillHidden === true &&
behind.seen.framesBehind <= 2 &&
behind.beats > 100 &&
behind.made?.failure === null &&
behindFile?.ok &&
behindFile.frames >= 120 &&
behindFile.gaps.length === 0,
JSON.stringify({
seen: behind.seen,
beats: behind.beats,
made: behind.made,
frames: behindFile?.frames,
gaps: behindFile?.gaps.slice(0, 5),
}),
);
check(
'and its picture and sound are those in front: colours every 30 frames, the blip as green begins',
Boolean(behindFile) &&
behindFile.colourOrder.includes('RGBW') &&
behindFile.turned.slice(1).every((at, i) => at - behindFile.turned[i] === 30) &&
behindFile.blipAt !== null &&
Math.abs(behindFile.blipAt) <= 20,
JSON.stringify({
turns: behindFile?.turns,
blipAtMs: behindFile?.blipAt,
toneDuring: behindFile?.toneDuring,
}),
);
const live = await recordBehind('cp68 behind live', true);
check(
'and in real time the run goes on behind another tab, and the recording with it',
live.seen.hidden &&
live.seen.stillHidden === true &&
live.seen.framesBehind <= 2 &&
live.beats > 60 &&
live.made?.failure === null &&
live.file?.ok &&
// Half its frames at least: in front of the others it has about 60 % here, the encoder in WSL
// being slow; standing still behind them it had none.
live.file.frames >= live.made.durationMs * 0.03,
JSON.stringify({ seen: live.seen, beats: live.beats, made: live.made, frames: live.file?.frames }),
);
await mutate(`
const p = __gm.state.project;
p.assets = p.assets.filter((a) => !a.id.startsWith('cp68-') && a.id !== ${JSON.stringify(madeId)});