Keep a recording going behind another browser tab
Chrome stops calling requestAnimationFrame for a hidden tab, and Pixi's ticker hangs on it. The run, the drawing and the recording all stood still as soon as the user switched to another tab. A plain run keeps standing still there, so a forgotten game does not play on unheard. - backgroundBeat.ts ticks the editor from a worker at the recording's fps while a recording goes and the tab is hidden, or the page has not drawn a frame of its own for 250 ms. Once the page draws again, the beat keeps out of the way. - The worker looks every 4 ms whether a tick is due. A 16 ms interval came in bursts behind a busy page, and the recording threw away the second tick of each burst. - backgroundTimers.ts and timerWorker.ts give the short waits of a recording timers in a worker. Chrome slows the page's own timers to once a second in a hidden tab. This covers the waits while saving and the wait for a video's picture after a seek. - Behind a tab Chrome does not call requestVideoFrameCallback, but the picture is there at seeked. The wait there is 10 ms instead of 150 ms while the tab is hidden. - The beat runs for Start recording, the exported game and Start recording... in the main menu. Tests: - tsc, eslint, Prettier clean - tests/viewport: 374 green, 3 new in background-beat.test.ts - e2e 68k (3 checks) on melektaus-atx with the tab really hidden: not in real time all 120 frames, colour changes every 30 frames, the blip 8 ms from green; in real time 97 of 120 frames (73 with the tab in front) - Counter-check without the beat: all three 68k checks red, 3 frames in 60 s not in real time, 7 frames in real time - Gate on melektaus-atx: green, 311 files, 4821 tests Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
87b2ee9402
commit
abb8fa51d8
10 files changed
+474
-94
No files matched your search
@@ -47,7 +47,7 @@ npm run build # writes dist/ and regenerates index.html; the folder is wh
|
||||
The browser suite and the three example games need a machine with a GPU:
|
||||
|
||||
```bash
|
||||
node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2083 checks (GM_STEPS=20 for one step)
|
||||
node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2086 checks (GM_STEPS=20 for one step)
|
||||
# every tool signs in first – see the Accounts page
|
||||
node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs
|
||||
node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
Take,
|
||||
type TakeInfo,
|
||||
} from '../viewport/RecordingTake';
|
||||
import { BackgroundBeat } from '../viewport/backgroundBeat';
|
||||
|
||||
/** What the editor recorder needs of the editor. */
|
||||
export interface EditorRecorderHost {
|
||||
@@ -44,6 +45,8 @@ export interface EditorRecorderHost {
|
||||
export class EditorRecorder {
|
||||
private take: Take | null = null;
|
||||
private source: CanvasSource | null = null;
|
||||
/** Ticks the editor on while it is behind another tab and records. */
|
||||
private beat: BackgroundBeat | null = null;
|
||||
/** Every recording of this page, newest last (test hook). */
|
||||
private readonly takes: Take[] = [];
|
||||
private seen = -1;
|
||||
@@ -96,6 +99,8 @@ export class EditorRecorder {
|
||||
this.host.showBadge(true);
|
||||
// After the render loop (`UPDATE_PRIORITY.LOW`): the canvas holds this tick's picture then.
|
||||
app.ticker.add(this.tick, this, UPDATE_PRIORITY.UTILITY);
|
||||
this.beat ??= new BackgroundBeat(app.ticker);
|
||||
this.beat.start(settings.fps);
|
||||
requestRender();
|
||||
}
|
||||
|
||||
@@ -112,6 +117,7 @@ export class EditorRecorder {
|
||||
this.source = null;
|
||||
const app = ui().app;
|
||||
app.ticker.remove(this.tick, this);
|
||||
this.beat?.stop();
|
||||
if (this.screenResolution !== null) {
|
||||
app.renderer.resize(Math.round(app.screen.width), Math.round(app.screen.height), this.screenResolution);
|
||||
this.screenResolution = null;
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
} from '../core/recording/recording';
|
||||
import type { AudioSwitch } from '../runtime/audioSwitch';
|
||||
import { ui } from '../ui';
|
||||
import { BackgroundBeat } from './backgroundBeat';
|
||||
import { PictureReader } from './PictureReader';
|
||||
import { type FrameSource, type MadeRecording, Take, type TakeHost, type TakeInfo } from './RecordingTake';
|
||||
import { type HeardVideo, SteadySound } from './SteadySound';
|
||||
@@ -46,6 +47,8 @@ export class GameRecorder implements RecordingSink {
|
||||
private paused = false;
|
||||
/** Every recording of this page, newest last (test hook). */
|
||||
private readonly takes: Take[] = [];
|
||||
/** Ticks the page on while it is behind another tab and records. */
|
||||
private beat: BackgroundBeat | null = null;
|
||||
|
||||
constructor(private readonly host: RecordingHost) {}
|
||||
|
||||
@@ -86,6 +89,7 @@ export class GameRecorder implements RecordingSink {
|
||||
host,
|
||||
() => {
|
||||
if (this.take === take) this.take = null;
|
||||
if (!this.take) this.beat?.stop();
|
||||
if (!steady) return;
|
||||
videoLibrary.setStepped(false);
|
||||
audio?.restore();
|
||||
@@ -97,6 +101,8 @@ export class GameRecorder implements RecordingSink {
|
||||
if (this.paused) take.setPaused(true);
|
||||
if (steady) videoLibrary.setStepped(true);
|
||||
if (audio && sound) audio.divert(sound.library);
|
||||
this.beat ??= new BackgroundBeat(ui().app.ticker);
|
||||
this.beat.start(settings.fps);
|
||||
return '';
|
||||
}
|
||||
|
||||
@@ -142,8 +148,13 @@ export class GameRecorder implements RecordingSink {
|
||||
this.stop();
|
||||
}
|
||||
|
||||
info(): { recording: boolean; takes: TakeInfo[] } {
|
||||
return { recording: this.isRecording, takes: this.takes.map((take) => take.info()) };
|
||||
/** `beats`: the ticks given behind another tab, of every recording so far. */
|
||||
info(): { recording: boolean; takes: TakeInfo[]; beats: number } {
|
||||
return {
|
||||
recording: this.isRecording,
|
||||
takes: this.takes.map((take) => take.info()),
|
||||
beats: this.beat?.ticks ?? 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
videoBitrate,
|
||||
vp9Codec,
|
||||
} from '../core/recording/recording';
|
||||
import { after, pause } from './backgroundTimers';
|
||||
import { soundLibrary } from './SoundLibrary';
|
||||
import type { HeardVideo, SteadySound } from './SteadySound';
|
||||
|
||||
@@ -623,7 +624,7 @@ export class Take {
|
||||
for (;;) {
|
||||
this.source.collect((frame) => this.encode(frame));
|
||||
if (!this.source.pending || performance.now() > until) return;
|
||||
await new Promise((resolve) => setTimeout(resolve, 4));
|
||||
await pause(4);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -640,15 +641,16 @@ export class Take {
|
||||
this.soundLimit = Math.round(pace.seconds * tap.rate);
|
||||
sound.renderTo(pace.seconds);
|
||||
const until = performance.now() + LAST_SOUND_MS;
|
||||
while (sound.made < pace.seconds && performance.now() < until)
|
||||
await new Promise((resolve) => setTimeout(resolve, 4));
|
||||
while (sound.made < pace.seconds && performance.now() < until) await pause(4);
|
||||
}
|
||||
return new Promise<void>((resolve) => {
|
||||
const timer = setTimeout(done, LAST_SOUND_MS);
|
||||
function done(): void {
|
||||
clearTimeout(timer);
|
||||
let waits = true;
|
||||
const done = (): void => {
|
||||
if (!waits) return;
|
||||
waits = false;
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
after(LAST_SOUND_MS, done);
|
||||
this.soundEnded = done;
|
||||
tap.node.port.postMessage('stop');
|
||||
}).finally(() => {
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
import { type Ticker, UPDATE_PRIORITY } from 'pixi.js';
|
||||
import { every } from './backgroundTimers';
|
||||
|
||||
/**
|
||||
* How long the page may go without a frame of its own before the beat takes over. A window behind
|
||||
* others stops the frames too, and need not say the page is hidden.
|
||||
*/
|
||||
export const STALL_MS = 250;
|
||||
/**
|
||||
* How often the beat looks whether a tick is due. A worker's interval of a whole frame came in
|
||||
* bursts behind a busy page, two ticks at once and then none, and a recording in real time missed
|
||||
* the frames between.
|
||||
*/
|
||||
const POLL_MS = 4;
|
||||
|
||||
/** Whether the beat ticks now: the page is hidden, or has drawn no frame of its own for a while. */
|
||||
export function beatDue(hidden: boolean, sinceOwnFrame: number): boolean {
|
||||
return hidden || sinceOwnFrame > STALL_MS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Keeps a ticker going while the page draws no frames of its own. Behind another tab Chrome stops
|
||||
* `requestAnimationFrame`, and with it the run, the drawing and a recording of them; a worker's beat
|
||||
* ticks it instead, at the recording's rate. Once the page draws again, the beat stands aside.
|
||||
*/
|
||||
export class BackgroundBeat {
|
||||
private stopBeat: (() => void) | null = null;
|
||||
private ownFrameAt = 0;
|
||||
private period = 0;
|
||||
private nextAt = 0;
|
||||
private driving = false;
|
||||
/** The ticks the beat gave (test hook). */
|
||||
ticks = 0;
|
||||
|
||||
constructor(private readonly ticker: Ticker) {}
|
||||
|
||||
private readonly seen = (): void => {
|
||||
if (!this.driving) this.ownFrameAt = performance.now();
|
||||
};
|
||||
|
||||
start(fps: number): void {
|
||||
this.stop();
|
||||
this.ownFrameAt = performance.now();
|
||||
this.period = 1000 / fps;
|
||||
this.ticker.add(this.seen, undefined, UPDATE_PRIORITY.UTILITY);
|
||||
this.stopBeat = every(POLL_MS, () => this.beat());
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
if (!this.stopBeat) return;
|
||||
this.stopBeat();
|
||||
this.stopBeat = null;
|
||||
this.ticker.remove(this.seen);
|
||||
}
|
||||
|
||||
private beat(): void {
|
||||
const now = performance.now();
|
||||
if (!beatDue(document.hidden, now - this.ownFrameAt)) {
|
||||
this.nextAt = now;
|
||||
return;
|
||||
}
|
||||
if (now < this.nextAt) return;
|
||||
// A frame on from the last; a page that fell behind starts afresh rather than catching up at once.
|
||||
this.nextAt = Math.max(this.nextAt + this.period, now + this.period / 2);
|
||||
this.driving = true;
|
||||
try {
|
||||
this.ticker.update(now);
|
||||
} finally {
|
||||
this.driving = false;
|
||||
}
|
||||
this.ticks += 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import TimerWorker from './timerWorker.ts?worker';
|
||||
|
||||
/**
|
||||
* Timers for what has to go on behind another tab, a recording above all. Chrome holds a hidden
|
||||
* page's own timers to once a second, and a chain of them to once a minute after five; a worker's
|
||||
* keep their pace (#332 found it for the music, see `wake.ts`). Where there are no workers, the
|
||||
* page's own.
|
||||
*/
|
||||
let worker: Worker | null = null;
|
||||
const waiting = new Map<number, () => void>();
|
||||
let lastId = 0;
|
||||
|
||||
function timerWorker(): Worker | null {
|
||||
if (typeof Worker === 'undefined') return null;
|
||||
if (!worker) {
|
||||
worker = new TimerWorker();
|
||||
worker.onmessage = (event: MessageEvent<number>) => waiting.get(event.data)?.();
|
||||
}
|
||||
return worker;
|
||||
}
|
||||
|
||||
/** Calls `fn` every `ms` until the function it returns is called. */
|
||||
export function every(ms: number, fn: () => void): () => void {
|
||||
const timers = timerWorker();
|
||||
if (!timers) {
|
||||
const timer = setInterval(fn, ms);
|
||||
return () => clearInterval(timer);
|
||||
}
|
||||
const id = ++lastId;
|
||||
waiting.set(id, fn);
|
||||
timers.postMessage({ id, ms, repeat: true });
|
||||
return () => {
|
||||
if (!waiting.delete(id)) return;
|
||||
timers.postMessage({ id, ms: 0, repeat: false });
|
||||
};
|
||||
}
|
||||
|
||||
/** Calls `fn` once, `ms` from now. */
|
||||
export function after(ms: number, fn: () => void): void {
|
||||
const timers = timerWorker();
|
||||
if (!timers) {
|
||||
setTimeout(fn, ms);
|
||||
return;
|
||||
}
|
||||
const id = ++lastId;
|
||||
waiting.set(id, () => {
|
||||
waiting.delete(id);
|
||||
fn();
|
||||
});
|
||||
timers.postMessage({ id, ms: Math.max(1, ms), repeat: false });
|
||||
}
|
||||
|
||||
/** Resolves `ms` from now. */
|
||||
export function pause(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => after(ms, resolve));
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
/**
|
||||
* The timers of `backgroundTimers.ts`: a worker's keep their pace in a hidden tab, where the page's
|
||||
* own are held to once a second. `{ id, ms, repeat }` starts one, `{ id, ms: 0 }` ends it; each time
|
||||
* it fires, its id comes back.
|
||||
*/
|
||||
const timers = new Map<number, ReturnType<typeof setTimeout>>();
|
||||
|
||||
self.onmessage = (event: MessageEvent<{ id: number; ms: number; repeat: boolean }>) => {
|
||||
const { id, ms, repeat } = event.data;
|
||||
const running = timers.get(id);
|
||||
if (running !== undefined) {
|
||||
clearTimeout(running);
|
||||
clearInterval(running);
|
||||
timers.delete(id);
|
||||
}
|
||||
if (ms <= 0) return;
|
||||
if (repeat)
|
||||
timers.set(
|
||||
id,
|
||||
setInterval(() => self.postMessage(id), ms),
|
||||
);
|
||||
else
|
||||
timers.set(
|
||||
id,
|
||||
setTimeout(() => {
|
||||
timers.delete(id);
|
||||
self.postMessage(id);
|
||||
}, ms),
|
||||
);
|
||||
};
|
||||
@@ -1,3 +1,5 @@
|
||||
import { after } from './backgroundTimers';
|
||||
|
||||
/**
|
||||
* Seeking a `<video>` that is asked for a new moment faster than it can decode one, as while
|
||||
* scrubbing. Every seek set while one still runs aborts it, and a 1080p video moved by the mouse
|
||||
@@ -9,6 +11,8 @@
|
||||
const SEEK_SLACK = 1 / 120;
|
||||
/** How long after `seeked` a seek counts as landed when Chrome hands over no frame for it. */
|
||||
const LAND_MS = 150;
|
||||
/** The same behind another tab, where Chrome hands over no frame at all and the picture is there at `seeked`. */
|
||||
const HIDDEN_LAND_MS = 10;
|
||||
|
||||
/** One element's seeks, see `seekVideo` and `chaseSeeks`. */
|
||||
export interface SeekChase {
|
||||
@@ -76,6 +80,8 @@ export function chaseSeeks(
|
||||
};
|
||||
if (early !== null) return land(early);
|
||||
element.requestVideoFrameCallback((_now, frame) => land(frame.mediaTime));
|
||||
setTimeout(() => land(null), LAND_MS);
|
||||
// A worker's timer: behind another tab Chrome holds the page's own to once a second, and a
|
||||
// recording not in real time waits for every landing.
|
||||
after(typeof document !== 'undefined' && document.hidden ? HIDDEN_LAND_MS : LAND_MS, () => land(null));
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
import type { Ticker } from 'pixi.js';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { BackgroundBeat, beatDue, STALL_MS } from '../../src/viewport/backgroundBeat';
|
||||
|
||||
const page = { hidden: false };
|
||||
|
||||
/** A ticker that runs its listeners on `update`, as Pixi's does; `frame` is one the page drew itself. */
|
||||
function fakeTicker() {
|
||||
const listeners = new Set<() => void>();
|
||||
const ticker = {
|
||||
updates: 0,
|
||||
add: (fn: () => void) => listeners.add(fn),
|
||||
remove: (fn: () => void) => listeners.delete(fn),
|
||||
update: () => {
|
||||
ticker.updates += 1;
|
||||
for (const fn of listeners) fn();
|
||||
},
|
||||
frame: () => {
|
||||
for (const fn of listeners) fn();
|
||||
},
|
||||
};
|
||||
return ticker;
|
||||
}
|
||||
|
||||
describe('beatDue', () => {
|
||||
it('ticks a hidden page, and a page that has drawn no frame of its own for a while', () => {
|
||||
expect(beatDue(true, 0)).toBe(true);
|
||||
expect(beatDue(false, 16)).toBe(false);
|
||||
expect(beatDue(false, STALL_MS)).toBe(false);
|
||||
expect(beatDue(false, STALL_MS + 1)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('BackgroundBeat', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval', 'performance'] });
|
||||
page.hidden = false;
|
||||
vi.stubGlobal('document', {
|
||||
get hidden() {
|
||||
return page.hidden;
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it('stands aside while the page draws, and ticks at the rate asked for once it is hidden', () => {
|
||||
const ticker = fakeTicker();
|
||||
const beat = new BackgroundBeat(ticker as unknown as Ticker);
|
||||
beat.start(60);
|
||||
for (let i = 0; i < 30; i++) {
|
||||
vi.advanceTimersByTime(16);
|
||||
ticker.frame();
|
||||
}
|
||||
expect(ticker.updates).toBe(0);
|
||||
page.hidden = true;
|
||||
vi.advanceTimersByTime(1600);
|
||||
expect(ticker.updates).toBe(96);
|
||||
expect(beat.ticks).toBe(96);
|
||||
beat.stop();
|
||||
vi.advanceTimersByTime(1600);
|
||||
expect(ticker.updates).toBe(96);
|
||||
});
|
||||
|
||||
it('takes over a page that stopped drawing without saying it is hidden, until it draws again', () => {
|
||||
const ticker = fakeTicker();
|
||||
const beat = new BackgroundBeat(ticker as unknown as Ticker);
|
||||
beat.start(30);
|
||||
vi.advanceTimersByTime(STALL_MS - 10);
|
||||
expect(ticker.updates).toBe(0);
|
||||
// Its own ticks do not count as the page's frames: it goes on at every beat.
|
||||
vi.advanceTimersByTime(1000);
|
||||
expect(ticker.updates).toBe(30);
|
||||
const given = ticker.updates;
|
||||
ticker.frame();
|
||||
vi.advanceTimersByTime(200);
|
||||
expect(ticker.updates).toBe(given);
|
||||
beat.stop();
|
||||
});
|
||||
});
|
||||
+196
-83
@@ -15,7 +15,8 @@ import { MAKE_OBJECT } from './lib/scaffold.mjs';
|
||||
* and then the video plays at once. A recording that runs meanwhile takes no frame. When the item
|
||||
* ends, its video stops and goes quiet with it. A sound item is planned once the clock runs, timed
|
||||
* to the cut. Recorded not in real time from a slowed editor, the file still has every frame, the
|
||||
* video's picture at each, and its sound and the sound item's in step.
|
||||
* video's picture at each, and its sound and the sound item's in step. Behind another tab, where the
|
||||
* page draws no frames, a recording goes on all the same, not in real time and in real time.
|
||||
*
|
||||
* Everything it adds carries a `cp68` id and goes again at the end.
|
||||
*/
|
||||
@@ -509,92 +510,102 @@ export async function composition(t) {
|
||||
const steadyLines = (await json('__gm.console.lines()'))
|
||||
.map((line) => line.text)
|
||||
.filter((text) => text.includes('"cp68 steady"'));
|
||||
const original = await fetch(
|
||||
`${t.baseUrl}data/videos/${steadyUpload.file.replace(/\.mp4$/, '.source.mkv')}`,
|
||||
);
|
||||
const file = { frames: 0, gaps: [], turns: [], toneBefore: null, toneDuring: null, blipAt: null };
|
||||
if (original.ok) {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'cp68-'));
|
||||
const mkv = join(dir, 'steady.mkv');
|
||||
writeFileSync(mkv, new Uint8Array(await original.arrayBuffer()));
|
||||
const pts = execFileSync('ffprobe', [
|
||||
'-v',
|
||||
'error',
|
||||
'-select_streams',
|
||||
'v:0',
|
||||
'-show_entries',
|
||||
'packet=pts_time',
|
||||
'-of',
|
||||
'csv=p=0',
|
||||
mkv,
|
||||
])
|
||||
.toString()
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map(Number)
|
||||
.sort((a, b) => a - b);
|
||||
file.frames = pts.length;
|
||||
file.gaps = pts.slice(1).flatMap((at, i) => (Math.abs(at - pts[i] - 1 / 60) > 0.0015 ? [i + 1] : []));
|
||||
// Each frame as 8x4 pixels: the colour of the video item covering it, or of what lies behind.
|
||||
const pixels = execFileSync(
|
||||
'ffmpeg',
|
||||
[
|
||||
/**
|
||||
* What ffmpeg reads from the original of a recording: its frames, where they are not 1/60 s
|
||||
* apart, where the video's colour turns, the video's tone and where the blip lies (from green).
|
||||
*/
|
||||
const readRecording = async (upload) => {
|
||||
const original = await fetch(`${t.baseUrl}data/videos/${upload.file.replace(/\.mp4$/, '.source.mkv')}`);
|
||||
const file = { frames: 0, gaps: [], turns: [], toneBefore: null, toneDuring: null, blipAt: null };
|
||||
if (original.ok) {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'cp68-'));
|
||||
const mkv = join(dir, 'steady.mkv');
|
||||
writeFileSync(mkv, new Uint8Array(await original.arrayBuffer()));
|
||||
const pts = execFileSync('ffprobe', [
|
||||
'-v',
|
||||
'error',
|
||||
'-i',
|
||||
'-select_streams',
|
||||
'v:0',
|
||||
'-show_entries',
|
||||
'packet=pts_time',
|
||||
'-of',
|
||||
'csv=p=0',
|
||||
mkv,
|
||||
'-map',
|
||||
'0:v',
|
||||
'-vf',
|
||||
'scale=8:4',
|
||||
'-f',
|
||||
'rawvideo',
|
||||
'-pix_fmt',
|
||||
'rgb24',
|
||||
'-',
|
||||
],
|
||||
{ maxBuffer: 64 * 1024 * 1024 },
|
||||
);
|
||||
const colours = [];
|
||||
for (let at = 0; at + 96 <= pixels.length; at += 96) {
|
||||
const [r, g, b] = [0, 1, 2].map((c) => pixels[at + 13 * 3 + c]);
|
||||
colours.push(r > 180 && g > 180 && b > 180 ? 'W' : r > 150 ? 'R' : g > 150 ? 'G' : b > 150 ? 'B' : '-');
|
||||
])
|
||||
.toString()
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map(Number)
|
||||
.sort((a, b) => a - b);
|
||||
file.frames = pts.length;
|
||||
file.gaps = pts.slice(1).flatMap((at, i) => (Math.abs(at - pts[i] - 1 / 60) > 0.0015 ? [i + 1] : []));
|
||||
// Each frame as 8x4 pixels: the colour of the video item covering it, or of what lies behind.
|
||||
const pixels = execFileSync(
|
||||
'ffmpeg',
|
||||
[
|
||||
'-v',
|
||||
'error',
|
||||
'-i',
|
||||
mkv,
|
||||
'-map',
|
||||
'0:v',
|
||||
'-vf',
|
||||
'scale=8:4',
|
||||
'-f',
|
||||
'rawvideo',
|
||||
'-pix_fmt',
|
||||
'rgb24',
|
||||
'-',
|
||||
],
|
||||
{ maxBuffer: 64 * 1024 * 1024 },
|
||||
);
|
||||
const colours = [];
|
||||
for (let at = 0; at + 96 <= pixels.length; at += 96) {
|
||||
const [r, g, b] = [0, 1, 2].map((c) => pixels[at + 13 * 3 + c]);
|
||||
colours.push(
|
||||
r > 180 && g > 180 && b > 180 ? 'W' : r > 150 ? 'R' : g > 150 ? 'G' : b > 150 ? 'B' : '-',
|
||||
);
|
||||
}
|
||||
file.turns = colours.flatMap((colour, i) =>
|
||||
i === 0 || colour !== colours[i - 1] ? [`${i}${colour}`] : [],
|
||||
);
|
||||
const sound = execFileSync(
|
||||
'ffmpeg',
|
||||
['-v', 'error', '-i', mkv, '-map', '0:a', '-ac', '1', '-ar', '48000', '-f', 'f32le', '-'],
|
||||
{ maxBuffer: 64 * 1024 * 1024 },
|
||||
);
|
||||
const samples = new Float32Array(sound.buffer, sound.byteOffset, sound.length / 4);
|
||||
const rms = (from, to) => {
|
||||
let sum = 0;
|
||||
const a = Math.max(0, Math.round(from * 48000));
|
||||
const b = Math.min(samples.length, Math.round(to * 48000));
|
||||
for (let i = a; i < b; i++) sum += samples[i] * samples[i];
|
||||
return b > a ? Math.sqrt(sum / (b - a)) : 0;
|
||||
};
|
||||
const red = Number.parseInt(file.turns.find((turn) => turn.endsWith('R')) ?? '-1', 10);
|
||||
const green = Number.parseInt(file.turns.find((turn) => turn.endsWith('G')) ?? '-1', 10);
|
||||
if (red >= 0 && green > red) {
|
||||
file.toneBefore = red > 6 ? rms(0, (red - 3) / 60) : 0;
|
||||
file.toneDuring = rms((red + 3) / 60, (green - 3) / 60);
|
||||
// The blip is loud over the tone of the video: the first 2 ms that are.
|
||||
for (let at = red / 60; at < (green + 30) / 60; at += 0.002)
|
||||
if (rms(at, at + 0.002) > 3 * file.toneDuring + 0.05) {
|
||||
file.blipAt = Math.round((at - green / 60) * 1000);
|
||||
break;
|
||||
}
|
||||
}
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
file.turns = colours.flatMap((colour, i) =>
|
||||
i === 0 || colour !== colours[i - 1] ? [`${i}${colour}`] : [],
|
||||
);
|
||||
const sound = execFileSync(
|
||||
'ffmpeg',
|
||||
['-v', 'error', '-i', mkv, '-map', '0:a', '-ac', '1', '-ar', '48000', '-f', 'f32le', '-'],
|
||||
{ maxBuffer: 64 * 1024 * 1024 },
|
||||
);
|
||||
const samples = new Float32Array(sound.buffer, sound.byteOffset, sound.length / 4);
|
||||
const rms = (from, to) => {
|
||||
let sum = 0;
|
||||
const a = Math.max(0, Math.round(from * 48000));
|
||||
const b = Math.min(samples.length, Math.round(to * 48000));
|
||||
for (let i = a; i < b; i++) sum += samples[i] * samples[i];
|
||||
return b > a ? Math.sqrt(sum / (b - a)) : 0;
|
||||
};
|
||||
const red = Number.parseInt(file.turns.find((turn) => turn.endsWith('R')) ?? '-1', 10);
|
||||
const green = Number.parseInt(file.turns.find((turn) => turn.endsWith('G')) ?? '-1', 10);
|
||||
if (red >= 0 && green > red) {
|
||||
file.toneBefore = red > 6 ? rms(0, (red - 3) / 60) : 0;
|
||||
file.toneDuring = rms((red + 3) / 60, (green - 3) / 60);
|
||||
// The blip is loud over the tone of the video: the first 2 ms that are.
|
||||
for (let at = red / 60; at < (green + 30) / 60; at += 0.002)
|
||||
if (rms(at, at + 0.002) > 3 * file.toneDuring + 0.05) {
|
||||
file.blipAt = Math.round((at - green / 60) * 1000);
|
||||
break;
|
||||
}
|
||||
}
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
// The video's four colours one after the other, whatever lay behind it before and after.
|
||||
const colourOrder = file.turns.map((turn) => turn.at(-1)).join('');
|
||||
const turned = file.turns
|
||||
.slice(colourOrder.indexOf('RGBW'), colourOrder.indexOf('RGBW') + 4)
|
||||
.map((turn) => Number.parseInt(turn, 10));
|
||||
// The video's four colours one after the other, whatever lay behind it before and after.
|
||||
const colourOrder = file.turns.map((turn) => turn.at(-1)).join('');
|
||||
const turned = file.turns
|
||||
.slice(colourOrder.indexOf('RGBW'), colourOrder.indexOf('RGBW') + 4)
|
||||
.map((turn) => Number.parseInt(turn, 10));
|
||||
return { ...file, ok: original.ok, status: original.status, colourOrder, turned };
|
||||
};
|
||||
const file = await readRecording(steadyUpload);
|
||||
const original = { ok: file.ok, status: file.status };
|
||||
const { colourOrder, turned } = file;
|
||||
check(
|
||||
'a recording not in real time has every frame, 1/60 s apart, though the editor drew about 20 a second',
|
||||
original.ok &&
|
||||
@@ -630,6 +641,108 @@ export async function composition(t) {
|
||||
}),
|
||||
);
|
||||
|
||||
// 68k. Behind another tab Chrome draws no frames, and the run stood still and the recording with
|
||||
// it. While it records, a worker's beat ticks the page instead: not in real time the file has
|
||||
// every frame, the video's picture at each and the blip in step; in real time the run goes on
|
||||
// at its pace and the recording with it.
|
||||
const tabs = 'http://127.0.0.1:9333/json';
|
||||
const ownTab = (await t.cdp.send('Target.getTargetInfo')).targetInfo.targetId;
|
||||
const recordBehind = async (name, realTime) => {
|
||||
await mutate(`
|
||||
const g = __gm.state.project.assets.find((a) => a.id === 'cp68-director').graph;
|
||||
Object.assign(g.nodes.find((n) => n.id === 'cp68-record').params, { name: ${JSON.stringify(name)}, realTime: ${realTime} });`);
|
||||
const before = await json('__gm.recording()');
|
||||
const take = `__gm.recording().takes[${before.takes.length}]`;
|
||||
await evaluate(`(() => {
|
||||
window.__cp68frames = 0;
|
||||
window.__cp68count = true;
|
||||
const count = () => {
|
||||
if (!window.__cp68count) return;
|
||||
window.__cp68frames++;
|
||||
requestAnimationFrame(count);
|
||||
};
|
||||
requestAnimationFrame(count);
|
||||
})()`);
|
||||
await play();
|
||||
const other = await (await fetch(`${tabs}/new?about:blank`, { method: 'PUT' })).json();
|
||||
const seen = { hidden: false, stillHidden: null, framesBehind: null, error: '' };
|
||||
try {
|
||||
await fetch(`${tabs}/activate/${other.id}`);
|
||||
await waitUntil('document.hidden', { label: 'the tab went behind another' });
|
||||
seen.hidden = true;
|
||||
const framesAt = await json('window.__cp68frames');
|
||||
await waitUntil(`${take}?.state === 'done' || ${take}?.state === 'failed'`, {
|
||||
timeout: 60_000,
|
||||
label: `"${name}" is saved behind another tab`,
|
||||
});
|
||||
seen.framesBehind = (await json('window.__cp68frames')) - framesAt;
|
||||
seen.stillHidden = await json('document.hidden');
|
||||
} catch (error) {
|
||||
seen.error = String(error?.message ?? error);
|
||||
} finally {
|
||||
await fetch(`${tabs}/activate/${ownTab}`);
|
||||
await fetch(`${tabs}/close/${other.id}`);
|
||||
}
|
||||
await waitUntil('!document.hidden', { label: 'the tab is in front again' });
|
||||
await evaluate('window.__cp68count = false; 0');
|
||||
await stop();
|
||||
const made = await json(take);
|
||||
if (made?.state !== 'done') return { made, seen, beats: 0, file: null };
|
||||
await waitUntil(
|
||||
`__gm.recording().uploads.length > ${before.uploads.length} && ['kept', 'failed'].includes(__gm.recording().uploads.at(-1).state)`,
|
||||
{ timeout: 30_000, label: 'the recording is uploaded' },
|
||||
);
|
||||
const after = await json('__gm.recording()');
|
||||
return { made, seen, beats: after.beats - before.beats, file: await readRecording(after.uploads.at(-1)) };
|
||||
};
|
||||
const behind = await recordBehind('cp68 behind', false);
|
||||
const behindFile = behind.file;
|
||||
check(
|
||||
'behind another tab, where the page draws no frame, a recording not in real time still has every frame',
|
||||
behind.seen.hidden &&
|
||||
behind.seen.stillHidden === true &&
|
||||
behind.seen.framesBehind <= 2 &&
|
||||
behind.beats > 100 &&
|
||||
behind.made?.failure === null &&
|
||||
behindFile?.ok &&
|
||||
behindFile.frames >= 120 &&
|
||||
behindFile.gaps.length === 0,
|
||||
JSON.stringify({
|
||||
seen: behind.seen,
|
||||
beats: behind.beats,
|
||||
made: behind.made,
|
||||
frames: behindFile?.frames,
|
||||
gaps: behindFile?.gaps.slice(0, 5),
|
||||
}),
|
||||
);
|
||||
check(
|
||||
'and its picture and sound are those in front: colours every 30 frames, the blip as green begins',
|
||||
Boolean(behindFile) &&
|
||||
behindFile.colourOrder.includes('RGBW') &&
|
||||
behindFile.turned.slice(1).every((at, i) => at - behindFile.turned[i] === 30) &&
|
||||
behindFile.blipAt !== null &&
|
||||
Math.abs(behindFile.blipAt) <= 20,
|
||||
JSON.stringify({
|
||||
turns: behindFile?.turns,
|
||||
blipAtMs: behindFile?.blipAt,
|
||||
toneDuring: behindFile?.toneDuring,
|
||||
}),
|
||||
);
|
||||
const live = await recordBehind('cp68 behind live', true);
|
||||
check(
|
||||
'and in real time the run goes on behind another tab, and the recording with it',
|
||||
live.seen.hidden &&
|
||||
live.seen.stillHidden === true &&
|
||||
live.seen.framesBehind <= 2 &&
|
||||
live.beats > 60 &&
|
||||
live.made?.failure === null &&
|
||||
live.file?.ok &&
|
||||
// Half its frames at least: in front of the others it has about 60 % here, the encoder in WSL
|
||||
// being slow; standing still behind them it had none.
|
||||
live.file.frames >= live.made.durationMs * 0.03,
|
||||
JSON.stringify({ seen: live.seen, beats: live.beats, made: live.made, frames: live.file?.frames }),
|
||||
);
|
||||
|
||||
await mutate(`
|
||||
const p = __gm.state.project;
|
||||
p.assets = p.assets.filter((a) => !a.id.startsWith('cp68-') && a.id !== ${JSON.stringify(madeId)});
|
||||
|
||||
Reference in new issue
Block a user