Keep a recording going behind another browser tab
Chrome stops calling requestAnimationFrame for a hidden tab, and Pixi's ticker hangs on it. The run, the drawing and the recording all stood still as soon as the user switched to another tab. A plain run keeps standing still there, so a forgotten game does not play on unheard. - backgroundBeat.ts ticks the editor from a worker at the recording's fps while a recording goes and the tab is hidden, or the page has not drawn a frame of its own for 250 ms. Once the page draws again, the beat keeps out of the way. - The worker looks every 4 ms whether a tick is due. A 16 ms interval came in bursts behind a busy page, and the recording threw away the second tick of each burst. - backgroundTimers.ts and timerWorker.ts give the short waits of a recording timers in a worker. Chrome slows the page's own timers to once a second in a hidden tab. This covers the waits while saving and the wait for a video's picture after a seek. - Behind a tab Chrome does not call requestVideoFrameCallback, but the picture is there at seeked. The wait there is 10 ms instead of 150 ms while the tab is hidden. - The beat runs for Start recording, the exported game and Start recording... in the main menu. Tests: - tsc, eslint, Prettier clean - tests/viewport: 374 green, 3 new in background-beat.test.ts - e2e 68k (3 checks) on melektaus-atx with the tab really hidden: not in real time all 120 frames, colour changes every 30 frames, the blip 8 ms from green; in real time 97 of 120 frames (73 with the tab in front) - Counter-check without the beat: all three 68k checks red, 3 frames in 60 s not in real time, 7 frames in real time - Gate on melektaus-atx: green, 311 files, 4821 tests Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
87b2ee9402
commit
abb8fa51d8
10 files changed
+474
-94
No files matched your search
@@ -47,7 +47,7 @@ npm run build # writes dist/ and regenerates index.html; the folder is wh
|
|||||||
The browser suite and the three example games need a machine with a GPU:
|
The browser suite and the three example games need a machine with a GPU:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2083 checks (GM_STEPS=20 for one step)
|
node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2086 checks (GM_STEPS=20 for one step)
|
||||||
# every tool signs in first – see the Accounts page
|
# every tool signs in first – see the Accounts page
|
||||||
node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs
|
node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs
|
||||||
node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs
|
node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ import {
|
|||||||
Take,
|
Take,
|
||||||
type TakeInfo,
|
type TakeInfo,
|
||||||
} from '../viewport/RecordingTake';
|
} from '../viewport/RecordingTake';
|
||||||
|
import { BackgroundBeat } from '../viewport/backgroundBeat';
|
||||||
|
|
||||||
/** What the editor recorder needs of the editor. */
|
/** What the editor recorder needs of the editor. */
|
||||||
export interface EditorRecorderHost {
|
export interface EditorRecorderHost {
|
||||||
@@ -44,6 +45,8 @@ export interface EditorRecorderHost {
|
|||||||
export class EditorRecorder {
|
export class EditorRecorder {
|
||||||
private take: Take | null = null;
|
private take: Take | null = null;
|
||||||
private source: CanvasSource | null = null;
|
private source: CanvasSource | null = null;
|
||||||
|
/** Ticks the editor on while it is behind another tab and records. */
|
||||||
|
private beat: BackgroundBeat | null = null;
|
||||||
/** Every recording of this page, newest last (test hook). */
|
/** Every recording of this page, newest last (test hook). */
|
||||||
private readonly takes: Take[] = [];
|
private readonly takes: Take[] = [];
|
||||||
private seen = -1;
|
private seen = -1;
|
||||||
@@ -96,6 +99,8 @@ export class EditorRecorder {
|
|||||||
this.host.showBadge(true);
|
this.host.showBadge(true);
|
||||||
// After the render loop (`UPDATE_PRIORITY.LOW`): the canvas holds this tick's picture then.
|
// After the render loop (`UPDATE_PRIORITY.LOW`): the canvas holds this tick's picture then.
|
||||||
app.ticker.add(this.tick, this, UPDATE_PRIORITY.UTILITY);
|
app.ticker.add(this.tick, this, UPDATE_PRIORITY.UTILITY);
|
||||||
|
this.beat ??= new BackgroundBeat(app.ticker);
|
||||||
|
this.beat.start(settings.fps);
|
||||||
requestRender();
|
requestRender();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -112,6 +117,7 @@ export class EditorRecorder {
|
|||||||
this.source = null;
|
this.source = null;
|
||||||
const app = ui().app;
|
const app = ui().app;
|
||||||
app.ticker.remove(this.tick, this);
|
app.ticker.remove(this.tick, this);
|
||||||
|
this.beat?.stop();
|
||||||
if (this.screenResolution !== null) {
|
if (this.screenResolution !== null) {
|
||||||
app.renderer.resize(Math.round(app.screen.width), Math.round(app.screen.height), this.screenResolution);
|
app.renderer.resize(Math.round(app.screen.width), Math.round(app.screen.height), this.screenResolution);
|
||||||
this.screenResolution = null;
|
this.screenResolution = null;
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import {
|
|||||||
} from '../core/recording/recording';
|
} from '../core/recording/recording';
|
||||||
import type { AudioSwitch } from '../runtime/audioSwitch';
|
import type { AudioSwitch } from '../runtime/audioSwitch';
|
||||||
import { ui } from '../ui';
|
import { ui } from '../ui';
|
||||||
|
import { BackgroundBeat } from './backgroundBeat';
|
||||||
import { PictureReader } from './PictureReader';
|
import { PictureReader } from './PictureReader';
|
||||||
import { type FrameSource, type MadeRecording, Take, type TakeHost, type TakeInfo } from './RecordingTake';
|
import { type FrameSource, type MadeRecording, Take, type TakeHost, type TakeInfo } from './RecordingTake';
|
||||||
import { type HeardVideo, SteadySound } from './SteadySound';
|
import { type HeardVideo, SteadySound } from './SteadySound';
|
||||||
@@ -46,6 +47,8 @@ export class GameRecorder implements RecordingSink {
|
|||||||
private paused = false;
|
private paused = false;
|
||||||
/** Every recording of this page, newest last (test hook). */
|
/** Every recording of this page, newest last (test hook). */
|
||||||
private readonly takes: Take[] = [];
|
private readonly takes: Take[] = [];
|
||||||
|
/** Ticks the page on while it is behind another tab and records. */
|
||||||
|
private beat: BackgroundBeat | null = null;
|
||||||
|
|
||||||
constructor(private readonly host: RecordingHost) {}
|
constructor(private readonly host: RecordingHost) {}
|
||||||
|
|
||||||
@@ -86,6 +89,7 @@ export class GameRecorder implements RecordingSink {
|
|||||||
host,
|
host,
|
||||||
() => {
|
() => {
|
||||||
if (this.take === take) this.take = null;
|
if (this.take === take) this.take = null;
|
||||||
|
if (!this.take) this.beat?.stop();
|
||||||
if (!steady) return;
|
if (!steady) return;
|
||||||
videoLibrary.setStepped(false);
|
videoLibrary.setStepped(false);
|
||||||
audio?.restore();
|
audio?.restore();
|
||||||
@@ -97,6 +101,8 @@ export class GameRecorder implements RecordingSink {
|
|||||||
if (this.paused) take.setPaused(true);
|
if (this.paused) take.setPaused(true);
|
||||||
if (steady) videoLibrary.setStepped(true);
|
if (steady) videoLibrary.setStepped(true);
|
||||||
if (audio && sound) audio.divert(sound.library);
|
if (audio && sound) audio.divert(sound.library);
|
||||||
|
this.beat ??= new BackgroundBeat(ui().app.ticker);
|
||||||
|
this.beat.start(settings.fps);
|
||||||
return '';
|
return '';
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -142,8 +148,13 @@ export class GameRecorder implements RecordingSink {
|
|||||||
this.stop();
|
this.stop();
|
||||||
}
|
}
|
||||||
|
|
||||||
info(): { recording: boolean; takes: TakeInfo[] } {
|
/** `beats`: the ticks given behind another tab, of every recording so far. */
|
||||||
return { recording: this.isRecording, takes: this.takes.map((take) => take.info()) };
|
info(): { recording: boolean; takes: TakeInfo[]; beats: number } {
|
||||||
|
return {
|
||||||
|
recording: this.isRecording,
|
||||||
|
takes: this.takes.map((take) => take.info()),
|
||||||
|
beats: this.beat?.ticks ?? 0,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ import {
|
|||||||
videoBitrate,
|
videoBitrate,
|
||||||
vp9Codec,
|
vp9Codec,
|
||||||
} from '../core/recording/recording';
|
} from '../core/recording/recording';
|
||||||
|
import { after, pause } from './backgroundTimers';
|
||||||
import { soundLibrary } from './SoundLibrary';
|
import { soundLibrary } from './SoundLibrary';
|
||||||
import type { HeardVideo, SteadySound } from './SteadySound';
|
import type { HeardVideo, SteadySound } from './SteadySound';
|
||||||
|
|
||||||
@@ -623,7 +624,7 @@ export class Take {
|
|||||||
for (;;) {
|
for (;;) {
|
||||||
this.source.collect((frame) => this.encode(frame));
|
this.source.collect((frame) => this.encode(frame));
|
||||||
if (!this.source.pending || performance.now() > until) return;
|
if (!this.source.pending || performance.now() > until) return;
|
||||||
await new Promise((resolve) => setTimeout(resolve, 4));
|
await pause(4);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -640,15 +641,16 @@ export class Take {
|
|||||||
this.soundLimit = Math.round(pace.seconds * tap.rate);
|
this.soundLimit = Math.round(pace.seconds * tap.rate);
|
||||||
sound.renderTo(pace.seconds);
|
sound.renderTo(pace.seconds);
|
||||||
const until = performance.now() + LAST_SOUND_MS;
|
const until = performance.now() + LAST_SOUND_MS;
|
||||||
while (sound.made < pace.seconds && performance.now() < until)
|
while (sound.made < pace.seconds && performance.now() < until) await pause(4);
|
||||||
await new Promise((resolve) => setTimeout(resolve, 4));
|
|
||||||
}
|
}
|
||||||
return new Promise<void>((resolve) => {
|
return new Promise<void>((resolve) => {
|
||||||
const timer = setTimeout(done, LAST_SOUND_MS);
|
let waits = true;
|
||||||
function done(): void {
|
const done = (): void => {
|
||||||
clearTimeout(timer);
|
if (!waits) return;
|
||||||
|
waits = false;
|
||||||
resolve();
|
resolve();
|
||||||
}
|
};
|
||||||
|
after(LAST_SOUND_MS, done);
|
||||||
this.soundEnded = done;
|
this.soundEnded = done;
|
||||||
tap.node.port.postMessage('stop');
|
tap.node.port.postMessage('stop');
|
||||||
}).finally(() => {
|
}).finally(() => {
|
||||||
|
|||||||
@@ -0,0 +1,73 @@
|
|||||||
|
import { type Ticker, UPDATE_PRIORITY } from 'pixi.js';
|
||||||
|
import { every } from './backgroundTimers';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* How long the page may go without a frame of its own before the beat takes over. A window behind
|
||||||
|
* others stops the frames too, and need not say the page is hidden.
|
||||||
|
*/
|
||||||
|
export const STALL_MS = 250;
|
||||||
|
/**
|
||||||
|
* How often the beat looks whether a tick is due. A worker's interval of a whole frame came in
|
||||||
|
* bursts behind a busy page, two ticks at once and then none, and a recording in real time missed
|
||||||
|
* the frames between.
|
||||||
|
*/
|
||||||
|
const POLL_MS = 4;
|
||||||
|
|
||||||
|
/** Whether the beat ticks now: the page is hidden, or has drawn no frame of its own for a while. */
|
||||||
|
export function beatDue(hidden: boolean, sinceOwnFrame: number): boolean {
|
||||||
|
return hidden || sinceOwnFrame > STALL_MS;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Keeps a ticker going while the page draws no frames of its own. Behind another tab Chrome stops
|
||||||
|
* `requestAnimationFrame`, and with it the run, the drawing and a recording of them; a worker's beat
|
||||||
|
* ticks it instead, at the recording's rate. Once the page draws again, the beat stands aside.
|
||||||
|
*/
|
||||||
|
export class BackgroundBeat {
|
||||||
|
private stopBeat: (() => void) | null = null;
|
||||||
|
private ownFrameAt = 0;
|
||||||
|
private period = 0;
|
||||||
|
private nextAt = 0;
|
||||||
|
private driving = false;
|
||||||
|
/** The ticks the beat gave (test hook). */
|
||||||
|
ticks = 0;
|
||||||
|
|
||||||
|
constructor(private readonly ticker: Ticker) {}
|
||||||
|
|
||||||
|
private readonly seen = (): void => {
|
||||||
|
if (!this.driving) this.ownFrameAt = performance.now();
|
||||||
|
};
|
||||||
|
|
||||||
|
start(fps: number): void {
|
||||||
|
this.stop();
|
||||||
|
this.ownFrameAt = performance.now();
|
||||||
|
this.period = 1000 / fps;
|
||||||
|
this.ticker.add(this.seen, undefined, UPDATE_PRIORITY.UTILITY);
|
||||||
|
this.stopBeat = every(POLL_MS, () => this.beat());
|
||||||
|
}
|
||||||
|
|
||||||
|
stop(): void {
|
||||||
|
if (!this.stopBeat) return;
|
||||||
|
this.stopBeat();
|
||||||
|
this.stopBeat = null;
|
||||||
|
this.ticker.remove(this.seen);
|
||||||
|
}
|
||||||
|
|
||||||
|
private beat(): void {
|
||||||
|
const now = performance.now();
|
||||||
|
if (!beatDue(document.hidden, now - this.ownFrameAt)) {
|
||||||
|
this.nextAt = now;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (now < this.nextAt) return;
|
||||||
|
// A frame on from the last; a page that fell behind starts afresh rather than catching up at once.
|
||||||
|
this.nextAt = Math.max(this.nextAt + this.period, now + this.period / 2);
|
||||||
|
this.driving = true;
|
||||||
|
try {
|
||||||
|
this.ticker.update(now);
|
||||||
|
} finally {
|
||||||
|
this.driving = false;
|
||||||
|
}
|
||||||
|
this.ticks += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
import TimerWorker from './timerWorker.ts?worker';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Timers for what has to go on behind another tab, a recording above all. Chrome holds a hidden
|
||||||
|
* page's own timers to once a second, and a chain of them to once a minute after five; a worker's
|
||||||
|
* keep their pace (#332 found it for the music, see `wake.ts`). Where there are no workers, the
|
||||||
|
* page's own.
|
||||||
|
*/
|
||||||
|
let worker: Worker | null = null;
|
||||||
|
const waiting = new Map<number, () => void>();
|
||||||
|
let lastId = 0;
|
||||||
|
|
||||||
|
function timerWorker(): Worker | null {
|
||||||
|
if (typeof Worker === 'undefined') return null;
|
||||||
|
if (!worker) {
|
||||||
|
worker = new TimerWorker();
|
||||||
|
worker.onmessage = (event: MessageEvent<number>) => waiting.get(event.data)?.();
|
||||||
|
}
|
||||||
|
return worker;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Calls `fn` every `ms` until the function it returns is called. */
|
||||||
|
export function every(ms: number, fn: () => void): () => void {
|
||||||
|
const timers = timerWorker();
|
||||||
|
if (!timers) {
|
||||||
|
const timer = setInterval(fn, ms);
|
||||||
|
return () => clearInterval(timer);
|
||||||
|
}
|
||||||
|
const id = ++lastId;
|
||||||
|
waiting.set(id, fn);
|
||||||
|
timers.postMessage({ id, ms, repeat: true });
|
||||||
|
return () => {
|
||||||
|
if (!waiting.delete(id)) return;
|
||||||
|
timers.postMessage({ id, ms: 0, repeat: false });
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Calls `fn` once, `ms` from now. */
|
||||||
|
export function after(ms: number, fn: () => void): void {
|
||||||
|
const timers = timerWorker();
|
||||||
|
if (!timers) {
|
||||||
|
setTimeout(fn, ms);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const id = ++lastId;
|
||||||
|
waiting.set(id, () => {
|
||||||
|
waiting.delete(id);
|
||||||
|
fn();
|
||||||
|
});
|
||||||
|
timers.postMessage({ id, ms: Math.max(1, ms), repeat: false });
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Resolves `ms` from now. */
|
||||||
|
export function pause(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => after(ms, resolve));
|
||||||
|
}
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
/**
|
||||||
|
* The timers of `backgroundTimers.ts`: a worker's keep their pace in a hidden tab, where the page's
|
||||||
|
* own are held to once a second. `{ id, ms, repeat }` starts one, `{ id, ms: 0 }` ends it; each time
|
||||||
|
* it fires, its id comes back.
|
||||||
|
*/
|
||||||
|
const timers = new Map<number, ReturnType<typeof setTimeout>>();
|
||||||
|
|
||||||
|
self.onmessage = (event: MessageEvent<{ id: number; ms: number; repeat: boolean }>) => {
|
||||||
|
const { id, ms, repeat } = event.data;
|
||||||
|
const running = timers.get(id);
|
||||||
|
if (running !== undefined) {
|
||||||
|
clearTimeout(running);
|
||||||
|
clearInterval(running);
|
||||||
|
timers.delete(id);
|
||||||
|
}
|
||||||
|
if (ms <= 0) return;
|
||||||
|
if (repeat)
|
||||||
|
timers.set(
|
||||||
|
id,
|
||||||
|
setInterval(() => self.postMessage(id), ms),
|
||||||
|
);
|
||||||
|
else
|
||||||
|
timers.set(
|
||||||
|
id,
|
||||||
|
setTimeout(() => {
|
||||||
|
timers.delete(id);
|
||||||
|
self.postMessage(id);
|
||||||
|
}, ms),
|
||||||
|
);
|
||||||
|
};
|
||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import { after } from './backgroundTimers';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Seeking a `<video>` that is asked for a new moment faster than it can decode one, as while
|
* Seeking a `<video>` that is asked for a new moment faster than it can decode one, as while
|
||||||
* scrubbing. Every seek set while one still runs aborts it, and a 1080p video moved by the mouse
|
* scrubbing. Every seek set while one still runs aborts it, and a 1080p video moved by the mouse
|
||||||
@@ -9,6 +11,8 @@
|
|||||||
const SEEK_SLACK = 1 / 120;
|
const SEEK_SLACK = 1 / 120;
|
||||||
/** How long after `seeked` a seek counts as landed when Chrome hands over no frame for it. */
|
/** How long after `seeked` a seek counts as landed when Chrome hands over no frame for it. */
|
||||||
const LAND_MS = 150;
|
const LAND_MS = 150;
|
||||||
|
/** The same behind another tab, where Chrome hands over no frame at all and the picture is there at `seeked`. */
|
||||||
|
const HIDDEN_LAND_MS = 10;
|
||||||
|
|
||||||
/** One element's seeks, see `seekVideo` and `chaseSeeks`. */
|
/** One element's seeks, see `seekVideo` and `chaseSeeks`. */
|
||||||
export interface SeekChase {
|
export interface SeekChase {
|
||||||
@@ -76,6 +80,8 @@ export function chaseSeeks(
|
|||||||
};
|
};
|
||||||
if (early !== null) return land(early);
|
if (early !== null) return land(early);
|
||||||
element.requestVideoFrameCallback((_now, frame) => land(frame.mediaTime));
|
element.requestVideoFrameCallback((_now, frame) => land(frame.mediaTime));
|
||||||
setTimeout(() => land(null), LAND_MS);
|
// A worker's timer: behind another tab Chrome holds the page's own to once a second, and a
|
||||||
|
// recording not in real time waits for every landing.
|
||||||
|
after(typeof document !== 'undefined' && document.hidden ? HIDDEN_LAND_MS : LAND_MS, () => land(null));
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
import type { Ticker } from 'pixi.js';
|
||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
import { BackgroundBeat, beatDue, STALL_MS } from '../../src/viewport/backgroundBeat';
|
||||||
|
|
||||||
|
const page = { hidden: false };
|
||||||
|
|
||||||
|
/** A ticker that runs its listeners on `update`, as Pixi's does; `frame` is one the page drew itself. */
|
||||||
|
function fakeTicker() {
|
||||||
|
const listeners = new Set<() => void>();
|
||||||
|
const ticker = {
|
||||||
|
updates: 0,
|
||||||
|
add: (fn: () => void) => listeners.add(fn),
|
||||||
|
remove: (fn: () => void) => listeners.delete(fn),
|
||||||
|
update: () => {
|
||||||
|
ticker.updates += 1;
|
||||||
|
for (const fn of listeners) fn();
|
||||||
|
},
|
||||||
|
frame: () => {
|
||||||
|
for (const fn of listeners) fn();
|
||||||
|
},
|
||||||
|
};
|
||||||
|
return ticker;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('beatDue', () => {
|
||||||
|
it('ticks a hidden page, and a page that has drawn no frame of its own for a while', () => {
|
||||||
|
expect(beatDue(true, 0)).toBe(true);
|
||||||
|
expect(beatDue(false, 16)).toBe(false);
|
||||||
|
expect(beatDue(false, STALL_MS)).toBe(false);
|
||||||
|
expect(beatDue(false, STALL_MS + 1)).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('BackgroundBeat', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval', 'performance'] });
|
||||||
|
page.hidden = false;
|
||||||
|
vi.stubGlobal('document', {
|
||||||
|
get hidden() {
|
||||||
|
return page.hidden;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.useRealTimers();
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('stands aside while the page draws, and ticks at the rate asked for once it is hidden', () => {
|
||||||
|
const ticker = fakeTicker();
|
||||||
|
const beat = new BackgroundBeat(ticker as unknown as Ticker);
|
||||||
|
beat.start(60);
|
||||||
|
for (let i = 0; i < 30; i++) {
|
||||||
|
vi.advanceTimersByTime(16);
|
||||||
|
ticker.frame();
|
||||||
|
}
|
||||||
|
expect(ticker.updates).toBe(0);
|
||||||
|
page.hidden = true;
|
||||||
|
vi.advanceTimersByTime(1600);
|
||||||
|
expect(ticker.updates).toBe(96);
|
||||||
|
expect(beat.ticks).toBe(96);
|
||||||
|
beat.stop();
|
||||||
|
vi.advanceTimersByTime(1600);
|
||||||
|
expect(ticker.updates).toBe(96);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('takes over a page that stopped drawing without saying it is hidden, until it draws again', () => {
|
||||||
|
const ticker = fakeTicker();
|
||||||
|
const beat = new BackgroundBeat(ticker as unknown as Ticker);
|
||||||
|
beat.start(30);
|
||||||
|
vi.advanceTimersByTime(STALL_MS - 10);
|
||||||
|
expect(ticker.updates).toBe(0);
|
||||||
|
// Its own ticks do not count as the page's frames: it goes on at every beat.
|
||||||
|
vi.advanceTimersByTime(1000);
|
||||||
|
expect(ticker.updates).toBe(30);
|
||||||
|
const given = ticker.updates;
|
||||||
|
ticker.frame();
|
||||||
|
vi.advanceTimersByTime(200);
|
||||||
|
expect(ticker.updates).toBe(given);
|
||||||
|
beat.stop();
|
||||||
|
});
|
||||||
|
});
|
||||||
+196
-83
@@ -15,7 +15,8 @@ import { MAKE_OBJECT } from './lib/scaffold.mjs';
|
|||||||
* and then the video plays at once. A recording that runs meanwhile takes no frame. When the item
|
* and then the video plays at once. A recording that runs meanwhile takes no frame. When the item
|
||||||
* ends, its video stops and goes quiet with it. A sound item is planned once the clock runs, timed
|
* ends, its video stops and goes quiet with it. A sound item is planned once the clock runs, timed
|
||||||
* to the cut. Recorded not in real time from a slowed editor, the file still has every frame, the
|
* to the cut. Recorded not in real time from a slowed editor, the file still has every frame, the
|
||||||
* video's picture at each, and its sound and the sound item's in step.
|
* video's picture at each, and its sound and the sound item's in step. Behind another tab, where the
|
||||||
|
* page draws no frames, a recording goes on all the same, not in real time and in real time.
|
||||||
*
|
*
|
||||||
* Everything it adds carries a `cp68` id and goes again at the end.
|
* Everything it adds carries a `cp68` id and goes again at the end.
|
||||||
*/
|
*/
|
||||||
@@ -509,92 +510,102 @@ export async function composition(t) {
|
|||||||
const steadyLines = (await json('__gm.console.lines()'))
|
const steadyLines = (await json('__gm.console.lines()'))
|
||||||
.map((line) => line.text)
|
.map((line) => line.text)
|
||||||
.filter((text) => text.includes('"cp68 steady"'));
|
.filter((text) => text.includes('"cp68 steady"'));
|
||||||
const original = await fetch(
|
/**
|
||||||
`${t.baseUrl}data/videos/${steadyUpload.file.replace(/\.mp4$/, '.source.mkv')}`,
|
* What ffmpeg reads from the original of a recording: its frames, where they are not 1/60 s
|
||||||
);
|
* apart, where the video's colour turns, the video's tone and where the blip lies (from green).
|
||||||
const file = { frames: 0, gaps: [], turns: [], toneBefore: null, toneDuring: null, blipAt: null };
|
*/
|
||||||
if (original.ok) {
|
const readRecording = async (upload) => {
|
||||||
const dir = mkdtempSync(join(tmpdir(), 'cp68-'));
|
const original = await fetch(`${t.baseUrl}data/videos/${upload.file.replace(/\.mp4$/, '.source.mkv')}`);
|
||||||
const mkv = join(dir, 'steady.mkv');
|
const file = { frames: 0, gaps: [], turns: [], toneBefore: null, toneDuring: null, blipAt: null };
|
||||||
writeFileSync(mkv, new Uint8Array(await original.arrayBuffer()));
|
if (original.ok) {
|
||||||
const pts = execFileSync('ffprobe', [
|
const dir = mkdtempSync(join(tmpdir(), 'cp68-'));
|
||||||
'-v',
|
const mkv = join(dir, 'steady.mkv');
|
||||||
'error',
|
writeFileSync(mkv, new Uint8Array(await original.arrayBuffer()));
|
||||||
'-select_streams',
|
const pts = execFileSync('ffprobe', [
|
||||||
'v:0',
|
|
||||||
'-show_entries',
|
|
||||||
'packet=pts_time',
|
|
||||||
'-of',
|
|
||||||
'csv=p=0',
|
|
||||||
mkv,
|
|
||||||
])
|
|
||||||
.toString()
|
|
||||||
.trim()
|
|
||||||
.split('\n')
|
|
||||||
.map(Number)
|
|
||||||
.sort((a, b) => a - b);
|
|
||||||
file.frames = pts.length;
|
|
||||||
file.gaps = pts.slice(1).flatMap((at, i) => (Math.abs(at - pts[i] - 1 / 60) > 0.0015 ? [i + 1] : []));
|
|
||||||
// Each frame as 8x4 pixels: the colour of the video item covering it, or of what lies behind.
|
|
||||||
const pixels = execFileSync(
|
|
||||||
'ffmpeg',
|
|
||||||
[
|
|
||||||
'-v',
|
'-v',
|
||||||
'error',
|
'error',
|
||||||
'-i',
|
'-select_streams',
|
||||||
|
'v:0',
|
||||||
|
'-show_entries',
|
||||||
|
'packet=pts_time',
|
||||||
|
'-of',
|
||||||
|
'csv=p=0',
|
||||||
mkv,
|
mkv,
|
||||||
'-map',
|
])
|
||||||
'0:v',
|
.toString()
|
||||||
'-vf',
|
.trim()
|
||||||
'scale=8:4',
|
.split('\n')
|
||||||
'-f',
|
.map(Number)
|
||||||
'rawvideo',
|
.sort((a, b) => a - b);
|
||||||
'-pix_fmt',
|
file.frames = pts.length;
|
||||||
'rgb24',
|
file.gaps = pts.slice(1).flatMap((at, i) => (Math.abs(at - pts[i] - 1 / 60) > 0.0015 ? [i + 1] : []));
|
||||||
'-',
|
// Each frame as 8x4 pixels: the colour of the video item covering it, or of what lies behind.
|
||||||
],
|
const pixels = execFileSync(
|
||||||
{ maxBuffer: 64 * 1024 * 1024 },
|
'ffmpeg',
|
||||||
);
|
[
|
||||||
const colours = [];
|
'-v',
|
||||||
for (let at = 0; at + 96 <= pixels.length; at += 96) {
|
'error',
|
||||||
const [r, g, b] = [0, 1, 2].map((c) => pixels[at + 13 * 3 + c]);
|
'-i',
|
||||||
colours.push(r > 180 && g > 180 && b > 180 ? 'W' : r > 150 ? 'R' : g > 150 ? 'G' : b > 150 ? 'B' : '-');
|
mkv,
|
||||||
|
'-map',
|
||||||
|
'0:v',
|
||||||
|
'-vf',
|
||||||
|
'scale=8:4',
|
||||||
|
'-f',
|
||||||
|
'rawvideo',
|
||||||
|
'-pix_fmt',
|
||||||
|
'rgb24',
|
||||||
|
'-',
|
||||||
|
],
|
||||||
|
{ maxBuffer: 64 * 1024 * 1024 },
|
||||||
|
);
|
||||||
|
const colours = [];
|
||||||
|
for (let at = 0; at + 96 <= pixels.length; at += 96) {
|
||||||
|
const [r, g, b] = [0, 1, 2].map((c) => pixels[at + 13 * 3 + c]);
|
||||||
|
colours.push(
|
||||||
|
r > 180 && g > 180 && b > 180 ? 'W' : r > 150 ? 'R' : g > 150 ? 'G' : b > 150 ? 'B' : '-',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
file.turns = colours.flatMap((colour, i) =>
|
||||||
|
i === 0 || colour !== colours[i - 1] ? [`${i}${colour}`] : [],
|
||||||
|
);
|
||||||
|
const sound = execFileSync(
|
||||||
|
'ffmpeg',
|
||||||
|
['-v', 'error', '-i', mkv, '-map', '0:a', '-ac', '1', '-ar', '48000', '-f', 'f32le', '-'],
|
||||||
|
{ maxBuffer: 64 * 1024 * 1024 },
|
||||||
|
);
|
||||||
|
const samples = new Float32Array(sound.buffer, sound.byteOffset, sound.length / 4);
|
||||||
|
const rms = (from, to) => {
|
||||||
|
let sum = 0;
|
||||||
|
const a = Math.max(0, Math.round(from * 48000));
|
||||||
|
const b = Math.min(samples.length, Math.round(to * 48000));
|
||||||
|
for (let i = a; i < b; i++) sum += samples[i] * samples[i];
|
||||||
|
return b > a ? Math.sqrt(sum / (b - a)) : 0;
|
||||||
|
};
|
||||||
|
const red = Number.parseInt(file.turns.find((turn) => turn.endsWith('R')) ?? '-1', 10);
|
||||||
|
const green = Number.parseInt(file.turns.find((turn) => turn.endsWith('G')) ?? '-1', 10);
|
||||||
|
if (red >= 0 && green > red) {
|
||||||
|
file.toneBefore = red > 6 ? rms(0, (red - 3) / 60) : 0;
|
||||||
|
file.toneDuring = rms((red + 3) / 60, (green - 3) / 60);
|
||||||
|
// The blip is loud over the tone of the video: the first 2 ms that are.
|
||||||
|
for (let at = red / 60; at < (green + 30) / 60; at += 0.002)
|
||||||
|
if (rms(at, at + 0.002) > 3 * file.toneDuring + 0.05) {
|
||||||
|
file.blipAt = Math.round((at - green / 60) * 1000);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rmSync(dir, { recursive: true, force: true });
|
||||||
}
|
}
|
||||||
file.turns = colours.flatMap((colour, i) =>
|
// The video's four colours one after the other, whatever lay behind it before and after.
|
||||||
i === 0 || colour !== colours[i - 1] ? [`${i}${colour}`] : [],
|
const colourOrder = file.turns.map((turn) => turn.at(-1)).join('');
|
||||||
);
|
const turned = file.turns
|
||||||
const sound = execFileSync(
|
.slice(colourOrder.indexOf('RGBW'), colourOrder.indexOf('RGBW') + 4)
|
||||||
'ffmpeg',
|
.map((turn) => Number.parseInt(turn, 10));
|
||||||
['-v', 'error', '-i', mkv, '-map', '0:a', '-ac', '1', '-ar', '48000', '-f', 'f32le', '-'],
|
return { ...file, ok: original.ok, status: original.status, colourOrder, turned };
|
||||||
{ maxBuffer: 64 * 1024 * 1024 },
|
};
|
||||||
);
|
const file = await readRecording(steadyUpload);
|
||||||
const samples = new Float32Array(sound.buffer, sound.byteOffset, sound.length / 4);
|
const original = { ok: file.ok, status: file.status };
|
||||||
const rms = (from, to) => {
|
const { colourOrder, turned } = file;
|
||||||
let sum = 0;
|
|
||||||
const a = Math.max(0, Math.round(from * 48000));
|
|
||||||
const b = Math.min(samples.length, Math.round(to * 48000));
|
|
||||||
for (let i = a; i < b; i++) sum += samples[i] * samples[i];
|
|
||||||
return b > a ? Math.sqrt(sum / (b - a)) : 0;
|
|
||||||
};
|
|
||||||
const red = Number.parseInt(file.turns.find((turn) => turn.endsWith('R')) ?? '-1', 10);
|
|
||||||
const green = Number.parseInt(file.turns.find((turn) => turn.endsWith('G')) ?? '-1', 10);
|
|
||||||
if (red >= 0 && green > red) {
|
|
||||||
file.toneBefore = red > 6 ? rms(0, (red - 3) / 60) : 0;
|
|
||||||
file.toneDuring = rms((red + 3) / 60, (green - 3) / 60);
|
|
||||||
// The blip is loud over the tone of the video: the first 2 ms that are.
|
|
||||||
for (let at = red / 60; at < (green + 30) / 60; at += 0.002)
|
|
||||||
if (rms(at, at + 0.002) > 3 * file.toneDuring + 0.05) {
|
|
||||||
file.blipAt = Math.round((at - green / 60) * 1000);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
rmSync(dir, { recursive: true, force: true });
|
|
||||||
}
|
|
||||||
// The video's four colours one after the other, whatever lay behind it before and after.
|
|
||||||
const colourOrder = file.turns.map((turn) => turn.at(-1)).join('');
|
|
||||||
const turned = file.turns
|
|
||||||
.slice(colourOrder.indexOf('RGBW'), colourOrder.indexOf('RGBW') + 4)
|
|
||||||
.map((turn) => Number.parseInt(turn, 10));
|
|
||||||
check(
|
check(
|
||||||
'a recording not in real time has every frame, 1/60 s apart, though the editor drew about 20 a second',
|
'a recording not in real time has every frame, 1/60 s apart, though the editor drew about 20 a second',
|
||||||
original.ok &&
|
original.ok &&
|
||||||
@@ -630,6 +641,108 @@ export async function composition(t) {
|
|||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// 68k. Behind another tab Chrome draws no frames, and the run stood still and the recording with
|
||||||
|
// it. While it records, a worker's beat ticks the page instead: not in real time the file has
|
||||||
|
// every frame, the video's picture at each and the blip in step; in real time the run goes on
|
||||||
|
// at its pace and the recording with it.
|
||||||
|
const tabs = 'http://127.0.0.1:9333/json';
|
||||||
|
const ownTab = (await t.cdp.send('Target.getTargetInfo')).targetInfo.targetId;
|
||||||
|
const recordBehind = async (name, realTime) => {
|
||||||
|
await mutate(`
|
||||||
|
const g = __gm.state.project.assets.find((a) => a.id === 'cp68-director').graph;
|
||||||
|
Object.assign(g.nodes.find((n) => n.id === 'cp68-record').params, { name: ${JSON.stringify(name)}, realTime: ${realTime} });`);
|
||||||
|
const before = await json('__gm.recording()');
|
||||||
|
const take = `__gm.recording().takes[${before.takes.length}]`;
|
||||||
|
await evaluate(`(() => {
|
||||||
|
window.__cp68frames = 0;
|
||||||
|
window.__cp68count = true;
|
||||||
|
const count = () => {
|
||||||
|
if (!window.__cp68count) return;
|
||||||
|
window.__cp68frames++;
|
||||||
|
requestAnimationFrame(count);
|
||||||
|
};
|
||||||
|
requestAnimationFrame(count);
|
||||||
|
})()`);
|
||||||
|
await play();
|
||||||
|
const other = await (await fetch(`${tabs}/new?about:blank`, { method: 'PUT' })).json();
|
||||||
|
const seen = { hidden: false, stillHidden: null, framesBehind: null, error: '' };
|
||||||
|
try {
|
||||||
|
await fetch(`${tabs}/activate/${other.id}`);
|
||||||
|
await waitUntil('document.hidden', { label: 'the tab went behind another' });
|
||||||
|
seen.hidden = true;
|
||||||
|
const framesAt = await json('window.__cp68frames');
|
||||||
|
await waitUntil(`${take}?.state === 'done' || ${take}?.state === 'failed'`, {
|
||||||
|
timeout: 60_000,
|
||||||
|
label: `"${name}" is saved behind another tab`,
|
||||||
|
});
|
||||||
|
seen.framesBehind = (await json('window.__cp68frames')) - framesAt;
|
||||||
|
seen.stillHidden = await json('document.hidden');
|
||||||
|
} catch (error) {
|
||||||
|
seen.error = String(error?.message ?? error);
|
||||||
|
} finally {
|
||||||
|
await fetch(`${tabs}/activate/${ownTab}`);
|
||||||
|
await fetch(`${tabs}/close/${other.id}`);
|
||||||
|
}
|
||||||
|
await waitUntil('!document.hidden', { label: 'the tab is in front again' });
|
||||||
|
await evaluate('window.__cp68count = false; 0');
|
||||||
|
await stop();
|
||||||
|
const made = await json(take);
|
||||||
|
if (made?.state !== 'done') return { made, seen, beats: 0, file: null };
|
||||||
|
await waitUntil(
|
||||||
|
`__gm.recording().uploads.length > ${before.uploads.length} && ['kept', 'failed'].includes(__gm.recording().uploads.at(-1).state)`,
|
||||||
|
{ timeout: 30_000, label: 'the recording is uploaded' },
|
||||||
|
);
|
||||||
|
const after = await json('__gm.recording()');
|
||||||
|
return { made, seen, beats: after.beats - before.beats, file: await readRecording(after.uploads.at(-1)) };
|
||||||
|
};
|
||||||
|
const behind = await recordBehind('cp68 behind', false);
|
||||||
|
const behindFile = behind.file;
|
||||||
|
check(
|
||||||
|
'behind another tab, where the page draws no frame, a recording not in real time still has every frame',
|
||||||
|
behind.seen.hidden &&
|
||||||
|
behind.seen.stillHidden === true &&
|
||||||
|
behind.seen.framesBehind <= 2 &&
|
||||||
|
behind.beats > 100 &&
|
||||||
|
behind.made?.failure === null &&
|
||||||
|
behindFile?.ok &&
|
||||||
|
behindFile.frames >= 120 &&
|
||||||
|
behindFile.gaps.length === 0,
|
||||||
|
JSON.stringify({
|
||||||
|
seen: behind.seen,
|
||||||
|
beats: behind.beats,
|
||||||
|
made: behind.made,
|
||||||
|
frames: behindFile?.frames,
|
||||||
|
gaps: behindFile?.gaps.slice(0, 5),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
check(
|
||||||
|
'and its picture and sound are those in front: colours every 30 frames, the blip as green begins',
|
||||||
|
Boolean(behindFile) &&
|
||||||
|
behindFile.colourOrder.includes('RGBW') &&
|
||||||
|
behindFile.turned.slice(1).every((at, i) => at - behindFile.turned[i] === 30) &&
|
||||||
|
behindFile.blipAt !== null &&
|
||||||
|
Math.abs(behindFile.blipAt) <= 20,
|
||||||
|
JSON.stringify({
|
||||||
|
turns: behindFile?.turns,
|
||||||
|
blipAtMs: behindFile?.blipAt,
|
||||||
|
toneDuring: behindFile?.toneDuring,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const live = await recordBehind('cp68 behind live', true);
|
||||||
|
check(
|
||||||
|
'and in real time the run goes on behind another tab, and the recording with it',
|
||||||
|
live.seen.hidden &&
|
||||||
|
live.seen.stillHidden === true &&
|
||||||
|
live.seen.framesBehind <= 2 &&
|
||||||
|
live.beats > 60 &&
|
||||||
|
live.made?.failure === null &&
|
||||||
|
live.file?.ok &&
|
||||||
|
// Half its frames at least: in front of the others it has about 60 % here, the encoder in WSL
|
||||||
|
// being slow; standing still behind them it had none.
|
||||||
|
live.file.frames >= live.made.durationMs * 0.03,
|
||||||
|
JSON.stringify({ seen: live.seen, beats: live.beats, made: live.made, frames: live.file?.frames }),
|
||||||
|
);
|
||||||
|
|
||||||
await mutate(`
|
await mutate(`
|
||||||
const p = __gm.state.project;
|
const p = __gm.state.project;
|
||||||
p.assets = p.assets.filter((a) => !a.id.startsWith('cp68-') && a.id !== ${JSON.stringify(madeId)});
|
p.assets = p.assets.filter((a) => !a.id.startsWith('cp68-') && a.id !== ${JSON.stringify(madeId)});
|
||||||
|
|||||||
Reference in new issue
Block a user