diff --git a/README.md b/README.md index 21ee1a05..880a7ea2 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ npm run build # writes dist/ and regenerates index.html; the folder is wh The browser suite and the three example games need a machine with a GPU: ```bash -node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2086 checks (GM_STEPS=20 for one step) +node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2088 checks (GM_STEPS=20 for one step) # every tool signs in first – see the Accounts page node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs diff --git a/src/viewport/VideoLibrary.ts b/src/viewport/VideoLibrary.ts index 68cca560..4e677151 100644 --- a/src/viewport/VideoLibrary.ts +++ b/src/viewport/VideoLibrary.ts @@ -23,12 +23,15 @@ import { type VideoPlay, } from '../core/model/video'; import { videoDrivenOf, videoLevelOf, videoTimeOf } from '../core/properties/videos'; +import { every } from './backgroundTimers'; import { FileLibrary } from './FileLibrary'; import { soundLibrary } from './SoundLibrary'; import type { HeardVideo } from './SteadySound'; import { followVideo, PICTURE_NUDGE, + pictureKept, + pictureLost, pictureShown, readSpeed, type ShownPicture, @@ -53,6 +56,11 @@ const IDLE_MS = 2000; * itself otherwise, heard, until it is let go. A free one (#420) plays on from there once asked again. */ const HUSH_MS = 100; +/** How often a player opened behind another tab looks for its first picture, see `openVideo`. */ +const HIDDEN_PICTURE_MS = 20; +/** `HTMLMediaElement`'s: let go (see `drop`), and with the picture of where it stands. */ +const NETWORK_EMPTY = 0; +const HAVE_CURRENT_DATA = 2; /** Seconds a driven video's level takes to get most of the way to a new one, see `hear`. */ const LEVEL_GLIDE = 0.015; @@ -390,9 +398,13 @@ export class VideoLibrary { player.element, drive, () => player.gone, - (shown) => { + (shown, inFront) => { player.texture?.source.update(); - if (shown !== null && player.element.paused) drive.shown = pictureShown(drive.shown, shown); + const { element } = player; + if (!element.paused) return; + if (shown !== null) drive.shown = pictureShown(drive.shown, shown, element.currentTime); + else + drive.shown = inFront ? pictureKept(drive.shown, element.currentTime) : pictureLost(drive.shown); }, ); else { @@ -494,9 +506,21 @@ export function openVideo(url: string, at: number, onPicture: (texture: Texture) element.addEventListener( 'loadeddata', () => { - element.requestVideoFrameCallback(() => - onPicture(new Texture({ source: new VideoSource({ resource: element, autoPlay: false }) })), - ); + let made = false; + const make = () => { + if (made) return; + made = true; + stopLooking(); + onPicture(new Texture({ source: new VideoSource({ resource: element, autoPlay: false }) })); + }; + // Behind another tab Chrome hands over no frame, and a player opened there stayed empty until + // the tab was in front again: a recording not in real time waited 3 s for each of its frames + // and took them without it (#423). Its first picture is there all the same once it has data. + const stopLooking = every(HIDDEN_PICTURE_MS, () => { + if (element.networkState === NETWORK_EMPTY) stopLooking(); + else if (document.hidden && element.readyState >= HAVE_CURRENT_DATA) make(); + }); + element.requestVideoFrameCallback(make); void element.play().catch(() => {}); }, { once: true }, diff --git a/src/viewport/videoFollow.ts b/src/viewport/videoFollow.ts index 949cc302..1a28ad67 100644 --- a/src/viewport/videoFollow.ts +++ b/src/viewport/videoFollow.ts @@ -98,22 +98,49 @@ export function followVideo({ target, speed, current, end }: FollowState): Follo } /** - * The picture a held video shows (#423): where it begins in the file, and how long a picture of - * the video lasts, as learnt from the pictures shown so far; null while not known. + * The picture a held video shows (#423): where it begins in the file, null while not known, and the + * latest time sought that it is known to show. `span` is how long a picture of the video lasts, as + * learnt from the pictures shown so far; null while not known. */ export interface ShownPicture { - at: number; + at: number | null; + reach: number; span: number | null; } /** - * The picture shown once Chrome hands over the one at `at`: the shortest step between two - * pictures seen is how long one lasts. A seek that steps one picture on teaches it at once. + * The picture shown once Chrome hands over the one at `at` for a seek to `sought`. A step between + * two pictures teaches how long one lasts only when the time sought moved at most half that step + * since the picture before was known to show: then no picture can lie between them. A span learnt + * from a longer step was too long, so seeks were left out within it and no shorter step came to + * correct it: the video stood for several frames of a recording. Such steps came after a time + * behind another tab, a jump, or a seek that stepped two pictures on. */ -export function pictureShown(before: ShownPicture | null, at: number): ShownPicture { - const step = before ? Math.abs(at - before.at) : 0; +export function pictureShown(before: ShownPicture | null, at: number, sought: number): ShownPicture { const span = before?.span ?? null; - return { at, span: step > 1e-6 && (span === null || step < span) ? step : span }; + if (!before || before.at === null) return { at, reach: sought, span }; + const step = Math.abs(at - before.at); + if (step <= 1e-6) return { at: before.at, reach: Math.max(before.reach, sought), span }; + // A tenth of a millisecond to spare: Chrome hands over a picture's time in microseconds. + const learns = Math.abs(sought - before.reach) <= step / 2 + 1e-4 && (span === null || step < span); + return { at, reach: sought, span: learns ? step : span }; +} + +/** + * A seek to `sought` that brought no frame though the page is in front: Chrome hands over none for + * a seek within the picture shown, so it still shows. + */ +export function pictureKept(before: ShownPicture | null, sought: number): ShownPicture | null { + if (!before || before.at === null) return before; + return { ...before, reach: Math.max(before.reach, sought) }; +} + +/** + * A seek behind another tab, where Chrome hands over no frame at all: which picture shows is not + * known, how long one lasts still is. + */ +export function pictureLost(before: ShownPicture | null): ShownPicture | null { + return before && { at: null, reach: before.reach, span: before.span }; } /** @@ -122,7 +149,7 @@ export function pictureShown(before: ShownPicture | null, at: number): ShownPict * Where the next picture begins belongs to that one, see `PICTURE_NUDGE`. */ export function showsPictureOf(shown: ShownPicture | null, target: number): boolean { - if (!shown || shown.span === null) return false; + if (!shown || shown.at === null || shown.span === null) return false; return target >= shown.at - 1e-6 && target < shown.at + shown.span - 1e-6; } diff --git a/src/viewport/videoSeek.ts b/src/viewport/videoSeek.ts index c81c7aa0..83ea202c 100644 --- a/src/viewport/videoSeek.ts +++ b/src/viewport/videoSeek.ts @@ -13,6 +13,18 @@ const SEEK_SLACK = 1 / 120; const LAND_MS = 150; /** The same behind another tab, where Chrome hands over no frame at all and the picture is there at `seeked`. */ const HIDDEN_LAND_MS = 10; +/** + * How long the page has to be in front again before a seek with no frame counts as one within the + * picture shown: Chrome may take a moment to hand over frames again. + */ +const FRONT_SETTLE_MS = 500; + +/** When the page came in front last, by `performance.now()`. */ +let frontSince = 0; +if (typeof document !== 'undefined' && typeof document.addEventListener === 'function') + document.addEventListener('visibilitychange', () => { + if (!document.hidden) frontSince = performance.now(); + }); /** One element's seeks, see `seekVideo` and `chaseSeeks`. */ export interface SeekChase { @@ -42,13 +54,15 @@ export function seekVideo(element: HTMLVideoElement, chase: SeekChase, time: num * loads a paused video's picture only then, kept the one before it for a frame. `landed` is where * the caller loads it again, told the time of that picture in the file. Chrome hands over no frame * for some seeks, though – one within the picture shown, for one – and a player waiting for one - * stayed empty until the next jump; after `LAND_MS` it lands without. + * stayed empty until the next jump; after `LAND_MS` it lands without. `inFront` tells whether the + * page was in front all the while, and for a moment before: behind another tab Chrome hands over + * no frame for any seek. */ export function chaseSeeks( element: HTMLVideoElement, chase: SeekChase, gone: () => boolean, - landed: (shown: number | null) => void, + landed: (shown: number | null, inFront: boolean) => void, ): void { let round = 0; // A held video may hand over the picture of a seek before `seeked`: a callback asked for only @@ -73,15 +87,21 @@ export function chaseSeeks( } if (!chase.landing) return; const now = ++round; + const behind = isHidden(); + const settled = !behind && performance.now() - frontSince >= FRONT_SETTLE_MS; const land = (shown: number | null) => { if (gone() || now !== round || element.seeking || !chase.landing) return; - landed(shown); + landed(shown, settled && !isHidden()); chase.landing = false; }; if (early !== null) return land(early); element.requestVideoFrameCallback((_now, frame) => land(frame.mediaTime)); // A worker's timer: behind another tab Chrome holds the page's own to once a second, and a // recording not in real time waits for every landing. - after(typeof document !== 'undefined' && document.hidden ? HIDDEN_LAND_MS : LAND_MS, () => land(null)); + after(behind ? HIDDEN_LAND_MS : LAND_MS, () => land(null)); }); } + +function isHidden(): boolean { + return typeof document !== 'undefined' && document.hidden; +} diff --git a/tests/viewport/video-follow.test.ts b/tests/viewport/video-follow.test.ts index 67b9ad32..3b79dc30 100644 --- a/tests/viewport/video-follow.test.ts +++ b/tests/viewport/video-follow.test.ts @@ -2,6 +2,8 @@ import { describe, expect, it } from 'vitest'; import { followVideo, PICTURE_NUDGE, + pictureKept, + pictureLost, pictureShown, readSpeed, SEEK_DRIFT, @@ -133,32 +135,62 @@ describe('following the asked time', () => { describe('the picture a held video shows (#423)', () => { it('learns how long a picture lasts from the pictures Chrome hands over', () => { - let shown = pictureShown(null, 0); - expect(shown).toEqual({ at: 0, span: null }); - shown = pictureShown(shown, 0); + // A video at 30 fps sought at 60: every other seek stays within the picture shown. + let shown = pictureShown(null, 0, 0.0005); + expect(shown).toEqual({ at: 0, reach: 0.0005, span: null }); + shown = pictureShown(shown, 0, 1 / 60); + expect(shown).toEqual({ at: 0, reach: 1 / 60, span: null }); + shown = pictureShown(shown, 1 / 30, 2 / 60); + expect(shown.span).toBeCloseTo(1 / 30, 12); + // A jump back or far on teaches nothing. + shown = pictureShown(shown, 1, 1.0005); + expect(shown).toEqual({ at: 1, reach: 1.0005, span: shown.span }); + expect(shown.span).toBeCloseTo(1 / 30, 12); + }); + + it('learns nothing from a step two pictures long, which would hold the video for good', () => { + // Sought one frame of 60 on, but from a picture known only at its start: two pictures on came. + let shown = pictureShown(null, 0, 0.0005); + shown = pictureShown(shown, 2 / 30, 2 / 30 + 0.0005); expect(shown.span).toBeNull(); - shown = pictureShown(shown, 2 / 30); - expect(shown.span).toBeCloseTo(2 / 30, 12); - shown = pictureShown(shown, 3 / 30); - expect(shown.span).toBeCloseTo(1 / 30, 12); - // A jump back or far on teaches nothing shorter. - shown = pictureShown(shown, 1); - expect(shown).toEqual({ at: 1, span: shown.span }); + // Learnt so, every seek within the two pictures was left out and none came to correct it. + expect(showsPictureOf(shown, 3 / 30)).toBe(false); + // A seek within the picture brings no frame in front of the others: it still shows, up to there. + shown = pictureKept(shown, 2 / 30 + 1 / 60 + 0.0005)!; + shown = pictureShown(shown, 3 / 30, 3 / 30 + 0.0005); expect(shown.span).toBeCloseTo(1 / 30, 12); }); + it('forgets the picture shown behind another tab, and keeps how long one lasts', () => { + const learnt = { at: 0.5, reach: 0.51, span: 1 / 30 }; + const lost = pictureLost(learnt)!; + expect(lost.span).toBe(1 / 30); + expect(showsPictureOf(lost, 0.51)).toBe(false); + expect(pictureKept(lost, 0.6)).toEqual(lost); + // The first picture handed over in front again is where it stands; the step to it teaches nothing. + const back = pictureShown(lost, 1.5, 1.5005); + expect(back).toEqual({ at: 1.5, reach: 1.5005, span: 1 / 30 }); + // Known only behind a tab, the span is not learnt from a picture far from the one before. + expect(pictureShown(pictureLost({ at: 0.5, reach: 0.5, span: null }), 0.9, 0.9)).toEqual({ + at: 0.9, + reach: 0.9, + span: null, + }); + expect(pictureLost(null)).toBeNull(); + }); + it('knows a seek within the picture shown brings no other; where the next begins is the next', () => { // As measured: frames of a video at 30 fps sought at 60. - const shown = { at: 1 / 30, span: 1 / 30 }; + const shown = { at: 1 / 30, reach: 1 / 30, span: 1 / 30 }; expect(showsPictureOf(shown, 1 / 30)).toBe(true); expect(showsPictureOf(shown, 1.5 / 30)).toBe(true); expect(showsPictureOf(shown, 2 / 30 - 0.0005)).toBe(true); expect(showsPictureOf(shown, 4 / 60)).toBe(false); expect(showsPictureOf(shown, 5 / 60)).toBe(false); expect(showsPictureOf(shown, 0.02)).toBe(false); - expect(showsPictureOf({ at: 1 / 30, span: null }, 1.5 / 30)).toBe(false); + expect(showsPictureOf({ at: 1 / 30, reach: 1 / 30, span: null }, 1.5 / 30)).toBe(false); expect(showsPictureOf(null, 0)).toBe(false); // A seek half a millisecond on stays within its picture, the shortest a video has (60 fps). - expect(showsPictureOf({ at: 0.5, span: 1 / 60 }, 0.5 + PICTURE_NUDGE)).toBe(true); + expect(showsPictureOf({ at: 0.5, reach: 0.5, span: 1 / 60 }, 0.5 + PICTURE_NUDGE)).toBe(true); }); }); diff --git a/tests/viewport/video-library.test.ts b/tests/viewport/video-library.test.ts index 40363968..532edda3 100644 --- a/tests/viewport/video-library.test.ts +++ b/tests/viewport/video-library.test.ts @@ -213,6 +213,44 @@ describe('VideoLibrary driven players (#404)', () => { expect(videos.showsAsked(key)).toBe(true); }); + it('gives a player opened behind another tab its first picture, with no frame handed over (#423)', async () => { + vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval'] }); + const page = { hidden: true }; + const events = new Map void>(); + const element = { + ...fakeVideo(), + seeking: false, + ended: false, + duration: 2, + playbackRate: 1, + readyState: 4, + networkState: 1, + addEventListener: (type: string, listener: () => void) => events.set(type, listener), + // Behind another tab Chrome calls none of these. + requestVideoFrameCallback: () => 0, + }; + vi.stubGlobal('document', { + get hidden() { + return page.hidden; + }, + addEventListener: () => {}, + createElement: () => element, + }); + const videos = await freshLibrary(); + const key = videos.wear(REF, { videoClock: 'driven', videoTime: 0.5 }, 'view1:a', 'game').file; + events.get('loadeddata')!(); + vi.advanceTimersByTime(100); + const behind = videos.hasPicture(key); + // In front Chrome hands over the frame, and the player waits for it. + page.hidden = false; + const front = videos.wear(REF, { videoClock: 'driven', videoTime: 0.5 }, 'view1:b', 'game').file; + events.get('loadeddata')!(); + vi.advanceTimersByTime(100); + vi.useRealTimers(); + expect(behind).toBe(true); + expect(videos.hasPicture(front)).toBe(false); + }); + it('holds a driven player quiet as soon as its wearer is gone, as an ended composition item', async () => { vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval', 'performance'] }); const events = new Map void>(); diff --git a/tools/e2e/68-composition.mjs b/tools/e2e/68-composition.mjs index d16daecd..05d2eb98 100644 --- a/tools/e2e/68-composition.mjs +++ b/tools/e2e/68-composition.mjs @@ -16,7 +16,9 @@ import { MAKE_OBJECT } from './lib/scaffold.mjs'; * ends, its video stops and goes quiet with it. A sound item is planned once the clock runs, timed * to the cut. Recorded not in real time from a slowed editor, the file still has every frame, the * video's picture at each, and its sound and the sound item's in step. Behind another tab, where the - * page draws no frames, a recording goes on all the same, not in real time and in real time. + * page draws no frames, a recording goes on all the same, not in real time and in real time. A video + * item that begins there is in it from its first frame, and switched between the tabs while it + * records, its video still turns in step. * * Everything it adds carries a `cp68` id and goes again at the end. */ @@ -647,7 +649,8 @@ export async function composition(t) { // at its pace and the recording with it. const tabs = 'http://127.0.0.1:9333/json'; const ownTab = (await t.cdp.send('Target.getTargetInfo')).targetInfo.targetId; - const recordBehind = async (name, realTime) => { + /** Records with the tab behind another, or `flipEvery` ms in front and behind in turn. */ + const recordBehind = async (name, realTime, flipEvery = 0) => { await mutate(` const g = __gm.state.project.assets.find((a) => a.id === 'cp68-director').graph; Object.assign(g.nodes.find((n) => n.id === 'cp68-record').params, { name: ${JSON.stringify(name)}, realTime: ${realTime} });`); @@ -670,6 +673,13 @@ export async function composition(t) { await fetch(`${tabs}/activate/${other.id}`); await waitUntil('document.hidden', { label: 'the tab went behind another' }); seen.hidden = true; + seen.flips = 0; + for (let front = false; flipEvery > 0 && seen.flips < 200; seen.flips++) { + if (['done', 'failed'].includes(await json(`${take}?.state`))) break; + await new Promise((resolve) => setTimeout(resolve, flipEvery)); + front = !front; + await fetch(`${tabs}/activate/${front ? ownTab : other.id}`); + } const framesAt = await json('window.__cp68frames'); await waitUntil(`${take}?.state === 'done' || ${take}?.state === 'failed'`, { timeout: 60_000, @@ -743,6 +753,60 @@ export async function composition(t) { JSON.stringify({ seen: live.seen, beats: live.beats, made: live.made, frames: live.file?.frames }), ); + // 68l. A video item that begins behind another tab: Chrome hands over no frame there, and its + // player, opened there, waited for one until the tab was in front again. Every frame meanwhile + // waited 3 s and was taken without the video, and without its sound. And switched between the + // tabs while it records: a picture learnt too long from a step across the time behind held + // the video for several frames, its colours turned late. + const exact = (file) => JSON.stringify(file?.turns) === JSON.stringify(['0R', '30G', '60B', '90W']); + // A frame late at most: a seek just after the tab came in front once landed with the picture before + // (1 run in 5). Learnt too long, the picture held 2 to 6 frames late. + const nearly = (file) => + file?.turns.length === 4 && + file.turns.every( + (turn, i) => turn.endsWith('RGBW'[i]) && Math.abs(Number.parseInt(turn, 10) - 30 * i) <= 1, + ); + await mutate(` + const c = __gm.state.project.assets.find((a) => a.id === 'cp68-cut'); + const v = c.objects.find((o) => o.id === 'cp68-v'); + c.objects.push({ ...structuredClone(v), id: 'cp68-v2', name: 'Video 68 b' }); + c.items.find((i) => i.id === 'cp68-v').length = 108; + c.items.push({ id: 'cp68-v2', track: 'cp68-front', start: 108, length: 12, offset: 108 });`); + const lateCut = await recordBehind('cp68 cut behind', false); + check( + 'a video item that begins behind another tab is in the recording from its first frame on', + lateCut.seen.stillHidden === true && + lateCut.made?.failure === null && + lateCut.file?.ok && + lateCut.file.frames >= 120 && + lateCut.file.gaps.length === 0 && + exact(lateCut.file) && + lateCut.made.tookMs < 15_000, + JSON.stringify({ + seen: lateCut.seen, + made: lateCut.made, + frames: lateCut.file?.frames, + turns: lateCut.file?.turns, + }), + ); + const flip = await recordBehind('cp68 flip', false, 350); + check( + 'switched between the tabs every 350 ms, its colours turn every 30 frames, a frame late at most', + flip.seen.flips >= 4 && + flip.made?.failure === null && + flip.file?.ok && + flip.file.gaps.length === 0 && + nearly(flip.file) && + flip.file.blipAt !== null && + Math.abs(flip.file.blipAt) <= 20, + JSON.stringify({ + seen: flip.seen, + made: flip.made, + turns: flip.file?.turns, + blipAtMs: flip.file?.blipAt, + }), + ); + await mutate(` const p = __gm.state.project; p.assets = p.assets.filter((a) => !a.id.startsWith('cp68-') && a.id !== ${JSON.stringify(madeId)});