From 43fd22a7633a204d20f3902901b4cd8b21b78037 Mon Sep 17 00:00:00 2001 From: MelekTaus Date: Sun, 11 Oct 2026 15:37:16 +0200 Subject: [PATCH] Keep a driven video ready while its sprite is off (#404) A cut between two videos flashed white: the sprite switched on had no picture yet, because its player only opened then or had been dropped meanwhile. A driven video now waits muted at its time while its sprite is off, in the editor and in a run. - A disabled sprite with Video clock Driven asks for its player without drawing it. A Free video still opens when shown. - A run's draw list keeps disabled driven sprites, marked as off. Before, collectWorldItems dropped every disabled object, so the game view never held a player for them. - VideoLibrary.wear() counts as asking. The sweeper dropped a held player after 2 to 4 s and it was loading again at the cut. - The speed of the asked time is read over a window that grows to 500 ms. Over 100 ms one 1/60 s step more or less read as 17 % off, the playback rate jumped and the sound dropped out. - A stream feeds every packet it has loaded to the decoder at once and waits once. One round trip per 20 ms packet let the decoder fall behind on a busy main thread, which made e2e 34 flaky in CI. Tests: - Unit streaming, video-library, video-follow: green, 4 new - e2e 34 + 66 (atx): ALL PASSED, stream skipped 0, 66m new - e2e 66 (atx), two runs: ALL PASSED, 25 checks, 66n new - Gate (atx): CHECK EXIT 0, 297 files, 4557 tests - Counter-checks: without the draw list fix 66n finds no player, without the wear() fix and with the old speed reading the new unit tests turn red Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- src/viewport/VideoLibrary.ts | 5 ++- src/viewport/WorldViewport.ts | 12 +++++-- src/viewport/items.ts | 17 +++++++--- src/viewport/streaming.ts | 20 ++++++++---- src/viewport/videoFollow.ts | 45 ++++++++++++++++--------- tests/viewport/fakeStream.ts | 13 +++++--- tests/viewport/streaming.test.ts | 19 +++++++++++ tests/viewport/video-follow.test.ts | 49 +++++++++++++++++++++++++--- tests/viewport/video-library.test.ts | 17 ++++++++++ tools/e2e/66-video.mjs | 43 ++++++++++++++++++++++++ 11 files changed, 202 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 3fc5d9b1..7d564d9a 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ npm run build # writes dist/ and regenerates index.html; the folder is wh The browser suite and the three example games need a machine with a GPU: ```bash -node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2000 checks (GM_STEPS=20 for one step) +node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2002 checks (GM_STEPS=20 for one step) # every tool signs in first – see the Accounts page node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs diff --git a/src/viewport/VideoLibrary.ts b/src/viewport/VideoLibrary.ts index 78f7442b..c5704c12 100644 --- a/src/viewport/VideoLibrary.ts +++ b/src/viewport/VideoLibrary.ts @@ -92,7 +92,9 @@ export class VideoLibrary { /** * What a sprite wears (#404): with its clock driven, a player of its own that shows the picture * at its Video time; otherwise the ref as it is. `wearer` names the sprite where it is drawn, so - * the same object in two views drives two players. Asked every frame, like `get`. + * the same object in two views drives two players. Asked every frame, like `get`, and it counts + * as an ask: a sprite switched off wears its player without drawing it, and one let go for that + * loaded again when it was switched back on, white until then. */ wear(ref: TextureRef, props: PropertyBag, wearer: string, heard: boolean): TextureRef { if (!videoDrivenOf(props) || !isVideoKey(ref.file)) return ref; @@ -100,6 +102,7 @@ export class VideoLibrary { const time = videoTimeOf(props); const player = this.players.get(key) ?? this.start(key, time); if (!player?.drive) return ref; + player.asked = performance.now(); const drive = player.drive; drive.time = time; drive.heard = heard; diff --git a/src/viewport/WorldViewport.ts b/src/viewport/WorldViewport.ts index 4022322f..768dd7fe 100644 --- a/src/viewport/WorldViewport.ts +++ b/src/viewport/WorldViewport.ts @@ -754,6 +754,9 @@ export abstract class WorldViewport extends W const ghost = this.ghostSprites.get(item.id); if (ghost) disposeSpriteMesh(ghost); this.ghostSprites.delete(item.id); + // A driven video (#404) is kept ready at its Video time while its sprite is switched off or out + // of view: switched on, it showed its plain colour until the video had loaded. + if (item.type === 'sprite' && this.wearVideo(item, false) !== item) this.animated = true; return; } if (!sprite) { @@ -781,14 +784,17 @@ export abstract class WorldViewport extends W syncSpriteMesh(ghost, drawn, size, textureLibrary, shaders); } - /** The item with the player of its own a driven video (#404) has, per view and object. */ - private wearVideo(item: DrawItem): DrawItem { + /** + * The item with the player of its own a driven video (#404) has, per view and object. One that is + * not `seen` is never heard. + */ + private wearVideo(item: DrawItem, seen = true): DrawItem { if (!item.texture) return item; const texture = videoLibrary.wear( item.texture, item.props, `${this.wearer}:${item.id}`, - this.hearsVideo(), + seen && this.hearsVideo(), ); return texture === item.texture ? item : { ...item, texture }; } diff --git a/src/viewport/items.ts b/src/viewport/items.ts index 13fbcfa6..401a5cba 100644 --- a/src/viewport/items.ts +++ b/src/viewport/items.ts @@ -34,6 +34,7 @@ import { type ClipRegion, clipRegions } from '../core/properties/clip'; import { colliderFrame } from '../core/properties/collider'; import { enabledAlongChain } from '../core/properties/enabled'; import { fontOf } from '../core/properties/fonts'; +import { videoDrivenOf } from '../core/properties/videos'; import { SORT_WITH_PARENT_KEY } from '../core/properties/PropertySchema'; import { resolveSceneObjectProps } from '../core/properties/resolve'; import { editorTransform } from '../core/properties/transform'; @@ -275,7 +276,11 @@ function passedOnTo( return passed ? at : null; } -/** The running world: scene content only, since symbols are editor-only. */ +/** + * The running world: scene content only, since symbols are editor-only. A switched-off object is + * left out, except a sprite whose video is driven (#404): the view keeps its player ready, so a + * cut to it shows the picture at once rather than its plain colour while the video loads. + */ export function collectWorldItems(world: World): DrawList { const placed = new Map( world.entities.map((e) => { @@ -286,8 +291,12 @@ export function collectWorldItems(world: World): DrawList { // Worked out before the symbols go: a clip area is one, and still cuts what is below it. const regions = clipRegions([...placed.values()], (p) => (p.e.parent ? placed.get(p.e.parent) : null)); const items = [...placed.values()] - .filter(({ e }) => !isSymbolType(e.type) && isHandleEnabled(e)) - .map(({ e, m, z }) => ({ + .map((p) => ({ ...p, enabled: isHandleEnabled(p.e) })) + .filter( + ({ e, enabled }) => + !isSymbolType(e.type) && (enabled || (e.type === 'sprite' && videoDrivenOf(e.props))), + ) + .map(({ e, m, z, enabled }) => ({ id: e.objectId, type: e.type, name: e.name, @@ -298,7 +307,7 @@ export function collectWorldItems(world: World): DrawList { props: e.props, parentId: e.parent?.objectId ?? '', sortsWith: passedOnTo(e.parent ?? null, (p) => p.parent ?? null)?.objectId ?? '', - enabled: true, + enabled, // Resolved per frame on purpose: a `Properties` node or an animation may swap `texture`. texture: wearsPicture(e.type) ? resolveTexture(world.project, e.props) : null, font: e.type === 'text' ? fontFileOf(world.project, fontOf(e.props)) : '', diff --git a/src/viewport/streaming.ts b/src/viewport/streaming.ts index a3d60c24..4f168ce5 100644 --- a/src/viewport/streaming.ts +++ b/src/viewport/streaming.ts @@ -144,9 +144,16 @@ class StreamDecoder { this.spare = []; while (have < end) { await file.reach(have); - const packet = file.reader.packets[this.next]; - if (!packet) break; - this.feed(packet); + const packets = file.reader.packets; + if (this.next >= packets.length) break; + // All of it that is fetched goes in at once: waiting on the page for each packet of 20 ms, a + // busy page decoded slower than it played and the player went short (CI tasks 638, 640, 651). + let upTo = have; + while (this.next < packets.length && upTo < end) { + const packet = packets[this.next]!; + this.feed(packet); + upTo = packet.at + packet.samples; + } const got = await this.take(); parts.push(got); have += got[0]?.length ?? 0; @@ -200,10 +207,9 @@ class StreamDecoder { this.decoder = decoder; // Every packet that ends before where it is asked for settles it and is thrown away; `read` cuts // the rest off the one it starts in. - while (this.next < packets.length && packets[this.next]!.at + packets[this.next]!.samples <= sample) { + while (this.next < packets.length && packets[this.next]!.at + packets[this.next]!.samples <= sample) this.feed(packets[this.next]!); - await this.take(); - } + await this.take(); this.outAt = packets[this.next]?.at ?? this.outAt; } @@ -220,7 +226,7 @@ class StreamDecoder { this.fed += 1; } - /** The samples of the packet handed over last, once the decoder has given them back. */ + /** The samples of the packets handed over since the last call, once the decoder has given them back. */ private async take(): Promise { while (this.given < this.fed && !this.failure) await new Promise((resolve) => { diff --git a/src/viewport/videoFollow.ts b/src/viewport/videoFollow.ts index 12a0caf8..b91255c5 100644 --- a/src/viewport/videoFollow.ts +++ b/src/viewport/videoFollow.ts @@ -17,34 +17,49 @@ const MAX_NUDGE = 0.1; const HELD_SLACK = 1 / 120; /** Around 1x a driven video is heard; slowed or sped up it is quiet. */ const HEARD_SPEED = 0.1; -/** The stretch of wall time the speed of the asked time is measured over, in milliseconds. */ +/** + * The wall time, in milliseconds, before the speed of the asked time is read at all, and how long + * a time that does not move takes to count as standing. + */ export const SPEED_WINDOW_MS = 100; +/** How far back the window reaches once it has grown, in milliseconds. */ +export const SPEED_SPAN_MS = 500; /** - * The speed of the asked time, measured over `SPEED_WINDOW_MS` rather than from frame to frame: a - * game steps its animations at its own rate, so between two drawn frames the time may not move at - * all and then move twice. + * The speed of the asked time, measured over a window rather than from frame to frame: a game + * steps its animations at its own rate, so between two drawn frames the time may not move at all + * and then move twice. Those steps are 1/60 s, one more or less in a window of 100 ms read as 17 % + * too fast or too slow, and the video was nudged and fell quiet in turn; the window therefore + * grows to `SPEED_SPAN_MS` and slides on from there. */ export interface SpeedReading { - /** Where the asked time stood when the window began, and when that was. */ - from: number; - since: number; - last: number; + /** Every asked time that moved within the window, oldest first, and when it was asked. */ + samples: { time: number; at: number }[]; speed: number; } /** * The next reading after the wearer asked for `time` at `now`. A jump (a loop going round, a key * set to another time) starts the window over and keeps the speed: playing on through a loop is - * right, and a still picture stays still. + * right, and a still picture stays still. While the time stands, the window waits at its latest + * frame, so a start is measured from where it began and not across the wait before it. */ export function readSpeed(reading: SpeedReading | null, time: number, now: number): SpeedReading { - if (!reading) return { from: time, since: now, last: time, speed: 0 }; - if (Math.abs(time - reading.last) > SEEK_DRIFT) - return { from: time, since: now, last: time, speed: reading.speed }; - const span = now - reading.since; - if (span < SPEED_WINDOW_MS) return { ...reading, last: time }; - return { from: time, since: now, last: time, speed: ((time - reading.from) * 1000) / span }; + if (!reading) return { samples: [{ time, at: now }], speed: 0 }; + const last = reading.samples[reading.samples.length - 1]!; + if (Math.abs(time - last.time) > SEEK_DRIFT) return { samples: [{ time, at: now }], speed: reading.speed }; + if (time === last.time) { + const standing = reading.speed === 0 && reading.samples.length === 1; + return standing || now - last.at >= SPEED_WINDOW_MS + ? { samples: [{ time, at: now }], speed: 0 } + : reading; + } + const samples = [...reading.samples, { time, at: now }]; + while (samples.length > 2 && now - samples[1]!.at >= SPEED_SPAN_MS) samples.shift(); + const first = samples[0]!; + const span = now - first.at; + if (span < SPEED_WINDOW_MS) return { samples, speed: reading.speed }; + return { samples, speed: ((time - first.time) * 1000) / span }; } export interface FollowState { diff --git a/tests/viewport/fakeStream.ts b/tests/viewport/fakeStream.ts index e356e9b9..52815e24 100644 --- a/tests/viewport/fakeStream.ts +++ b/tests/viewport/fakeStream.ts @@ -57,8 +57,11 @@ function answer(status: number, bytes: Uint8Array, contentRange: string | null) /** Every decoder made, with the timestamps of what it was given and whether it was closed. */ export const decoders: { fed: number[]; closed: boolean; channels: number }[] = []; -/** An `AudioDecoder` that gives back, for each packet, 960 samples counting up from where it starts. */ -export function stubDecoder(): void { +/** + * An `AudioDecoder` that gives back, for each packet, 960 samples counting up from where it starts; + * with `delayMs`, each on a task that long after, as on a busy page. + */ +export function stubDecoder(delayMs = 0): void { decoders.length = 0; vi.stubGlobal( 'EncodedAudioChunk', @@ -84,7 +87,7 @@ export function stubDecoder(): void { decode(chunk: { timestamp: number }) { const at = Math.round((chunk.timestamp * 48000) / 1e6); this.made.fed.push(at); - queueMicrotask(() => { + const give = () => { if (this.made.closed) return; this.init.output({ numberOfFrames: 960, @@ -94,7 +97,9 @@ export function stubDecoder(): void { }, close: () => {}, }); - }); + }; + if (delayMs > 0) setTimeout(give, delayMs); + else queueMicrotask(give); } close() { this.state = 'closed'; diff --git a/tests/viewport/streaming.test.ts b/tests/viewport/streaming.test.ts index e0c0c40b..11e373f3 100644 --- a/tests/viewport/streaming.test.ts +++ b/tests/viewport/streaming.test.ts @@ -93,6 +93,25 @@ describe('a copy streamed', () => { expect(samples()[3 * 48000 - 1]).toBe(PRE_SKIP + 5 * 48000 - 1); }); + it('keeps ahead where the decoder answers each packet 20 ms late, as on a busy page', async () => { + const copy = fakeCopy(10); + serveCopies({ [URL]: copy.bytes }); + const file = new StreamFile(URL, copy.frames); + await file.ready(); + stubDecoder(20); + vi.useFakeTimers(); + try { + const player = fakeNode(); + feedOf(file, { from: 0, until: copy.frames, loop: null }, player); + // Packet by packet that took 130 answers, 2.6 s: longer than the 2 s head lasts (CI task 651). + await vi.advanceTimersByTimeAsync(200); + expect(player.samples()).toHaveLength(4 * 48000); + expect(player.samples().every((value, i) => value === PRE_SKIP + i)).toBe(true); + } finally { + vi.useRealTimers(); + } + }); + it('jumps in mid-way after settling the decoder on what comes before', async () => { const copy = fakeCopy(10); serveCopies({ [URL]: copy.bytes }); diff --git a/tests/viewport/video-follow.test.ts b/tests/viewport/video-follow.test.ts index 57685eba..fdd346e2 100644 --- a/tests/viewport/video-follow.test.ts +++ b/tests/viewport/video-follow.test.ts @@ -1,5 +1,11 @@ import { describe, expect, it } from 'vitest'; -import { followVideo, readSpeed, SEEK_DRIFT, SPEED_WINDOW_MS } from '../../src/viewport/videoFollow'; +import { + followVideo, + readSpeed, + SEEK_DRIFT, + SPEED_SPAN_MS, + SPEED_WINDOW_MS, +} from '../../src/viewport/videoFollow'; /** * How a driven video keeps up with the time its wearer asks for (#404): it plays along while that @@ -10,7 +16,7 @@ describe('the speed of the asked time', () => { let reading = readSpeed(null, 0, 0); // A game steps its animation at its own rate: here a frame without a step, then two in one. for (const [time, now] of [ - [0, 16], + [0.017, 17], [0.033, 33], [0.05, 50], [0.05, 66], @@ -22,14 +28,47 @@ describe('the speed of the asked time', () => { expect(reading.speed).toBeCloseTo(1, 5); }); + it('reads steps of 1/60 s on uneven frames as 1x within 5 % once the window has grown', () => { + let reading = readSpeed(null, 0, 0); + const read: number[] = []; + let now = 0; + let seed = 7; + while (now < 3000) { + // Frames 8 to 25 ms apart, in no order a window could line up with. + seed = (seed * 48271) % 2147483647; + now += 8 + (seed % 18); + reading = readSpeed(reading, Math.floor((now * 60) / 1000) / 60, now); + if (now > SPEED_SPAN_MS) read.push(reading.speed); + } + expect(read.length).toBeGreaterThan(100); + expect(Math.min(...read)).toBeGreaterThan(0.95); + expect(Math.max(...read)).toBeLessThan(1.05); + }); + + it('measures a start from where the time began to move, not across the wait before it', () => { + let reading = readSpeed(null, 0.5, 0); + const start = 16 * 187; + for (let now = 16; now <= start; now += 16) reading = readSpeed(reading, 0.5, now); + for (let now = start + 16; now <= start + 208; now += 16) + reading = readSpeed(reading, 0.5 + (now - start) / 1000, now); + expect(reading.speed).toBeCloseTo(1, 5); + }); + it('keeps the speed through a jump and measures again from there', () => { - const playing = { from: 3.9, since: 900, last: 3.98, speed: 1 }; + const playing = { + samples: [ + { time: 3.6, at: 600 }, + { time: 3.98, at: 980 }, + ], + speed: 1, + }; const turned = readSpeed(playing, 0.01, 990); - expect(turned).toEqual({ from: 0.01, since: 990, last: 0.01, speed: 1 }); + expect(turned).toEqual({ samples: [{ time: 0.01, at: 990 }], speed: 1 }); }); it('reads a still time as standing', () => { - const playing = { from: 2, since: 0, last: 2, speed: 1 }; + const playing = { samples: [{ time: 2, at: 0 }], speed: 1 }; + expect(readSpeed(playing, 2, SPEED_WINDOW_MS / 2).speed).toBe(1); expect(readSpeed(playing, 2, SPEED_WINDOW_MS).speed).toBe(0); }); }); diff --git a/tests/viewport/video-library.test.ts b/tests/viewport/video-library.test.ts index c1e866b0..3c9f78c7 100644 --- a/tests/viewport/video-library.test.ts +++ b/tests/viewport/video-library.test.ts @@ -8,7 +8,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; */ const KEY = `video:a1b2c3d4e5f6/${'c7'.repeat(32)}.mp4#0~0~1~1`; +let videosMade = 0; + function fakeVideo() { + videosMade += 1; return { muted: false, playsInline: false, @@ -83,6 +86,20 @@ describe('VideoLibrary sweeping', () => { vi.advanceTimersByTime(2200); expect(videos.debugInfo()).toEqual([]); }); + + it('keeps a driven player that is only worn, as by a sprite switched off', async () => { + const videos = await freshLibrary(); + const ref = { file: KEY, smooth: true, rect: null, sourceWidth: 320, sourceHeight: 180, slice: null }; + const props = { videoClock: 'driven', videoTime: 1.2 }; + videosMade = 0; + for (let ms = 0; ms < 6000; ms += 100) { + videos.wear(ref, props, 'view2:a', false); + vi.advanceTimersByTime(100); + } + // Let go and opened again would be a second video, loading while the sprite shows white. + expect(videosMade).toBe(1); + expect(videos.debugInfo().map(({ key }) => key)).toEqual([`${KEY}@view2:a`]); + }); }); describe('VideoLibrary driven players (#404)', () => { diff --git a/tools/e2e/66-video.mjs b/tools/e2e/66-video.mjs index 9f15b886..d0ecf7d0 100644 --- a/tools/e2e/66-video.mjs +++ b/tools/e2e/66-video.mjs @@ -383,6 +383,49 @@ export async function video(t) { ); await showTab('scene'); + // 66m. Switched off, a driven sprite keeps its player ready at its Video time, longer than a player + // nobody asks for lives: switched on, the first frame shows 1.2 s blue, not its plain colour. + // 4.5 s, as the sweeper looks only every 2 s and lets go of what is 2 s idle. + const switchLate = (on) => + mutate(` + const late = __gm.state.activeScene.objects.find((o) => o.id === '${LATE}'); + late.overrides = { ...late.overrides, videoTime: 1.2, enabled: ${on} };`); + await switchLate(false); + await sleep(4500); + const readied = (await json('__gm.videos()')).find((v) => v.key.includes(LATE)); + await switchLate(true); + await t.evaluate('__gm.afterFrames(1)'); + const firstShown = await colorAt(await json(`__gm.objectRect('${LATE}')`)); + check( + 'a driven sprite switched off keeps its video ready and shows its picture the frame it is switched on', + readied?.ready === true && + readied.paused && + readied.muted && + Math.abs(readied.time - 1.2) < 0.05 && + firstShown === COLORS.blue, + JSON.stringify({ readied, firstShown }), + ); + + // 66n. The same in a run: the world used to leave a switched-off object out of what the Game tab + // draws, so a cut to it loaded the video only then and showed white meanwhile. Nor did wearing + // the player count as asking for it: let go after 2 to 4 s, it was opened again. + await switchLate(false); + const sceneKeys = new Set((await json('__gm.videos()')).map((v) => v.key)); + await play(); + await sleep(4500); + const inRun = (await json('__gm.videos()')).filter((v) => v.key.includes(LATE) && !sceneKeys.has(v.key)); + await stop(); + await switchLate(true); + check( + 'in a run, a driven sprite switched off keeps its video ready too', + inRun.length === 1 && + inRun[0].ready && + inRun[0].paused && + inRun[0].muted && + Math.abs(inRun[0].time - 1.2) < 0.05, + JSON.stringify(inRun), + ); + await mutate(` const p = __gm.state.project; const gone = ['${OBJECT}', '${EARLY}', '${LATE}'];