diff --git a/README.md b/README.md index 75b6d2b8..1df03417 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ npm run build # writes dist/ and regenerates index.html; the folder is wh The browser suite and the three example games need a machine with a GPU: ```bash -node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2036 checks (GM_STEPS=20 for one step) +node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2038 checks (GM_STEPS=20 for one step) # every tool signs in first – see the Accounts page node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs diff --git a/src/core/recording/recording.ts b/src/core/recording/recording.ts index 12c68a87..86c62d5d 100644 --- a/src/core/recording/recording.ts +++ b/src/core/recording/recording.ts @@ -126,8 +126,10 @@ export class RecordingClock { return this.origin !== null; } + /** A pause before the first frame is no part of the recording's time. */ start(now: number): void { this.origin = now; + this.paused = 0; } pause(now: number): void { diff --git a/src/editor/debug/sceneHooks.ts b/src/editor/debug/sceneHooks.ts index 28b1499b..4fc9b564 100644 --- a/src/editor/debug/sceneHooks.ts +++ b/src/editor/debug/sceneHooks.ts @@ -113,6 +113,8 @@ export function sceneHooks({ state, editor }: HookContext) { gameScreen: (id: string) => gameView.objectScreen(id), gameRect: (id: string) => gameView.objectRect(id), gameDrawnRect: (id: string) => gameView.drawnRect(id), + /** The player of a driven video the Game tab draws for this object, not another view's. */ + gameVideo: (id: string) => gameView.videoOf(id), gameCenter: () => centreOf(gameView), /** What the main camera draws in the Game tab, back to front (#302). */ gameDrawOrder: () => gameView.drawOrder(), diff --git a/src/runtime/systems/CompositionSystem.ts b/src/runtime/systems/CompositionSystem.ts index 4b3dcce1..41f53d7f 100644 --- a/src/runtime/systems/CompositionSystem.ts +++ b/src/runtime/systems/CompositionSystem.ts @@ -38,13 +38,13 @@ interface Run { id: number; /** By item id. An item a behavior destroyed stays here until it ends, so it is not built again. */ live: Map; - /** Seconds the clock has stood so far for a video without its first picture, see `waits`. */ + /** Seconds the clock has stood so far for a video without its picture, see `waits`. */ waited: number; } /** What the view that shows a run knows of its driven videos (#404). */ export interface VideoPictures { - /** Whether it left the sprite `objectId` out, its video still without a first picture. */ + /** Whether it did not show the sprite `objectId`, its video still without the picture asked for. */ missing(objectId: string): boolean; } @@ -55,7 +55,7 @@ export interface VideoPictures { * frame's `finished`. The editor run plays none: it writes into the saved scene, where the objects of * a cut would stay behind. * - * While the view still waits for the first picture of a video an item brings in, the composition's + * While the view still waits for the picture of a video an item brings in, the composition's * clock stands, so the video starts at its first frame and not where the clock has got to meanwhile. */ export class CompositionSystem implements System { diff --git a/src/viewport/GameRecorder.ts b/src/viewport/GameRecorder.ts index 48ad08c8..2cf14860 100644 --- a/src/viewport/GameRecorder.ts +++ b/src/viewport/GameRecorder.ts @@ -1,4 +1,5 @@ import { Matrix, type RenderTexture } from 'pixi.js'; +import { VIDEO_WAIT_MS } from '../core/model/video'; import { preSkipOf } from '../core/music/ogg'; import { AVC_CODEC, colourOf, MatroskaWriter, VP9_CODEC } from '../core/recording/matroska'; import { @@ -34,6 +35,8 @@ const LAST_FRAMES_MS = 500; export interface RecordingView { picture(): { shown: PictureArea; whole: PictureArea } | null; drawPicture(transform: Matrix, target: RenderTexture): void; + /** Whether its last redraw left out a video sprite, the video still without its picture. */ + missesVideo(): boolean; } /** A recording once it is made. */ @@ -73,6 +76,8 @@ export interface TakeInfo { soundFrames: number; /** From the first frame to the end of the last, in ms. */ durationMs: number; + /** `performance.now()` when the last frame was taken, null before the first. */ + lastFrameAt: number | null; /** What the video encoder said its colour is, null while it said nothing. */ colour: VideoColorSpaceInit | null; bytes: number; @@ -165,6 +170,10 @@ class Take { private soundEnded: (() => void) | null = null; private stopAsked = false; private paused = false; + private held = false; + /** Since when the view lacks a video, see `holdForVideo`; null while it lacks none. */ + private heldSince: number | null = null; + private lastFrameAt: number | null = null; private failure: string | null = null; constructor( @@ -278,13 +287,37 @@ class Take { } setPaused(paused: boolean): void { - if (paused === this.paused) return; - this.paused = paused; + this.stand(() => (this.paused = paused)); + } + + /** + * A frame that lacks a video is not taken (#417): a composition's clock stands until its video has + * its picture, and the frames meanwhile were empty where the video goes. Like a pause, but the game + * plays on, so what it sounds then is left out too. At most `VIDEO_WAIT_MS` at a stretch, as long + * as the composition waits. + */ + private holdForVideo(view: RecordingView): void { const now = performance.now(); - if (paused) this.clock.pause(now); + if (view.missesVideo()) this.heldSince ??= now; + else this.heldSince = null; + const held = this.heldSince !== null && now - this.heldSince < VIDEO_WAIT_MS; + this.stand(() => (this.held = held)); + } + + /** The clock stands while paused or held, whichever came first and whichever ends last. */ + private stand(change: () => void): void { + const was = this.standing; + change(); + if (this.standing === was) return; + const now = performance.now(); + if (this.standing) this.clock.pause(now); else this.clock.resume(now); } + private get standing(): boolean { + return this.paused || this.held; + } + capture(view: RecordingView): void { if (this.state === 'waiting') { const picture = view.picture(); @@ -302,7 +335,9 @@ class Take { if (this.state !== 'recording' || !video || !reader) return; reader.collect((bytes, ms) => this.encode(bytes, ms)); // A frame taken may have reached the most a file holds and ended the recording. - if (!this.taking || this.paused) return; + if (!this.taking) return; + this.holdForVideo(view); + if (this.standing) return; const now = performance.now(); if (!this.clock.started) this.clock.start(now); const ms = this.clock.at(now); @@ -318,6 +353,7 @@ class Take { ); view.drawPicture(new Matrix(m.a, 0, 0, m.d, m.tx, m.ty), reader.texture); reader.read(ms); + this.lastFrameAt = now; } private get taking(): boolean { @@ -352,8 +388,9 @@ class Take { return; } // Before the first frame there is no time to put it at. Pieces keep coming during a pause only - // as long as they were on their way, so they belong before it and are kept. - if (!audio || !this.tap || this.state !== 'recording' || !this.clock.started) return; + // as long as they were on their way, so they belong before it and are kept. A hold leaves them + // out: the game plays on meanwhile. + if (!audio || !this.tap || this.state !== 'recording' || !this.clock.started || this.held) return; const rate = this.tap.rate; this.soundStart ??= Math.max(0, this.clock.at(performance.now()) - (frames / rate) * 1000); const planar = new Float32Array(frames * 2); @@ -511,6 +548,7 @@ class Take { dropped: this.dropped, soundFrames: this.soundFrames, durationMs: this.frames > 0 ? Math.round(this.lastMs + 1000 / this.settings.fps) : 0, + lastFrameAt: this.lastFrameAt, colour: this.colour, bytes: this.writer?.bytes ?? 0, failure: this.failure, diff --git a/src/viewport/WorldViewport.ts b/src/viewport/WorldViewport.ts index 6821369e..19c59498 100644 --- a/src/viewport/WorldViewport.ts +++ b/src/viewport/WorldViewport.ts @@ -125,7 +125,7 @@ export abstract class WorldViewport extends W private readonly videosShown = new Map(); /** Since when the view keeps its last frame for a driven video it brings in, see `holdsForVideo`. */ private heldSince: number | null = null; - /** The sprites the last redraw left out, their driven video still without its first picture. */ + /** The sprites the last redraw did not show, their driven video still without the picture asked for. */ private readonly videosLeftOut = new Set(); private cameraMoves = 0; private readonly stopFiles: (() => void)[]; @@ -506,7 +506,7 @@ export abstract class WorldViewport extends W */ private holdsForVideo(view: View, items: DrawItem[], shaders: ShaderSource): boolean { const bounds = worldBounds(view); - const waits = items.some((item) => { + const waiting = items.filter((item) => { const file = item.texture?.file; if (item.type !== 'sprite' || !item.enabled || !file || !isVideoKey(file)) return false; if (!videoDrivenOf(item.props)) return false; @@ -520,9 +520,12 @@ export abstract class WorldViewport extends W return videoLibrary.hasPicture(key) && !videoLibrary.showsAsked(key); }); const now = performance.now(); - if (!waits) this.heldSince = null; + if (waiting.length === 0) this.heldSince = null; else this.heldSince ??= now; - return waits && now - this.heldSince! < VIDEO_HOLD_MS; + const holds = waiting.length > 0 && now - this.heldSince! < VIDEO_HOLD_MS; + // The held frame does not show them either, so a composition waits for them meanwhile. + if (holds) for (const item of waiting) this.videosLeftOut.add(item.id); + return holds; } /** @@ -888,7 +891,7 @@ export abstract class WorldViewport extends W if (ghost) disposeSpriteMesh(ghost); this.ghostSprites.delete(item.id); this.videosShown.delete(item.id); - if (!videoLibrary.hasPicture(key)) this.videosLeftOut.add(item.id); + this.videosLeftOut.add(item.id); return; } if (key !== null) this.videosShown.set(item.id, key); @@ -926,14 +929,26 @@ export abstract class WorldViewport extends W return `${this.wearer}:${item.id}`; } + /** + * What the player of a driven video drawn here for `objectId` is doing (test hook): the one it + * showed last, else the newest it opened; null for none. + */ + videoOf(objectId: string): ReturnType[number] | null { + const mark = `@${this.videoWearer({ id: objectId })}`; + const players = videoLibrary.debugInfo().filter((player) => player.key.endsWith(mark)); + const shown = this.videosShown.get(objectId); + return players.find((player) => player.key === shown) ?? players[players.length - 1] ?? null; + } + /** Whether a driven video (#404) this view wears is still without its first picture. */ videosLoading(): boolean { return videoLibrary.loading(this.wearer); } /** - * Whether the last redraw left out a driven video (#404) it should show, its player still without - * a first picture. Unlike `videosLoading`, one opened ahead or switched off does not count. + * Whether the last redraw did not show a driven video (#404) it should show: its player has no + * picture yet, or still seeks to the one asked for, as after loading a clip that starts later. + * Unlike `videosLoading`, one opened ahead or switched off does not count. */ missesVideo(): boolean { return this.videosLeftOut.size > 0; diff --git a/src/viewport/videoFollow.ts b/src/viewport/videoFollow.ts index b91255c5..902a6f26 100644 --- a/src/viewport/videoFollow.ts +++ b/src/viewport/videoFollow.ts @@ -42,7 +42,8 @@ export interface SpeedReading { * The next reading after the wearer asked for `time` at `now`. A jump (a loop going round, a key * set to another time) starts the window over and keeps the speed: playing on through a loop is * right, and a still picture stays still. While the time stands, the window waits at its latest - * frame, so a start is measured from where it began and not across the wait before it. + * frame, so a start is measured from where it began and not across the wait before it. Until then + * a start counts as 1x. */ export function readSpeed(reading: SpeedReading | null, time: number, now: number): SpeedReading { if (!reading) return { samples: [{ time, at: now }], speed: 0 }; @@ -58,8 +59,13 @@ export function readSpeed(reading: SpeedReading | null, time: number, now: numbe while (samples.length > 2 && now - samples[1]!.at >= SPEED_SPAN_MS) samples.shift(); const first = samples[0]!; const span = now - first.at; - if (span < SPEED_WINDOW_MS) return { samples, speed: reading.speed }; - return { samples, speed: ((time - first.time) * 1000) / span }; + const pace = ((time - first.time) * 1000) / Math.max(span, 1); + if (span >= SPEED_WINDOW_MS) return { samples, speed: pace }; + // A start from standing plays at 1x until the window has its reading: held and sought frame by + // frame instead, the first pictures of a cut stuck and were skipped. A step faster than a video + // plays is a jump, not a start. + const starts = reading.speed === 0 && pace >= MIN_PLAY_SPEED && pace <= MAX_PLAY_SPEED; + return { samples, speed: starts ? 1 : reading.speed }; } export interface FollowState { diff --git a/tests/core/recording.test.ts b/tests/core/recording.test.ts index 988177b4..591b70ba 100644 --- a/tests/core/recording.test.ts +++ b/tests/core/recording.test.ts @@ -71,6 +71,14 @@ describe('RecordingClock', () => { clock.resume(8000); expect(clock.at(8000)).toBe(2000); }); + + it('leaves a hold before the first frame out of the time, not off it', () => { + const clock = new RecordingClock(); + clock.pause(0); + clock.resume(2000); + clock.start(2000); + expect(clock.at(2500)).toBe(500); + }); }); describe('FrameGate', () => { diff --git a/tests/viewport/video-follow.test.ts b/tests/viewport/video-follow.test.ts index fdd346e2..0db73327 100644 --- a/tests/viewport/video-follow.test.ts +++ b/tests/viewport/video-follow.test.ts @@ -23,11 +23,27 @@ describe('the speed of the asked time', () => { [0.083, 83], ] as const) reading = readSpeed(reading, time, now); - expect(reading.speed).toBe(0); + expect(reading.samples[0]).toEqual({ time: 0, at: 0 }); reading = readSpeed(reading, 0.1, SPEED_WINDOW_MS); expect(reading.speed).toBeCloseTo(1, 5); }); + it('counts a start from standing as 1x until the window has its reading', () => { + let reading = readSpeed(null, 0.8, 0); + reading = readSpeed(reading, 0.8, 16); + expect(reading.speed).toBe(0); + // The first step of 1/60 s after the wait: held and sought instead, the first pictures stuck. + reading = readSpeed(reading, 0.8 + 1 / 60, 33); + expect(reading.speed).toBe(1); + // Two steps in one frame are still a start. + expect(readSpeed(readSpeed(null, 0.8, 0), 0.8 + 2 / 60, 16).speed).toBe(1); + }); + + it('does not count a jump faster than a video plays, or a time running back, as a start', () => { + expect(readSpeed(readSpeed(null, 0.8, 0), 0.9, 16).speed).toBe(0); + expect(readSpeed(readSpeed(null, 0.8, 0), 0.8 - 1 / 60, 16).speed).toBe(0); + }); + it('reads steps of 1/60 s on uneven frames as 1x within 5 % once the window has grown', () => { let reading = readSpeed(null, 0, 0); const read: number[] = []; diff --git a/tools/e2e/68-composition.mjs b/tools/e2e/68-composition.mjs index 07fbd9a4..a238441a 100644 --- a/tools/e2e/68-composition.mjs +++ b/tools/e2e/68-composition.mjs @@ -8,7 +8,8 @@ import { MAKE_OBJECT } from './lib/scaffold.mjs'; * it: each item becomes an object below the placement while it lasts, the first track in front, a * clip moves the item it overlaps, and `On composition finished` hears the end, by which time no * item is left. Nothing of the run stays in the scene. An item's video is waited for: the clock - * stands until the Game tab has its first picture. + * stands until the Game tab has its first picture, for a clip that starts later the picture there, + * and then the video plays at once. A recording that runs meanwhile takes no frame. * * Everything it adds carries a `cp68` id and goes again at the end. */ @@ -23,7 +24,7 @@ const link = (id, from, fromPort, to, toPort) => ({ }); export async function composition(t) { - const { check, sleep, json, evaluate, mutate, play, stop, showTab } = t; + const { check, sleep, json, evaluate, mutate, press, play, stop, showTab, waitUntil } = t; await showTab('scene'); const graph = { @@ -192,7 +193,7 @@ export async function composition(t) { const loading = []; const step = () => { const time = ${clock}; - const player = __gm.videos().find((v) => v.key.endsWith(':${id('v')}')); + const player = __gm.gameVideo('${id('v')}'); if (player?.ready) return done(JSON.stringify({ loading, ready: time })); if (player) loading.push(time); if (!__gm.world() || loading.length > 300) return done(JSON.stringify({ loading, ready: null })); @@ -219,9 +220,109 @@ export async function composition(t) { ); await stop(); + // 68e. The item wears a clip that starts at 0.8 s: its player loads, then seeks back to 0.8. The + // clock stands until the Game tab draws that picture, and from its first step on the video + // plays – it was held and sought frame by frame for 0.1 s before, its first pictures stuck. await mutate(` const p = __gm.state.project; - p.assets = p.assets.filter((a) => !a.id.startsWith('cp68-')); + p.assets.find((a) => a.id === 'cp68-video').clips = [{ id: 'cp68-late', name: 'Late', props: { start: 0.8, end: 0 } }]; + p.assets.find((a) => a.id === 'cp68-screen').props.texture = 'cp68-late';`); + await play(); + const rows = JSON.parse( + await evaluate(`new Promise((done) => { + const rows = []; + const step = () => { + const player = __gm.gameVideo('${id('v')}'); + rows.push({ time: ${clock}, drawn: __gm.gameDrawnRect('${id('v')}') !== null, + at: player?.time ?? null, paused: player?.paused ?? null }); + if (rows.length >= 240 || !__gm.world()) return done(JSON.stringify(rows)); + requestAnimationFrame(step); + }; + step(); + })`), + ); + const first = rows.findIndex((row) => row.drawn); + const moving = rows.filter((row) => row.drawn && row.time > 0).slice(0, 3); + check( + 'a clip that starts later is waited for, then plays from its first step on', + first >= 0 && + rows.slice(0, first).every((row) => row.time === 0) && + Math.abs(rows[first].at - 0.8) < 0.02 && + moving.length === 3 && + moving.every((row) => row.paused === false), + JSON.stringify({ first, shown: rows[first], moving }), + ); + await stop(); + + // 68f. A recording runs when Play composition starts the cut: no frame is taken while the Game + // tab leaves the item's video out – it recorded the empty frames of the wait before. A clip + // that starts elsewhere, so a new player loads: the one of 68e is still there, at its picture. + await mutate(` + const video = __gm.state.project.assets.find((a) => a.id === 'cp68-video'); + video.clips.push({ id: 'cp68-later', name: 'Later', props: { start: 1.2, end: 0 } }); + __gm.state.project.assets.find((a) => a.id === 'cp68-screen').props.texture = 'cp68-later';`); + const flow = structuredClone(graph); + flow.nodes.push( + node('record', 'StartRecording', 300, -150, { name: 'cp68 take', resolution: 'view', fps: '60' }), + node('key', 'OnKeyDown', 0, 450, { key: 'p' }), + ); + flow.edges = flow.edges.filter((e) => e.id !== 'cp68-e1'); + flow.edges.push(link('e4', 'start', 'exec', 'record', 'exec'), link('e5', 'key', 'exec', 'play', 'exec')); + await mutate( + `__gm.state.project.assets.find((a) => a.id === 'cp68-director').graph = ${JSON.stringify(flow)};`, + ); + const { uploads, takes } = await json('__gm.recording()'); + const take = `__gm.recording().takes[${takes.length}]`; + await play(); + await waitUntil(`${take}?.state === 'recording' && ${take}.lastFrameAt !== null`, { + label: 'the recording takes frames', + }); + // Sampled every frame in the page, from before the key on. + await evaluate(`{ + const rows = (window.cp68Rows = []); + const step = () => { + const exists = (__gm.world()?.entities ?? []).some((e) => e.objectId === '${id('v')}'); + rows.push({ exists, drawn: __gm.gameDrawnRect('${id('v')}') !== null, taken: ${take}?.lastFrameAt ?? null }); + const drawn = rows.findIndex((row) => row.drawn); + if ((drawn < 0 || rows.length <= drawn + 20) && rows.length < 400) requestAnimationFrame(step); + }; + requestAnimationFrame(step); + }`); + await press('p'); + await waitUntil( + `window.cp68Rows.length >= 400 || (window.cp68Rows.some((row) => row.drawn) && window.cp68Rows.length > window.cp68Rows.findIndex((row) => row.drawn) + 20)`, + { label: 'the item is drawn' }, + ); + const takeRows = await json('window.cp68Rows'); + await stop(); + // The frames in which the item was there and its video still left out. + const missing = takeRows.filter((row) => row.exists && !row.drawn); + const shown = takeRows.findIndex((row) => row.drawn); + check( + 'a recording takes no frame while a composition waits for its video, and goes on after', + missing.length >= 2 && + missing.every((row) => row.taken === missing[0].taken) && + shown >= 0 && + takeRows.at(-1).taken > missing.at(-1).taken, + JSON.stringify({ + missing: missing.length, + takenWhileMissing: new Set(missing.map((row) => row.taken)).size - 1, + shown, + after: + takeRows.at(-1).taken !== null && missing.length > 0 + ? Math.round(takeRows.at(-1).taken - missing.at(-1).taken) + : null, + }), + ); + await waitUntil( + `__gm.recording().uploads.length > ${uploads.length} && ['added', 'failed'].includes(__gm.recording().uploads.at(-1).state)`, + { timeout: 60_000, label: 'the recording is in the project' }, + ); + const madeId = (await json('__gm.recording()')).uploads.at(-1).assetId ?? ''; + + await mutate(` + const p = __gm.state.project; + p.assets = p.assets.filter((a) => !a.id.startsWith('cp68-') && a.id !== ${JSON.stringify(madeId)}); const scene = __gm.state.activeScene; scene.objects = scene.objects.filter((o) => !o.id.startsWith('cp68-'));`); }