From e191dc4b2ed48391bf61c0425bb1f4b2a929b14e Mon Sep 17 00:00:00 2001 From: MelekTaus Date: Sun, 11 Oct 2026 21:11:05 +0200 Subject: [PATCH] Record the game view from a behavior into a video asset (#405) A game can now film itself. A behavior starts and stops a take, and the editor turns it into a video asset with the run's picture and the sound the player hears. The exported game hands the file over as a download, since there is no project to keep it in. - New action nodes `Start recording` (Name, Resolution, FPS) and `Stop recording`. An empty name becomes `Recording `. - Resolution is `Game view`, `720p` or `1080p`. For the fixed sizes the height is set and the width follows the camera. - The picture is the camera only, without hints, overlays or the letterbox. It is drawn into a render texture of the target size, so a smaller window still gives full 1080p. - `PictureReader` reads the texture back through pixel buffers with a fence. The editor never waits on the GPU, and a frame with all three buffers busy is dropped and counted. - Nothing of the take reaches the visible canvas. A first version drew tiles into it and copied them out, and with the low-latency canvas that midway state flickered on screen. - The sound is tapped behind the master limiter in the FX worklet and encoded as Opus. - H.264 is chosen when the browser has it, VP9 otherwise. A small own Matroska writer packs both tracks, no new dependency. It names the frame rate and BT.709 so ffmpeg keeps both. - A second `Start recording` is ignored with a warning in the log. A take goes on through `Load scene` and a restart, pauses with the editor and is saved when the run ends. It stops at 2 GB. - The editor uploads the take on the existing video route and makes the asset once the run is over. The video tab gets a Download button for the converted MP4. - `downloadBlob` in `src/ui` replaces the song editor's own copy. Findings: - `On start` runs before the game screen knows the world. A take therefore fixes its size on the first frame that has a camera. - Without a frame rate in the file, ffmpeg turned 30 fps into 59.94. - The worker's poster JPEG is off in colour for every upload (red 255 comes out as 229, BT.601). Not touched here. Tests: - Unit recording + matroska: 21 green, plus the node catalog cases. The Matroska test reads the file back with its own EBML reader. - e2e 67 (new) on melektaus-atx: ALL PASSED. 1080p30 from a 1280x800 window gives a 1920x1080 MP4 with sound and no dropped frame. Every 4th pixel is checked, a green quarter catches an upside-down picture. A take ended by Esc is saved with the timestamp name. - e2e 60j: the exported game downloads an `.mkv` with EBML magic. - The flicker is gone on the user's machine without `?vsync`. - Gate on melektaus-atx: CHECK EXIT 0, 299 files, 4586 tests. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- src/core/graph/NodeContext.ts | 5 + src/core/graph/nodes/action/recordingNodes.ts | 66 +++ src/core/graph/nodes/index.ts | 3 + src/core/recording/matroska.ts | 374 ++++++++++++ src/core/recording/recording.ts | 177 ++++++ src/editor/Editor.ts | 14 + src/editor/PlaySession.ts | 43 +- src/editor/assetImport.ts | 2 +- src/editor/debug/videoHooks.ts | 2 + src/editor/recordingUploads.ts | 84 +++ src/editor/song/SongEditor.ts | 9 +- src/editor/video/VideoEditor.ts | 17 +- src/editor/videoImport.ts | 7 +- src/editor/viewport/GameViewport.ts | 19 +- src/player/GameScreen.ts | 6 +- src/player/Player.ts | 25 +- src/runtime/GameRun.ts | 3 + src/runtime/systems/BehaviorSystem.ts | 10 +- src/runtime/systems/behaviorContext.ts | 14 + src/ui/download.ts | 14 + src/ui/index.ts | 1 + src/viewport/GameRecorder.ts | 532 ++++++++++++++++++ src/viewport/PictureReader.ts | 102 ++++ src/viewport/SoundLibrary.ts | 19 + src/viewport/VideoLibrary.ts | 7 +- src/viewport/WorldViewport.ts | 61 +- src/viewport/fxWorklet.ts | 49 ++ tests/core/matroska.test.ts | 202 +++++++ tests/core/node-catalog.test.ts | 15 + tests/core/recording.test.ts | 132 +++++ tests/helpers.ts | 6 + tools/e2e/60-release.mjs | 30 +- tools/e2e/67-recording.mjs | 183 ++++++ tools/e2e/index.mjs | 2 + 35 files changed, 2211 insertions(+), 26 deletions(-) create mode 100644 src/core/graph/nodes/action/recordingNodes.ts create mode 100644 src/core/recording/matroska.ts create mode 100644 src/core/recording/recording.ts create mode 100644 src/editor/recordingUploads.ts create mode 100644 src/ui/download.ts create mode 100644 src/viewport/GameRecorder.ts create mode 100644 src/viewport/PictureReader.ts create mode 100644 tests/core/matroska.test.ts create mode 100644 tests/core/recording.test.ts create mode 100644 tools/e2e/67-recording.mjs diff --git a/README.md b/README.md index 4c1927c1..41fffb44 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ npm run build # writes dist/ and regenerates index.html; the folder is wh The browser suite and the three example games need a machine with a GPU: ```bash -node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2010 checks (GM_STEPS=20 for one step) +node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2016 checks (GM_STEPS=20 for one step) # every tool signs in first – see the Accounts page node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs diff --git a/src/core/graph/NodeContext.ts b/src/core/graph/NodeContext.ts index b93395e5..f2f14d3d 100644 --- a/src/core/graph/NodeContext.ts +++ b/src/core/graph/NodeContext.ts @@ -2,6 +2,7 @@ import type { AnimationPlayer, EaseShare } from '../model/animation'; import type { Mat } from '../model/matrix'; import type { Position, PropertyBag } from '../model/Properties'; import type { FxParams } from '../soundfx/nodes'; +import type { RecordingSettings } from '../recording/recording'; import { type ChainTransform, composeAlongChain } from '../model/walk'; import { enabledAlongChain } from '../properties/enabled'; import { localMatrixOf, positionOf, transformOf } from '../properties/transform'; @@ -391,4 +392,8 @@ export interface NodeContext { isPaused(): boolean; /** From the next frame on, only behaviors that run while paused are ticked. */ setPaused(paused: boolean): void; + /** Starts recording the game view (#405); where it cannot, the log says why. */ + startRecording(settings: RecordingSettings): void; + /** Ends the recording and saves it; without one, only the log hears of it. */ + stopRecording(): void; } diff --git a/src/core/graph/nodes/action/recordingNodes.ts b/src/core/graph/nodes/action/recordingNodes.ts new file mode 100644 index 00000000..03e7dc65 --- /dev/null +++ b/src/core/graph/nodes/action/recordingNodes.ts @@ -0,0 +1,66 @@ +import { RECORDING_FPS, RECORDING_RESOLUTIONS, recordingSettingsOf } from '../../../recording/recording'; +import type { NodeDef, PortDef } from '../../NodeRegistry'; +import { asString } from '../../values'; +import { exec } from '../shared'; + +const choicePort = ( + key: string, + label: string, + options: readonly { value: string; label: string }[], +): PortDef => ({ + key, + label: key, + type: 'string', + default: options[0]!.value, + field: { key, label, kind: 'enum', default: options[0]!.value, options: [...options] }, +}); + +export const StartRecording: NodeDef = { + type: 'StartRecording', + label: 'Start recording', + category: 'action', + description: + 'Records the game view as a video, with what the master plays: only the picture of the camera, ' + + 'none of what lies over it. "name" is what the recording is saved as; left empty, it is named ' + + 'after the moment it starts. "resolution" Game view records at the size the picture has on ' + + 'screen when it starts, 720p and 1080p at that height with the width of the camera. The recording ' + + 'goes on through Load scene and Restart game, holds while the editor pauses the run, and ends with ' + + 'Stop recording or with the run. In the editor it becomes a video asset of the project, once the ' + + 'server has converted it; a released game downloads it as an MKV file. A second start while one ' + + 'records is ignored and says so in the log.', + inputs: [ + exec(), + { + key: 'name', + label: 'name', + type: 'string', + field: { key: 'name', label: 'Name', kind: 'text', default: '' }, + }, + choicePort('resolution', 'Resolution', RECORDING_RESOLUTIONS), + choicePort('fps', 'FPS', RECORDING_FPS), + ], + outputs: [exec()], + params: [], + run: (ctx, inputs) => { + ctx.startRecording( + recordingSettingsOf(asString(inputs.name), asString(inputs.resolution), asString(inputs.fps)), + ); + return { next: 'exec' }; + }, +}; + +export const StopRecording: NodeDef = { + type: 'StopRecording', + label: 'Stop recording', + category: 'action', + description: + 'Ends the recording Start recording began and saves it. Without one running it does nothing but ' + + 'say so in the log.', + inputs: [exec()], + outputs: [exec()], + params: [], + run: (ctx) => { + ctx.stopRecording(); + return { next: 'exec' }; + }, +}; diff --git a/src/core/graph/nodes/index.ts b/src/core/graph/nodes/index.ts index d7728595..1fa31105 100644 --- a/src/core/graph/nodes/index.ts +++ b/src/core/graph/nodes/index.ts @@ -33,6 +33,7 @@ import { StopSong } from './action/StopSong'; import { SetZ } from './action/SetZ'; import { Spawn } from './action/Spawn'; import { StopGame } from './action/StopGame'; +import { StartRecording, StopRecording } from './action/recordingNodes'; import { StopSound } from './action/StopSound'; import { CellToWorld, SetTile, TileAt, TileLiteral, WorldToCell } from './action/tileNodes'; import { Abs } from './data/Abs'; @@ -323,6 +324,8 @@ export const BUILT_IN_NODES: readonly NodeDef[] = [ Log, StopGame, RestartGame, + StartRecording, + StopRecording, PauseGame, ResumeGame, IsGamePaused, diff --git a/src/core/recording/matroska.ts b/src/core/recording/matroska.ts new file mode 100644 index 00000000..ca35d6bc --- /dev/null +++ b/src/core/recording/matroska.ts @@ -0,0 +1,374 @@ +/** + * Matroska (`.mkv`), the file a recording of the game view is made into (#405): the encoded frames + * of a `VideoEncoder` and the Opus packets of an `AudioEncoder`, in clusters as they come. Only the + * head waits for the end – it holds what the encoders say about themselves once they ran, and the + * length – so the clusters can leave the writer long before. + */ +import { OPUS_RATE } from '../music/ogg'; + +export const AVC_CODEC = 'V_MPEG4/ISO/AVC'; +export const VP9_CODEC = 'V_VP9'; +export const OPUS_CODEC = 'A_OPUS'; + +/** Milliseconds per tick of a block's time: the usual 1 ms, so a cluster spans 32 s at most. */ +const TIMESTAMP_SCALE = 1_000_000; +/** What a cluster may span before the next one starts, inside the ±32767 ticks of a block. */ +const CLUSTER_SPAN_MS = 30_000; +const CLUSTER_MOST_BYTES = 8 * 1024 * 1024; +/** How far one track may run ahead of a silent other before its blocks are written anyway. */ +const INTERLEAVE_WAIT_MS = 1000; +/** What Opus asks a player to decode before a point it seeks to (RFC 7845: 80 ms). */ +const OPUS_PRE_ROLL_NS = 80_000_000; + +const ID = { + ebml: 0x1a45dfa3, + ebmlVersion: 0x4286, + ebmlReadVersion: 0x42f7, + ebmlMaxIdLength: 0x42f2, + ebmlMaxSizeLength: 0x42f3, + docType: 0x4282, + docTypeVersion: 0x4287, + docTypeReadVersion: 0x4285, + segment: 0x18538067, + info: 0x1549a966, + timestampScale: 0x2ad7b1, + duration: 0x4489, + muxingApp: 0x4d80, + writingApp: 0x5741, + tracks: 0x1654ae6b, + trackEntry: 0xae, + trackNumber: 0xd7, + trackUid: 0x73c5, + trackType: 0x83, + flagLacing: 0x9c, + codecId: 0x86, + codecPrivate: 0x63a2, + defaultDuration: 0x23e383, + codecDelay: 0x56aa, + seekPreRoll: 0x56bb, + video: 0xe0, + pixelWidth: 0xb0, + pixelHeight: 0xba, + colour: 0x55b0, + matrixCoefficients: 0x55b1, + range: 0x55b9, + transferCharacteristics: 0x55ba, + primaries: 0x55bb, + audio: 0xe1, + samplingFrequency: 0xb5, + channels: 0x9f, + cluster: 0x1f43b675, + timestamp: 0xe7, + simpleBlock: 0xa3, +} as const; + +export interface MatroskaVideo { + codec: typeof AVC_CODEC | typeof VP9_CODEC; + width: number; + height: number; + /** H.264 needs its `avcC`, the `description` the encoder hands out with its first frame. */ + private: Uint8Array | null; + /** The frame rate asked for: without it a player guesses one from the times of the frames. */ + fps: number; + colour: MatroskaColour; +} + +/** How the pixels are to be read, by the numbers of ITU-T H.273 that Matroska takes. */ +export interface MatroskaColour { + matrix: number; + transfer: number; + primaries: number; + fullRange: boolean; +} + +/** What Chrome's encoders make of a canvas when they do not say: BT.709 in the broadcast range. */ +const BT709: MatroskaColour = { matrix: 1, transfer: 1, primaries: 1, fullRange: false }; + +const MATRICES: Record = { rgb: 0, bt709: 1, bt470bg: 5, smpte170m: 6, 'bt2020-ncl': 9 }; +const TRANSFERS: Record = { + bt709: 1, + smpte170m: 6, + linear: 8, + 'iec61966-2-1': 13, + pq: 16, + hlg: 18, +}; +const PRIMARIES: Record = { bt709: 1, bt470bg: 5, smpte170m: 6, bt2020: 9, smpte432: 12 }; + +/** + * The colour a `VideoColorSpaceInit` names, as a WebCodecs encoder reports it; what it leaves out is + * taken from BT.709. Without it a player falls back on its own guess, and ffmpeg's is BT.601: a red + * of 255 came back as 233. + */ +export function colourOf( + space: { + matrix?: string | null; + transfer?: string | null; + primaries?: string | null; + fullRange?: boolean | null; + } | null, +): MatroskaColour { + return { + matrix: MATRICES[space?.matrix ?? ''] ?? BT709.matrix, + transfer: TRANSFERS[space?.transfer ?? ''] ?? BT709.transfer, + primaries: PRIMARIES[space?.primaries ?? ''] ?? BT709.primaries, + fullRange: space?.fullRange ?? BT709.fullRange, + }; +} + +export interface MatroskaAudio { + channels: number; + /** The OpusHead, the encoder's `description`. */ + private: Uint8Array | null; + /** Samples at 48 kHz the decoder throws away at the start, as the OpusHead says. */ + preSkip: number; +} + +export interface MatroskaHead { + video: MatroskaVideo; + audio: MatroskaAudio | null; + durationMs: number; + app: string; +} + +/** The track a block belongs to: the video is always the first, the sound the second. */ +export type MatroskaTrack = 1 | 2; + +interface Block { + track: MatroskaTrack; + ms: number; + key: boolean; + data: Uint8Array; +} + +/** + * Takes the frames of both tracks as the encoders hand them out and writes them in clusters, in the + * order of their time. The two tracks arrive apart, so a block waits until the other track has come + * as far, or until it is `INTERLEAVE_WAIT_MS` behind what that track already brought. + */ +export class MatroskaWriter { + private readonly waiting: [Block[], Block[]] = [[], []]; + /** The latest time each track has brought, null before its first block. */ + private readonly latest: [number | null, number | null] = [null, null]; + private cluster: { ms: number; blocks: Uint8Array[]; bytes: number } | null = null; + private done: Uint8Array[] = []; + private written = 0; + + /** `tracks` 1 is a recording without sound. */ + constructor(private readonly tracks: 1 | 2) {} + + /** Bytes of the clusters so far, the open one included: what the file is without its head. */ + get bytes(): number { + return this.written + (this.cluster?.bytes ?? 0); + } + + add(track: MatroskaTrack, timestampUs: number, key: boolean, data: Uint8Array): void { + const ms = Math.round(timestampUs / 1000); + this.waiting[track - 1]!.push({ track, ms, key, data }); + const latest = this.latest[track - 1] ?? null; + this.latest[track - 1] = latest === null ? ms : Math.max(latest, ms); + this.interleave(false); + } + + /** The clusters finished since the last call, in file order. */ + take(): Uint8Array[] { + const done = this.done; + this.done = []; + return done; + } + + /** Writes what still waits and closes the last cluster; the head goes in front of everything taken. */ + finish(head: MatroskaHead): { head: Uint8Array; rest: Uint8Array[] } { + this.interleave(true); + this.closeCluster(); + return { head: headOf(head, this.written), rest: this.take() }; + } + + private interleave(all: boolean): void { + for (;;) { + const next = this.nextBlock(all); + if (!next) return; + this.write(next); + } + } + + /** The earliest waiting block, if it may be written yet. */ + private nextBlock(all: boolean): Block | null { + const [first, second] = this.waiting; + const a = first[0]; + const b = this.tracks === 2 ? second[0] : undefined; + const earliest = a && b ? (b.ms < a.ms ? b : a) : (a ?? b); + if (!earliest) return null; + if (!all && this.tracks === 2) { + const other = this.latest[earliest.track === 1 ? 1 : 0]; + const own = this.latest[earliest.track - 1] ?? earliest.ms; + const caughtUp = other !== null && other >= earliest.ms; + if (!caughtUp && own - earliest.ms < INTERLEAVE_WAIT_MS) return null; + } + this.waiting[earliest.track - 1]!.shift(); + return earliest; + } + + private write(block: Block): void { + const open = this.cluster; + const startsCluster = + !open || + (block.track === 1 && block.key && open.blocks.length > 0) || + Math.abs(block.ms - open.ms) > CLUSTER_SPAN_MS || + open.bytes > CLUSTER_MOST_BYTES; + if (startsCluster) { + this.closeCluster(); + this.cluster = { ms: block.ms, blocks: [], bytes: 0 }; + } + const cluster = this.cluster!; + const header = new Uint8Array(4); + header[0] = 0x80 | block.track; + new DataView(header.buffer).setInt16(1, block.ms - cluster.ms); + header[3] = block.key ? 0x80 : 0; + const element = concat([ + idBytes(ID.simpleBlock), + sizeBytes(header.length + block.data.length), + header, + block.data, + ]); + cluster.blocks.push(element); + cluster.bytes += element.length; + } + + private closeCluster(): void { + const cluster = this.cluster; + this.cluster = null; + if (!cluster || cluster.blocks.length === 0) return; + const body = [uint(ID.timestamp, Math.max(0, cluster.ms)), ...cluster.blocks]; + const bytes = master(ID.cluster, body); + this.done.push(bytes); + this.written += bytes.length; + } +} + +/** The EBML header, and the segment's own head with its size, info and tracks. */ +function headOf(head: MatroskaHead, clusterBytes: number): Uint8Array { + const ebml = master(ID.ebml, [ + uint(ID.ebmlVersion, 1), + uint(ID.ebmlReadVersion, 1), + uint(ID.ebmlMaxIdLength, 4), + uint(ID.ebmlMaxSizeLength, 8), + text(ID.docType, 'matroska'), + uint(ID.docTypeVersion, 4), + uint(ID.docTypeReadVersion, 2), + ]); + const info = master(ID.info, [ + uint(ID.timestampScale, TIMESTAMP_SCALE), + float(ID.duration, head.durationMs), + text(ID.muxingApp, head.app), + text(ID.writingApp, head.app), + ]); + const { video, audio } = head; + const entries = [ + master(ID.trackEntry, [ + uint(ID.trackNumber, 1), + uint(ID.trackUid, 1), + uint(ID.trackType, 1), + uint(ID.flagLacing, 0), + text(ID.codecId, video.codec), + ...(video.private ? [binary(ID.codecPrivate, video.private)] : []), + uint(ID.defaultDuration, Math.round(1e9 / video.fps)), + master(ID.video, [ + uint(ID.pixelWidth, video.width), + uint(ID.pixelHeight, video.height), + master(ID.colour, [ + uint(ID.matrixCoefficients, video.colour.matrix), + uint(ID.range, video.colour.fullRange ? 2 : 1), + uint(ID.transferCharacteristics, video.colour.transfer), + uint(ID.primaries, video.colour.primaries), + ]), + ]), + ]), + ]; + if (audio) + entries.push( + master(ID.trackEntry, [ + uint(ID.trackNumber, 2), + uint(ID.trackUid, 2), + uint(ID.trackType, 2), + uint(ID.flagLacing, 0), + text(ID.codecId, OPUS_CODEC), + ...(audio.private ? [binary(ID.codecPrivate, audio.private)] : []), + uint(ID.codecDelay, Math.round((audio.preSkip * 1e9) / OPUS_RATE)), + uint(ID.seekPreRoll, OPUS_PRE_ROLL_NS), + master(ID.audio, [float(ID.samplingFrequency, OPUS_RATE), uint(ID.channels, audio.channels)]), + ]), + ); + const tracks = master(ID.tracks, entries); + const segmentSize = info.length + tracks.length + clusterBytes; + return concat([ebml, idBytes(ID.segment), sizeBytes(segmentSize, 8), info, tracks]); +} + +/** An element id as it is written: its own marker bits are part of the number. */ +function idBytes(id: number): Uint8Array { + const length = id > 0xffffff ? 4 : id > 0xffff ? 3 : id > 0xff ? 2 : 1; + const out = new Uint8Array(length); + for (let i = 0; i < length; i++) out[i] = Math.floor(id / 256 ** (length - 1 - i)) % 256; + return out; +} + +/** A size as an EBML variable-length integer, in the fewest bytes or in `length` of them. */ +function sizeBytes(size: number, length = sizeLength(size)): Uint8Array { + const out = new Uint8Array(length); + let rest = size; + for (let i = length - 1; i >= 0; i--) { + out[i] = rest % 256; + rest = Math.floor(rest / 256); + } + out[0]! |= 0x80 >> (length - 1); + return out; +} + +/** All ones is "unknown", so a size needs one value more than its bits would hold. */ +function sizeLength(size: number): number { + let length = 1; + while (size >= 2 ** (7 * length) - 1) length++; + return length; +} + +function element(id: number, data: Uint8Array): Uint8Array { + return concat([idBytes(id), sizeBytes(data.length), data]); +} + +function master(id: number, children: readonly Uint8Array[]): Uint8Array { + return element(id, concat(children)); +} + +function uint(id: number, value: number): Uint8Array { + const bytes: number[] = []; + let rest = Math.max(0, Math.round(value)); + do { + bytes.unshift(rest % 256); + rest = Math.floor(rest / 256); + } while (rest > 0); + return element(id, Uint8Array.from(bytes)); +} + +function float(id: number, value: number): Uint8Array { + const data = new Uint8Array(8); + new DataView(data.buffer).setFloat64(0, value); + return element(id, data); +} + +function text(id: number, value: string): Uint8Array { + return element(id, new TextEncoder().encode(value)); +} + +function binary(id: number, value: Uint8Array): Uint8Array { + return element(id, value); +} + +function concat(parts: readonly Uint8Array[]): Uint8Array { + const out = new Uint8Array(parts.reduce((sum, part) => sum + part.length, 0)); + let at = 0; + for (const part of parts) { + out.set(part, at); + at += part.length; + } + return out; +} diff --git a/src/core/recording/recording.ts b/src/core/recording/recording.ts new file mode 100644 index 00000000..12c68a87 --- /dev/null +++ b/src/core/recording/recording.ts @@ -0,0 +1,177 @@ +/** + * Recording the game view (#405): what `Start recording` asks for, and the rules a recording is made + * by – its size, its name, which H.264 it is, when a frame is taken and where the picture lands. + * The browser's side, encoders and canvas, is `viewport/GameRecorder.ts`. + */ + +export type RecordingResolution = 'view' | '720p' | '1080p'; + +export interface RecordingSettings { + /** Empty: named after the moment it starts (`recordingName`). */ + name: string; + resolution: RecordingResolution; + fps: 30 | 60; +} + +export const RECORDING_RESOLUTIONS: readonly { value: RecordingResolution; label: string }[] = [ + { value: 'view', label: 'Game view' }, + { value: '720p', label: '720p' }, + { value: '1080p', label: '1080p' }, +]; + +export const RECORDING_FPS = [ + { value: '60', label: '60' }, + { value: '30', label: '30' }, +]; + +/** What a run hands its graphs to record with; each answer is a line for the log, '' for none. */ +export interface RecordingSink { + start(settings: RecordingSettings): string; + stop(): string; +} + +/** The upload takes 2 GB (`MAX_VIDEO_BYTES`); a recording stops a little before, its head still to come. */ +export const RECORDING_MOST_BYTES = 2048 * 1024 * 1024 - 16 * 1024 * 1024; + +/** What `Game view` records at most: a 4K picture, larger ones are scaled down to it. */ +const MOST_WIDTH = 3840; +const MOST_HEIGHT = 2160; +/** Bits per pixel of a frame: 10 Mbit/s at 1080p60. Flat game graphics need less than film. */ +const BITS_PER_PIXEL = 0.08; +const KEY_FRAME_SECONDS = 2; + +export function recordingSettingsOf( + name: string, + resolution: string, + fps: string | number, +): RecordingSettings { + const known = RECORDING_RESOLUTIONS.find((r) => r.value === resolution); + return { name: name.trim(), resolution: known?.value ?? 'view', fps: Number(fps) === 30 ? 30 : 60 }; +} + +/** The name a recording is saved under: the one asked for, else `Recording 2026-10-11 16:42:05`. */ +export function recordingName(asked: string, at: Date): string { + if (asked.trim()) return asked.trim(); + const two = (n: number) => String(n).padStart(2, '0'); + const day = `${at.getFullYear()}-${two(at.getMonth() + 1)}-${two(at.getDate())}`; + return `Recording ${day} ${two(at.getHours())}:${two(at.getMinutes())}:${two(at.getSeconds())}`; +} + +/** + * The size of the frames, even on both sides as the encoders want it. `shown` is the camera's + * picture as it lies on screen and `whole` all of it, both in device pixels: they differ only while + * a fixed scale cuts the picture to the view. `Game view` records what is shown at its size; 720p + * and 1080p hold the height and take the width from the camera. + */ +export function recordingSize( + resolution: RecordingResolution, + shown: { w: number; h: number }, + whole: { w: number; h: number }, +): { w: number; h: number } { + if (resolution === 'view') { + const fit = Math.min(1, MOST_WIDTH / shown.w, MOST_HEIGHT / shown.h); + return { w: even(shown.w * fit), h: even(shown.h * fit) }; + } + const h = resolution === '720p' ? 720 : 1080; + return { w: even((h * whole.w) / whole.h), h }; +} + +function even(value: number): number { + return Math.max(2, Math.round(value / 2) * 2); +} + +/** H.264 levels by the frame size and macroblocks a second they allow (ITU-T H.264, table A-1). */ +const AVC_LEVELS: readonly { level: number; frame: number; rate: number }[] = [ + { level: 0x1f, frame: 3600, rate: 108_000 }, + { level: 0x20, frame: 5120, rate: 216_000 }, + { level: 0x28, frame: 8192, rate: 245_760 }, + { level: 0x2a, frame: 8704, rate: 522_240 }, + { level: 0x32, frame: 22_080, rate: 589_824 }, + { level: 0x33, frame: 36_864, rate: 983_040 }, + { level: 0x34, frame: 36_864, rate: 2_073_600 }, + { level: 0x3c, frame: 139_264, rate: 4_177_920 }, +]; + +/** The codec string of H.264 High at the lowest level that takes these frames. */ +export function avcCodec(w: number, h: number, fps: number): string { + const frame = Math.ceil(w / 16) * Math.ceil(h / 16); + const found = AVC_LEVELS.find((l) => frame <= l.frame && frame * fps <= l.rate) ?? AVC_LEVELS.at(-1)!; + return `avc1.6400${found.level.toString(16).padStart(2, '0')}`; +} + +/** VP9 profile 0 at 8 bits; level 4.1 takes 1080p60, 5.1 4K60. */ +export function vp9Codec(w: number, h: number): string { + return w * h <= 2_228_224 ? 'vp09.00.41.08' : 'vp09.00.51.08'; +} + +export function videoBitrate(w: number, h: number, fps: number): number { + return Math.round(w * h * fps * BITS_PER_PIXEL); +} + +/** Whether the `index`th frame taken (from 0) starts a group, so a player can seek to it. */ +export function isKeyFrame(index: number, fps: number): boolean { + return index % (fps * KEY_FRAME_SECONDS) === 0; +} + +/** + * The time of a recording in milliseconds, from its first frame: real time, minus what the editor + * held it paused. A game that pauses itself is still seen, so that is not taken out. + */ +export class RecordingClock { + private origin: number | null = null; + private pausedAt: number | null = null; + private paused = 0; + + get started(): boolean { + return this.origin !== null; + } + + start(now: number): void { + this.origin = now; + } + + pause(now: number): void { + this.pausedAt ??= now; + } + + resume(now: number): void { + if (this.pausedAt === null) return; + this.paused += now - this.pausedAt; + this.pausedAt = null; + } + + at(now: number): number { + if (this.origin === null) return 0; + return Math.max(0, (this.pausedAt ?? now) - this.origin - this.paused); + } +} + +/** + * Which drawn frames a recording takes: about `fps` a second whatever the screen draws. A frame a + * little early still counts, so 60 at 60 Hz takes every one however unevenly they come. + */ +export class FrameGate { + private next = 0; + + constructor(private readonly fps: number) {} + + take(ms: number): boolean { + const period = 1000 / this.fps; + if (ms < this.next - period / 4) return false; + this.next = Math.max(this.next + period, ms + period / 2); + return true; + } +} + +/** + * Where the view's local CSS pixels land when `area` of it (a centre and a size) fills a frame of + * `size` pixels: the recording draws into a texture of that size, whatever the window's. + */ +export function pictureTransform( + area: { x: number; y: number; w: number; h: number }, + size: { w: number; h: number }, +): { a: number; d: number; tx: number; ty: number } { + const sx = size.w / area.w; + const sy = size.h / area.h; + return { a: sx, d: sy, tx: size.w / 2 - area.x * sx, ty: size.h / 2 - area.y * sy }; +} diff --git a/src/editor/Editor.ts b/src/editor/Editor.ts index c183c6a5..6c06f309 100644 --- a/src/editor/Editor.ts +++ b/src/editor/Editor.ts @@ -76,6 +76,8 @@ import { HierarchyPanel } from './panels/HierarchyPanel'; import { TilemapPanel } from './panels/TilemapPanel'; import { Toolbar } from './panels/Toolbar'; import { type PlayHost, PlaySession, type WorldSnapshot } from './PlaySession'; +import { type RecordingUpload, RecordingUploads } from './recordingUploads'; +import type { MadeRecording } from '../viewport/GameRecorder'; import { ProjectSession } from './ProjectSession'; import { ParticlesEditor } from './particles/ParticlesEditor'; import { ShaderEditor } from './shader/ShaderEditor'; @@ -313,6 +315,7 @@ export class Editor implements PlayHost { /** Frame rate, refreshed twice a second; null until the first measurement. */ fps: number | null = null; private readonly play: PlaySession; + private readonly recordings: RecordingUploads; private readonly editorRun: EditorRun; private readonly projects: ProjectSession; private readonly leftColumn: VStack; @@ -400,6 +403,7 @@ export class Editor implements PlayHost { this.hierarchy = new HierarchyPanel(state); this.inspector = new Inspector(state); this.tilemapPanel = new TilemapPanel(state); + this.recordings = new RecordingUploads({ state, videos }); this.play = new PlaySession(state, this); this.editorRun = new EditorRun(state, { // The game view is the one place where no key means anything to the editor, so that is where @@ -1341,6 +1345,7 @@ export class Editor implements PlayHost { this.editorRun.tick(step); this.viewport.animate(sceneDraws); this.gameView.animate(gameDraws); + this.play.recordFrame(); this.console.refresh(); if (this.shows('animation')) this.animationEditor.tick(step); if (this.shows('tile')) this.tileEditor.tick(step); @@ -1480,4 +1485,13 @@ export class Editor implements PlayHost { performance: this.play.performance(), }; } + + /** The recordings of the game view (#405) and what became of them on the way into the project. */ + recordingInfo(): ReturnType & { uploads: RecordingUpload[] } { + return { ...this.play.recordingInfo(), uploads: this.recordings.info() }; + } + + recordingSaved(recording: MadeRecording): void { + this.recordings.add(recording); + } } diff --git a/src/editor/PlaySession.ts b/src/editor/PlaySession.ts index cf5432a4..e397afce 100644 --- a/src/editor/PlaySession.ts +++ b/src/editor/PlaySession.ts @@ -14,8 +14,9 @@ import { gameClock } from '../core/time'; import { loadPhysics, readyPhysics } from '../runtime/physics/engine'; import { Input } from '../runtime/systems/Input'; import type { World } from '../runtime/World'; -import { ui } from '../ui'; +import { onFrame, ui } from '../ui'; import { soundLibrary } from '../viewport/SoundLibrary'; +import { GameRecorder, type MadeRecording } from '../viewport/GameRecorder'; import { VIDEO_WAIT_MS, videoLibrary } from '../viewport/VideoLibrary'; import { FrameMeter, type FrameTimes } from './frameMeter'; import { saveArea } from './saves'; @@ -55,6 +56,8 @@ export interface PlayHost { showGame(): void; /** Leave the game view if it is the one on screen. */ leaveGame(): void; + /** A recording of the game view is made (#405): it goes into the project. */ + recordingSaved(recording: MadeRecording): void; } export class PlaySession { @@ -74,11 +77,21 @@ export class PlaySession { * frames of a cut showed white without them. */ private videoWait: number | null = null; + /** Records the game view for `Start recording` (#405); one for the editor, so it outlives a restart. */ + private readonly recorder: GameRecorder; constructor( private readonly state: EditorState, private readonly host: PlayHost, ) { + this.recorder = new GameRecorder({ + view: () => host.gameView, + log: (line) => { + this.world?.log.push(line); + console.warn('[recording]', line); + }, + saved: (recording) => host.recordingSaved(recording), + }); // An object picked during a run is looked at now: the hierarchy has to list it to show it picked. state.on('selection', () => { if (this.world && this.panels.holdsStructure) this.follow(this.panels.flush()); @@ -121,6 +134,7 @@ export class PlaySession { // on mid-note, rather than playing to their end over a still image. soundLibrary.setSuspended(this.state.paused); videoLibrary.setPaused(this.state.paused); + this.recorder.setPaused(this.state.paused); ui().focus.blur(); this.state.setNotice(this.playNotice(), 'quiet'); } @@ -173,6 +187,7 @@ export class PlaySession { physics, clock: gameClock, saves: saveArea(runtime.id, 'play'), + recorder: this.recorder, }; this.run = GameRun.start(runtime, scene.id, this.input, services); videoLibrary.startRun(); @@ -237,7 +252,12 @@ export class PlaySession { : 'No camera is switched on – nothing is rendered · Esc stops'; } - stop(): void { + /** `restarting`: a `Restart game` plays on at once, and a recording goes on with it. */ + stop(restarting = false): void { + if (!restarting) { + this.recorder.end(); + this.recorder.setPaused(false); + } // The run's seconds end with it: the editor goes back to the picture of a freshly loaded scene. gameClock.reset(); // And its sounds end with it, looping music included – nothing may outlive the world. @@ -282,7 +302,7 @@ export class PlaySession { return; } if (request === 'restart') { - this.stop(); + this.stop(true); this.play(); return; } @@ -302,6 +322,23 @@ export class PlaySession { this.follow(world.paused && !wasPaused ? this.panels.flush() : update); } + /** + * Called once everything that draws the game view this frame is queued: a recording takes the + * frame after all of it, behind another tab too. + */ + recordFrame(): void { + if (!this.recorder.isRecording || !this.world) return; + onFrame('recording', () => { + if (!this.host.gameView.visible) this.host.gameView.redrawHidden(); + this.recorder.capture(); + }); + } + + /** What became of the recordings (#405, test hook). */ + recordingInfo(): ReturnType { + return this.recorder.info(); + } + /** Brings the panels up to date as far as `update` says; the game's own picture never waits. */ private follow(update: PanelUpdate): void { if (update.structure) { diff --git a/src/editor/assetImport.ts b/src/editor/assetImport.ts index 16b215b1..1e3400bf 100644 --- a/src/editor/assetImport.ts +++ b/src/editor/assetImport.ts @@ -73,7 +73,7 @@ async function importFile( * what the asset is called. The server stores by content hash, so a re-imported file comes back * as the one the project already references; two assets on one file would disagree about it. */ -function adopt( +export function adopt( kind: ImportKind, uploaded: U, nameBase: string, diff --git a/src/editor/debug/videoHooks.ts b/src/editor/debug/videoHooks.ts index 42f49765..8384ee82 100644 --- a/src/editor/debug/videoHooks.ts +++ b/src/editor/debug/videoHooks.ts @@ -36,5 +36,7 @@ export function videoHooks({ state, editor }: HookContext) { videoPickerRow: (file: string) => currentVideoPicker()?.rowGlobal(file) ?? null, /** What each player behind a sprite is doing: its time, whether it runs and is heard. */ videos: () => videoLibrary.debugInfo(), + /** The recordings of the game view (#405): each take, and its way into the project. */ + recording: () => editor.recordingInfo(), }; } diff --git a/src/editor/recordingUploads.ts b/src/editor/recordingUploads.ts new file mode 100644 index 00000000..b47928e3 --- /dev/null +++ b/src/editor/recordingUploads.ts @@ -0,0 +1,84 @@ +import type { UploadedVideo } from '../core/store/VideoStore'; +import type { MadeRecording } from '../viewport/GameRecorder'; +import { adoptVideo, type VideoImportDeps } from './videoImport'; + +/** How often the server is asked whether a recording is converted, as the video picker does. */ +const POLL_MS = 1000; + +type UploadState = 'uploading' | 'converting' | 'added' | 'elsewhere' | 'failed'; + +/** What became of a recording on its way into the project (test hook). */ +export interface RecordingUpload { + name: string; + state: UploadState; + /** The name the server gave it, once it is up. */ + file: string; + assetId: string; + bytes: number; + error: string; +} + +/** + * Brings a recording of the game view into the project (#405): up to the server, which converts it + * like any video, then an asset named after the recording once it is ready. Only while the editor + * is not playing: a run plays a copy of the project that is thrown away, and so would the asset be. + */ +export class RecordingUploads { + private readonly uploads: RecordingUpload[] = []; + + constructor(private readonly deps: VideoImportDeps) {} + + add(recording: MadeRecording): void { + const upload: RecordingUpload = { + name: recording.name, + state: 'uploading', + file: '', + assetId: '', + bytes: recording.blob.size, + error: '', + }; + this.uploads.push(upload); + void this.bring(recording, upload).catch((error: unknown) => { + upload.state = 'failed'; + upload.error = error instanceof Error ? error.message : String(error); + this.deps.state.setNotice(`Could not save the recording "${recording.name}": ${upload.error}`, 'error'); + }); + } + + info(): RecordingUpload[] { + return this.uploads.map((upload) => ({ ...upload })); + } + + private async bring(recording: MadeRecording, upload: RecordingUpload): Promise { + const { state, videos } = this.deps; + const projectId = state.project.id; + state.setNotice(`Uploading the recording "${recording.name}"…`); + let video: UploadedVideo = await videos.upload( + recording.blob, + 'application/octet-stream', + `${recording.name}.mkv`, + ); + upload.file = video.file; + upload.state = 'converting'; + while (video.status !== 'ready') { + if (video.status === 'failed') throw new Error(video.error ?? 'the server could not convert it'); + await wait(POLL_MS); + video = (await videos.list()).find((v) => v.file === upload.file) ?? video; + } + while (state.mode !== 'edit') await wait(POLL_MS); + if (state.project.id !== projectId) { + upload.state = 'elsewhere'; + state.setNotice(`The recording "${recording.name}" is ready – the video picker has it.`); + return; + } + const outcome = adoptVideo(video, recording.name, this.deps); + if (!outcome.assetId) throw new Error(outcome.error ?? 'the project did not take it'); + upload.state = 'added'; + upload.assetId = outcome.assetId; + state.setNotice(`Added the recording "${recording.name}" to the project.`); + } +} + +function wait(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/src/editor/song/SongEditor.ts b/src/editor/song/SongEditor.ts index 40f39a9e..9407ffbd 100644 --- a/src/editor/song/SongEditor.ts +++ b/src/editor/song/SongEditor.ts @@ -26,7 +26,7 @@ import { findSound } from '../../core/model/sound'; import { textOf } from '../../core/model/Properties'; import { assetFxOf } from '../../core/properties/assetFx'; import type { SoundStore } from '../../core/store/SoundStore'; -import { Button, Dialog, Dropdown, Label, Menu, ui, VStack, type Widget } from '../../ui'; +import { Button, Dialog, downloadBlob, Dropdown, Label, Menu, ui, VStack, type Widget } from '../../ui'; import { Music } from '../../viewport/music'; import { MASTER_MIX } from '../../core/music/mixer'; import { soundLibrary } from '../../viewport/SoundLibrary'; @@ -596,10 +596,5 @@ export class SongEditor extends AssetTabEditor { /** Hands the take to the browser as a file, for when the server would not take it. */ function downloadWav(wav: Uint8Array, name: string): void { - const url = URL.createObjectURL(new Blob([wav], { type: 'audio/wav' })); - const link = document.createElement('a'); - link.href = url; - link.download = `${name}.wav`; - link.click(); - setTimeout(() => URL.revokeObjectURL(url), 60_000); + downloadBlob(new Blob([wav], { type: 'audio/wav' }), `${name}.wav`); } diff --git a/src/editor/video/VideoEditor.ts b/src/editor/video/VideoEditor.ts index a66e7348..7629283a 100644 --- a/src/editor/video/VideoEditor.ts +++ b/src/editor/video/VideoEditor.ts @@ -1,5 +1,6 @@ import type { Asset, VideoAsset } from '../../core/model/Asset'; -import { Button } from '../../ui'; +import { Button, downloadFile } from '../../ui'; +import { videoFileUrl } from '../../viewport/VideoLibrary'; import { AssetTabEditor } from '../AssetTabEditor'; import { TOOLBAR_ICON_SVG } from '../panels/toolbarIcons'; import type { EditorState } from '../state'; @@ -15,6 +16,8 @@ const INFO_WIDTH = 220; export class VideoEditor extends AssetTabEditor { /** Public for the check scripts. */ readonly playButton: Button; + /** Saves the MP4 the server made (#405): a recording of the game view is had this way. */ + readonly downloadButton: Button; constructor(state: EditorState) { super(state, { infoWidth: INFO_WIDTH }); @@ -24,10 +27,14 @@ export class VideoEditor extends AssetTabEditor { tooltip: 'Play the picked clip, else the whole video (Space)', onClick: () => this.togglePlay(), }); + this.downloadButton = new Button('Download', { + tooltip: 'Save the video as an MP4 file', + onClick: () => this.download(), + }); const canvas = new VideoCanvas(state); canvas.onTogglePlay = () => this.togglePlay(); canvas.onStopped = () => this.syncPlayButton(); - this.mount(canvas, [this.playButton]); + this.mount(canvas, [this.playButton, this.downloadButton]); state.on('load', () => this.canvas.stop()); state.on('selection', () => this.refresh()); } @@ -50,6 +57,11 @@ export class VideoEditor extends AssetTabEditor { this.syncPlayButton(); } + private download(): void { + const asset = this.asset(); + if (asset?.file) downloadFile(videoFileUrl(asset.file), `${asset.name}.mp4`); + } + /** Only called while the tab is visible. */ tick(): void { this.canvas.tick(); @@ -72,6 +84,7 @@ export class VideoEditor extends AssetTabEditor { protected override syncExtras(asset: VideoAsset | null): void { this.fitButton.setEnabled(Boolean(asset?.file)); + this.downloadButton.setEnabled(Boolean(asset?.file)); this.syncPlayButton(); } diff --git a/src/editor/videoImport.ts b/src/editor/videoImport.ts index 15dea713..dad887eb 100644 --- a/src/editor/videoImport.ts +++ b/src/editor/videoImport.ts @@ -1,6 +1,6 @@ import { CreateVideoCommand } from '../core/commands/assetCommands'; import type { UploadedVideo, VideoStore } from '../core/store/VideoStore'; -import { type ImportKind, type ImportOutcome, importStored } from './assetImport'; +import { adopt, type ImportKind, type ImportOutcome, importStored } from './assetImport'; import type { EditorState } from './state'; /** What the file dialog offers and a drop accepts; ffmpeg on the server has the last word. */ @@ -44,6 +44,11 @@ export function importStoredVideo(video: UploadedVideo, deps: VideoImportDeps): return importStored(VIDEO_KIND, video, deps); } +/** Makes an asset named `name` of a video the server has converted, without opening it (#405). */ +export function adoptVideo(video: UploadedVideo, name: string, deps: VideoImportDeps): ImportOutcome { + return adopt(VIDEO_KIND, video, name, name, deps); +} + /** * Sends the files up one after the other and says how it went; what is done already is made an * asset at once. The rest is converted on the server – `waiting` gets their names, for the picker diff --git a/src/editor/viewport/GameViewport.ts b/src/editor/viewport/GameViewport.ts index 70551f78..7e7c3cc9 100644 --- a/src/editor/viewport/GameViewport.ts +++ b/src/editor/viewport/GameViewport.ts @@ -2,6 +2,7 @@ * The scene seen through its active camera at the chosen scale, letterboxed: the running world while * playing, the model as a still preview otherwise. */ +import type { Container } from 'pixi.js'; import { isSymbolType } from '../../core/model/AssetTypes'; import type { EditorState } from '../state'; import type { World } from '../../runtime/World'; @@ -44,6 +45,8 @@ export class GameViewport extends WorldViewport { /** The model collected once per redraw of the preview: view, cameras and items all read it. */ private redrawFrame: Frame | null = null; private redrawing = false; + /** Set while a recording draws the view behind another tab (#405). */ + private forced = false; constructor(state: EditorState) { super(state); @@ -119,7 +122,7 @@ export class GameViewport extends WorldViewport { * otherwise run on every selection change behind another tab. Showing the tab redraws it. */ override redraw(): void { - if (!this.visible) return; + if (!this.visible && !this.forced) return; const start = performance.now(); this.redrawing = true; try { @@ -131,6 +134,16 @@ export class GameViewport extends WorldViewport { this.drawTime += performance.now() - start; } + /** A redraw while the tab is hidden: a recording goes on behind another tab (#405). */ + redrawHidden(): void { + this.forced = true; + try { + this.redraw(); + } finally { + this.forced = false; + } + } + get scaleMode(): GameScale { return this.state.gameScale; } @@ -207,6 +220,10 @@ export class GameViewport extends WorldViewport { return Theme.colors.background; } + protected override chrome(): Container[] { + return [this.hint, this.editorRunCaption, this.fullscreenButton, this.performanceOverlay]; + } + protected override beforeItems(view: View | null): void { this.onScaleShown?.(view ? this.factor : null); this.hint.visible = view === null; diff --git a/src/player/GameScreen.ts b/src/player/GameScreen.ts index 132c5f04..3054c104 100644 --- a/src/player/GameScreen.ts +++ b/src/player/GameScreen.ts @@ -1,4 +1,4 @@ -import type { TextStyle } from 'pixi.js'; +import type { Container, TextStyle } from 'pixi.js'; import type { World } from '../runtime/World'; import { Theme, textStyle, ui, crispText } from '../ui'; import { type ActiveCamera, cameraOfWorld, overlaysOfWorld } from '../viewport/activeCamera'; @@ -119,6 +119,10 @@ export class GameScreen extends WorldViewport { return Theme.colors.background; } + protected override chrome(): Container[] { + return [this.caption, this.hint, this.fullscreenButton]; + } + protected override beforeItems(view: View | null): void { this.caption.visible = this.running === null; this.caption.position.set(this.w / 2, this.h / 2); diff --git a/src/player/Player.ts b/src/player/Player.ts index f24e8b03..6cb14a5c 100644 --- a/src/player/Player.ts +++ b/src/player/Player.ts @@ -5,7 +5,8 @@ import { GameRun } from '../runtime/GameRun'; import { loadPhysics, readyPhysics } from '../runtime/physics/engine'; import { behaviorError } from '../runtime/systems/BehaviorSystem'; import { Input } from '../runtime/systems/Input'; -import { requestRender } from '../ui'; +import { downloadBlob, requestRender } from '../ui'; +import { GameRecorder } from '../viewport/GameRecorder'; import { soundLibrary } from '../viewport/SoundLibrary'; import { VIDEO_WAIT_MS, videoLibrary } from '../viewport/VideoLibrary'; import type { GameScreen } from './GameScreen'; @@ -20,6 +21,8 @@ export class Player { private readonly input = new Input(); /** Until when a run waits at its start for its driven videos (#404); see `PlaySession`. */ private videoWait: number | null = null; + /** `Start recording` (#405): a released game has no project to add to, so the file is downloaded. */ + private readonly recorder: GameRecorder; constructor( private readonly project: Project, @@ -27,6 +30,14 @@ export class Player { private readonly saves: SaveArea, ) { screen.onStart = () => this.start(); + this.recorder = new GameRecorder({ + view: () => screen, + log: (line) => { + this.run?.world.log.push(line); + console.warn('[recording]', line); + }, + saved: (recording) => downloadBlob(recording.blob, `${recording.name}.mkv`), + }); } get isRunning(): boolean { @@ -59,6 +70,7 @@ export class Player { physics, clock: gameClock, saves: this.saves, + recorder: this.recorder, }); videoLibrary.startRun(); this.input.attach(); @@ -67,9 +79,11 @@ export class Player { if (this.run.world.stopRequested) this.stop(); } - stop(): void { + /** `restarting`: a `Restart game` plays on at once, and a recording goes on with it. */ + stop(restarting = false): void { const run = this.run; if (!run) return; + if (!restarting) this.recorder.end(); this.run = null; this.videoWait = null; this.input.detach(); @@ -98,14 +112,17 @@ export class Player { const request = run.tick(step, timeStep(dt, gameClock.scale, false)); if (request === 'stop') this.stop(); else if (request === 'restart') { - this.stop(); + this.stop(true); this.start(); } else if (request === 'scene') { this.screen.setWorld(run.world); this.waitForVideos(); if (run.world.stopRequested) this.stop(); } - if (this.run) this.screen.redraw(); + if (this.run) { + this.screen.redraw(); + this.recorder.capture(); + } requestRender(); } diff --git a/src/runtime/GameRun.ts b/src/runtime/GameRun.ts index 5af68138..0dbb9b91 100644 --- a/src/runtime/GameRun.ts +++ b/src/runtime/GameRun.ts @@ -6,6 +6,7 @@ import type { SaveArea } from '../core/graph/savedValues'; import type { SceneAsset } from '../core/model/Asset'; import { findScene, type Project } from '../core/model/Project'; import { mixOf } from '../core/music/mixer'; +import type { RecordingSink } from '../core/recording/recording'; import type { AudioSink } from './systems/Audio'; import { addGameSystems, type BehaviorSystem, type TimeScaled } from './systems/BehaviorSystem'; import type { Input } from './systems/Input'; @@ -18,6 +19,8 @@ export interface RunServices { physics: Rapier; clock: TimeScaled; saves: SaveArea; + /** Outlives a `Load scene` and a restart, so a recording goes on through both (#405). */ + recorder?: RecordingSink; } /** What a scene holds of objects, which is all a run writes into it. */ diff --git a/src/runtime/systems/BehaviorSystem.ts b/src/runtime/systems/BehaviorSystem.ts index 2332c37d..e3d509bb 100644 --- a/src/runtime/systems/BehaviorSystem.ts +++ b/src/runtime/systems/BehaviorSystem.ts @@ -55,6 +55,7 @@ import { CollisionSystem } from './CollisionSystem'; import type { CollisionSource } from './solids'; import { AnimationSystem } from './AnimationSystem'; import type { Input } from './Input'; +import type { RecordingSink } from '../../core/recording/recording'; import type { System } from './System'; export type { TimeScaled } from './behaviorContext'; @@ -115,6 +116,8 @@ export interface GameSystemOptions { clock?: TimeScaled | null; /** Where persistent Variables assets are read from at the start and written to as they change (#166). */ saves?: SaveArea | null; + /** Where `Start recording` records to (#405): only a played game has one. */ + recorder?: RecordingSink | null; } /** Collisions and animations are ticked first, so behaviors see this frame's events. */ @@ -130,11 +133,12 @@ export function addGameSystems( physics = null, clock = null, saves = null, + recorder = null, }: GameSystemOptions = {}, ): BehaviorSystem { const collisions = physics ? new PhysicsSystem(physics) : new CollisionSystem(); const animations = new AnimationSystem(); - const sources = { collisions, animations, audio, clock, saves }; + const sources = { collisions, animations, audio, clock, saves, recorder }; const behaviors = new BehaviorSystem(project, input, sources, shouldRun); if (carried) behaviors.carryOver(carried); if (kept) { @@ -155,6 +159,7 @@ export interface EventSources { audio?: AudioSink | null; clock?: TimeScaled | null; saves?: SaveArea | null; + recorder?: RecordingSink | null; } /** A behavior's `runWhilePaused` switch: whether it is ticked while Pause game holds the world. */ @@ -198,6 +203,7 @@ export class BehaviorSystem implements System { private readonly audio: AudioSink; private readonly clock: TimeScaled | null; private readonly saves: SaveArea | null; + private readonly recorder: RecordingSink | null; /** What was last written to `saves`, so a frame that changed nothing writes nothing. */ private lastSaved = ''; @@ -216,6 +222,7 @@ export class BehaviorSystem implements System { this.audio = sources.audio ?? SILENT_AUDIO; this.clock = sources.clock ?? null; this.saves = sources.saves ?? null; + this.recorder = sources.recorder ?? null; } /** The values `global` declarations share this run – read-only to callers, rebuilt by `start`. */ @@ -750,6 +757,7 @@ export class BehaviorSystem implements System { collisions: this.collisions, audio: this.audio, clock: this.clock, + recorder: this.recorder, shared: this.sharedState, stateChanged: (machineId, from, to) => this.pending.push({ machineId, from, to, source: instance }), }); diff --git a/src/runtime/systems/behaviorContext.ts b/src/runtime/systems/behaviorContext.ts index adf45b2a..87e2306a 100644 --- a/src/runtime/systems/behaviorContext.ts +++ b/src/runtime/systems/behaviorContext.ts @@ -42,6 +42,7 @@ import type { World } from '../World'; import type { AudioSink, NoteTiming } from './Audio'; import type { Instance } from './behaviorPulses'; import type { Input } from './Input'; +import type { RecordingSink } from '../../core/recording/recording'; import { bodyOf, type CollisionSource, layerHolderOf, NOT_BLOCKED } from './solids'; export interface TimeScaled { @@ -55,6 +56,8 @@ export interface ContextSources { collisions: CollisionSource | null; audio: AudioSink; clock: TimeScaled | null; + /** Where a recording of the game view goes (#405); only a played game has one. */ + recorder: RecordingSink | null; shared: SharedState; /** A graph moved one of this instance's machines; its events follow between exec flows. */ stateChanged(machineId: string, from: string, to: string): void; @@ -265,9 +268,20 @@ export function behaviorContext( setPaused: (paused) => { world.paused = paused; }, + startRecording: (settings) => tell(world, run.recorder?.start(settings) ?? NO_RECORDER), + stopRecording: () => tell(world, run.recorder?.stop() ?? NO_RECORDER), }; } +const NO_RECORDER = 'Recording works only while the game is played, not in the editor run'; + +/** A line of the host for the log, as the `Log` node writes one; nothing for an empty one. */ +function tell(world: World, line: string): void { + if (!line) return; + world.log.push(line); + console.warn('[recording]', line); +} + /** * What the graph asked for, as the sink understands it: the asset's own volume scaled by the * node's, and its loop flag unless the node overrules it either way. The asset is what a sound diff --git a/src/ui/download.ts b/src/ui/download.ts new file mode 100644 index 00000000..058f769e --- /dev/null +++ b/src/ui/download.ts @@ -0,0 +1,14 @@ +/** Hands a file at `href` to the browser to save as `fileName`; the page stays where it is. */ +export function downloadFile(href: string, fileName: string): void { + const link = document.createElement('a'); + link.href = href; + link.download = fileName; + link.click(); +} + +/** Hands bytes made in the page to the browser as a file. */ +export function downloadBlob(blob: Blob, fileName: string): void { + const url = URL.createObjectURL(blob); + downloadFile(url, fileName); + setTimeout(() => URL.revokeObjectURL(url), 60_000); +} diff --git a/src/ui/index.ts b/src/ui/index.ts index 7ded094e..6e7e5f59 100644 --- a/src/ui/index.ts +++ b/src/ui/index.ts @@ -44,6 +44,7 @@ export { type GridOptions, } from './draw'; export { clamp, centreY, formatBytes, formatNumber, NUMBER_DECIMALS, snapToStep, zoomFactor } from './math'; +export { downloadBlob, downloadFile } from './download'; export { activeDrag, type DragPayload, endDrag, onDragChange, startDrag } from './drag'; export { Label, truncate } from './widgets/Label'; export { Button, type ButtonOptions } from './widgets/Button'; diff --git a/src/viewport/GameRecorder.ts b/src/viewport/GameRecorder.ts new file mode 100644 index 00000000..48ad08c8 --- /dev/null +++ b/src/viewport/GameRecorder.ts @@ -0,0 +1,532 @@ +import { Matrix, type RenderTexture } from 'pixi.js'; +import { preSkipOf } from '../core/music/ogg'; +import { AVC_CODEC, colourOf, MatroskaWriter, VP9_CODEC } from '../core/recording/matroska'; +import { + avcCodec, + FrameGate, + isKeyFrame, + pictureTransform, + RECORDING_MOST_BYTES, + RecordingClock, + recordingName, + type RecordingSettings, + type RecordingSink, + recordingSize, + videoBitrate, + vp9Codec, +} from '../core/recording/recording'; +import { ui } from '../ui'; +import { PictureReader } from './PictureReader'; +import { soundLibrary } from './SoundLibrary'; +import type { PictureArea } from './WorldViewport'; + +const OPUS_BITRATE = 160_000; +/** Frames the video encoder may have waiting before the next is dropped instead of piling up. */ +const MOST_QUEUED = 6; +/** Encoded bytes held as loose pieces before they are folded into a Blob, out of the page's heap. */ +const FOLD_BYTES = 16 * 1024 * 1024; +/** How long a stop waits for the last sound the worklet holds; a suspended context sends none. */ +const LAST_SOUND_MS = 300; +/** How long a stop waits for the frames still on their way back from the GPU. */ +const LAST_FRAMES_MS = 500; + +/** The view a recording takes its frames from (`WorldViewport`). */ +export interface RecordingView { + picture(): { shown: PictureArea; whole: PictureArea } | null; + drawPicture(transform: Matrix, target: RenderTexture): void; +} + +/** A recording once it is made. */ +export interface MadeRecording { + name: string; + blob: Blob; + durationMs: number; + width: number; + height: number; +} + +/** Where the recorder looks, and what it tells. */ +export interface RecordingHost { + view(): RecordingView | null; + /** A line for the log of the run that plays. */ + log(line: string): void; + saved(recording: MadeRecording): void; +} + +/** + * `waiting` for a picture to take its size from: `On start` runs before the view is handed the + * world, and a game may switch its camera on later. + */ +type TakeState = 'waiting' | 'starting' | 'recording' | 'finishing' | 'done' | 'failed'; + +/** What a check reads of a recording (test hook). */ +export interface TakeInfo { + name: string; + state: TakeState; + width: number; + height: number; + fps: number; + codec: string; + sound: boolean; + frames: number; + dropped: number; + soundFrames: number; + /** From the first frame to the end of the last, in ms. */ + durationMs: number; + /** What the video encoder said its colour is, null while it said nothing. */ + colour: VideoColorSpaceInit | null; + bytes: number; + failure: string | null; +} + +/** + * Records the game view (#405): `Start recording` and `Stop recording` reach it through the run, the + * host hands it the view and draws a frame for it each tick (`capture`). One recording at a time; it + * outlives the world, so a `Load scene` or a restart goes on in the same file. + */ +export class GameRecorder implements RecordingSink { + private take: Take | null = null; + private paused = false; + /** Every recording of this page, newest last (test hook). */ + private readonly takes: Take[] = []; + + constructor(private readonly host: RecordingHost) {} + + get isRecording(): boolean { + return this.take !== null; + } + + start(settings: RecordingSettings): string { + if (this.take) + return `Start recording is ignored: "${this.take.name}" is recording already – Stop recording ends it`; + const take = new Take(recordingName(settings.name, new Date()), settings, this.host, () => { + if (this.take === take) this.take = null; + }); + this.take = take; + this.takes.push(take); + if (this.paused) take.setPaused(true); + return ''; + } + + stop(): string { + if (!this.take) return 'Stop recording: nothing is recording'; + this.take.finish(''); + this.take = null; + return ''; + } + + /** The editor holds the run: the recording holds with it, and its time leaves the pause out. */ + setPaused(paused: boolean): void { + this.paused = paused; + this.take?.setPaused(paused); + } + + /** The view has drawn this frame: the recording takes it, if it is due. */ + capture(): void { + const view = this.take && this.host.view(); + if (view) this.take!.capture(view); + } + + /** The run is over: a recording still going is saved, not lost. */ + end(): void { + this.stop(); + } + + info(): { recording: boolean; takes: TakeInfo[] } { + return { recording: this.isRecording, takes: this.takes.map((take) => take.info()) }; + } +} + +/** One recording, from its encoders to its file. */ +class Take { + private state: TakeState = 'waiting'; + private size = { w: 0, h: 0 }; + private readonly clock = new RecordingClock(); + private readonly gate: FrameGate; + private video: VideoEncoder | null = null; + private audio: AudioEncoder | null = null; + private tap: { node: AudioWorkletNode; rate: number; release(): void } | null = null; + private writer: MatroskaWriter | null = null; + private codec = ''; + private videoHead: Uint8Array | null = null; + private colour: VideoColorSpaceInit | null = null; + private audioHead: Uint8Array | null = null; + private reader: PictureReader | null = null; + private toldNoCamera = false; + private folded: Blob[] = []; + private loose: Uint8Array[] = []; + private looseBytes = 0; + private frames = 0; + private dropped = 0; + private lastMs = 0; + private soundFrames = 0; + /** The recording's time of the first sample heard, in ms; null before it. */ + private soundStart: number | null = null; + private soundEnded: (() => void) | null = null; + private stopAsked = false; + private paused = false; + private failure: string | null = null; + + constructor( + readonly name: string, + private readonly settings: RecordingSettings, + private readonly host: RecordingHost, + private readonly gone: () => void, + ) { + this.gate = new FrameGate(settings.fps); + } + + /** + * Takes the size from the first picture there is and sets up the encoders; until they are ready + * frames are not taken, and a stop asked meanwhile ends it after. + */ + private open(picture: { shown: PictureArea; whole: PictureArea }): void { + const ratio = ui().app.renderer.resolution; + const device = (area: PictureArea) => ({ w: area.w * ratio, h: area.h * ratio }); + this.size = recordingSize(this.settings.resolution, device(picture.shown), device(picture.whole)); + this.state = 'starting'; + try { + this.reader = new PictureReader(this.size.w, this.size.h); + } catch (error) { + this.fail(error instanceof Error ? error.message : String(error)); + return; + } + void this.openEncoders().then( + () => { + if (this.state !== 'starting') return; + this.state = 'recording'; + if (this.stopAsked) this.finish(''); + }, + (error: unknown) => this.fail(error instanceof Error ? error.message : String(error)), + ); + } + + private async openEncoders(): Promise { + if (typeof VideoEncoder === 'undefined') + throw new Error('this browser cannot encode video – recording needs Chrome'); + const { w, h } = this.size; + const fps = this.settings.fps; + const base = { width: w, height: h, framerate: fps, bitrate: videoBitrate(w, h, fps) }; + const configs: VideoEncoderConfig[] = [ + { ...base, codec: avcCodec(w, h, fps), avc: { format: 'avc' }, latencyMode: 'realtime' }, + { ...base, codec: vp9Codec(w, h), latencyMode: 'realtime' }, + ]; + let config: VideoEncoderConfig | null = null; + for (const candidate of configs) + if ( + await VideoEncoder.isConfigSupported(candidate).then( + (s) => s.supported === true, + () => false, + ) + ) { + config = candidate; + break; + } + if (!config) throw new Error(`this browser cannot encode ${w}x${h} at ${fps} fps`); + this.codec = config.codec; + await this.openSound(); + const writer = new MatroskaWriter(this.audio ? 2 : 1); + this.writer = writer; + const video = new VideoEncoder({ + output: (chunk, meta) => { + const head = meta?.decoderConfig?.description; + if (head && !this.videoHead) this.videoHead = copyOf(head); + this.colour ??= meta?.decoderConfig?.colorSpace ?? null; + writer.add(1, chunk.timestamp, chunk.type === 'key', chunkBytes(chunk)); + this.collect(); + }, + error: (error) => this.fail(error.message), + }); + video.configure(config); + this.video = video; + } + + /** The master behind the limiter, as Opus; without it the recording goes on silent and says so. */ + private async openSound(): Promise { + const tap = await soundLibrary.outputTap().catch(() => null); + const config: AudioEncoderConfig = { + codec: 'opus', + sampleRate: tap?.rate ?? 48000, + numberOfChannels: 2, + bitrate: OPUS_BITRATE, + }; + const supported = + tap !== null && + typeof AudioEncoder !== 'undefined' && + (await AudioEncoder.isConfigSupported(config).then( + (s) => s.supported === true, + () => false, + )); + if (!tap || !supported) { + tap?.release(); + this.host.log(`Recording "${this.name}" has no sound: this browser cannot record it`); + return; + } + const audio = new AudioEncoder({ + output: (chunk, meta) => { + const head = meta?.decoderConfig?.description; + if (head && !this.audioHead) this.audioHead = copyOf(head); + this.writer?.add(2, chunk.timestamp, true, chunkBytes(chunk)); + this.collect(); + }, + error: (error) => this.fail(error.message), + }); + audio.configure(config); + this.audio = audio; + this.tap = tap; + tap.node.port.onmessage = (event: MessageEvent) => this.heard(event.data); + } + + setPaused(paused: boolean): void { + if (paused === this.paused) return; + this.paused = paused; + const now = performance.now(); + if (paused) this.clock.pause(now); + else this.clock.resume(now); + } + + capture(view: RecordingView): void { + if (this.state === 'waiting') { + const picture = view.picture(); + if (picture) this.open(picture); + else if (!this.toldNoCamera) { + this.toldNoCamera = true; + this.host.log( + `Recording "${this.name}" waits: no camera is switched on, so there is nothing to record`, + ); + } + return; + } + const video = this.video; + const reader = this.reader; + if (this.state !== 'recording' || !video || !reader) return; + reader.collect((bytes, ms) => this.encode(bytes, ms)); + // A frame taken may have reached the most a file holds and ended the recording. + if (!this.taking || this.paused) return; + const now = performance.now(); + if (!this.clock.started) this.clock.start(now); + const ms = this.clock.at(now); + if (!this.gate.take(ms)) return; + const picture = view.picture(); + if (!picture || reader.busy || video.encodeQueueSize >= MOST_QUEUED) { + this.dropped++; + return; + } + const m = pictureTransform( + this.settings.resolution === 'view' ? picture.shown : picture.whole, + this.size, + ); + view.drawPicture(new Matrix(m.a, 0, 0, m.d, m.tx, m.ty), reader.texture); + reader.read(ms); + } + + private get taking(): boolean { + return this.state === 'recording'; + } + + /** A frame back from the GPU, into the video encoder. */ + private encode(bytes: Uint8Array, ms: number): void { + const video = this.video; + if (!video || video.state !== 'configured') return; + const frame = new VideoFrame(bytes, { + format: 'RGBX', + codedWidth: this.size.w, + codedHeight: this.size.h, + timestamp: Math.round(ms * 1000), + duration: Math.round(1e6 / this.settings.fps), + }); + video.encode(frame, { keyFrame: isKeyFrame(this.frames, this.settings.fps) }); + frame.close(); + this.frames++; + this.lastMs = ms; + if (this.taking && (this.writer?.bytes ?? 0) > RECORDING_MOST_BYTES) + this.finish(`Recording "${this.name}" reached 2 GB, the most the server takes, and was stopped`); + } + + /** A piece of the master from the worklet; an empty one is the last. */ + private heard(sides: Float32Array[]): void { + const audio = this.audio; + const frames = sides[0]?.length ?? 0; + if (frames === 0) { + this.soundEnded?.(); + return; + } + // Before the first frame there is no time to put it at. Pieces keep coming during a pause only + // as long as they were on their way, so they belong before it and are kept. + if (!audio || !this.tap || this.state !== 'recording' || !this.clock.started) return; + const rate = this.tap.rate; + this.soundStart ??= Math.max(0, this.clock.at(performance.now()) - (frames / rate) * 1000); + const planar = new Float32Array(frames * 2); + planar.set(sides[0]!, 0); + planar.set(sides[1] ?? sides[0]!, frames); + const data = new AudioData({ + format: 'f32-planar', + sampleRate: rate, + numberOfFrames: frames, + numberOfChannels: 2, + timestamp: Math.round(this.soundStart * 1000 + (this.soundFrames / rate) * 1e6), + data: planar, + }); + audio.encode(data); + data.close(); + this.soundFrames += frames; + } + + /** Ends the recording and saves what it has; `why` goes to the log when it is not the node's stop. */ + finish(why: string): void { + if (why) this.host.log(why); + if (this.state === 'waiting') { + this.state = 'done'; + this.gone(); + this.host.log(`Recording "${this.name}" ended before its first frame, so nothing was saved`); + return; + } + if (this.state === 'starting') { + this.stopAsked = true; + return; + } + if (this.state !== 'recording') return; + this.state = 'finishing'; + this.gone(); + void this.save().catch((error: unknown) => + this.fail(error instanceof Error ? error.message : String(error)), + ); + } + + private async save(): Promise { + await Promise.all([this.lastFrames(), this.lastSound()]); + if (this.video?.state === 'configured') await this.video.flush(); + if (this.audio?.state === 'configured') await this.audio.flush(); + this.close(); + const writer = this.writer; + if (!writer || this.frames === 0) { + this.state = 'done'; + this.host.log(`Recording "${this.name}" ended before its first frame, so nothing was saved`); + return; + } + const frame = 1000 / this.settings.fps; + const { head, rest } = writer.finish({ + video: { + codec: this.codec.startsWith('avc1') ? AVC_CODEC : VP9_CODEC, + width: this.size.w, + height: this.size.h, + private: this.videoHead, + fps: this.settings.fps, + colour: colourOf(this.colour), + }, + audio: this.audio + ? { + channels: 2, + private: this.audioHead, + preSkip: preSkipOf(this.audioHead ?? new Uint8Array()) ?? 0, + } + : null, + durationMs: this.lastMs + frame, + app: 'game-maker', + }); + this.collect(); + const blob = new Blob([head, ...this.folded, ...this.loose, ...rest] as BlobPart[], { + type: 'video/x-matroska', + }); + this.folded = []; + this.loose = []; + this.state = 'done'; + this.host.saved({ + name: this.name, + blob, + durationMs: this.lastMs + frame, + width: this.size.w, + height: this.size.h, + }); + } + + /** Takes the frames still on their way back from the GPU, a moment at most. */ + private async lastFrames(): Promise { + const until = performance.now() + LAST_FRAMES_MS; + for (;;) { + this.reader?.collect((bytes, ms) => this.encode(bytes, ms)); + if (!this.reader?.pending || performance.now() > until) return; + await new Promise((resolve) => setTimeout(resolve, 4)); + } + } + + /** Asks the worklet for what it still holds and waits for it, a moment at most. */ + private lastSound(): Promise { + const tap = this.tap; + if (!tap) return Promise.resolve(); + return new Promise((resolve) => { + const timer = setTimeout(done, LAST_SOUND_MS); + function done(): void { + clearTimeout(timer); + resolve(); + } + this.soundEnded = done; + tap.node.port.postMessage('stop'); + }).finally(() => { + tap.release(); + this.tap = null; + }); + } + + /** Moves the clusters the writer finished into the file, folding them into Blobs as they pile up. */ + private collect(): void { + for (const part of this.writer?.take() ?? []) { + this.loose.push(part); + this.looseBytes += part.length; + } + if (this.looseBytes < FOLD_BYTES) return; + this.folded.push(new Blob(this.loose as BlobPart[])); + this.loose = []; + this.looseBytes = 0; + } + + private close(): void { + for (const encoder of [this.video, this.audio]) + if (encoder && encoder.state !== 'closed') encoder.close(); + this.reader?.destroy(); + this.reader = null; + } + + private fail(reason: string): void { + if (this.state === 'done' || this.state === 'failed') return; + this.failure = reason; + this.state = 'failed'; + this.tap?.release(); + this.tap = null; + this.close(); + this.gone(); + this.host.log(`Recording "${this.name}" failed: ${reason}`); + } + + info(): TakeInfo { + return { + name: this.name, + state: this.state, + width: this.size.w, + height: this.size.h, + fps: this.settings.fps, + codec: this.codec, + sound: this.audio !== null, + frames: this.frames, + dropped: this.dropped, + soundFrames: this.soundFrames, + durationMs: this.frames > 0 ? Math.round(this.lastMs + 1000 / this.settings.fps) : 0, + colour: this.colour, + bytes: this.writer?.bytes ?? 0, + failure: this.failure, + }; + } +} + +function chunkBytes(chunk: EncodedVideoChunk | EncodedAudioChunk): Uint8Array { + const data = new Uint8Array(chunk.byteLength); + chunk.copyTo(data); + return data; +} + +function copyOf(source: AllowSharedBufferSource): Uint8Array { + const view = ArrayBuffer.isView(source) + ? new Uint8Array(source.buffer, source.byteOffset, source.byteLength) + : new Uint8Array(source); + return view.slice(); +} diff --git a/src/viewport/PictureReader.ts b/src/viewport/PictureReader.ts new file mode 100644 index 00000000..c4426501 --- /dev/null +++ b/src/viewport/PictureReader.ts @@ -0,0 +1,102 @@ +import { RenderTexture, type WebGLRenderer } from 'pixi.js'; +import { ui } from '../ui'; + +/** Frames on their way back from the GPU at once; while all are, a frame due is dropped. */ +const MOST_IN_FLIGHT = 3; + +interface InFlight { + buffer: WebGLBuffer; + fence: WebGLSync; + ms: number; +} + +/** + * Hands a recording its frames without drawing into the canvas on screen (#405): the low latency + * canvas shows what is drawn into it at once, so a frame drawn there flickered in its corner. The + * view is drawn into `texture` instead and read back through pixel buffers, so the main thread does + * not wait for the GPU: a frame comes out of `collect` a tick or two after it was drawn. + */ +export class PictureReader { + readonly texture: RenderTexture; + private readonly gl: WebGL2RenderingContext; + private readonly free: WebGLBuffer[] = []; + private readonly inFlight: InFlight[] = []; + private readonly bytes: Uint8Array; + + constructor( + readonly w: number, + readonly h: number, + ) { + const renderer = ui().app.renderer as WebGLRenderer; + const gl = renderer.gl as WebGL2RenderingContext | undefined; + if (!gl || typeof gl.fenceSync !== 'function') throw new Error('recording needs WebGL 2'); + this.gl = gl; + this.texture = RenderTexture.create({ width: w, height: h, resolution: 1, antialias: true }); + for (let i = 0; i < MOST_IN_FLIGHT; i++) { + const buffer = gl.createBuffer(); + gl.bindBuffer(gl.PIXEL_PACK_BUFFER, buffer); + gl.bufferData(gl.PIXEL_PACK_BUFFER, w * h * 4, gl.STREAM_READ); + this.free.push(buffer); + } + gl.bindBuffer(gl.PIXEL_PACK_BUFFER, null); + this.bytes = new Uint8Array(w * h * 4); + } + + /** Whether a frame drawn now would have no buffer to go back in. */ + get busy(): boolean { + return this.free.length === 0; + } + + get pending(): number { + return this.inFlight.length; + } + + /** Starts reading back what `texture` holds now, the frame of recording time `ms`. */ + read(ms: number): void { + const buffer = this.free.pop(); + if (!buffer) return; + const gl = this.gl; + const renderer = ui().app.renderer as WebGLRenderer; + const target = renderer.renderTarget.getRenderTarget(this.texture); + // Resolves the multisampled picture into the texture; a second time is only a copy. + renderer.renderTarget.adaptor.finishRenderPass(target); + const gpu = renderer.renderTarget.getGpuRenderTarget(target); + // Through the adaptor, so Pixi knows which framebuffer is bound. + renderer.renderTarget.adaptor.bindFramebuffer(gpu.resolveTargetFramebuffer); + gl.bindBuffer(gl.PIXEL_PACK_BUFFER, buffer); + gl.readPixels(0, 0, this.w, this.h, gl.RGBA, gl.UNSIGNED_BYTE, 0); + gl.bindBuffer(gl.PIXEL_PACK_BUFFER, null); + const fence = gl.fenceSync(gl.SYNC_GPU_COMMANDS_COMPLETE, 0)!; + gl.flush(); + this.inFlight.push({ buffer, fence, ms }); + } + + /** + * Hands over each frame the GPU has finished, oldest first, its rows top first as Pixi draws a + * texture. The bytes are reused for the next frame, so `take` copies what it keeps. + */ + collect(take: (bytes: Uint8Array, ms: number) => void): void { + const gl = this.gl; + while (this.inFlight.length > 0) { + const next = this.inFlight[0]!; + if (gl.clientWaitSync(next.fence, 0, 0) === gl.TIMEOUT_EXPIRED) return; + gl.deleteSync(next.fence); + gl.bindBuffer(gl.PIXEL_PACK_BUFFER, next.buffer); + gl.getBufferSubData(gl.PIXEL_PACK_BUFFER, 0, this.bytes); + gl.bindBuffer(gl.PIXEL_PACK_BUFFER, null); + this.inFlight.shift(); + this.free.push(next.buffer); + take(this.bytes, next.ms); + } + } + + destroy(): void { + const gl = this.gl; + for (const { buffer, fence } of this.inFlight.splice(0)) { + gl.deleteSync(fence); + gl.deleteBuffer(buffer); + } + for (const buffer of this.free.splice(0)) gl.deleteBuffer(buffer); + this.texture.destroy(true); + } +} diff --git a/src/viewport/SoundLibrary.ts b/src/viewport/SoundLibrary.ts index 6553d91f..9f2ebefa 100644 --- a/src/viewport/SoundLibrary.ts +++ b/src/viewport/SoundLibrary.ts @@ -614,6 +614,25 @@ export class SoundLibrary implements AudioSink { return readMeter((this.guardMeter ??= this.tapMeter(guard.limiter))); } + /** + * A node that hears what leaves the output, limiter and all (#405): a recording of the game view + * takes its sound there, in pieces of both sides (`record-tap`). Null without a live output. + */ + async outputTap(): Promise<{ node: AudioWorkletNode; rate: number; release(): void } | null> { + const context = this.audio(); + if (!(context instanceof AudioContext) || !(await this.fxWorklet())) return null; + const guard = this.guard; + if (!guard) return null; + const node = new AudioWorkletNode(context, 'record-tap', { + numberOfOutputs: 0, + channelCount: 2, + channelCountMode: 'explicit', + }); + const source = guard.limiter ?? guard.input; + source.connect(node); + return { node, rate: context.sampleRate, release: () => source.disconnect(node) }; + } + /** Lets every level meter go. */ unmeter(): void { for (const bus of this.buses.values()) this.dropMeter(bus); diff --git a/src/viewport/VideoLibrary.ts b/src/viewport/VideoLibrary.ts index f5ad2c80..c76f732b 100644 --- a/src/viewport/VideoLibrary.ts +++ b/src/viewport/VideoLibrary.ts @@ -27,6 +27,11 @@ import { soundLibrary } from './SoundLibrary'; import { followVideo, readSpeed, type SpeedReading } from './videoFollow'; const VIDEO_BASE = '/game-maker/data/videos'; + +/** Where the MP4 of a video asset's `file` is served. */ +export function videoFileUrl(file: string): string { + return `${VIDEO_BASE}/${file}`; +} /** How close to its end a clip counts as there: half a frame at 30 fps. */ const END_SLACK = 1 / 60; /** A player nothing has asked for in this long is let go: no sprite wears it any more. */ @@ -281,7 +286,7 @@ export class VideoLibrary { const at = drive ? drivenTimeOf(play, time, NaN) : play.start; const player: Player = { play, - element: openVideo(`${VIDEO_BASE}/${play.file}`, at, (texture) => { + element: openVideo(videoFileUrl(play.file), at, (texture) => { if (player.gone) return texture.destroy(true); player.texture = texture; if (drive) this.follow(player, drive); diff --git a/src/viewport/WorldViewport.ts b/src/viewport/WorldViewport.ts index bc0ad813..2bdfd373 100644 --- a/src/viewport/WorldViewport.ts +++ b/src/viewport/WorldViewport.ts @@ -1,4 +1,12 @@ -import { Container, type FederatedPointerEvent, Graphics, type Mesh, Texture } from 'pixi.js'; +import { + Container, + type FederatedPointerEvent, + Graphics, + type Matrix, + type Mesh, + type RenderTexture, + Texture, +} from 'pixi.js'; import { isSymbolType } from '../core/model/AssetTypes'; import { type Extent, scaleOf } from '../core/model/matrix'; import type { Vec2 } from '../core/model/Properties'; @@ -51,6 +59,14 @@ export interface Frame { background: number; } +/** A part of the view by its centre and size, in local CSS pixels. */ +export interface PictureArea { + x: number; + y: number; + w: number; + h: number; +} + export abstract class WorldViewport extends Widget { protected readonly bg = new Graphics(); protected readonly content = new Container(); @@ -512,6 +528,49 @@ export abstract class WorldViewport extends W return (shading && shadedPicture(shading, shaders, ppu, true)?.grid) || view.grid; } + /** + * The camera's picture as the view lays it out (#405), in local CSS pixels by its centre: the part + * on screen, and all of it, which is larger only while a fixed scale cuts it to the view. Null + * without a camera. + */ + picture(): { shown: PictureArea; whole: PictureArea } | null { + const view = this.currentView(); + const camera = view && this.mainCamera(); + if (!view || !camera) return null; + const { rect } = view; + const frame = cameraFrame(camera.props); + const x = rect.x + rect.w / 2; + const y = rect.y + rect.h / 2; + return { + shown: { x, y, w: rect.w, h: rect.h }, + whole: { x, y, w: frame.w * view.scale, h: frame.h * view.scale }, + }; + } + + /** + * Draws the view into `target` mapped by `transform`, without what lies over the picture + * (`chrome`): a recording reads it from there (#405). Never into the canvas on screen, which shows + * at once what is drawn into it while its context is low latency. Drawn even while its tab is + * hidden, so a recording goes on behind another tab. + */ + drawPicture(transform: Matrix, target: RenderTexture): void { + const shown = this.chrome().filter((part) => part.visible); + for (const part of shown) part.visible = false; + const visible = this.visible; + this.visible = true; + try { + ui().app.renderer.render({ container: this, target, transform, clear: true, clearColor: [0, 0, 0, 1] }); + } finally { + this.visible = visible; + for (const part of shown) part.visible = true; + } + } + + /** What the view lays over the game's picture and a recording leaves out. */ + protected chrome(): Container[] { + return []; + } + /** Whether the game view is at a fixed scale rather than fit: the only one overlays snap at. */ protected fixedScale(): boolean { return false; diff --git a/src/viewport/fxWorklet.ts b/src/viewport/fxWorklet.ts index 659ed22c..60d17f66 100644 --- a/src/viewport/fxWorklet.ts +++ b/src/viewport/fxWorklet.ts @@ -391,3 +391,52 @@ class ChannelProbe extends AudioWorkletProcessor { } registerProcessor('channel-probe', ChannelProbe); + +/** Frames a recording's sound is handed over in: about a tenth of a second. */ +const RECORD_BLOCK = 4096; + +/** + * Hands what reaches it to the page, both sides, `RECORD_BLOCK` frames at a time (#405): the master + * as it leaves the limiter, for a recording of the game view. Told to stop, it sends what it holds + * and an empty piece after it, the sign that nothing more comes. + */ +class RecordTap extends Stoppable { + private readonly sides = [new Float32Array(RECORD_BLOCK), new Float32Array(RECORD_BLOCK)]; + private filled = 0; + + constructor() { + super(); + this.port.onmessage = () => { + this.send(); + this.port.postMessage([new Float32Array(0), new Float32Array(0)]); + this.stop(); + }; + } + + process(inputs: Float32Array[][]): boolean { + if (this.stopped) return false; + const input = inputs[0] ?? []; + const left = input[0]; + const frames = left?.length ?? 128; + this.sides.forEach((side, c) => { + const data = input[c] ?? left; + if (data) side.set(data, this.filled); + else side.fill(0, this.filled, this.filled + frames); + }); + this.filled += frames; + if (this.filled + frames > RECORD_BLOCK) this.send(); + return true; + } + + private send(): void { + if (this.filled === 0) return; + const out = this.sides.map((side) => side.slice(0, this.filled)); + this.filled = 0; + this.port.postMessage( + out, + out.map((side) => side.buffer), + ); + } +} + +registerProcessor('record-tap', RecordTap); diff --git a/tests/core/matroska.test.ts b/tests/core/matroska.test.ts new file mode 100644 index 00000000..daa90b9c --- /dev/null +++ b/tests/core/matroska.test.ts @@ -0,0 +1,202 @@ +import { describe, expect, it } from 'vitest'; +import { AVC_CODEC, colourOf, type MatroskaHead, MatroskaWriter } from '../../src/core/recording/matroska'; + +/** One element as read back: its id in hex, its bytes, and what it holds where it is a master. */ +interface Read { + id: string; + data: Uint8Array; + children: Read[]; +} + +const MASTERS = new Set([ + '1a45dfa3', + '18538067', + '1549a966', + '1654ae6b', + 'ae', + 'e0', + '55b0', + 'e1', + '1f43b675', +]); + +function vint(bytes: Uint8Array, at: number, keepMarker: boolean): { value: number; length: number } { + const first = bytes[at]!; + let length = 1; + while (length <= 8 && !(first & (0x80 >> (length - 1)))) length++; + let value = keepMarker ? first : first & (0xff >> length); + for (let i = 1; i < length; i++) value = value * 256 + bytes[at + i]!; + return { value, length }; +} + +function readAll(bytes: Uint8Array, from = 0, to = bytes.length): Read[] { + const out: Read[] = []; + let at = from; + while (at < to) { + const id = vint(bytes, at, true); + const size = vint(bytes, at + id.length, false); + const start = at + id.length + size.length; + const data = bytes.subarray(start, start + size.value); + const hex = id.value.toString(16); + out.push({ id: hex, data, children: MASTERS.has(hex) ? readAll(bytes, start, start + size.value) : [] }); + at = start + size.value; + } + return out; +} + +const find = (list: Read[], id: string) => list.find((e) => e.id === id); +const uintOf = (e: Read | undefined) => (e ? e.data.reduce((v, b) => v * 256 + b, 0) : NaN); +const textOf = (e: Read | undefined) => new TextDecoder().decode(e?.data); + +/** Track, time relative to its cluster, key flag and payload of each block, with the cluster's time. */ +function blocksOf( + file: Uint8Array, +): { cluster: number; track: number; ms: number; key: boolean; data: number[] }[] { + const segment = find(readAll(file), '18538067')!; + return segment.children + .filter((e) => e.id === '1f43b675') + .flatMap((cluster) => { + const start = uintOf(find(cluster.children, 'e7')); + return cluster.children + .filter((e) => e.id === 'a3') + .map((block) => ({ + cluster: start, + track: block.data[0]! & 0x7f, + ms: start + new DataView(block.data.buffer, block.data.byteOffset + 1, 2).getInt16(0), + key: (block.data[3]! & 0x80) !== 0, + data: [...block.data.subarray(4)], + })); + }); +} + +const HEAD: MatroskaHead = { + video: { + codec: AVC_CODEC, + width: 1280, + height: 720, + private: Uint8Array.from([1, 100, 0, 31]), + fps: 30, + colour: colourOf({ matrix: 'bt709', transfer: 'iec61966-2-1', primaries: 'bt709', fullRange: false }), + }, + audio: { channels: 2, private: new TextEncoder().encode('OpusHead'), preSkip: 312 }, + durationMs: 1234.5, + app: 'game-maker', +}; + +/** The whole file: the head, then the clusters handed out before (`earlier`) and at the end. */ +function fileOf(writer: MatroskaWriter, head: MatroskaHead = HEAD, earlier: Uint8Array[] = []): Uint8Array { + const taken = writer.take(); + const { head: bytes, rest } = writer.finish(head); + const parts = [bytes, ...earlier, ...taken, ...rest]; + const out = new Uint8Array(parts.reduce((n, p) => n + p.length, 0)); + let at = 0; + for (const part of parts) { + out.set(part, at); + at += part.length; + } + return out; +} + +describe('MatroskaWriter (#405)', () => { + it('writes a head that says what the tracks are and how long the file is', () => { + const writer = new MatroskaWriter(2); + writer.add(1, 0, true, Uint8Array.from([9])); + const file = fileOf(writer); + const [ebml, segment] = readAll(file); + expect(textOf(find(ebml!.children, '4282'))).toBe('matroska'); + // The segment's size covers exactly the rest of the file. + expect(segment!.data.byteOffset + segment!.data.length).toBe(file.length); + const info = find(segment!.children, '1549a966')!; + expect(uintOf(find(info.children, '2ad7b1'))).toBe(1_000_000); + const duration = find(info.children, '4489')!; + expect(new DataView(duration.data.buffer, duration.data.byteOffset, 8).getFloat64(0)).toBe(1234.5); + const [video, audio] = find(segment!.children, '1654ae6b')!.children; + expect(textOf(find(video!.children, '86'))).toBe('V_MPEG4/ISO/AVC'); + expect([...find(video!.children, '63a2')!.data]).toEqual([1, 100, 0, 31]); + const picture = find(video!.children, 'e0')!; + expect([uintOf(find(picture.children, 'b0')), uintOf(find(picture.children, 'ba'))]).toEqual([1280, 720]); + // A frame lasts a 30th of a second, in nanoseconds, so nothing has to guess the rate. + expect(uintOf(find(video!.children, '23e383'))).toBe(33_333_333); + // BT.709 pixels in the broadcast range, sRGB's transfer. + const colour = find(picture.children, '55b0')!.children; + expect(['55b1', '55b9', '55ba', '55bb'].map((id) => uintOf(find(colour, id)))).toEqual([1, 1, 13, 1]); + expect(textOf(find(audio!.children, '86'))).toBe('A_OPUS'); + expect(textOf(find(audio!.children, '63a2'))).toBe('OpusHead'); + // 312 samples at 48 kHz, in nanoseconds. + expect(uintOf(find(audio!.children, '56aa'))).toBe(6_500_000); + expect(uintOf(find(find(audio!.children, 'e1')!.children, '9f'))).toBe(2); + }); + + it('takes the colour from what the encoder says, BT.709 where it says nothing', () => { + expect(colourOf(null)).toEqual({ matrix: 1, transfer: 1, primaries: 1, fullRange: false }); + expect(colourOf({ matrix: 'smpte170m', fullRange: true })).toEqual({ + matrix: 6, + transfer: 1, + primaries: 1, + fullRange: true, + }); + }); + + it('writes the blocks of both tracks in the order of their time, whatever order they come in', () => { + const writer = new MatroskaWriter(2); + writer.add(1, 0, true, Uint8Array.from([1])); + writer.add(1, 16_667, false, Uint8Array.from([2])); + writer.add(1, 33_333, false, Uint8Array.from([3])); + writer.add(2, 0, true, Uint8Array.from([10])); + writer.add(2, 20_000, true, Uint8Array.from([11])); + writer.add(2, 40_000, true, Uint8Array.from([12])); + const blocks = blocksOf(fileOf(writer)); + expect(blocks.map((b) => [b.track, b.ms])).toEqual([ + [1, 0], + [2, 0], + [1, 17], + [2, 20], + [1, 33], + [2, 40], + ]); + expect(blocks.map((b) => b.data[0])).toEqual([1, 10, 2, 11, 3, 12]); + }); + + it('hands out a cluster once the next key frame starts another', () => { + const writer = new MatroskaWriter(1); + writer.add(1, 0, true, Uint8Array.from([1])); + writer.add(1, 1_000_000, false, Uint8Array.from([2])); + expect(writer.take()).toEqual([]); + writer.add(1, 2_000_000, true, Uint8Array.from([3])); + const first = writer.take(); + expect(first).toHaveLength(1); + const blocks = blocksOf(fileOf(writer, HEAD, first)); + expect(blocks.map((b) => [b.cluster, b.ms, b.key])).toEqual([ + [0, 0, true], + [0, 1000, false], + [2000, 2000, true], + ]); + }); + + it('starts a new cluster before a block would be too far from its start to be told', () => { + const writer = new MatroskaWriter(1); + writer.add(1, 0, true, Uint8Array.from([1])); + writer.add(1, 40_000_000, false, Uint8Array.from([2])); + const blocks = blocksOf(fileOf(writer)); + expect(blocks.map((b) => [b.cluster, b.ms])).toEqual([ + [0, 0], + [40_000, 40_000], + ]); + }); + + it('does not hold the video back for ever when no sound comes', () => { + const writer = new MatroskaWriter(2); + for (let i = 0; i < 90; i++) writer.add(1, i * 33_333, i % 30 === 0, Uint8Array.from([i])); + // Three seconds of video, no sound: what is more than a second behind has gone into clusters. + expect(writer.take().length).toBeGreaterThan(0); + }); + + it('counts its bytes, the open cluster included, and leaves out a missing sound track', () => { + const writer = new MatroskaWriter(1); + writer.add(1, 0, true, new Uint8Array(100)); + expect(writer.bytes).toBeGreaterThan(100); + const file = fileOf(writer, { ...HEAD, audio: null }); + const segment = find(readAll(file), '18538067')!; + expect(find(segment.children, '1654ae6b')!.children).toHaveLength(1); + }); +}); diff --git a/tests/core/node-catalog.test.ts b/tests/core/node-catalog.test.ts index 6c078210..bd12cf3e 100644 --- a/tests/core/node-catalog.test.ts +++ b/tests/core/node-catalog.test.ts @@ -2393,6 +2393,21 @@ const CASES: Record = { ], StopGame: [{ name: 'ends play and the flow', next: null, after: (ctx) => expect(ctx.stopped).toBe(true) }], RestartGame: [{ name: 'ends the flow – the world restarts before the next tick', next: null }], + StartRecording: [ + { + name: 'asks for a recording and flows on', + inputs: { name: 'Take', resolution: '1080p', fps: '60' }, + next: 'exec', + after: (ctx) => expect(ctx.recordings).toEqual([{ name: 'Take', resolution: '1080p', fps: 60 }]), + }, + ], + StopRecording: [ + { + name: 'asks to stop and flows on', + next: 'exec', + after: (ctx) => expect(ctx.recordings).toEqual(['stop']), + }, + ], PauseGame: [ { name: 'pauses the game and flows on', next: 'exec', after: (ctx) => expect(ctx.paused).toBe(true) }, ], diff --git a/tests/core/recording.test.ts b/tests/core/recording.test.ts new file mode 100644 index 00000000..988177b4 --- /dev/null +++ b/tests/core/recording.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, it } from 'vitest'; +import { + avcCodec, + FrameGate, + isKeyFrame, + pictureTransform, + RecordingClock, + recordingName, + recordingSettingsOf, + recordingSize, +} from '../../src/core/recording/recording'; +import { runDef, fakeContext } from '../helpers'; + +describe('recording settings (#405)', () => { + it('names a recording after its start when no name is given', () => { + const at = new Date(2026, 9, 11, 16, 42, 5); + expect(recordingName('', at)).toBe('Recording 2026-10-11 16:42:05'); + expect(recordingName(' ', at)).toBe('Recording 2026-10-11 16:42:05'); + expect(recordingName(' Boss fight ', at)).toBe('Boss fight'); + }); + + it('reads what the node hands over, an unknown choice as the default', () => { + expect(recordingSettingsOf(' Run ', '1080p', '30')).toEqual({ + name: 'Run', + resolution: '1080p', + fps: 30, + }); + expect(recordingSettingsOf('', 'huge', 25)).toEqual({ name: '', resolution: 'view', fps: 60 }); + }); + + it('records the game view at its size on screen, even on both sides', () => { + expect(recordingSize('view', { w: 1279, h: 721 }, { w: 1279, h: 721 })).toEqual({ w: 1280, h: 722 }); + // A 5K screen is held to 4K. + expect(recordingSize('view', { w: 5120, h: 2880 }, { w: 5120, h: 2880 })).toEqual({ w: 3840, h: 2160 }); + }); + + it('holds the height of 720p and 1080p and takes the width from the camera', () => { + expect(recordingSize('1080p', { w: 800, h: 450 }, { w: 800, h: 450 })).toEqual({ w: 1920, h: 1080 }); + expect(recordingSize('720p', { w: 400, h: 400 }, { w: 400, h: 400 })).toEqual({ w: 720, h: 720 }); + // 4:3 at 720p, and the camera's whole picture rules where the screen cuts it. + expect(recordingSize('720p', { w: 300, h: 400 }, { w: 640, h: 480 })).toEqual({ w: 960, h: 720 }); + }); + + it('picks the lowest H.264 level that takes the frames', () => { + expect(avcCodec(1280, 720, 30)).toBe('avc1.64001f'); + expect(avcCodec(1920, 1080, 30)).toBe('avc1.640028'); + expect(avcCodec(1920, 1080, 60)).toBe('avc1.64002a'); + expect(avcCodec(2560, 1440, 60)).toBe('avc1.640033'); + expect(avcCodec(3840, 2160, 60)).toBe('avc1.640034'); + }); + + it('starts a group every two seconds', () => { + expect([0, 1, 119, 120, 240].map((i) => isKeyFrame(i, 60))).toEqual([true, false, false, true, true]); + expect(isKeyFrame(60, 30)).toBe(true); + }); +}); + +describe('RecordingClock', () => { + it('counts from the start and leaves the editor pause out', () => { + const clock = new RecordingClock(); + expect(clock.at(500)).toBe(0); + clock.start(1000); + expect(clock.at(1500)).toBe(500); + clock.pause(2000); + expect(clock.at(5000)).toBe(1000); + clock.resume(6000); + expect(clock.at(6500)).toBe(1500); + // Pausing twice and resuming once still takes out the whole pause. + clock.pause(7000); + clock.pause(7500); + clock.resume(8000); + expect(clock.at(8000)).toBe(2000); + }); +}); + +describe('FrameGate', () => { + const taken = (fps: number, frameMs: number, seconds: number, jitter = 0) => { + const gate = new FrameGate(fps); + let count = 0; + for (let i = 0; i * frameMs < seconds * 1000; i++) { + const wobble = jitter ? ((i * 7919) % 11) / 10 - 0.5 : 0; + if (gate.take(Math.max(0, i * frameMs + wobble * jitter))) count++; + } + return count; + }; + + it('takes every frame at 60 on a 60 Hz screen, however unevenly they come', () => { + expect(taken(60, 1000 / 60, 10)).toBe(600); + expect(taken(60, 1000 / 60, 10, 6)).toBe(600); + }); + + it('takes every other at 30', () => { + expect(taken(30, 1000 / 60, 10)).toBe(300); + }); + + it('takes about 60 a second from a 144 Hz screen', () => { + expect(Math.abs(taken(60, 1000 / 144, 10) - 600)).toBeLessThanOrEqual(6); + }); +}); + +describe('pictureTransform', () => { + it('maps the picture onto the whole frame', () => { + // A picture of 640x360 CSS pixels centred at (400, 300), recorded at 1080p. + const area = { x: 400, y: 300, w: 640, h: 360 }; + const m = pictureTransform(area, { w: 1920, h: 1080 }); + const at = (x: number, y: number) => ({ x: m.a * x + m.tx, y: m.d * y + m.ty }); + expect(at(80, 120)).toEqual({ x: 0, y: 0 }); + expect(at(720, 480)).toEqual({ x: 1920, y: 1080 }); + expect(at(400, 300)).toEqual({ x: 960, y: 540 }); + }); +}); + +describe('the recording nodes', () => { + it('Start recording hands its settings over and flows on', () => { + const ctx = fakeContext(); + const result = runDef('StartRecording', ctx, { name: 'Intro', resolution: '720p', fps: '30' }, {}); + expect(result.next).toBe('exec'); + expect(ctx.recordings).toEqual([{ name: 'Intro', resolution: '720p', fps: 30 }]); + }); + + it('Start recording asks for the game view at 60 by default', () => { + const ctx = fakeContext(); + runDef('StartRecording', ctx, {}, {}); + expect(ctx.recordings).toEqual([{ name: '', resolution: 'view', fps: 60 }]); + }); + + it('Stop recording asks for the stop and flows on', () => { + const ctx = fakeContext(); + expect(runDef('StopRecording', ctx, {}, {}).next).toBe('exec'); + expect(ctx.recordings).toEqual(['stop']); + }); +}); diff --git a/tests/helpers.ts b/tests/helpers.ts index 9bc3cf6c..c15ccf80 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -32,6 +32,7 @@ import type { Position, PropertyBag, Vec2 } from '../src/core/model/Properties'; import { createEmptyProject, findScene, type Project, ROUTING_ID } from '../src/core/model/Project'; import { settleRouting } from '../src/core/music/projectRouting'; import { MASTER_BUS } from '../src/core/properties/buses'; +import type { RecordingSettings } from '../src/core/recording/recording'; import type { SceneObject } from '../src/core/model/SceneObject'; import { deserializeProject, serializeProject } from '../src/core/store/serialize'; import type { AudioSink } from '../src/runtime/systems/Audio'; @@ -291,6 +292,8 @@ export type FakeContext = NodeContext & { /** The time scale and the pause the context holds, as the game nodes left them. */ scale: number; paused: boolean; + /** What the recording nodes asked for (#405), in order: the settings of a start, `'stop'` for a stop. */ + recordings: (RecordingSettings | 'stop')[]; }; /** A NodeContext double: self/owner handles, `speed = 100`, dt 0.5, moves applied directly, `log` and `stop` recorded. */ @@ -411,6 +414,9 @@ export function fakeContext(overrides: Partial = {}): FakeContext { setPaused: (paused) => { ctx.paused = paused; }, + recordings: [], + startRecording: (settings) => ctx.recordings.push(settings), + stopRecording: () => ctx.recordings.push('stop'), ...overrides, }; return ctx; diff --git a/tools/e2e/60-release.mjs b/tools/e2e/60-release.mjs index d0aed3fb..a7e74898 100644 --- a/tools/e2e/60-release.mjs +++ b/tools/e2e/60-release.mjs @@ -13,7 +13,7 @@ const BACKGROUND = '#22252c'; /** * A small game of its own: a red crate in the middle of the camera that the arrow keys move, a - * `Log` on start, and S stopping the game. + * `Log` and a recording on start, and S stopping the game. */ function game(id) { const mover = { @@ -37,10 +37,15 @@ function game(id) { nodes: [ node('c-start', 'OnStart', 0, 0), node('c-log', 'Log', 260, 0, { message: 'released game started' }), + node('c-record', 'StartRecording', 520, 0, { name: 'Released 60', fps: '30' }), node('c-key', 'OnKeyDown', 0, 160, { key: 's' }), node('c-stop', 'StopGame', 260, 160), ], - edges: [edge('c1', 'c-start', 'exec', 'c-log', 'exec'), edge('c2', 'c-key', 'exec', 'c-stop', 'exec')], + edges: [ + edge('c1', 'c-start', 'exec', 'c-log', 'exec'), + edge('c2', 'c-key', 'exec', 'c-stop', 'exec'), + edge('c3', 'c-log', 'exec', 'c-record', 'exec'), + ], }; return { formatVersion: 2, @@ -226,6 +231,9 @@ export async function release(t) { ); // 60d. A click starts it in the start scene, zoomed to Auto: a whole number of pixels per unit. + // What the page hands the browser to download is caught here instead (60j). + await t.evaluate(`window.__downloads = []; + HTMLAnchorElement.prototype.click = function () { window.__downloads.push({ name: this.download, href: this.href }); };`); const size = { w: await json('innerWidth'), h: await json('innerHeight') }; await click(size.w / 2, size.h / 2); await waitUntil('__gameMakerPlay.running()', { label: 'the game runs' }); @@ -286,6 +294,24 @@ export async function release(t) { JSON.stringify({ moved, screen: await player('screen()') }), ); + // 60j (#405). The game recorded itself from its start: ending the run saved the recording, and a + // released game, with no project to put it in, hands it to the browser as a Matroska file. + await waitUntil('window.__downloads.length > 0', { label: 'the recording is downloaded' }); + const downloads = await json('window.__downloads'); + const file = JSON.parse( + await t.evaluate(`fetch(${JSON.stringify(downloads[0]?.href ?? '')}) + .then((r) => r.arrayBuffer()) + .then((b) => JSON.stringify({ bytes: b.byteLength, head: [...new Uint8Array(b.slice(0, 4))] }))`), + ); + check( + 'a released game downloads its recording as an MKV file once the run ends', + downloads.length === 1 && + downloads[0].name === 'Released 60.mkv' && + file.bytes > 1000 && + JSON.stringify(file.head) === JSON.stringify([0x1a, 0x45, 0xdf, 0xa3]), + JSON.stringify({ downloads: downloads.map((d) => d.name), file }), + ); + // 60h (#267). A click on the corner button makes the page fullscreen and does not reach the // game – it would start it again; a second click leaves fullscreen. const corner = (await player('fullscreen()')).button; diff --git a/tools/e2e/67-recording.mjs b/tools/e2e/67-recording.mjs new file mode 100644 index 00000000..b3f8c29c --- /dev/null +++ b/tools/e2e/67-recording.mjs @@ -0,0 +1,183 @@ +import { MAKE_OBJECT } from './lib/scaffold.mjs'; + +/** + * Recording the game view (#405). A behavior starts a recording at 1080p and 30 fps when the run + * starts, and V stops it: the server converts the file like any upload, and once the run is over a + * video asset of that name is in the project, 1920×1080 with sound, its picture the red sprite that + * covers the camera. G starts another at the size of the game view; a second G is ignored and says + * so in the log, and Esc ends the run, which saves that one too, named after when it started. + * + * Everything it adds carries an `rc67` id or is one of the two video assets, and goes again at the end. + */ +const CONTROL = 'rc67-control'; +const COVER = 'rc67-cover'; +const NAME = 'rc67 take'; + +const node = (id, type, x, y, params = {}) => ({ id: `rc67-${id}`, type, x, y, params }); +const edge = (id, from, to) => ({ + id: `rc67-${id}`, + from: { node: `rc67-${from}`, port: 'exec' }, + to: { node: `rc67-${to}`, port: 'exec' }, +}); + +export async function recording(t) { + const { check, sleep, json, evaluate, mutate, press, play, stop, showTab, waitUntil } = t; + await showTab('scene'); + + const graph = { + nodes: [ + node('start', 'OnStart', 0, 0), + node('record', 'StartRecording', 300, 0, { name: NAME, resolution: '1080p', fps: '30' }), + node('v', 'OnKeyDown', 0, 200, { key: 'v' }), + node('stop', 'StopRecording', 300, 200), + node('g', 'OnKeyDown', 0, 400, { key: 'g' }), + node('again', 'StartRecording', 300, 400), + ], + edges: [edge('e1', 'start', 'record'), edge('e2', 'v', 'stop'), edge('e3', 'g', 'again')], + }; + // Red over the whole camera picture, blue over its right half and green over its lower left + // quarter, in front of what earlier steps left there: a frame drawn at the wrong place, or read + // back upside down, shows the wrong colour. + await mutate(` + const p = __gm.state.project; + const scene = __gm.state.activeScene; + const cam = scene.objects.find((o) => o.id === 'o3').position; + p.assets.push({ id: '${CONTROL}', type: 'behavior', name: 'Recorder 67', props: { enabled: true, runInEditor: false }, variables: [], graph: ${JSON.stringify(graph)} }); + ${MAKE_OBJECT} + const sprite = p.assets.find((a) => a.type === 'sprite'); + const cover = make('${COVER}', sprite.id, '', 0, 'Cover 67'); + Object.assign(cover.position, { x: cam.x, y: cam.y, z: 1000 }); + cover.overrides = { width: 900, height: 600, color: 0xff0000, texture: '' }; + const right = make('rc67-right', sprite.id, '', 0, 'Right 67'); + Object.assign(right.position, { x: cam.x + 225, y: cam.y, z: 1001 }); + right.overrides = { width: 450, height: 600, color: 0x0000ff, texture: '' }; + const low = make('rc67-low', sprite.id, '', 0, 'Low 67'); + Object.assign(low.position, { x: cam.x - 225, y: cam.y + 150, z: 1001 }); + low.overrides = { width: 450, height: 300, color: 0x00ff00, texture: '' }; + const logic = make('rc67-logic', '${CONTROL}', '', 0, 'Recorder 67'); + Object.assign(logic.position, { x: cam.x + 400, y: cam.y }); + scene.objects.push(cover, right, low, logic); + `); + const recordings = () => json('__gm.recording()'); + + // 67a. Recorded from the start of the run, at 1080p and 30 fps, with the master's sound. The window + // is smaller than 1080p: the frame is drawn into a texture of its own (`PictureReader`). + await play(); + await sleep(1200); + await press('v'); + await waitUntil(`__gm.recording().takes[0]?.state === 'done'`, { label: 'the first take is saved' }); + const first = (await recordings()).takes[0]; + check( + 'Start recording records the game view at the size asked, with sound, and Stop recording saves it', + first.name === NAME && + first.width === 1920 && + first.height === 1080 && + first.fps === 30 && + first.sound && + first.soundFrames > 0 && + first.frames >= 25 && + first.frames <= 45 && + first.failure === null, + JSON.stringify(first), + ); + + // 67b. A second start while one records is ignored, and says so in the log. + await press('g'); + await sleep(300); + await press('g'); + await sleep(900); + const log = (await json('__gm.world()'))?.log ?? []; + const during = await recordings(); + check( + 'a second Start recording while one records is ignored and says so in the log', + during.takes.length === 2 && + log.some((line) => line.includes('Start recording is ignored') && line.includes(during.takes[1].name)), + JSON.stringify({ log, takes: during.takes.map((take) => take.name) }), + ); + await stop(); + + // 67c. Ending the run saved the second take; both became video assets once the server had them. + await waitUntil( + `__gm.recording().uploads.length === 2 && __gm.recording().uploads.every((u) => u.state === 'added' || u.state === 'failed')`, + { + timeout: 60_000, + label: 'both recordings are in the project', + }, + ); + const { takes, uploads } = await recordings(); + const second = takes[1]; + const assets = await json(`__gm.state.project.assets + .filter((a) => a.type === 'video' && ${JSON.stringify(uploads.map((u) => u.assetId))}.includes(a.id)) + .map((a) => ({ id: a.id, name: a.name, file: a.file, width: a.width, height: a.height, duration: a.duration, fps: a.fps, audio: a.audio }))`); + const named = assets.find((a) => a.name === NAME); + check( + 'the recording becomes a video asset of its name: 1920×1080, its length, with sound', + named !== undefined && + named.width === 1920 && + named.height === 1080 && + named.audio === true && + Math.abs(named.duration - first.durationMs / 1000) < 0.1 && + named.fps === 30, + JSON.stringify({ assets, uploads }), + ); + const timed = assets.find((a) => a.name !== NAME); + check( + 'ending the run saves a recording still going, named after when it started, at the game view size', + second.state === 'done' && + /^Recording \d{4}-\d\d-\d\d \d\d:\d\d:\d\d$/.test(second.name) && + timed?.name === second.name && + timed.width === second.width && + timed.height === second.height && + second.width % 2 === 0 && + second.fps === 60, + JSON.stringify({ second, timed }), + ); + + // 67d. The picture is the camera's: red at the top left, green below it and blue on the right, + // none of what the view lays over it, the right way up, and as red as it was drawn; every + // 4th pixel is read. + // From the MP4 as Chrome decodes it: the poster is no witness, ffmpeg cuts it with BT.601 + // whatever the file says (a red of 255 comes out 229). + const colorsOf = (file) => + evaluate(`new Promise((resolve) => { + const video = document.createElement('video'); + video.muted = true; + video.onerror = () => resolve('null'); + video.src = '/game-maker/data/videos/${file}'; + video.play().then(() => video.requestVideoFrameCallback(() => { + video.pause(); + const canvas = document.createElement('canvas'); + canvas.width = video.videoWidth; + canvas.height = video.videoHeight; + const pen = canvas.getContext('2d'); + pen.drawImage(video, 0, 0); + const { data } = pen.getImageData(0, 0, canvas.width, canvas.height); + const off = []; + for (let y = 2; y < canvas.height; y += 4) + for (let x = 2; x < canvas.width; x += 4) { + const i = (y * canvas.width + x) * 4; + const edge = canvas.width / 2; + const middle = canvas.height / 2; + if (Math.abs(x - edge) < canvas.width / 100 || Math.abs(y - middle) < canvas.height / 100) continue; + const hot = x >= edge ? 2 : y >= middle ? 1 : 0; + const rgb = [data[i], data[i + 1], data[i + 2]]; + if (rgb.some((v, c) => (c === hot ? v < 245 : v > 12))) off.push([x, y, ...rgb]); + } + resolve(JSON.stringify({ w: canvas.width, h: canvas.height, off: off.length, first: off.slice(0, 5) })); + }), () => resolve('null')); + })`); + const colors = `[${await colorsOf(named?.file ?? '')},${await colorsOf(timed?.file ?? '')}]`; + const red = JSON.parse(colors); + check( + 'the recorded picture is the camera picture, the right way up, as red as it was drawn', + red.every((frame) => frame !== null && frame.w > 0 && frame.off === 0), + colors, + ); + + await mutate(` + const p = __gm.state.project; + const made = ${JSON.stringify(assets.map((a) => a.id))}; + p.assets = p.assets.filter((a) => a.id !== '${CONTROL}' && !made.includes(a.id)); + const scene = __gm.state.activeScene; + scene.objects = scene.objects.filter((o) => !o.id.startsWith('rc67-'));`); +} diff --git a/tools/e2e/index.mjs b/tools/e2e/index.mjs index 8161bf41..3e0a64d6 100644 --- a/tools/e2e/index.mjs +++ b/tools/e2e/index.mjs @@ -69,6 +69,7 @@ import { renderLayers } from './63-render-layers.mjs'; import { pattern } from './64-pattern.mjs'; import { song } from './65-song.mjs'; import { video } from './66-video.mjs'; +import { recording } from './67-recording.mjs'; export const STEPS = [ hierarchy, @@ -137,4 +138,5 @@ export const STEPS = [ pattern, song, video, + recording, ];