Record the game view from a behavior into a video asset (#405)
A game can now film itself. A behavior starts and stops a take, and the editor turns it into a video asset with the run's picture and the sound the player hears. The exported game hands the file over as a download, since there is no project to keep it in. - New action nodes `Start recording` (Name, Resolution, FPS) and `Stop recording`. An empty name becomes `Recording <timestamp>`. - Resolution is `Game view`, `720p` or `1080p`. For the fixed sizes the height is set and the width follows the camera. - The picture is the camera only, without hints, overlays or the letterbox. It is drawn into a render texture of the target size, so a smaller window still gives full 1080p. - `PictureReader` reads the texture back through pixel buffers with a fence. The editor never waits on the GPU, and a frame with all three buffers busy is dropped and counted. - Nothing of the take reaches the visible canvas. A first version drew tiles into it and copied them out, and with the low-latency canvas that midway state flickered on screen. - The sound is tapped behind the master limiter in the FX worklet and encoded as Opus. - H.264 is chosen when the browser has it, VP9 otherwise. A small own Matroska writer packs both tracks, no new dependency. It names the frame rate and BT.709 so ffmpeg keeps both. - A second `Start recording` is ignored with a warning in the log. A take goes on through `Load scene` and a restart, pauses with the editor and is saved when the run ends. It stops at 2 GB. - The editor uploads the take on the existing video route and makes the asset once the run is over. The video tab gets a Download button for the converted MP4. - `downloadBlob` in `src/ui` replaces the song editor's own copy. Findings: - `On start` runs before the game screen knows the world. A take therefore fixes its size on the first frame that has a camera. - Without a frame rate in the file, ffmpeg turned 30 fps into 59.94. - The worker's poster JPEG is off in colour for every upload (red 255 comes out as 229, BT.601). Not touched here. Tests: - Unit recording + matroska: 21 green, plus the node catalog cases. The Matroska test reads the file back with its own EBML reader. - e2e 67 (new) on melektaus-atx: ALL PASSED. 1080p30 from a 1280x800 window gives a 1920x1080 MP4 with sound and no dropped frame. Every 4th pixel is checked, a green quarter catches an upside-down picture. A take ended by Esc is saved with the timestamp name. - e2e 60j: the exported game downloads an `.mkv` with EBML magic. - The flicker is gone on the user's machine without `?vsync`. - Gate on melektaus-atx: CHECK EXIT 0, 299 files, 4586 tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
adminandClaude Opus 5.5 committed
1 parent
36ffd9a61a
commit
e191dc4b2e
35 files changed
+2211
-26
No files matched your search
@@ -0,0 +1,202 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { AVC_CODEC, colourOf, type MatroskaHead, MatroskaWriter } from '../../src/core/recording/matroska';
|
||||
|
||||
/** One element as read back: its id in hex, its bytes, and what it holds where it is a master. */
|
||||
interface Read {
|
||||
id: string;
|
||||
data: Uint8Array;
|
||||
children: Read[];
|
||||
}
|
||||
|
||||
const MASTERS = new Set([
|
||||
'1a45dfa3',
|
||||
'18538067',
|
||||
'1549a966',
|
||||
'1654ae6b',
|
||||
'ae',
|
||||
'e0',
|
||||
'55b0',
|
||||
'e1',
|
||||
'1f43b675',
|
||||
]);
|
||||
|
||||
function vint(bytes: Uint8Array, at: number, keepMarker: boolean): { value: number; length: number } {
|
||||
const first = bytes[at]!;
|
||||
let length = 1;
|
||||
while (length <= 8 && !(first & (0x80 >> (length - 1)))) length++;
|
||||
let value = keepMarker ? first : first & (0xff >> length);
|
||||
for (let i = 1; i < length; i++) value = value * 256 + bytes[at + i]!;
|
||||
return { value, length };
|
||||
}
|
||||
|
||||
function readAll(bytes: Uint8Array, from = 0, to = bytes.length): Read[] {
|
||||
const out: Read[] = [];
|
||||
let at = from;
|
||||
while (at < to) {
|
||||
const id = vint(bytes, at, true);
|
||||
const size = vint(bytes, at + id.length, false);
|
||||
const start = at + id.length + size.length;
|
||||
const data = bytes.subarray(start, start + size.value);
|
||||
const hex = id.value.toString(16);
|
||||
out.push({ id: hex, data, children: MASTERS.has(hex) ? readAll(bytes, start, start + size.value) : [] });
|
||||
at = start + size.value;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const find = (list: Read[], id: string) => list.find((e) => e.id === id);
|
||||
const uintOf = (e: Read | undefined) => (e ? e.data.reduce((v, b) => v * 256 + b, 0) : NaN);
|
||||
const textOf = (e: Read | undefined) => new TextDecoder().decode(e?.data);
|
||||
|
||||
/** Track, time relative to its cluster, key flag and payload of each block, with the cluster's time. */
|
||||
function blocksOf(
|
||||
file: Uint8Array,
|
||||
): { cluster: number; track: number; ms: number; key: boolean; data: number[] }[] {
|
||||
const segment = find(readAll(file), '18538067')!;
|
||||
return segment.children
|
||||
.filter((e) => e.id === '1f43b675')
|
||||
.flatMap((cluster) => {
|
||||
const start = uintOf(find(cluster.children, 'e7'));
|
||||
return cluster.children
|
||||
.filter((e) => e.id === 'a3')
|
||||
.map((block) => ({
|
||||
cluster: start,
|
||||
track: block.data[0]! & 0x7f,
|
||||
ms: start + new DataView(block.data.buffer, block.data.byteOffset + 1, 2).getInt16(0),
|
||||
key: (block.data[3]! & 0x80) !== 0,
|
||||
data: [...block.data.subarray(4)],
|
||||
}));
|
||||
});
|
||||
}
|
||||
|
||||
const HEAD: MatroskaHead = {
|
||||
video: {
|
||||
codec: AVC_CODEC,
|
||||
width: 1280,
|
||||
height: 720,
|
||||
private: Uint8Array.from([1, 100, 0, 31]),
|
||||
fps: 30,
|
||||
colour: colourOf({ matrix: 'bt709', transfer: 'iec61966-2-1', primaries: 'bt709', fullRange: false }),
|
||||
},
|
||||
audio: { channels: 2, private: new TextEncoder().encode('OpusHead'), preSkip: 312 },
|
||||
durationMs: 1234.5,
|
||||
app: 'game-maker',
|
||||
};
|
||||
|
||||
/** The whole file: the head, then the clusters handed out before (`earlier`) and at the end. */
|
||||
function fileOf(writer: MatroskaWriter, head: MatroskaHead = HEAD, earlier: Uint8Array[] = []): Uint8Array {
|
||||
const taken = writer.take();
|
||||
const { head: bytes, rest } = writer.finish(head);
|
||||
const parts = [bytes, ...earlier, ...taken, ...rest];
|
||||
const out = new Uint8Array(parts.reduce((n, p) => n + p.length, 0));
|
||||
let at = 0;
|
||||
for (const part of parts) {
|
||||
out.set(part, at);
|
||||
at += part.length;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
describe('MatroskaWriter (#405)', () => {
|
||||
it('writes a head that says what the tracks are and how long the file is', () => {
|
||||
const writer = new MatroskaWriter(2);
|
||||
writer.add(1, 0, true, Uint8Array.from([9]));
|
||||
const file = fileOf(writer);
|
||||
const [ebml, segment] = readAll(file);
|
||||
expect(textOf(find(ebml!.children, '4282'))).toBe('matroska');
|
||||
// The segment's size covers exactly the rest of the file.
|
||||
expect(segment!.data.byteOffset + segment!.data.length).toBe(file.length);
|
||||
const info = find(segment!.children, '1549a966')!;
|
||||
expect(uintOf(find(info.children, '2ad7b1'))).toBe(1_000_000);
|
||||
const duration = find(info.children, '4489')!;
|
||||
expect(new DataView(duration.data.buffer, duration.data.byteOffset, 8).getFloat64(0)).toBe(1234.5);
|
||||
const [video, audio] = find(segment!.children, '1654ae6b')!.children;
|
||||
expect(textOf(find(video!.children, '86'))).toBe('V_MPEG4/ISO/AVC');
|
||||
expect([...find(video!.children, '63a2')!.data]).toEqual([1, 100, 0, 31]);
|
||||
const picture = find(video!.children, 'e0')!;
|
||||
expect([uintOf(find(picture.children, 'b0')), uintOf(find(picture.children, 'ba'))]).toEqual([1280, 720]);
|
||||
// A frame lasts a 30th of a second, in nanoseconds, so nothing has to guess the rate.
|
||||
expect(uintOf(find(video!.children, '23e383'))).toBe(33_333_333);
|
||||
// BT.709 pixels in the broadcast range, sRGB's transfer.
|
||||
const colour = find(picture.children, '55b0')!.children;
|
||||
expect(['55b1', '55b9', '55ba', '55bb'].map((id) => uintOf(find(colour, id)))).toEqual([1, 1, 13, 1]);
|
||||
expect(textOf(find(audio!.children, '86'))).toBe('A_OPUS');
|
||||
expect(textOf(find(audio!.children, '63a2'))).toBe('OpusHead');
|
||||
// 312 samples at 48 kHz, in nanoseconds.
|
||||
expect(uintOf(find(audio!.children, '56aa'))).toBe(6_500_000);
|
||||
expect(uintOf(find(find(audio!.children, 'e1')!.children, '9f'))).toBe(2);
|
||||
});
|
||||
|
||||
it('takes the colour from what the encoder says, BT.709 where it says nothing', () => {
|
||||
expect(colourOf(null)).toEqual({ matrix: 1, transfer: 1, primaries: 1, fullRange: false });
|
||||
expect(colourOf({ matrix: 'smpte170m', fullRange: true })).toEqual({
|
||||
matrix: 6,
|
||||
transfer: 1,
|
||||
primaries: 1,
|
||||
fullRange: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('writes the blocks of both tracks in the order of their time, whatever order they come in', () => {
|
||||
const writer = new MatroskaWriter(2);
|
||||
writer.add(1, 0, true, Uint8Array.from([1]));
|
||||
writer.add(1, 16_667, false, Uint8Array.from([2]));
|
||||
writer.add(1, 33_333, false, Uint8Array.from([3]));
|
||||
writer.add(2, 0, true, Uint8Array.from([10]));
|
||||
writer.add(2, 20_000, true, Uint8Array.from([11]));
|
||||
writer.add(2, 40_000, true, Uint8Array.from([12]));
|
||||
const blocks = blocksOf(fileOf(writer));
|
||||
expect(blocks.map((b) => [b.track, b.ms])).toEqual([
|
||||
[1, 0],
|
||||
[2, 0],
|
||||
[1, 17],
|
||||
[2, 20],
|
||||
[1, 33],
|
||||
[2, 40],
|
||||
]);
|
||||
expect(blocks.map((b) => b.data[0])).toEqual([1, 10, 2, 11, 3, 12]);
|
||||
});
|
||||
|
||||
it('hands out a cluster once the next key frame starts another', () => {
|
||||
const writer = new MatroskaWriter(1);
|
||||
writer.add(1, 0, true, Uint8Array.from([1]));
|
||||
writer.add(1, 1_000_000, false, Uint8Array.from([2]));
|
||||
expect(writer.take()).toEqual([]);
|
||||
writer.add(1, 2_000_000, true, Uint8Array.from([3]));
|
||||
const first = writer.take();
|
||||
expect(first).toHaveLength(1);
|
||||
const blocks = blocksOf(fileOf(writer, HEAD, first));
|
||||
expect(blocks.map((b) => [b.cluster, b.ms, b.key])).toEqual([
|
||||
[0, 0, true],
|
||||
[0, 1000, false],
|
||||
[2000, 2000, true],
|
||||
]);
|
||||
});
|
||||
|
||||
it('starts a new cluster before a block would be too far from its start to be told', () => {
|
||||
const writer = new MatroskaWriter(1);
|
||||
writer.add(1, 0, true, Uint8Array.from([1]));
|
||||
writer.add(1, 40_000_000, false, Uint8Array.from([2]));
|
||||
const blocks = blocksOf(fileOf(writer));
|
||||
expect(blocks.map((b) => [b.cluster, b.ms])).toEqual([
|
||||
[0, 0],
|
||||
[40_000, 40_000],
|
||||
]);
|
||||
});
|
||||
|
||||
it('does not hold the video back for ever when no sound comes', () => {
|
||||
const writer = new MatroskaWriter(2);
|
||||
for (let i = 0; i < 90; i++) writer.add(1, i * 33_333, i % 30 === 0, Uint8Array.from([i]));
|
||||
// Three seconds of video, no sound: what is more than a second behind has gone into clusters.
|
||||
expect(writer.take().length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('counts its bytes, the open cluster included, and leaves out a missing sound track', () => {
|
||||
const writer = new MatroskaWriter(1);
|
||||
writer.add(1, 0, true, new Uint8Array(100));
|
||||
expect(writer.bytes).toBeGreaterThan(100);
|
||||
const file = fileOf(writer, { ...HEAD, audio: null });
|
||||
const segment = find(readAll(file), '18538067')!;
|
||||
expect(find(segment.children, '1654ae6b')!.children).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -2393,6 +2393,21 @@ const CASES: Record<string, Case[]> = {
|
||||
],
|
||||
StopGame: [{ name: 'ends play and the flow', next: null, after: (ctx) => expect(ctx.stopped).toBe(true) }],
|
||||
RestartGame: [{ name: 'ends the flow – the world restarts before the next tick', next: null }],
|
||||
StartRecording: [
|
||||
{
|
||||
name: 'asks for a recording and flows on',
|
||||
inputs: { name: 'Take', resolution: '1080p', fps: '60' },
|
||||
next: 'exec',
|
||||
after: (ctx) => expect(ctx.recordings).toEqual([{ name: 'Take', resolution: '1080p', fps: 60 }]),
|
||||
},
|
||||
],
|
||||
StopRecording: [
|
||||
{
|
||||
name: 'asks to stop and flows on',
|
||||
next: 'exec',
|
||||
after: (ctx) => expect(ctx.recordings).toEqual(['stop']),
|
||||
},
|
||||
],
|
||||
PauseGame: [
|
||||
{ name: 'pauses the game and flows on', next: 'exec', after: (ctx) => expect(ctx.paused).toBe(true) },
|
||||
],
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
avcCodec,
|
||||
FrameGate,
|
||||
isKeyFrame,
|
||||
pictureTransform,
|
||||
RecordingClock,
|
||||
recordingName,
|
||||
recordingSettingsOf,
|
||||
recordingSize,
|
||||
} from '../../src/core/recording/recording';
|
||||
import { runDef, fakeContext } from '../helpers';
|
||||
|
||||
describe('recording settings (#405)', () => {
|
||||
it('names a recording after its start when no name is given', () => {
|
||||
const at = new Date(2026, 9, 11, 16, 42, 5);
|
||||
expect(recordingName('', at)).toBe('Recording 2026-10-11 16:42:05');
|
||||
expect(recordingName(' ', at)).toBe('Recording 2026-10-11 16:42:05');
|
||||
expect(recordingName(' Boss fight ', at)).toBe('Boss fight');
|
||||
});
|
||||
|
||||
it('reads what the node hands over, an unknown choice as the default', () => {
|
||||
expect(recordingSettingsOf(' Run ', '1080p', '30')).toEqual({
|
||||
name: 'Run',
|
||||
resolution: '1080p',
|
||||
fps: 30,
|
||||
});
|
||||
expect(recordingSettingsOf('', 'huge', 25)).toEqual({ name: '', resolution: 'view', fps: 60 });
|
||||
});
|
||||
|
||||
it('records the game view at its size on screen, even on both sides', () => {
|
||||
expect(recordingSize('view', { w: 1279, h: 721 }, { w: 1279, h: 721 })).toEqual({ w: 1280, h: 722 });
|
||||
// A 5K screen is held to 4K.
|
||||
expect(recordingSize('view', { w: 5120, h: 2880 }, { w: 5120, h: 2880 })).toEqual({ w: 3840, h: 2160 });
|
||||
});
|
||||
|
||||
it('holds the height of 720p and 1080p and takes the width from the camera', () => {
|
||||
expect(recordingSize('1080p', { w: 800, h: 450 }, { w: 800, h: 450 })).toEqual({ w: 1920, h: 1080 });
|
||||
expect(recordingSize('720p', { w: 400, h: 400 }, { w: 400, h: 400 })).toEqual({ w: 720, h: 720 });
|
||||
// 4:3 at 720p, and the camera's whole picture rules where the screen cuts it.
|
||||
expect(recordingSize('720p', { w: 300, h: 400 }, { w: 640, h: 480 })).toEqual({ w: 960, h: 720 });
|
||||
});
|
||||
|
||||
it('picks the lowest H.264 level that takes the frames', () => {
|
||||
expect(avcCodec(1280, 720, 30)).toBe('avc1.64001f');
|
||||
expect(avcCodec(1920, 1080, 30)).toBe('avc1.640028');
|
||||
expect(avcCodec(1920, 1080, 60)).toBe('avc1.64002a');
|
||||
expect(avcCodec(2560, 1440, 60)).toBe('avc1.640033');
|
||||
expect(avcCodec(3840, 2160, 60)).toBe('avc1.640034');
|
||||
});
|
||||
|
||||
it('starts a group every two seconds', () => {
|
||||
expect([0, 1, 119, 120, 240].map((i) => isKeyFrame(i, 60))).toEqual([true, false, false, true, true]);
|
||||
expect(isKeyFrame(60, 30)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('RecordingClock', () => {
|
||||
it('counts from the start and leaves the editor pause out', () => {
|
||||
const clock = new RecordingClock();
|
||||
expect(clock.at(500)).toBe(0);
|
||||
clock.start(1000);
|
||||
expect(clock.at(1500)).toBe(500);
|
||||
clock.pause(2000);
|
||||
expect(clock.at(5000)).toBe(1000);
|
||||
clock.resume(6000);
|
||||
expect(clock.at(6500)).toBe(1500);
|
||||
// Pausing twice and resuming once still takes out the whole pause.
|
||||
clock.pause(7000);
|
||||
clock.pause(7500);
|
||||
clock.resume(8000);
|
||||
expect(clock.at(8000)).toBe(2000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('FrameGate', () => {
|
||||
const taken = (fps: number, frameMs: number, seconds: number, jitter = 0) => {
|
||||
const gate = new FrameGate(fps);
|
||||
let count = 0;
|
||||
for (let i = 0; i * frameMs < seconds * 1000; i++) {
|
||||
const wobble = jitter ? ((i * 7919) % 11) / 10 - 0.5 : 0;
|
||||
if (gate.take(Math.max(0, i * frameMs + wobble * jitter))) count++;
|
||||
}
|
||||
return count;
|
||||
};
|
||||
|
||||
it('takes every frame at 60 on a 60 Hz screen, however unevenly they come', () => {
|
||||
expect(taken(60, 1000 / 60, 10)).toBe(600);
|
||||
expect(taken(60, 1000 / 60, 10, 6)).toBe(600);
|
||||
});
|
||||
|
||||
it('takes every other at 30', () => {
|
||||
expect(taken(30, 1000 / 60, 10)).toBe(300);
|
||||
});
|
||||
|
||||
it('takes about 60 a second from a 144 Hz screen', () => {
|
||||
expect(Math.abs(taken(60, 1000 / 144, 10) - 600)).toBeLessThanOrEqual(6);
|
||||
});
|
||||
});
|
||||
|
||||
describe('pictureTransform', () => {
|
||||
it('maps the picture onto the whole frame', () => {
|
||||
// A picture of 640x360 CSS pixels centred at (400, 300), recorded at 1080p.
|
||||
const area = { x: 400, y: 300, w: 640, h: 360 };
|
||||
const m = pictureTransform(area, { w: 1920, h: 1080 });
|
||||
const at = (x: number, y: number) => ({ x: m.a * x + m.tx, y: m.d * y + m.ty });
|
||||
expect(at(80, 120)).toEqual({ x: 0, y: 0 });
|
||||
expect(at(720, 480)).toEqual({ x: 1920, y: 1080 });
|
||||
expect(at(400, 300)).toEqual({ x: 960, y: 540 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('the recording nodes', () => {
|
||||
it('Start recording hands its settings over and flows on', () => {
|
||||
const ctx = fakeContext();
|
||||
const result = runDef('StartRecording', ctx, { name: 'Intro', resolution: '720p', fps: '30' }, {});
|
||||
expect(result.next).toBe('exec');
|
||||
expect(ctx.recordings).toEqual([{ name: 'Intro', resolution: '720p', fps: 30 }]);
|
||||
});
|
||||
|
||||
it('Start recording asks for the game view at 60 by default', () => {
|
||||
const ctx = fakeContext();
|
||||
runDef('StartRecording', ctx, {}, {});
|
||||
expect(ctx.recordings).toEqual([{ name: '', resolution: 'view', fps: 60 }]);
|
||||
});
|
||||
|
||||
it('Stop recording asks for the stop and flows on', () => {
|
||||
const ctx = fakeContext();
|
||||
expect(runDef('StopRecording', ctx, {}, {}).next).toBe('exec');
|
||||
expect(ctx.recordings).toEqual(['stop']);
|
||||
});
|
||||
});
|
||||
@@ -32,6 +32,7 @@ import type { Position, PropertyBag, Vec2 } from '../src/core/model/Properties';
|
||||
import { createEmptyProject, findScene, type Project, ROUTING_ID } from '../src/core/model/Project';
|
||||
import { settleRouting } from '../src/core/music/projectRouting';
|
||||
import { MASTER_BUS } from '../src/core/properties/buses';
|
||||
import type { RecordingSettings } from '../src/core/recording/recording';
|
||||
import type { SceneObject } from '../src/core/model/SceneObject';
|
||||
import { deserializeProject, serializeProject } from '../src/core/store/serialize';
|
||||
import type { AudioSink } from '../src/runtime/systems/Audio';
|
||||
@@ -291,6 +292,8 @@ export type FakeContext = NodeContext & {
|
||||
/** The time scale and the pause the context holds, as the game nodes left them. */
|
||||
scale: number;
|
||||
paused: boolean;
|
||||
/** What the recording nodes asked for (#405), in order: the settings of a start, `'stop'` for a stop. */
|
||||
recordings: (RecordingSettings | 'stop')[];
|
||||
};
|
||||
|
||||
/** A NodeContext double: self/owner handles, `speed = 100`, dt 0.5, moves applied directly, `log` and `stop` recorded. */
|
||||
@@ -411,6 +414,9 @@ export function fakeContext(overrides: Partial<NodeContext> = {}): FakeContext {
|
||||
setPaused: (paused) => {
|
||||
ctx.paused = paused;
|
||||
},
|
||||
recordings: [],
|
||||
startRecording: (settings) => ctx.recordings.push(settings),
|
||||
stopRecording: () => ctx.recordings.push('stop'),
|
||||
...overrides,
|
||||
};
|
||||
return ctx;
|
||||
|
||||
Reference in new issue
Block a user