Record the game view from a behavior into a video asset (#405)
A game can now film itself. A behavior starts and stops a take, and the editor turns it into a video asset with the run's picture and the sound the player hears. The exported game hands the file over as a download, since there is no project to keep it in. - New action nodes `Start recording` (Name, Resolution, FPS) and `Stop recording`. An empty name becomes `Recording <timestamp>`. - Resolution is `Game view`, `720p` or `1080p`. For the fixed sizes the height is set and the width follows the camera. - The picture is the camera only, without hints, overlays or the letterbox. It is drawn into a render texture of the target size, so a smaller window still gives full 1080p. - `PictureReader` reads the texture back through pixel buffers with a fence. The editor never waits on the GPU, and a frame with all three buffers busy is dropped and counted. - Nothing of the take reaches the visible canvas. A first version drew tiles into it and copied them out, and with the low-latency canvas that midway state flickered on screen. - The sound is tapped behind the master limiter in the FX worklet and encoded as Opus. - H.264 is chosen when the browser has it, VP9 otherwise. A small own Matroska writer packs both tracks, no new dependency. It names the frame rate and BT.709 so ffmpeg keeps both. - A second `Start recording` is ignored with a warning in the log. A take goes on through `Load scene` and a restart, pauses with the editor and is saved when the run ends. It stops at 2 GB. - The editor uploads the take on the existing video route and makes the asset once the run is over. The video tab gets a Download button for the converted MP4. - `downloadBlob` in `src/ui` replaces the song editor's own copy. Findings: - `On start` runs before the game screen knows the world. A take therefore fixes its size on the first frame that has a camera. - Without a frame rate in the file, ffmpeg turned 30 fps into 59.94. - The worker's poster JPEG is off in colour for every upload (red 255 comes out as 229, BT.601). Not touched here. Tests: - Unit recording + matroska: 21 green, plus the node catalog cases. The Matroska test reads the file back with its own EBML reader. - e2e 67 (new) on melektaus-atx: ALL PASSED. 1080p30 from a 1280x800 window gives a 1920x1080 MP4 with sound and no dropped frame. Every 4th pixel is checked, a green quarter catches an upside-down picture. A take ended by Esc is saved with the timestamp name. - e2e 60j: the exported game downloads an `.mkv` with EBML magic. - The flicker is gone on the user's machine without `?vsync`. - Gate on melektaus-atx: CHECK EXIT 0, 299 files, 4586 tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
36ffd9a61a
commit
e191dc4b2e
35 files changed
+2211
-26
No files matched your search
@@ -14,8 +14,9 @@ import { gameClock } from '../core/time';
|
||||
import { loadPhysics, readyPhysics } from '../runtime/physics/engine';
|
||||
import { Input } from '../runtime/systems/Input';
|
||||
import type { World } from '../runtime/World';
|
||||
import { ui } from '../ui';
|
||||
import { onFrame, ui } from '../ui';
|
||||
import { soundLibrary } from '../viewport/SoundLibrary';
|
||||
import { GameRecorder, type MadeRecording } from '../viewport/GameRecorder';
|
||||
import { VIDEO_WAIT_MS, videoLibrary } from '../viewport/VideoLibrary';
|
||||
import { FrameMeter, type FrameTimes } from './frameMeter';
|
||||
import { saveArea } from './saves';
|
||||
@@ -55,6 +56,8 @@ export interface PlayHost {
|
||||
showGame(): void;
|
||||
/** Leave the game view if it is the one on screen. */
|
||||
leaveGame(): void;
|
||||
/** A recording of the game view is made (#405): it goes into the project. */
|
||||
recordingSaved(recording: MadeRecording): void;
|
||||
}
|
||||
|
||||
export class PlaySession {
|
||||
@@ -74,11 +77,21 @@ export class PlaySession {
|
||||
* frames of a cut showed white without them.
|
||||
*/
|
||||
private videoWait: number | null = null;
|
||||
/** Records the game view for `Start recording` (#405); one for the editor, so it outlives a restart. */
|
||||
private readonly recorder: GameRecorder;
|
||||
|
||||
constructor(
|
||||
private readonly state: EditorState,
|
||||
private readonly host: PlayHost,
|
||||
) {
|
||||
this.recorder = new GameRecorder({
|
||||
view: () => host.gameView,
|
||||
log: (line) => {
|
||||
this.world?.log.push(line);
|
||||
console.warn('[recording]', line);
|
||||
},
|
||||
saved: (recording) => host.recordingSaved(recording),
|
||||
});
|
||||
// An object picked during a run is looked at now: the hierarchy has to list it to show it picked.
|
||||
state.on('selection', () => {
|
||||
if (this.world && this.panels.holdsStructure) this.follow(this.panels.flush());
|
||||
@@ -121,6 +134,7 @@ export class PlaySession {
|
||||
// on mid-note, rather than playing to their end over a still image.
|
||||
soundLibrary.setSuspended(this.state.paused);
|
||||
videoLibrary.setPaused(this.state.paused);
|
||||
this.recorder.setPaused(this.state.paused);
|
||||
ui().focus.blur();
|
||||
this.state.setNotice(this.playNotice(), 'quiet');
|
||||
}
|
||||
@@ -173,6 +187,7 @@ export class PlaySession {
|
||||
physics,
|
||||
clock: gameClock,
|
||||
saves: saveArea(runtime.id, 'play'),
|
||||
recorder: this.recorder,
|
||||
};
|
||||
this.run = GameRun.start(runtime, scene.id, this.input, services);
|
||||
videoLibrary.startRun();
|
||||
@@ -237,7 +252,12 @@ export class PlaySession {
|
||||
: 'No camera is switched on – nothing is rendered · Esc stops';
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
/** `restarting`: a `Restart game` plays on at once, and a recording goes on with it. */
|
||||
stop(restarting = false): void {
|
||||
if (!restarting) {
|
||||
this.recorder.end();
|
||||
this.recorder.setPaused(false);
|
||||
}
|
||||
// The run's seconds end with it: the editor goes back to the picture of a freshly loaded scene.
|
||||
gameClock.reset();
|
||||
// And its sounds end with it, looping music included – nothing may outlive the world.
|
||||
@@ -282,7 +302,7 @@ export class PlaySession {
|
||||
return;
|
||||
}
|
||||
if (request === 'restart') {
|
||||
this.stop();
|
||||
this.stop(true);
|
||||
this.play();
|
||||
return;
|
||||
}
|
||||
@@ -302,6 +322,23 @@ export class PlaySession {
|
||||
this.follow(world.paused && !wasPaused ? this.panels.flush() : update);
|
||||
}
|
||||
|
||||
/**
|
||||
* Called once everything that draws the game view this frame is queued: a recording takes the
|
||||
* frame after all of it, behind another tab too.
|
||||
*/
|
||||
recordFrame(): void {
|
||||
if (!this.recorder.isRecording || !this.world) return;
|
||||
onFrame('recording', () => {
|
||||
if (!this.host.gameView.visible) this.host.gameView.redrawHidden();
|
||||
this.recorder.capture();
|
||||
});
|
||||
}
|
||||
|
||||
/** What became of the recordings (#405, test hook). */
|
||||
recordingInfo(): ReturnType<GameRecorder['info']> {
|
||||
return this.recorder.info();
|
||||
}
|
||||
|
||||
/** Brings the panels up to date as far as `update` says; the game's own picture never waits. */
|
||||
private follow(update: PanelUpdate): void {
|
||||
if (update.structure) {
|
||||
|
||||
Reference in new issue
Block a user