Files
feuerwerk/src/editor/PlaySession.ts
T
adminandClaude Opus 5.5 e191dc4b2e
game-maker / gate (push) Successful in 4m14s
game-maker / suite (push) Successful in 17m7s
Record the game view from a behavior into a video asset (#405)
A game can now film itself. A behavior starts and stops a take, and
the editor turns it into a video asset with the run's picture and the
sound the player hears. The exported game hands the file over as a
download, since there is no project to keep it in.

- New action nodes `Start recording` (Name, Resolution, FPS) and
  `Stop recording`. An empty name becomes `Recording <timestamp>`.
- Resolution is `Game view`, `720p` or `1080p`. For the fixed sizes
  the height is set and the width follows the camera.
- The picture is the camera only, without hints, overlays or the
  letterbox. It is drawn into a render texture of the target size,
  so a smaller window still gives full 1080p.
- `PictureReader` reads the texture back through pixel buffers with
  a fence. The editor never waits on the GPU, and a frame with all
  three buffers busy is dropped and counted.
- Nothing of the take reaches the visible canvas. A first version
  drew tiles into it and copied them out, and with the low-latency
  canvas that midway state flickered on screen.
- The sound is tapped behind the master limiter in the FX worklet
  and encoded as Opus.
- H.264 is chosen when the browser has it, VP9 otherwise. A small
  own Matroska writer packs both tracks, no new dependency. It names
  the frame rate and BT.709 so ffmpeg keeps both.
- A second `Start recording` is ignored with a warning in the log.
  A take goes on through `Load scene` and a restart, pauses with the
  editor and is saved when the run ends. It stops at 2 GB.
- The editor uploads the take on the existing video route and makes
  the asset once the run is over. The video tab gets a Download
  button for the converted MP4.
- `downloadBlob` in `src/ui` replaces the song editor's own copy.

Findings:
- `On start` runs before the game screen knows the world. A take
  therefore fixes its size on the first frame that has a camera.
- Without a frame rate in the file, ffmpeg turned 30 fps into 59.94.
- The worker's poster JPEG is off in colour for every upload (red
  255 comes out as 229, BT.601). Not touched here.

Tests:
- Unit recording + matroska: 21 green, plus the node catalog cases.
  The Matroska test reads the file back with its own EBML reader.
- e2e 67 (new) on melektaus-atx: ALL PASSED. 1080p30 from a
  1280x800 window gives a 1920x1080 MP4 with sound and no dropped
  frame. Every 4th pixel is checked, a green quarter catches an
  upside-down picture. A take ended by Esc is saved with the
  timestamp name.
- e2e 60j: the exported game downloads an `.mkv` with EBML magic.
- The flicker is gone on the user's machine without `?vsync`.
- Gate on melektaus-atx: CHECK EXIT 0, 299 files, 4586 tests.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-11 21:11:05 +02:00

427 lines
16 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* Owns everything that only exists while the project is playing: the world, its systems, the
* keyboard and the per-frame step. Play never touches the saved model – the world is built
* from the runtime copy `state.enterPlay()` swaps in and `exitPlay()` throws away.
*/
import { emptySharedState, type SharedState, worldPositionOf } from '../core/graph';
import { namedValue } from '../core/graph/values';
import { GameRun, type RunServices } from '../runtime/GameRun';
import { AnimationSystem } from '../runtime/systems/AnimationSystem';
import { type BehaviorSystem, behaviorError, type Trace } from '../runtime/systems/BehaviorSystem';
import type { AnimationPlayer } from '../core/model/animation';
import type { PropertyBag } from '../core/model/Properties';
import { gameClock } from '../core/time';
import { loadPhysics, readyPhysics } from '../runtime/physics/engine';
import { Input } from '../runtime/systems/Input';
import type { World } from '../runtime/World';
import { onFrame, ui } from '../ui';
import { soundLibrary } from '../viewport/SoundLibrary';
import { GameRecorder, type MadeRecording } from '../viewport/GameRecorder';
import { VIDEO_WAIT_MS, videoLibrary } from '../viewport/VideoLibrary';
import { FrameMeter, type FrameTimes } from './frameMeter';
import { saveArea } from './saves';
import { PanelClock, panelPolicy, type PanelUpdate } from './playThrottle';
import type { GameViewport } from './viewport/GameViewport';
import type { GraphEditor } from './graph/GraphEditor';
import type { LiveMachineState } from './graph/GraphCanvas';
import type { StateMachineEditor } from './statemachine/StateMachineEditor';
import type { EditorState } from './state';
import { stopPreviews } from './music/transport';
/** Facts about the running world for the check scripts. */
export interface WorldSnapshot {
entities: {
objectId: string;
assetId: string;
name: string;
x: number;
y: number;
worldX: number;
worldY: number;
parent: string;
props: Record<string, unknown>;
animations: AnimationPlayer[];
}[];
/** What `global` declarations share this run – no entity carries it, so it is reported apart. */
shared: SharedState;
log: string[];
}
export interface PlayHost {
gameView: GameViewport;
/** The scene view: while a run holds the panels back, it still draws every frame it is on screen. */
viewport: { visible: boolean; redraw(): void };
graphEditor: GraphEditor;
stateMachineEditor: StateMachineEditor;
showGame(): void;
/** Leave the game view if it is the one on screen. */
leaveGame(): void;
/** A recording of the game view is made (#405): it goes into the project. */
recordingSaved(recording: MadeRecording): void;
}
export class PlaySession {
private run: GameRun | null = null;
private readonly input = new Input();
/** Set while Play waits for the physics engine, so a second press does not queue a second run. */
private waiting = false;
/** When the panels catch up with the run (#265). */
private readonly panels = new PanelClock();
/** How often the panels were brought up to date this run (test hook). */
private updates = { structure: 0, values: 0 };
/** Where the frames of this run go (#275). */
private readonly meter = new FrameMeter();
/**
* Until when the run waits at its start for the Game tab's driven videos (#404), by
* `performance.now()`; null once it goes. Its players are opened only at Play, and the first
* frames of a cut showed white without them.
*/
private videoWait: number | null = null;
/** Records the game view for `Start recording` (#405); one for the editor, so it outlives a restart. */
private readonly recorder: GameRecorder;
constructor(
private readonly state: EditorState,
private readonly host: PlayHost,
) {
this.recorder = new GameRecorder({
view: () => host.gameView,
log: (line) => {
this.world?.log.push(line);
console.warn('[recording]', line);
},
saved: (recording) => host.recordingSaved(recording),
});
// An object picked during a run is looked at now: the hierarchy has to list it to show it picked.
state.on('selection', () => {
if (this.world && this.panels.holdsStructure) this.follow(this.panels.flush());
});
}
private get world(): World | null {
return this.run?.world ?? null;
}
private get behaviors(): BehaviorSystem | null {
return this.run?.behaviors ?? null;
}
get isPlaying(): boolean {
return this.world !== null;
}
get isPaused(): boolean {
return this.world !== null && this.state.paused;
}
/** Paused by a behavior (Pause game) rather than by the toolbar – the game's own pause. */
get isGamePaused(): boolean {
return this.world?.paused ?? false;
}
/** Held at its start until the Game tab has a picture of its driven videos (#404). */
get isWaiting(): boolean {
return this.videoWait !== null;
}
/** Pause or resume; does nothing while not playing. */
togglePause(): void {
if (!this.world) return;
this.state.setPaused(!this.state.paused);
// What you inspect while paused is exact, whatever the panels were holding back.
if (this.state.paused) this.follow(this.panels.flush());
// A held note is part of the picture a pause freezes: the sounds stop where they are and carry
// on mid-note, rather than playing to their end over a still image.
soundLibrary.setSuspended(this.state.paused);
videoLibrary.setPaused(this.state.paused);
this.recorder.setPaused(this.state.paused);
ui().focus.blur();
this.state.setNotice(this.playNotice(), 'quiet');
}
traceFor(assetId: string): Trace | null {
return this.behaviors?.traceFor(assetId) ?? null;
}
/** Which state each running instance of a machine holds; empty when nothing is playing. */
machineStates(machineId: string): LiveMachineState[] {
return this.behaviors?.machineStates(machineId) ?? [];
}
get time(): number {
return this.world?.time ?? 0;
}
/** The scene the run is in – not the editor's active one, which a `Load scene` leaves alone. */
get sceneId(): string {
return this.world?.sceneId ?? '';
}
syncFromModel(): void {
this.world?.syncFromModel();
this.behaviors?.syncFromModel();
}
/** The values a Variables asset shares in this run; null when nothing is playing. */
liveVariables(assetId: string): PropertyBag | null {
return this.behaviors?.liveVariables(assetId) ?? null;
}
play(): void {
const scene = this.state.sceneToPlay();
if (!scene) return;
// The editor loads the engine right after opening, so it is almost always there already; a game
// without it would collide differently, so Play waits for it rather than running without (#146).
const physics = readyPhysics();
if (!physics) {
this.waitForPhysics();
return;
}
ui().focus.blur();
stopPreviews();
// A run starts at second zero, whatever the editor's previews have been counting up to.
gameClock.reset();
const runtime = this.state.enterPlay();
const services: RunServices = {
audio: soundLibrary,
physics,
clock: gameClock,
saves: saveArea(runtime.id, 'play'),
recorder: this.recorder,
};
this.run = GameRun.start(runtime, scene.id, this.input, services);
videoLibrary.startRun();
this.updates = { structure: 0, values: 0 };
this.meter.reset();
this.timeSystems(this.run);
this.input.attach();
this.host.gameView.setWorld(this.run.world);
this.host.showGame();
this.waitForVideos();
this.state.setNotice(this.playNotice(), 'quiet');
if (this.run.world.stopRequested) this.stop();
}
/** A `Load scene` swapped the world (see `GameRun.loadScene`). */
private sceneLoaded(run: GameRun): void {
this.timeSystems(run);
this.host.gameView.setWorld(run.world);
this.waitForVideos();
this.state.setNotice(this.playNotice(), 'quiet');
if (run.world.stopRequested) this.stop();
}
private waitForPhysics(): void {
if (this.waiting) return;
this.waiting = true;
this.state.setNotice('Loading the physics engine…', 'quiet');
loadPhysics().then(
() => {
this.waiting = false;
if (this.state.mode === 'edit') this.play();
},
(error: unknown) => {
this.waiting = false;
const reason = error instanceof Error ? error.message : String(error);
console.error('[physics] Play needs the physics engine, which could not be loaded:', error);
this.state.setNotice(`Play needs the physics engine, which could not be loaded: ${reason}`, 'error');
},
);
}
/** The Game tab has drawn the new world once by now, which opened the players it shows. */
private waitForVideos(): void {
this.videoWait = this.host.gameView.videosLoading() ? performance.now() + VIDEO_WAIT_MS : null;
}
/** Whether the run still waits for its videos; the frame it stops waiting, the notice says so. */
private waitsForVideos(): boolean {
if (this.videoWait === null) return false;
if (performance.now() < this.videoWait && this.host.gameView.videosLoading()) return true;
this.videoWait = null;
this.state.setNotice(this.playNotice(), 'quiet');
return false;
}
private playNotice(): string {
if (this.videoWait !== null) return 'Loading the videos… · Esc stops';
if (this.state.paused) return 'Paused · Ctrl+Enter resumes · Esc stops';
if (this.world?.paused) return 'The game paused itself · Esc stops';
return this.world?.activeCamera()
? 'Playing · edits are not saved · Ctrl+Enter pauses · Esc stops'
: 'No camera is switched on – nothing is rendered · Esc stops';
}
/** `restarting`: a `Restart game` plays on at once, and a recording goes on with it. */
stop(restarting = false): void {
if (!restarting) {
this.recorder.end();
this.recorder.setPaused(false);
}
// The run's seconds end with it: the editor goes back to the picture of a freshly loaded scene.
gameClock.reset();
// And its sounds end with it, looping music included – nothing may outlive the world.
soundLibrary.setSuspended(false);
soundLibrary.stopAll();
videoLibrary.endRun();
this.videoWait = null;
const world = this.world;
this.run = null;
this.input.detach();
ui().focus.blur();
world?.destroy();
this.host.gameView.setWorld(null);
this.state.exitPlay();
this.host.leaveGame();
const lastError = world && behaviorError(world);
this.state.setNotice(lastError ?? '', 'error');
}
/**
* `step` is what the global clock moved this frame – already scaled and capped. `awake` is the
* step it would have moved without the game's own pause, which stops the clock but not the
* behaviors that run while paused.
*/
tick(step: number, awake: number): void {
const run = this.run;
if (!run) return;
const world = run.world;
if (this.state.paused || this.waitsForVideos()) {
// Keys and clicks that happen while paused are dropped instead of piling up – otherwise
// resuming would deliver every one of them in a single frame.
this.input.endFrame();
world.dropPointerEdges();
return;
}
const wasPaused = world.paused;
world.held = this.state.heldObjectIds;
const request = run.tick(step, awake);
if (world.paused !== wasPaused) this.state.setNotice(this.playNotice(), 'quiet');
if (request === 'stop') {
this.stop();
return;
}
if (request === 'restart') {
this.stop(true);
this.play();
return;
}
if (request === 'scene') {
this.sceneLoaded(run);
return;
}
// Mirror what the systems did into the runtime copy every frame, even while the panels wait: an
// edit made during the run lands on the model, and it must not land on an old picture of it.
const structure = world.syncToModel().structure;
const update = this.panels.frame(
panelPolicy(this.state.project.whilePlaying),
performance.now(),
structure,
);
// A game that pauses itself is looked at like one paused from the toolbar.
this.follow(world.paused && !wasPaused ? this.panels.flush() : update);
}
/**
* Called once everything that draws the game view this frame is queued: a recording takes the
* frame after all of it, behind another tab too.
*/
recordFrame(): void {
if (!this.recorder.isRecording || !this.world) return;
onFrame('recording', () => {
if (!this.host.gameView.visible) this.host.gameView.redrawHidden();
this.recorder.capture();
});
}
/** What became of the recordings (#405, test hook). */
recordingInfo(): ReturnType<GameRecorder['info']> {
return this.recorder.info();
}
/** Brings the panels up to date as far as `update` says; the game's own picture never waits. */
private follow(update: PanelUpdate): void {
if (update.structure) {
this.updates.structure++;
this.state.preview();
}
if (update.values) {
this.updates.values++;
this.state.runtimeTick();
} else {
// The views draw on `runtime`, and what is on screen of the game has to move every frame.
if (this.host.gameView.visible) this.host.gameView.redraw();
if (this.host.viewport.visible) this.host.viewport.redraw();
}
if (!update.graphs) return;
if (this.host.graphEditor.visible) this.host.graphEditor.canvas.redraw();
// The machine tab shows the state a run is in, so it repaints for the same reason the graph does.
if (this.host.stateMachineEditor.visible) this.host.stateMachineEditor.canvas.redraw();
}
/** The world tells the meter what each system took; a `Load scene` brings a world of its own. */
private timeSystems(run: GameRun): void {
run.world.timeSystem = (system, ms) => {
const area =
system === run.behaviors ? 'behaviors' : system instanceof AnimationSystem ? 'animation' : 'physics';
this.meter.addSystem(area, ms);
};
}
/**
* Closes the editor's frame for the meter (see `FrameMeter.endFrame`); true when a new average is
* ready. Nothing is measured while nothing plays.
*/
measureFrame(dt: number, busy: number, drawn: number, rendered: number): boolean {
return this.isPlaying && this.meter.endFrame(dt, busy, drawn, rendered);
}
/** How many entities run and where the frames went lately; null while nothing plays. */
performance(): { entities: number; times: FrameTimes | null } | null {
const world = this.world;
return world ? { entities: world.entities.length, times: this.meter.average } : null;
}
/** How often the panels caught up with this run, by kind (test hook). */
panelUpdates(): { structure: number; values: number } {
return { ...this.updates };
}
/** The check scripts get the run as JSON, so an object a bag holds is handed over as its name. */
snapshot(): WorldSnapshot | null {
const world = this.world;
if (!world) return null;
return {
entities: world.entities.map((e) => {
const w = worldPositionOf(e);
return {
objectId: e.objectId,
assetId: e.assetId,
name: e.name,
x: e.x,
y: e.y,
worldX: w.x,
worldY: w.y,
parent: e.parent?.objectId ?? '',
props: namedBag(e.props),
animations: structuredClone(e.animations ?? []),
};
}),
shared: namedShared(this.behaviors?.shared ?? emptySharedState()),
log: [...world.log],
};
}
}
function namedBag(bag: PropertyBag): Record<string, unknown> {
const out: Record<string, unknown> = {};
for (const [key, value] of Object.entries(bag)) out[key] = namedValue(value);
return out;
}
function namedShared(shared: SharedState): SharedState {
const variables: Record<string, PropertyBag> = {};
for (const [id, bag] of Object.entries(shared.variables)) variables[id] = namedBag(bag) as PropertyBag;
return { variables, states: shared.states };
}