Record the game view from a behavior into a video asset (#405)

A game can now film itself. A behavior starts and stops a take, and
the editor turns it into a video asset with the run's picture and the
sound the player hears. The exported game hands the file over as a
download, since there is no project to keep it in.

- New action nodes `Start recording` (Name, Resolution, FPS) and
  `Stop recording`. An empty name becomes `Recording <timestamp>`.
- Resolution is `Game view`, `720p` or `1080p`. For the fixed sizes
  the height is set and the width follows the camera.
- The picture is the camera only, without hints, overlays or the
  letterbox. It is drawn into a render texture of the target size,
  so a smaller window still gives full 1080p.
- `PictureReader` reads the texture back through pixel buffers with
  a fence. The editor never waits on the GPU, and a frame with all
  three buffers busy is dropped and counted.
- Nothing of the take reaches the visible canvas. A first version
  drew tiles into it and copied them out, and with the low-latency
  canvas that midway state flickered on screen.
- The sound is tapped behind the master limiter in the FX worklet
  and encoded as Opus.
- H.264 is chosen when the browser has it, VP9 otherwise. A small
  own Matroska writer packs both tracks, no new dependency. It names
  the frame rate and BT.709 so ffmpeg keeps both.
- A second `Start recording` is ignored with a warning in the log.
  A take goes on through `Load scene` and a restart, pauses with the
  editor and is saved when the run ends. It stops at 2 GB.
- The editor uploads the take on the existing video route and makes
  the asset once the run is over. The video tab gets a Download
  button for the converted MP4.
- `downloadBlob` in `src/ui` replaces the song editor's own copy.

Findings:
- `On start` runs before the game screen knows the world. A take
  therefore fixes its size on the first frame that has a camera.
- Without a frame rate in the file, ffmpeg turned 30 fps into 59.94.
- The worker's poster JPEG is off in colour for every upload (red
  255 comes out as 229, BT.601). Not touched here.

Tests:
- Unit recording + matroska: 21 green, plus the node catalog cases.
  The Matroska test reads the file back with its own EBML reader.
- e2e 67 (new) on melektaus-atx: ALL PASSED. 1080p30 from a
  1280x800 window gives a 1920x1080 MP4 with sound and no dropped
  frame. Every 4th pixel is checked, a green quarter catches an
  upside-down picture. A take ended by Esc is saved with the
  timestamp name.
- e2e 60j: the exported game downloads an `.mkv` with EBML magic.
- The flicker is gone on the user's machine without `?vsync`.
- Gate on melektaus-atx: CHECK EXIT 0, 299 files, 4586 tests.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/editor/Editor.ts

@@ -76,6 +76,8 @@ import { HierarchyPanel } from './panels/HierarchyPanel';
 import { TilemapPanel } from './panels/TilemapPanel';
 import { Toolbar } from './panels/Toolbar';
 import { type PlayHost, PlaySession, type WorldSnapshot } from './PlaySession';
+import { type RecordingUpload, RecordingUploads } from './recordingUploads';
+import type { MadeRecording } from '../viewport/GameRecorder';
 import { ProjectSession } from './ProjectSession';
 import { ParticlesEditor } from './particles/ParticlesEditor';
 import { ShaderEditor } from './shader/ShaderEditor';
@@ -313,6 +315,7 @@ export class Editor implements PlayHost {
   /** Frame rate, refreshed twice a second; null until the first measurement. */
   fps: number | null = null;
   private readonly play: PlaySession;
+  private readonly recordings: RecordingUploads;
   private readonly editorRun: EditorRun;
   private readonly projects: ProjectSession;
   private readonly leftColumn: VStack;
@@ -400,6 +403,7 @@ export class Editor implements PlayHost {
     this.hierarchy = new HierarchyPanel(state);
     this.inspector = new Inspector(state);
     this.tilemapPanel = new TilemapPanel(state);
+    this.recordings = new RecordingUploads({ state, videos });
     this.play = new PlaySession(state, this);
     this.editorRun = new EditorRun(state, {
       // The game view is the one place where no key means anything to the editor, so that is where
@@ -1341,6 +1345,7 @@ export class Editor implements PlayHost {
     this.editorRun.tick(step);
     this.viewport.animate(sceneDraws);
     this.gameView.animate(gameDraws);
+    this.play.recordFrame();
     this.console.refresh();
     if (this.shows('animation')) this.animationEditor.tick(step);
     if (this.shows('tile')) this.tileEditor.tick(step);
@@ -1480,4 +1485,13 @@ export class Editor implements PlayHost {
       performance: this.play.performance(),
     };
   }
+
+  /** The recordings of the game view (#405) and what became of them on the way into the project. */
+  recordingInfo(): ReturnType<PlaySession['recordingInfo']> & { uploads: RecordingUpload[] } {
+    return { ...this.play.recordingInfo(), uploads: this.recordings.info() };
+  }
+
+  recordingSaved(recording: MadeRecording): void {
+    this.recordings.add(recording);
+  }
 }

src/editor/PlaySession.ts

@@ -14,8 +14,9 @@ import { gameClock } from '../core/time';
 import { loadPhysics, readyPhysics } from '../runtime/physics/engine';
 import { Input } from '../runtime/systems/Input';
 import type { World } from '../runtime/World';
-import { ui } from '../ui';
+import { onFrame, ui } from '../ui';
 import { soundLibrary } from '../viewport/SoundLibrary';
+import { GameRecorder, type MadeRecording } from '../viewport/GameRecorder';
 import { VIDEO_WAIT_MS, videoLibrary } from '../viewport/VideoLibrary';
 import { FrameMeter, type FrameTimes } from './frameMeter';
 import { saveArea } from './saves';
@@ -55,6 +56,8 @@ export interface PlayHost {
   showGame(): void;
   /** Leave the game view if it is the one on screen. */
   leaveGame(): void;
+  /** A recording of the game view is made (#405): it goes into the project. */
+  recordingSaved(recording: MadeRecording): void;
 }
 
 export class PlaySession {
@@ -74,11 +77,21 @@ export class PlaySession {
    * frames of a cut showed white without them.
    */
   private videoWait: number | null = null;
+  /** Records the game view for `Start recording` (#405); one for the editor, so it outlives a restart. */
+  private readonly recorder: GameRecorder;
 
   constructor(
     private readonly state: EditorState,
     private readonly host: PlayHost,
   ) {
+    this.recorder = new GameRecorder({
+      view: () => host.gameView,
+      log: (line) => {
+        this.world?.log.push(line);
+        console.warn('[recording]', line);
+      },
+      saved: (recording) => host.recordingSaved(recording),
+    });
     // An object picked during a run is looked at now: the hierarchy has to list it to show it picked.
     state.on('selection', () => {
       if (this.world && this.panels.holdsStructure) this.follow(this.panels.flush());
@@ -121,6 +134,7 @@ export class PlaySession {
     // on mid-note, rather than playing to their end over a still image.
     soundLibrary.setSuspended(this.state.paused);
     videoLibrary.setPaused(this.state.paused);
+    this.recorder.setPaused(this.state.paused);
     ui().focus.blur();
     this.state.setNotice(this.playNotice(), 'quiet');
   }
@@ -173,6 +187,7 @@ export class PlaySession {
       physics,
       clock: gameClock,
       saves: saveArea(runtime.id, 'play'),
+      recorder: this.recorder,
     };
     this.run = GameRun.start(runtime, scene.id, this.input, services);
     videoLibrary.startRun();
@@ -237,7 +252,12 @@ export class PlaySession {
       : 'No camera is switched on – nothing is rendered · Esc stops';
   }
 
-  stop(): void {
+  /** `restarting`: a `Restart game` plays on at once, and a recording goes on with it. */
+  stop(restarting = false): void {
+    if (!restarting) {
+      this.recorder.end();
+      this.recorder.setPaused(false);
+    }
     // The run's seconds end with it: the editor goes back to the picture of a freshly loaded scene.
     gameClock.reset();
     // And its sounds end with it, looping music included – nothing may outlive the world.
@@ -282,7 +302,7 @@ export class PlaySession {
       return;
     }
     if (request === 'restart') {
-      this.stop();
+      this.stop(true);
       this.play();
       return;
     }
@@ -302,6 +322,23 @@ export class PlaySession {
     this.follow(world.paused && !wasPaused ? this.panels.flush() : update);
   }
 
+  /**
+   * Called once everything that draws the game view this frame is queued: a recording takes the
+   * frame after all of it, behind another tab too.
+   */
+  recordFrame(): void {
+    if (!this.recorder.isRecording || !this.world) return;
+    onFrame('recording', () => {
+      if (!this.host.gameView.visible) this.host.gameView.redrawHidden();
+      this.recorder.capture();
+    });
+  }
+
+  /** What became of the recordings (#405, test hook). */
+  recordingInfo(): ReturnType<GameRecorder['info']> {
+    return this.recorder.info();
+  }
+
   /** Brings the panels up to date as far as `update` says; the game's own picture never waits. */
   private follow(update: PanelUpdate): void {
     if (update.structure) {

src/editor/assetImport.ts

@@ -73,7 +73,7 @@ async function importFile<U extends UploadedFile, D extends ImportDeps>(
  * what the asset is called. The server stores by content hash, so a re-imported file comes back
  * as the one the project already references; two assets on one file would disagree about it.
  */
-function adopt<U extends UploadedFile, D extends ImportDeps>(
+export function adopt<U extends UploadedFile, D extends ImportDeps>(
   kind: ImportKind<U, D>,
   uploaded: U,
   nameBase: string,

src/editor/debug/videoHooks.ts

@@ -36,5 +36,7 @@ export function videoHooks({ state, editor }: HookContext) {
     videoPickerRow: (file: string) => currentVideoPicker()?.rowGlobal(file) ?? null,
     /** What each player behind a sprite is doing: its time, whether it runs and is heard. */
     videos: () => videoLibrary.debugInfo(),
+    /** The recordings of the game view (#405): each take, and its way into the project. */
+    recording: () => editor.recordingInfo(),
   };
 }

src/editor/recordingUploads.ts

@@ -0,0 +1,84 @@
+import type { UploadedVideo } from '../core/store/VideoStore';
+import type { MadeRecording } from '../viewport/GameRecorder';
+import { adoptVideo, type VideoImportDeps } from './videoImport';
+
+/** How often the server is asked whether a recording is converted, as the video picker does. */
+const POLL_MS = 1000;
+
+type UploadState = 'uploading' | 'converting' | 'added' | 'elsewhere' | 'failed';
+
+/** What became of a recording on its way into the project (test hook). */
+export interface RecordingUpload {
+  name: string;
+  state: UploadState;
+  /** The name the server gave it, once it is up. */
+  file: string;
+  assetId: string;
+  bytes: number;
+  error: string;
+}
+
+/**
+ * Brings a recording of the game view into the project (#405): up to the server, which converts it
+ * like any video, then an asset named after the recording once it is ready. Only while the editor
+ * is not playing: a run plays a copy of the project that is thrown away, and so would the asset be.
+ */
+export class RecordingUploads {
+  private readonly uploads: RecordingUpload[] = [];
+
+  constructor(private readonly deps: VideoImportDeps) {}
+
+  add(recording: MadeRecording): void {
+    const upload: RecordingUpload = {
+      name: recording.name,
+      state: 'uploading',
+      file: '',
+      assetId: '',
+      bytes: recording.blob.size,
+      error: '',
+    };
+    this.uploads.push(upload);
+    void this.bring(recording, upload).catch((error: unknown) => {
+      upload.state = 'failed';
+      upload.error = error instanceof Error ? error.message : String(error);
+      this.deps.state.setNotice(`Could not save the recording "${recording.name}": ${upload.error}`, 'error');
+    });
+  }
+
+  info(): RecordingUpload[] {
+    return this.uploads.map((upload) => ({ ...upload }));
+  }
+
+  private async bring(recording: MadeRecording, upload: RecordingUpload): Promise<void> {
+    const { state, videos } = this.deps;
+    const projectId = state.project.id;
+    state.setNotice(`Uploading the recording "${recording.name}"…`);
+    let video: UploadedVideo = await videos.upload(
+      recording.blob,
+      'application/octet-stream',
+      `${recording.name}.mkv`,
+    );
+    upload.file = video.file;
+    upload.state = 'converting';
+    while (video.status !== 'ready') {
+      if (video.status === 'failed') throw new Error(video.error ?? 'the server could not convert it');
+      await wait(POLL_MS);
+      video = (await videos.list()).find((v) => v.file === upload.file) ?? video;
+    }
+    while (state.mode !== 'edit') await wait(POLL_MS);
+    if (state.project.id !== projectId) {
+      upload.state = 'elsewhere';
+      state.setNotice(`The recording "${recording.name}" is ready – the video picker has it.`);
+      return;
+    }
+    const outcome = adoptVideo(video, recording.name, this.deps);
+    if (!outcome.assetId) throw new Error(outcome.error ?? 'the project did not take it');
+    upload.state = 'added';
+    upload.assetId = outcome.assetId;
+    state.setNotice(`Added the recording "${recording.name}" to the project.`);
+  }
+}
+
+function wait(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

src/editor/song/SongEditor.ts

@@ -26,7 +26,7 @@ import { findSound } from '../../core/model/sound';
 import { textOf } from '../../core/model/Properties';
 import { assetFxOf } from '../../core/properties/assetFx';
 import type { SoundStore } from '../../core/store/SoundStore';
-import { Button, Dialog, Dropdown, Label, Menu, ui, VStack, type Widget } from '../../ui';
+import { Button, Dialog, downloadBlob, Dropdown, Label, Menu, ui, VStack, type Widget } from '../../ui';
 import { Music } from '../../viewport/music';
 import { MASTER_MIX } from '../../core/music/mixer';
 import { soundLibrary } from '../../viewport/SoundLibrary';
@@ -596,10 +596,5 @@ export class SongEditor extends AssetTabEditor<SongAsset, SongTimeline> {
 
 /** Hands the take to the browser as a file, for when the server would not take it. */
 function downloadWav(wav: Uint8Array<ArrayBuffer>, name: string): void {
-  const url = URL.createObjectURL(new Blob([wav], { type: 'audio/wav' }));
-  const link = document.createElement('a');
-  link.href = url;
-  link.download = `${name}.wav`;
-  link.click();
-  setTimeout(() => URL.revokeObjectURL(url), 60_000);
+  downloadBlob(new Blob([wav], { type: 'audio/wav' }), `${name}.wav`);
 }

src/editor/video/VideoEditor.ts

@@ -1,5 +1,6 @@
 import type { Asset, VideoAsset } from '../../core/model/Asset';
-import { Button } from '../../ui';
+import { Button, downloadFile } from '../../ui';
+import { videoFileUrl } from '../../viewport/VideoLibrary';
 import { AssetTabEditor } from '../AssetTabEditor';
 import { TOOLBAR_ICON_SVG } from '../panels/toolbarIcons';
 import type { EditorState } from '../state';
@@ -15,6 +16,8 @@ const INFO_WIDTH = 220;
 export class VideoEditor extends AssetTabEditor<VideoAsset, VideoCanvas> {
   /** Public for the check scripts. */
   readonly playButton: Button;
+  /** Saves the MP4 the server made (#405): a recording of the game view is had this way. */
+  readonly downloadButton: Button;
 
   constructor(state: EditorState) {
     super(state, { infoWidth: INFO_WIDTH });
@@ -24,10 +27,14 @@ export class VideoEditor extends AssetTabEditor<VideoAsset, VideoCanvas> {
       tooltip: 'Play the picked clip, else the whole video (Space)',
       onClick: () => this.togglePlay(),
     });
+    this.downloadButton = new Button('Download', {
+      tooltip: 'Save the video as an MP4 file',
+      onClick: () => this.download(),
+    });
     const canvas = new VideoCanvas(state);
     canvas.onTogglePlay = () => this.togglePlay();
     canvas.onStopped = () => this.syncPlayButton();
-    this.mount(canvas, [this.playButton]);
+    this.mount(canvas, [this.playButton, this.downloadButton]);
     state.on('load', () => this.canvas.stop());
     state.on('selection', () => this.refresh());
   }
@@ -50,6 +57,11 @@ export class VideoEditor extends AssetTabEditor<VideoAsset, VideoCanvas> {
     this.syncPlayButton();
   }
 
+  private download(): void {
+    const asset = this.asset();
+    if (asset?.file) downloadFile(videoFileUrl(asset.file), `${asset.name}.mp4`);
+  }
+
   /** Only called while the tab is visible. */
   tick(): void {
     this.canvas.tick();
@@ -72,6 +84,7 @@ export class VideoEditor extends AssetTabEditor<VideoAsset, VideoCanvas> {
 
   protected override syncExtras(asset: VideoAsset | null): void {
     this.fitButton.setEnabled(Boolean(asset?.file));
+    this.downloadButton.setEnabled(Boolean(asset?.file));
     this.syncPlayButton();
   }
 

src/editor/videoImport.ts

@@ -1,6 +1,6 @@
 import { CreateVideoCommand } from '../core/commands/assetCommands';
 import type { UploadedVideo, VideoStore } from '../core/store/VideoStore';
-import { type ImportKind, type ImportOutcome, importStored } from './assetImport';
+import { adopt, type ImportKind, type ImportOutcome, importStored } from './assetImport';
 import type { EditorState } from './state';
 
 /** What the file dialog offers and a drop accepts; ffmpeg on the server has the last word. */
@@ -44,6 +44,11 @@ export function importStoredVideo(video: UploadedVideo, deps: VideoImportDeps):
   return importStored(VIDEO_KIND, video, deps);
 }
 
+/** Makes an asset named `name` of a video the server has converted, without opening it (#405). */
+export function adoptVideo(video: UploadedVideo, name: string, deps: VideoImportDeps): ImportOutcome {
+  return adopt(VIDEO_KIND, video, name, name, deps);
+}
+
 /**
  * Sends the files up one after the other and says how it went; what is done already is made an
  * asset at once. The rest is converted on the server – `waiting` gets their names, for the picker

src/editor/viewport/GameViewport.ts

@@ -2,6 +2,7 @@
  * The scene seen through its active camera at the chosen scale, letterboxed: the running world while
  * playing, the model as a still preview otherwise.
  */
+import type { Container } from 'pixi.js';
 import { isSymbolType } from '../../core/model/AssetTypes';
 import type { EditorState } from '../state';
 import type { World } from '../../runtime/World';
@@ -44,6 +45,8 @@ export class GameViewport extends WorldViewport<EditorState> {
   /** The model collected once per redraw of the preview: view, cameras and items all read it. */
   private redrawFrame: Frame | null = null;
   private redrawing = false;
+  /** Set while a recording draws the view behind another tab (#405). */
+  private forced = false;
 
   constructor(state: EditorState) {
     super(state);
@@ -119,7 +122,7 @@ export class GameViewport extends WorldViewport<EditorState> {
    * otherwise run on every selection change behind another tab. Showing the tab redraws it.
    */
   override redraw(): void {
-    if (!this.visible) return;
+    if (!this.visible && !this.forced) return;
     const start = performance.now();
     this.redrawing = true;
     try {
@@ -131,6 +134,16 @@ export class GameViewport extends WorldViewport<EditorState> {
     this.drawTime += performance.now() - start;
   }
 
+  /** A redraw while the tab is hidden: a recording goes on behind another tab (#405). */
+  redrawHidden(): void {
+    this.forced = true;
+    try {
+      this.redraw();
+    } finally {
+      this.forced = false;
+    }
+  }
+
   get scaleMode(): GameScale {
     return this.state.gameScale;
   }
@@ -207,6 +220,10 @@ export class GameViewport extends WorldViewport<EditorState> {
     return Theme.colors.background;
   }
 
+  protected override chrome(): Container[] {
+    return [this.hint, this.editorRunCaption, this.fullscreenButton, this.performanceOverlay];
+  }
+
   protected override beforeItems(view: View | null): void {
     this.onScaleShown?.(view ? this.factor : null);
     this.hint.visible = view === null;