diff --git a/README.md b/README.md index 1df03417..8b3238ee 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ npm run build # writes dist/ and regenerates index.html; the folder is wh The browser suite and the three example games need a machine with a GPU: ```bash -node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2038 checks (GM_STEPS=20 for one step) +node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2039 checks (GM_STEPS=20 for one step) # every tool signs in first – see the Accounts page node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs diff --git a/src/core/recording/cursors.ts b/src/core/recording/cursors.ts new file mode 100644 index 00000000..797f5313 --- /dev/null +++ b/src/core/recording/cursors.ts @@ -0,0 +1,59 @@ +/** + * The mouse pointer of an editor recording: the system draws it over the canvas, so the recording + * draws its own. One shape per CSS cursor the editor uses, as an SVG path in CSS pixels, filled dark + * and outlined light like a system cursor. `hot` is the point that sits on the pointer. + */ + +export interface CursorShape { + path: string; + hot: { x: number; y: number }; + /** Degrees the path is turned by around `hot`. */ + turn: number; +} + +const ARROW = 'M0 0 L0 16 L4.2 12.2 L7 18.6 L9.6 17.5 L6.9 11.3 L12 11.3 Z'; +const DOUBLE_ARROW = 'M-11 0 L-6 -5 L-6 -2 L6 -2 L6 -5 L11 0 L6 5 L6 2 L-6 2 L-6 5 Z'; +const MOVE = + 'M0 -11 L4 -7 L1.5 -7 L1.5 -1.5 L7 -1.5 L7 -4 L11 0 L7 4 L7 1.5 L1.5 1.5 L1.5 7 L4 7 L0 11 ' + + 'L-4 7 L-1.5 7 L-1.5 1.5 L-7 1.5 L-7 4 L-11 0 L-7 -4 L-7 -1.5 L-1.5 -1.5 L-1.5 -7 L-4 -7 Z'; +const CROSS = 'M-1 -10 L1 -10 L1 -1 L10 -1 L10 1 L1 1 L1 10 L-1 10 L-1 1 L-10 1 L-10 -1 L-1 -1 Z'; +const BEAM = 'M-4 -9 L4 -9 L4 -7.5 L1 -7.5 L1 7.5 L4 7.5 L4 9 L-4 9 L-4 7.5 L-1 7.5 L-1 -7.5 L-4 -7.5 Z'; +const HAND = + 'M5 0 C6 0 7 0.8 7 2 L7 7 C7.5 6.5 9 6.5 9.5 7.5 C10 7 11.5 7 12 8 C12.5 7.6 14 7.8 14 9.5 ' + + 'L14 14 C14 16.5 12.5 18 10 18 L7 18 C5.5 18 4.5 17.2 3.5 16 L0.6 11.8 C-0.2 10.6 1.2 9.2 ' + + '2.4 10.1 L3 10.6 L3 2 C3 0.8 4 0 5 0 Z'; +const OPEN_HAND = + 'M3 7 C3 5.5 5 5.5 5 7 L5 3 C5 1.5 7 1.5 7 3 L7 2 C7 0.5 9 0.5 9 2 L9 3 C9 1.5 11 1.5 11 3 ' + + 'L11 5 C11 3.5 13 3.5 13 5 L13 12 C13 15.5 11 18 8 18 L7 18 C5 18 4 17 3 15.5 L0.5 11.5 ' + + 'C-0.3 10.3 1.2 9 2.3 9.9 L3 10.5 Z'; +const FIST = + 'M3 8 C3 6.5 5 6.5 5 7.5 C5 6 7 6 7 7.2 C7 5.8 9 5.8 9 7.2 C9 6 11 6 11 7.5 C11 6.6 13 6.6 ' + + '13 8 L13 12 C13 15.5 11 18 8 18 L7 18 C5 18 4 17 3 15.5 L2 13 C1.5 11 3 10 3 10 Z'; + +const centred = (path: string, turn = 0): CursorShape => ({ path, hot: { x: 0, y: 0 }, turn }); + +const SHAPES: Record = { + default: { path: ARROW, hot: { x: 0, y: 0 }, turn: 0 }, + pointer: { path: HAND, hot: { x: 5, y: 0 }, turn: 0 }, + grab: { path: OPEN_HAND, hot: { x: 7, y: 9 }, turn: 0 }, + grabbing: { path: FIST, hot: { x: 7, y: 11 }, turn: 0 }, + move: centred(MOVE), + crosshair: centred(CROSS), + text: centred(BEAM), + 'ew-resize': centred(DOUBLE_ARROW), + 'col-resize': centred(DOUBLE_ARROW), + 'ns-resize': centred(DOUBLE_ARROW, 90), + 'row-resize': centred(DOUBLE_ARROW, 90), + 'nwse-resize': centred(DOUBLE_ARROW, 45), + 'nesw-resize': centred(DOUBLE_ARROW, -45), +}; + +/** + * The shape for a CSS `cursor` value as the canvas has it, null for `none`. A value with images + * (`url(…) 4 4, pointer`) is read by its keyword at the end; anything unknown is the arrow. + */ +export function cursorShape(css: string): CursorShape | null { + const keyword = css.split(',').at(-1)!.trim().toLowerCase(); + if (keyword === 'none') return null; + return SHAPES[keyword] ?? SHAPES.default!; +} diff --git a/src/core/recording/recording.ts b/src/core/recording/recording.ts index 86c62d5d..a88da139 100644 --- a/src/core/recording/recording.ts +++ b/src/core/recording/recording.ts @@ -80,6 +80,46 @@ function even(value: number): number { return Math.max(2, Math.round(value / 2) * 2); } +/** The editor is recorded at 60 fps, whatever the screen draws. */ +export const EDITOR_RECORDING_FPS = 60; + +/** + * The size of an editor recording: the whole canvas in device pixels, an odd row or column cut off, + * since the encoders want both sides even. + */ +export function editorRecordingSize(canvas: { w: number; h: number }): { w: number; h: number } { + const cut = (value: number) => Math.max(2, Math.floor(value / 2) * 2); + return { w: cut(canvas.w), h: cut(canvas.h) }; +} + +/** How long a recording runs, as the editor shows it: `0:07`, `12:45`, `1:02:03`. */ +export function elapsedText(ms: number): string { + const seconds = Math.max(0, Math.floor(ms / 1000)); + const two = (n: number) => String(n).padStart(2, '0'); + const h = Math.floor(seconds / 3600); + const m = Math.floor(seconds / 60) % 60; + const s = seconds % 60; + return h > 0 ? `${h}:${two(m)}:${two(s)}` : `${m}:${two(s)}`; +} + +/** + * Where the canvas lands in a frame of `frame` size. As it is at the start, it lands where it is, + * cut by the odd pixel. A window resized during the recording is fitted in whole and centred, the + * rest of the frame left dark. + */ +export function fitPicture( + canvas: { w: number; h: number }, + frame: { w: number; h: number }, +): { x: number; y: number; w: number; h: number } { + const cutW = canvas.w - frame.w; + const cutH = canvas.h - frame.h; + if (cutW >= 0 && cutW <= 1 && cutH >= 0 && cutH <= 1) return { x: 0, y: 0, w: canvas.w, h: canvas.h }; + const scale = Math.min(frame.w / canvas.w, frame.h / canvas.h); + const w = Math.round(canvas.w * scale); + const h = Math.round(canvas.h * scale); + return { x: Math.round((frame.w - w) / 2), y: Math.round((frame.h - h) / 2), w, h }; +} + /** H.264 levels by the frame size and macroblocks a second they allow (ITU-T H.264, table A-1). */ const AVC_LEVELS: readonly { level: number; frame: number; rate: number }[] = [ { level: 0x1f, frame: 3600, rate: 108_000 }, diff --git a/src/editor/Editor.ts b/src/editor/Editor.ts index 060869bf..35a3ceff 100644 --- a/src/editor/Editor.ts +++ b/src/editor/Editor.ts @@ -78,7 +78,8 @@ import { TilemapPanel } from './panels/TilemapPanel'; import { Toolbar } from './panels/Toolbar'; import { type PlayHost, PlaySession, type WorldSnapshot } from './PlaySession'; import { type RecordingUpload, RecordingUploads } from './recordingUploads'; -import type { MadeRecording } from '../viewport/GameRecorder'; +import { EditorRecorder } from './EditorRecorder'; +import type { MadeRecording } from '../viewport/RecordingTake'; import { ProjectSession } from './ProjectSession'; import { ParticlesEditor } from './particles/ParticlesEditor'; import { ShaderEditor } from './shader/ShaderEditor'; @@ -321,6 +322,8 @@ export class Editor implements PlayHost { fps: number | null = null; private readonly play: PlaySession; private readonly recordings: RecordingUploads; + /** Records the whole editor from the main menu. */ + private readonly editorRecorder: EditorRecorder; private readonly editorRun: EditorRun; private readonly projects: ProjectSession; private readonly leftColumn: VStack; @@ -413,6 +416,15 @@ export class Editor implements PlayHost { this.inspector = new Inspector(state); this.tilemapPanel = new TilemapPanel(state); this.recordings = new RecordingUploads({ state, videos }); + this.editorRecorder = new EditorRecorder({ + log: (line) => { + state.setNotice(line); + console.warn('[recording]', line); + }, + saved: (recording) => this.recordings.add(recording), + showBadge: (on) => this.toolbar.showRecording(on), + badgeRoom: () => this.toolbar.recordingRoom(), + }); this.play = new PlaySession(state, this); this.editorRun = new EditorRun(state, { // The game view is the one place where no key means anything to the editor, so that is where @@ -792,13 +804,19 @@ export class Editor implements PlayHost { } /** - * The main menu's last group (#179): the user management for an administrator, and the plugins - * for everybody – both open the one tab, which holds the plugins below the accounts. + * The main menu's last groups: recording the editor, then the user management for an + * administrator and the plugins for everybody (#179) – both open the one tab, which holds the + * plugins below the accounts. */ private managementEntries(): MenuEntry[] { const open = () => this.openDocument('management'); + const recorder = this.editorRecorder; return [ { separator: true }, + recorder.isRecording + ? { label: 'Stop recording', onSelect: () => recorder.stop() } + : { label: 'Start recording', onSelect: () => recorder.start() }, + { separator: true }, { label: 'General settings…', onSelect: () => this.openGeneralSettings() }, ...(this.usersPane ? [{ label: 'User management', onSelect: open }] : []), { label: 'Plugins', onSelect: open }, @@ -1323,6 +1341,7 @@ export class Editor implements PlayHost { await this.autosave.flush(); this.autosave.stop(); if (this.play.isPlaying) this.play.stop(); + this.editorRecorder.stop(); this.editorRun.shutDown(); await this.plugins?.stopAll(); for (const cleanup of this.cleanups.splice(0).reverse()) cleanup(); @@ -1511,9 +1530,19 @@ export class Editor implements PlayHost { }; } - /** The recordings of the game view (#405) and what became of them on the way into the project. */ - recordingInfo(): ReturnType & { uploads: RecordingUpload[] } { - return { ...this.play.recordingInfo(), uploads: this.recordings.info() }; + /** + * The recordings of the game view (#405), those of the editor, and what became of them on the way + * into the project. + */ + recordingInfo(): ReturnType & { + editor: ReturnType; + uploads: RecordingUpload[]; + } { + return { + ...this.play.recordingInfo(), + editor: this.editorRecorder.info(), + uploads: this.recordings.info(), + }; } recordingSaved(recording: MadeRecording): void { diff --git a/src/editor/EditorRecorder.ts b/src/editor/EditorRecorder.ts new file mode 100644 index 00000000..23d28114 --- /dev/null +++ b/src/editor/EditorRecorder.ts @@ -0,0 +1,281 @@ +import { Container, Graphics, UPDATE_PRIORITY } from 'pixi.js'; +import { cursorShape, type CursorShape } from '../core/recording/cursors'; +import { + EDITOR_RECORDING_FPS, + editorRecordingSize, + elapsedText, + fitPicture, + recordingName, +} from '../core/recording/recording'; +import { crispText, renderCount, requestRender, textStyle, Theme, ui } from '../ui'; +import { type FrameSource, Take, type TakeHost, type TakeInfo } from '../viewport/RecordingTake'; + +/** What the editor recorder needs of the editor. */ +export interface EditorRecorderHost extends TakeHost { + /** Makes room in the top bar for the badge while recording, or takes it away. */ + showBadge(on: boolean): void; + /** Where the badge goes, in CSS pixels. */ + badgeRoom(): { x: number; y: number; w: number; h: number }; +} + +/** + * Records the whole editor (main menu → Start recording): the canvas as it is on screen, at its size + * in device pixels and 60 fps, with the mouse pointer drawn in, since the system draws that over the + * canvas. What the editor draws is copied from the canvas right after each render, so recording costs + * no second drawing. The badge beside the menu is drawn onto the screen after that copy, so it is + * never in the recording. + */ +export class EditorRecorder { + private take: Take | null = null; + private source: CanvasSource | null = null; + /** Every recording of this page, newest last (test hook). */ + private readonly takes: Take[] = []; + private seen = -1; + private startedAt = 0; + private shownText = ''; + private readonly badge = new RecordingBadge(); + + constructor(private readonly host: EditorRecorderHost) {} + + get isRecording(): boolean { + return this.take !== null; + } + + start(): void { + if (this.take) return; + const app = ui().app; + const source = new CanvasSource(app.canvas); + const take = new Take(recordingName('', new Date()), EDITOR_RECORDING_FPS, source, this.host, () => { + if (this.take === take) this.end(); + }); + this.take = take; + this.source = source; + this.takes.push(take); + this.startedAt = performance.now(); + this.shownText = ''; + this.seen = -1; + this.host.showBadge(true); + // After the render loop (`UPDATE_PRIORITY.LOW`): the canvas holds this tick's picture then. + app.ticker.add(this.tick, this, UPDATE_PRIORITY.UTILITY); + requestRender(); + } + + stop(): void { + const take = this.take; + if (!take) return; + this.end(); + take.finish(''); + } + + private end(): void { + this.take = null; + this.source?.detach(); + this.source = null; + ui().app.ticker.remove(this.tick, this); + this.host.showBadge(false); + requestRender(); + } + + private tick(): void { + const take = this.take; + const source = this.source; + if (!take || !source) return; + const drawn = renderCount(); + if (drawn !== this.seen) { + this.seen = drawn; + source.copy(); + this.drawBadge(); + } + take.capture(); + // An idle editor draws nothing, and the time beside the menu has to move on. + if (elapsedText(performance.now() - this.startedAt) !== this.shownText) requestRender(); + } + + private drawBadge(): void { + this.shownText = elapsedText(performance.now() - this.startedAt); + this.badge.show(this.host.badgeRoom(), this.shownText); + const app = ui().app; + app.renderer.render({ container: this.badge, clear: false }); + // Pixi hit-tests the pointer against what it drew on screen last: that has to be the stage. + (app.renderer as unknown as { _lastObjectRendered: Container })._lastObjectRendered = app.stage; + } + + info(): { + recording: boolean; + takes: TakeInfo[]; + badge: { x: number; y: number; w: number; h: number } | null; + } { + return { + recording: this.isRecording, + takes: this.takes.map((take) => take.info()), + badge: this.isRecording ? this.host.badgeRoom() : null, + }; + } +} + +/** The red dot and the time beside the menu while the editor records. */ +class RecordingBadge extends Container { + private readonly dot = new Graphics(); + private readonly time = crispText({ text: '', style: textStyle('body') }); + + constructor() { + super(); + this.addChild(this.dot, this.time); + } + + show(room: { x: number; y: number; w: number; h: number }, text: string): void { + const radius = 5; + this.position.set(room.x, room.y); + this.dot + .clear() + .circle(radius + 2, room.h / 2, radius) + .fill(Theme.colors.danger); + this.time.text = text; + this.time.position.set(radius * 2 + 7, Math.round((room.h - this.time.height) / 2)); + } +} + +/** + * The canvas, copied after each render into a picture of the recording's size; a frame is that + * picture with the mouse pointer drawn over it. + */ +class CanvasSource implements FrameSource { + private readonly size: { w: number; h: number }; + private readonly picture: OffscreenCanvas; + private readonly picturePen: OffscreenCanvasRenderingContext2D; + private readonly frame: OffscreenCanvas; + private readonly framePen: OffscreenCanvasRenderingContext2D; + private readonly pointer = new PointerTrack(); + private readonly paths = new Map(); + private copied = false; + private ready: VideoFrame[] = []; + + constructor(private readonly canvas: HTMLCanvasElement) { + this.size = editorRecordingSize({ w: canvas.width, h: canvas.height }); + this.picture = new OffscreenCanvas(this.size.w, this.size.h); + this.picturePen = this.picture.getContext('2d', { alpha: false })!; + this.frame = new OffscreenCanvas(this.size.w, this.size.h); + this.framePen = this.frame.getContext('2d', { alpha: false })!; + this.pointer.attach(canvas); + } + + open(): { w: number; h: number } { + return this.size; + } + + /** The canvas holds what the editor just drew: it becomes the picture. */ + copy(): void { + const { canvas, picturePen: pen } = this; + const at = fitPicture({ w: canvas.width, h: canvas.height }, this.size); + pen.fillStyle = '#000'; + pen.fillRect(0, 0, this.size.w, this.size.h); + pen.drawImage(canvas, 0, 0, canvas.width, canvas.height, at.x, at.y, at.w, at.h); + this.copied = true; + } + + holds(): boolean { + return !this.copied; + } + + get busy(): boolean { + return false; + } + + get pending(): number { + return this.ready.length; + } + + take(ms: number): boolean { + if (!this.copied) return false; + this.framePen.drawImage(this.picture, 0, 0); + this.drawPointer(); + this.ready.push( + new VideoFrame(this.frame, { + timestamp: Math.round(ms * 1000), + duration: Math.round(1e6 / EDITOR_RECORDING_FPS), + }), + ); + return true; + } + + private drawPointer(): void { + const { canvas, pointer } = this; + const shape = cursorShape(canvas.style.cursor); + if (!pointer.inside || !shape) return; + const rect = canvas.getBoundingClientRect(); + if (rect.width === 0 || rect.height === 0) return; + const at = fitPicture({ w: canvas.width, h: canvas.height }, this.size); + const x = at.x + ((pointer.x - rect.left) / rect.width) * at.w; + const y = at.y + ((pointer.y - rect.top) / rect.height) * at.h; + const scale = at.w / rect.width; + let path = this.paths.get(shape); + if (!path) this.paths.set(shape, (path = new Path2D(shape.path))); + const pen = this.framePen; + pen.save(); + pen.translate(x, y); + pen.scale(scale, scale); + pen.rotate((shape.turn * Math.PI) / 180); + pen.translate(-shape.hot.x, -shape.hot.y); + pen.lineJoin = 'round'; + pen.lineWidth = 2; + pen.strokeStyle = '#fff'; + pen.stroke(path); + pen.fillStyle = '#000'; + pen.fill(path); + pen.restore(); + } + + collect(encode: (frame: VideoFrame) => void): void { + for (const frame of this.ready.splice(0)) encode(frame); + } + + /** The recording ends: the pointer is no longer followed, the frames taken still go out. */ + detach(): void { + this.pointer.detach(); + } + + close(): void { + this.pointer.detach(); + for (const frame of this.ready.splice(0)) frame.close(); + } +} + +/** Where the mouse is, in client pixels, and whether it is over the page at all. */ +class PointerTrack { + x = 0; + y = 0; + inside = false; + private attached = false; + + private readonly moved = (event: PointerEvent): void => { + this.x = event.clientX; + this.y = event.clientY; + this.inside = true; + }; + + private readonly left = (): void => { + this.inside = false; + }; + + /** Starts where Pixi last saw the pointer: the click on Start recording put it there. */ + attach(canvas: HTMLCanvasElement): void { + if (this.attached) return; + this.attached = true; + const rect = canvas.getBoundingClientRect(); + const at = ui().app.renderer.events.pointer.global; + this.x = rect.left + at.x; + this.y = rect.top + at.y; + this.inside = at.x > 0 || at.y > 0; + window.addEventListener('pointermove', this.moved, { capture: true, passive: true }); + window.addEventListener('pointerdown', this.moved, { capture: true, passive: true }); + document.documentElement.addEventListener('pointerleave', this.left); + } + + detach(): void { + if (!this.attached) return; + this.attached = false; + window.removeEventListener('pointermove', this.moved, { capture: true }); + window.removeEventListener('pointerdown', this.moved, { capture: true }); + document.documentElement.removeEventListener('pointerleave', this.left); + } +} diff --git a/src/editor/PlaySession.ts b/src/editor/PlaySession.ts index 9d658df2..7330b379 100644 --- a/src/editor/PlaySession.ts +++ b/src/editor/PlaySession.ts @@ -17,7 +17,8 @@ import { Input } from '../runtime/systems/Input'; import type { World } from '../runtime/World'; import { onFrame, ui } from '../ui'; import { soundLibrary } from '../viewport/SoundLibrary'; -import { GameRecorder, type MadeRecording } from '../viewport/GameRecorder'; +import { GameRecorder } from '../viewport/GameRecorder'; +import type { MadeRecording } from '../viewport/RecordingTake'; import { videoLibrary } from '../viewport/VideoLibrary'; import { FrameMeter, type FrameTimes } from './frameMeter'; import { saveArea } from './saves'; diff --git a/src/editor/panels/Toolbar.ts b/src/editor/panels/Toolbar.ts index 2baea181..7d35f9f7 100644 --- a/src/editor/panels/Toolbar.ts +++ b/src/editor/panels/Toolbar.ts @@ -27,6 +27,9 @@ const SAVE_LABELS: Record = { /** Square, because an icon has no reading direction that would ask for a wider box. */ const ICON_BUTTON = Theme.controlHeight; +/** Width beside the menu for the recording's dot and time. */ +const RECORDING_ROOM = 52; + type IconName = keyof typeof TOOLBAR_ICON_SVG; /** @@ -46,6 +49,8 @@ function iconButton(name: IconName, onClick: () => void, tooltip: () => string): export class Toolbar extends HStack { private readonly bg = new Graphics(); private readonly menuButton: Button; + /** The room between the menu and undo; it widens while the editor records (`showRecording`). */ + private readonly lead = new Spacer(Theme.padding); private readonly undoButton: Button; private readonly redoButton: Button; private readonly playButton: Button; @@ -74,7 +79,7 @@ export class Toolbar extends HStack { }); this.menuButton = iconButton('menu', () => actions.openMainMenu(this.menuButton), tip('menu')); this.add(this.menuButton); - this.add(new Spacer(Theme.padding)); + this.add(this.lead); this.undoButton = iconButton('undo', () => state.undo(), tip('undo')); this.redoButton = iconButton('redo', () => state.redo(), tip('redo')); this.add(this.undoButton); @@ -132,6 +137,21 @@ export class Toolbar extends HStack { }; } + /** + * Makes room beside the menu for what shows that the editor records, or takes it away again. The + * room is left empty here: what fills it is drawn over the screen only, not into the recording. + */ + showRecording(on: boolean): void { + this.lead.minWidth = Theme.padding + (on ? RECORDING_ROOM : 0); + this.relayout(); + } + + /** Where that room is on screen, in CSS pixels. */ + recordingRoom(): { x: number; y: number; w: number; h: number } { + const at = this.lead.getGlobalPosition(); + return { x: at.x, y: this.menuButton.getGlobalPosition().y, w: RECORDING_ROOM, h: this.menuButton.h }; + } + /** Frame rate readout; `null` while no measurement is available yet. */ setFps(fps: number | null): void { this.fps.setText(fps === null ? '– fps' : `${fps} fps`); diff --git a/src/editor/recordingUploads.ts b/src/editor/recordingUploads.ts index b47928e3..2cb867d0 100644 --- a/src/editor/recordingUploads.ts +++ b/src/editor/recordingUploads.ts @@ -1,5 +1,5 @@ import type { UploadedVideo } from '../core/store/VideoStore'; -import type { MadeRecording } from '../viewport/GameRecorder'; +import type { MadeRecording } from '../viewport/RecordingTake'; import { adoptVideo, type VideoImportDeps } from './videoImport'; /** How often the server is asked whether a recording is converted, as the video picker does. */ @@ -19,7 +19,7 @@ export interface RecordingUpload { } /** - * Brings a recording of the game view into the project (#405): up to the server, which converts it + * Brings a recording of the game view (#405) or of the editor into the project: up to the server, which converts it * like any video, then an asset named after the recording once it is ready. Only while the editor * is not playing: a run plays a copy of the project that is thrown away, and so would the asset be. */ diff --git a/src/viewport/GameRecorder.ts b/src/viewport/GameRecorder.ts index 2cf14860..20431f52 100644 --- a/src/viewport/GameRecorder.ts +++ b/src/viewport/GameRecorder.ts @@ -1,36 +1,16 @@ import { Matrix, type RenderTexture } from 'pixi.js'; -import { VIDEO_WAIT_MS } from '../core/model/video'; -import { preSkipOf } from '../core/music/ogg'; -import { AVC_CODEC, colourOf, MatroskaWriter, VP9_CODEC } from '../core/recording/matroska'; import { - avcCodec, - FrameGate, - isKeyFrame, pictureTransform, - RECORDING_MOST_BYTES, - RecordingClock, recordingName, type RecordingSettings, type RecordingSink, recordingSize, - videoBitrate, - vp9Codec, } from '../core/recording/recording'; import { ui } from '../ui'; import { PictureReader } from './PictureReader'; -import { soundLibrary } from './SoundLibrary'; +import { type FrameSource, Take, type TakeHost, type TakeInfo } from './RecordingTake'; import type { PictureArea } from './WorldViewport'; -const OPUS_BITRATE = 160_000; -/** Frames the video encoder may have waiting before the next is dropped instead of piling up. */ -const MOST_QUEUED = 6; -/** Encoded bytes held as loose pieces before they are folded into a Blob, out of the page's heap. */ -const FOLD_BYTES = 16 * 1024 * 1024; -/** How long a stop waits for the last sound the worklet holds; a suspended context sends none. */ -const LAST_SOUND_MS = 300; -/** How long a stop waits for the frames still on their way back from the GPU. */ -const LAST_FRAMES_MS = 500; - /** The view a recording takes its frames from (`WorldViewport`). */ export interface RecordingView { picture(): { shown: PictureArea; whole: PictureArea } | null; @@ -39,49 +19,9 @@ export interface RecordingView { missesVideo(): boolean; } -/** A recording once it is made. */ -export interface MadeRecording { - name: string; - blob: Blob; - durationMs: number; - width: number; - height: number; -} - /** Where the recorder looks, and what it tells. */ -export interface RecordingHost { +export interface RecordingHost extends TakeHost { view(): RecordingView | null; - /** A line for the log of the run that plays. */ - log(line: string): void; - saved(recording: MadeRecording): void; -} - -/** - * `waiting` for a picture to take its size from: `On start` runs before the view is handed the - * world, and a game may switch its camera on later. - */ -type TakeState = 'waiting' | 'starting' | 'recording' | 'finishing' | 'done' | 'failed'; - -/** What a check reads of a recording (test hook). */ -export interface TakeInfo { - name: string; - state: TakeState; - width: number; - height: number; - fps: number; - codec: string; - sound: boolean; - frames: number; - dropped: number; - soundFrames: number; - /** From the first frame to the end of the last, in ms. */ - durationMs: number; - /** `performance.now()` when the last frame was taken, null before the first. */ - lastFrameAt: number | null; - /** What the video encoder said its colour is, null while it said nothing. */ - colour: VideoColorSpaceInit | null; - bytes: number; - failure: string | null; } /** @@ -104,7 +44,9 @@ export class GameRecorder implements RecordingSink { start(settings: RecordingSettings): string { if (this.take) return `Start recording is ignored: "${this.take.name}" is recording already – Stop recording ends it`; - const take = new Take(recordingName(settings.name, new Date()), settings, this.host, () => { + const name = recordingName(settings.name, new Date()); + const source = new ViewSource(name, settings, this.host); + const take = new Take(name, settings.fps, source, this.host, () => { if (this.take === take) this.take = null; }); this.take = take; @@ -128,8 +70,7 @@ export class GameRecorder implements RecordingSink { /** The view has drawn this frame: the recording takes it, if it is due. */ capture(): void { - const view = this.take && this.host.view(); - if (view) this.take!.capture(view); + if (this.host.view()) this.take?.capture(); } /** The run is over: a recording still going is saved, not lost. */ @@ -142,429 +83,80 @@ export class GameRecorder implements RecordingSink { } } -/** One recording, from its encoders to its file. */ -class Take { - private state: TakeState = 'waiting'; - private size = { w: 0, h: 0 }; - private readonly clock = new RecordingClock(); - private readonly gate: FrameGate; - private video: VideoEncoder | null = null; - private audio: AudioEncoder | null = null; - private tap: { node: AudioWorkletNode; rate: number; release(): void } | null = null; - private writer: MatroskaWriter | null = null; - private codec = ''; - private videoHead: Uint8Array | null = null; - private colour: VideoColorSpaceInit | null = null; - private audioHead: Uint8Array | null = null; +/** + * The game view, drawn into a texture of the recording's size and read back from the GPU + * (`PictureReader`): the frames come out a tick or two after they are taken. + */ +class ViewSource implements FrameSource { private reader: PictureReader | null = null; + private size = { w: 0, h: 0 }; private toldNoCamera = false; - private folded: Blob[] = []; - private loose: Uint8Array[] = []; - private looseBytes = 0; - private frames = 0; - private dropped = 0; - private lastMs = 0; - private soundFrames = 0; - /** The recording's time of the first sample heard, in ms; null before it. */ - private soundStart: number | null = null; - private soundEnded: (() => void) | null = null; - private stopAsked = false; - private paused = false; - private held = false; - /** Since when the view lacks a video, see `holdForVideo`; null while it lacks none. */ - private heldSince: number | null = null; - private lastFrameAt: number | null = null; - private failure: string | null = null; constructor( - readonly name: string, + private readonly name: string, private readonly settings: RecordingSettings, private readonly host: RecordingHost, - private readonly gone: () => void, - ) { - this.gate = new FrameGate(settings.fps); - } + ) {} - /** - * Takes the size from the first picture there is and sets up the encoders; until they are ready - * frames are not taken, and a stop asked meanwhile ends it after. - */ - private open(picture: { shown: PictureArea; whole: PictureArea }): void { - const ratio = ui().app.renderer.resolution; - const device = (area: PictureArea) => ({ w: area.w * ratio, h: area.h * ratio }); - this.size = recordingSize(this.settings.resolution, device(picture.shown), device(picture.whole)); - this.state = 'starting'; - try { - this.reader = new PictureReader(this.size.w, this.size.h); - } catch (error) { - this.fail(error instanceof Error ? error.message : String(error)); - return; - } - void this.openEncoders().then( - () => { - if (this.state !== 'starting') return; - this.state = 'recording'; - if (this.stopAsked) this.finish(''); - }, - (error: unknown) => this.fail(error instanceof Error ? error.message : String(error)), - ); - } - - private async openEncoders(): Promise { - if (typeof VideoEncoder === 'undefined') - throw new Error('this browser cannot encode video – recording needs Chrome'); - const { w, h } = this.size; - const fps = this.settings.fps; - const base = { width: w, height: h, framerate: fps, bitrate: videoBitrate(w, h, fps) }; - const configs: VideoEncoderConfig[] = [ - { ...base, codec: avcCodec(w, h, fps), avc: { format: 'avc' }, latencyMode: 'realtime' }, - { ...base, codec: vp9Codec(w, h), latencyMode: 'realtime' }, - ]; - let config: VideoEncoderConfig | null = null; - for (const candidate of configs) - if ( - await VideoEncoder.isConfigSupported(candidate).then( - (s) => s.supported === true, - () => false, - ) - ) { - config = candidate; - break; - } - if (!config) throw new Error(`this browser cannot encode ${w}x${h} at ${fps} fps`); - this.codec = config.codec; - await this.openSound(); - const writer = new MatroskaWriter(this.audio ? 2 : 1); - this.writer = writer; - const video = new VideoEncoder({ - output: (chunk, meta) => { - const head = meta?.decoderConfig?.description; - if (head && !this.videoHead) this.videoHead = copyOf(head); - this.colour ??= meta?.decoderConfig?.colorSpace ?? null; - writer.add(1, chunk.timestamp, chunk.type === 'key', chunkBytes(chunk)); - this.collect(); - }, - error: (error) => this.fail(error.message), - }); - video.configure(config); - this.video = video; - } - - /** The master behind the limiter, as Opus; without it the recording goes on silent and says so. */ - private async openSound(): Promise { - const tap = await soundLibrary.outputTap().catch(() => null); - const config: AudioEncoderConfig = { - codec: 'opus', - sampleRate: tap?.rate ?? 48000, - numberOfChannels: 2, - bitrate: OPUS_BITRATE, - }; - const supported = - tap !== null && - typeof AudioEncoder !== 'undefined' && - (await AudioEncoder.isConfigSupported(config).then( - (s) => s.supported === true, - () => false, - )); - if (!tap || !supported) { - tap?.release(); - this.host.log(`Recording "${this.name}" has no sound: this browser cannot record it`); - return; - } - const audio = new AudioEncoder({ - output: (chunk, meta) => { - const head = meta?.decoderConfig?.description; - if (head && !this.audioHead) this.audioHead = copyOf(head); - this.writer?.add(2, chunk.timestamp, true, chunkBytes(chunk)); - this.collect(); - }, - error: (error) => this.fail(error.message), - }); - audio.configure(config); - this.audio = audio; - this.tap = tap; - tap.node.port.onmessage = (event: MessageEvent) => this.heard(event.data); - } - - setPaused(paused: boolean): void { - this.stand(() => (this.paused = paused)); - } - - /** - * A frame that lacks a video is not taken (#417): a composition's clock stands until its video has - * its picture, and the frames meanwhile were empty where the video goes. Like a pause, but the game - * plays on, so what it sounds then is left out too. At most `VIDEO_WAIT_MS` at a stretch, as long - * as the composition waits. - */ - private holdForVideo(view: RecordingView): void { - const now = performance.now(); - if (view.missesVideo()) this.heldSince ??= now; - else this.heldSince = null; - const held = this.heldSince !== null && now - this.heldSince < VIDEO_WAIT_MS; - this.stand(() => (this.held = held)); - } - - /** The clock stands while paused or held, whichever came first and whichever ends last. */ - private stand(change: () => void): void { - const was = this.standing; - change(); - if (this.standing === was) return; - const now = performance.now(); - if (this.standing) this.clock.pause(now); - else this.clock.resume(now); - } - - private get standing(): boolean { - return this.paused || this.held; - } - - capture(view: RecordingView): void { - if (this.state === 'waiting') { - const picture = view.picture(); - if (picture) this.open(picture); - else if (!this.toldNoCamera) { + open(): { w: number; h: number } | null { + const picture = this.host.view()?.picture(); + if (!picture) { + if (!this.toldNoCamera) { this.toldNoCamera = true; this.host.log( `Recording "${this.name}" waits: no camera is switched on, so there is nothing to record`, ); } - return; - } - const video = this.video; - const reader = this.reader; - if (this.state !== 'recording' || !video || !reader) return; - reader.collect((bytes, ms) => this.encode(bytes, ms)); - // A frame taken may have reached the most a file holds and ended the recording. - if (!this.taking) return; - this.holdForVideo(view); - if (this.standing) return; - const now = performance.now(); - if (!this.clock.started) this.clock.start(now); - const ms = this.clock.at(now); - if (!this.gate.take(ms)) return; - const picture = view.picture(); - if (!picture || reader.busy || video.encodeQueueSize >= MOST_QUEUED) { - this.dropped++; - return; + return null; } + const ratio = ui().app.renderer.resolution; + const device = (area: PictureArea) => ({ w: area.w * ratio, h: area.h * ratio }); + this.size = recordingSize(this.settings.resolution, device(picture.shown), device(picture.whole)); + this.reader = new PictureReader(this.size.w, this.size.h); + return this.size; + } + + holds(): boolean { + return this.host.view()?.missesVideo() ?? false; + } + + get busy(): boolean { + return this.reader?.busy ?? true; + } + + get pending(): number { + return this.reader?.pending ?? 0; + } + + take(ms: number): boolean { + const view = this.host.view(); + const picture = view?.picture(); + if (!view || !picture || !this.reader) return false; const m = pictureTransform( this.settings.resolution === 'view' ? picture.shown : picture.whole, this.size, ); - view.drawPicture(new Matrix(m.a, 0, 0, m.d, m.tx, m.ty), reader.texture); - reader.read(ms); - this.lastFrameAt = now; + view.drawPicture(new Matrix(m.a, 0, 0, m.d, m.tx, m.ty), this.reader.texture); + this.reader.read(ms); + return true; } - private get taking(): boolean { - return this.state === 'recording'; - } - - /** A frame back from the GPU, into the video encoder. */ - private encode(bytes: Uint8Array, ms: number): void { - const video = this.video; - if (!video || video.state !== 'configured') return; - const frame = new VideoFrame(bytes, { - format: 'RGBX', - codedWidth: this.size.w, - codedHeight: this.size.h, - timestamp: Math.round(ms * 1000), - duration: Math.round(1e6 / this.settings.fps), - }); - video.encode(frame, { keyFrame: isKeyFrame(this.frames, this.settings.fps) }); - frame.close(); - this.frames++; - this.lastMs = ms; - if (this.taking && (this.writer?.bytes ?? 0) > RECORDING_MOST_BYTES) - this.finish(`Recording "${this.name}" reached 2 GB, the most the server takes, and was stopped`); - } - - /** A piece of the master from the worklet; an empty one is the last. */ - private heard(sides: Float32Array[]): void { - const audio = this.audio; - const frames = sides[0]?.length ?? 0; - if (frames === 0) { - this.soundEnded?.(); - return; - } - // Before the first frame there is no time to put it at. Pieces keep coming during a pause only - // as long as they were on their way, so they belong before it and are kept. A hold leaves them - // out: the game plays on meanwhile. - if (!audio || !this.tap || this.state !== 'recording' || !this.clock.started || this.held) return; - const rate = this.tap.rate; - this.soundStart ??= Math.max(0, this.clock.at(performance.now()) - (frames / rate) * 1000); - const planar = new Float32Array(frames * 2); - planar.set(sides[0]!, 0); - planar.set(sides[1] ?? sides[0]!, frames); - const data = new AudioData({ - format: 'f32-planar', - sampleRate: rate, - numberOfFrames: frames, - numberOfChannels: 2, - timestamp: Math.round(this.soundStart * 1000 + (this.soundFrames / rate) * 1e6), - data: planar, - }); - audio.encode(data); - data.close(); - this.soundFrames += frames; - } - - /** Ends the recording and saves what it has; `why` goes to the log when it is not the node's stop. */ - finish(why: string): void { - if (why) this.host.log(why); - if (this.state === 'waiting') { - this.state = 'done'; - this.gone(); - this.host.log(`Recording "${this.name}" ended before its first frame, so nothing was saved`); - return; - } - if (this.state === 'starting') { - this.stopAsked = true; - return; - } - if (this.state !== 'recording') return; - this.state = 'finishing'; - this.gone(); - void this.save().catch((error: unknown) => - this.fail(error instanceof Error ? error.message : String(error)), + collect(encode: (frame: VideoFrame) => void): void { + this.reader?.collect((bytes, ms) => + encode( + new VideoFrame(bytes, { + format: 'RGBX', + codedWidth: this.size.w, + codedHeight: this.size.h, + timestamp: Math.round(ms * 1000), + duration: Math.round(1e6 / this.settings.fps), + }), + ), ); } - private async save(): Promise { - await Promise.all([this.lastFrames(), this.lastSound()]); - if (this.video?.state === 'configured') await this.video.flush(); - if (this.audio?.state === 'configured') await this.audio.flush(); - this.close(); - const writer = this.writer; - if (!writer || this.frames === 0) { - this.state = 'done'; - this.host.log(`Recording "${this.name}" ended before its first frame, so nothing was saved`); - return; - } - const frame = 1000 / this.settings.fps; - const { head, rest } = writer.finish({ - video: { - codec: this.codec.startsWith('avc1') ? AVC_CODEC : VP9_CODEC, - width: this.size.w, - height: this.size.h, - private: this.videoHead, - fps: this.settings.fps, - colour: colourOf(this.colour), - }, - audio: this.audio - ? { - channels: 2, - private: this.audioHead, - preSkip: preSkipOf(this.audioHead ?? new Uint8Array()) ?? 0, - } - : null, - durationMs: this.lastMs + frame, - app: 'game-maker', - }); - this.collect(); - const blob = new Blob([head, ...this.folded, ...this.loose, ...rest] as BlobPart[], { - type: 'video/x-matroska', - }); - this.folded = []; - this.loose = []; - this.state = 'done'; - this.host.saved({ - name: this.name, - blob, - durationMs: this.lastMs + frame, - width: this.size.w, - height: this.size.h, - }); - } - - /** Takes the frames still on their way back from the GPU, a moment at most. */ - private async lastFrames(): Promise { - const until = performance.now() + LAST_FRAMES_MS; - for (;;) { - this.reader?.collect((bytes, ms) => this.encode(bytes, ms)); - if (!this.reader?.pending || performance.now() > until) return; - await new Promise((resolve) => setTimeout(resolve, 4)); - } - } - - /** Asks the worklet for what it still holds and waits for it, a moment at most. */ - private lastSound(): Promise { - const tap = this.tap; - if (!tap) return Promise.resolve(); - return new Promise((resolve) => { - const timer = setTimeout(done, LAST_SOUND_MS); - function done(): void { - clearTimeout(timer); - resolve(); - } - this.soundEnded = done; - tap.node.port.postMessage('stop'); - }).finally(() => { - tap.release(); - this.tap = null; - }); - } - - /** Moves the clusters the writer finished into the file, folding them into Blobs as they pile up. */ - private collect(): void { - for (const part of this.writer?.take() ?? []) { - this.loose.push(part); - this.looseBytes += part.length; - } - if (this.looseBytes < FOLD_BYTES) return; - this.folded.push(new Blob(this.loose as BlobPart[])); - this.loose = []; - this.looseBytes = 0; - } - - private close(): void { - for (const encoder of [this.video, this.audio]) - if (encoder && encoder.state !== 'closed') encoder.close(); + close(): void { this.reader?.destroy(); this.reader = null; } - - private fail(reason: string): void { - if (this.state === 'done' || this.state === 'failed') return; - this.failure = reason; - this.state = 'failed'; - this.tap?.release(); - this.tap = null; - this.close(); - this.gone(); - this.host.log(`Recording "${this.name}" failed: ${reason}`); - } - - info(): TakeInfo { - return { - name: this.name, - state: this.state, - width: this.size.w, - height: this.size.h, - fps: this.settings.fps, - codec: this.codec, - sound: this.audio !== null, - frames: this.frames, - dropped: this.dropped, - soundFrames: this.soundFrames, - durationMs: this.frames > 0 ? Math.round(this.lastMs + 1000 / this.settings.fps) : 0, - lastFrameAt: this.lastFrameAt, - colour: this.colour, - bytes: this.writer?.bytes ?? 0, - failure: this.failure, - }; - } -} - -function chunkBytes(chunk: EncodedVideoChunk | EncodedAudioChunk): Uint8Array { - const data = new Uint8Array(chunk.byteLength); - chunk.copyTo(data); - return data; -} - -function copyOf(source: AllowSharedBufferSource): Uint8Array { - const view = ArrayBuffer.isView(source) - ? new Uint8Array(source.buffer, source.byteOffset, source.byteLength) - : new Uint8Array(source); - return view.slice(); } diff --git a/src/viewport/RecordingTake.ts b/src/viewport/RecordingTake.ts new file mode 100644 index 00000000..7d386e34 --- /dev/null +++ b/src/viewport/RecordingTake.ts @@ -0,0 +1,498 @@ +import { VIDEO_WAIT_MS } from '../core/model/video'; +import { preSkipOf } from '../core/music/ogg'; +import { AVC_CODEC, colourOf, MatroskaWriter, VP9_CODEC } from '../core/recording/matroska'; +import { + avcCodec, + FrameGate, + isKeyFrame, + RECORDING_MOST_BYTES, + RecordingClock, + videoBitrate, + vp9Codec, +} from '../core/recording/recording'; +import { soundLibrary } from './SoundLibrary'; + +const OPUS_BITRATE = 160_000; +/** Frames the video encoder may have waiting before the next is dropped instead of piling up. */ +const MOST_QUEUED = 6; +/** Encoded bytes held as loose pieces before they are folded into a Blob, out of the page's heap. */ +const FOLD_BYTES = 16 * 1024 * 1024; +/** How long a stop waits for the last sound the worklet holds; a suspended context sends none. */ +const LAST_SOUND_MS = 300; +/** How long a stop waits for the frames still on their way to the encoder. */ +const LAST_FRAMES_MS = 500; + +/** A recording once it is made. */ +export interface MadeRecording { + name: string; + blob: Blob; + durationMs: number; + width: number; + height: number; +} + +/** What a take tells. */ +export interface TakeHost { + /** A line for whoever started it: the log of the run, or the editor's notice. */ + log(line: string): void; + saved(recording: MadeRecording): void; +} + +/** + * Where a take gets its pictures: the game view (`GameRecorder`) or the whole editor + * (`EditorRecorder`). + */ +export interface FrameSource { + /** + * The size of the frames, even on both sides, and gets ready to take them; null while there is + * no picture to take the size from. Throws when it cannot take any. + */ + open(): { w: number; h: number } | null; + /** Whether the picture now lacks something still loading (#417): the frame is not taken. */ + holds(): boolean; + /** Whether a frame taken now would have nowhere to go. */ + readonly busy: boolean; + /** Frames taken that `collect` has not handed over yet. */ + readonly pending: number; + /** Takes the picture now as the frame of recording time `ms`; false when there was none. */ + take(ms: number): boolean; + /** Hands over the frames ready to encode, oldest first; whoever takes one closes it. */ + collect(encode: (frame: VideoFrame) => void): void; + close(): void; +} + +/** + * `waiting` for a picture to take its size from: `On start` runs before the view is handed the + * world, and a game may switch its camera on later. + */ +type TakeState = 'waiting' | 'starting' | 'recording' | 'finishing' | 'done' | 'failed'; + +/** What a check reads of a recording (test hook). */ +export interface TakeInfo { + name: string; + state: TakeState; + width: number; + height: number; + fps: number; + codec: string; + sound: boolean; + frames: number; + dropped: number; + soundFrames: number; + /** From the first frame to the end of the last, in ms. */ + durationMs: number; + /** `performance.now()` when the last frame was taken, null before the first. */ + lastFrameAt: number | null; + /** What the video encoder said its colour is, null while it said nothing. */ + colour: VideoColorSpaceInit | null; + bytes: number; + failure: string | null; +} + +/** One recording, from its encoders to its file (#405). */ +export class Take { + private state: TakeState = 'waiting'; + private size = { w: 0, h: 0 }; + private readonly clock = new RecordingClock(); + private readonly gate: FrameGate; + private video: VideoEncoder | null = null; + private audio: AudioEncoder | null = null; + private tap: { node: AudioWorkletNode; rate: number; release(): void } | null = null; + private writer: MatroskaWriter | null = null; + private codec = ''; + private videoHead: Uint8Array | null = null; + private colour: VideoColorSpaceInit | null = null; + private audioHead: Uint8Array | null = null; + private folded: Blob[] = []; + private loose: Uint8Array[] = []; + private looseBytes = 0; + private frames = 0; + private dropped = 0; + private lastMs = 0; + private soundFrames = 0; + /** The recording's time of the first sample heard, in ms; null before it. */ + private soundStart: number | null = null; + private soundEnded: (() => void) | null = null; + private stopAsked = false; + private paused = false; + private held = false; + /** Since when the source holds, see `holdForVideo`; null while it does not. */ + private heldSince: number | null = null; + private lastFrameAt: number | null = null; + private failure: string | null = null; + + constructor( + readonly name: string, + private readonly fps: number, + private readonly source: FrameSource, + private readonly host: TakeHost, + private readonly gone: () => void, + ) { + this.gate = new FrameGate(fps); + } + + /** + * Takes the size from the first picture there is and sets up the encoders; until they are ready + * frames are not taken, and a stop asked meanwhile ends it after. + */ + private open(): void { + let size: { w: number; h: number } | null; + try { + size = this.source.open(); + } catch (error) { + this.fail(error instanceof Error ? error.message : String(error)); + return; + } + if (!size) return; + this.size = size; + this.state = 'starting'; + void this.openEncoders().then( + () => { + if (this.state !== 'starting') return; + this.state = 'recording'; + if (this.stopAsked) this.finish(''); + }, + (error: unknown) => this.fail(error instanceof Error ? error.message : String(error)), + ); + } + + private async openEncoders(): Promise { + if (typeof VideoEncoder === 'undefined') + throw new Error('this browser cannot encode video – recording needs Chrome'); + const { w, h } = this.size; + const fps = this.fps; + const base = { width: w, height: h, framerate: fps, bitrate: videoBitrate(w, h, fps) }; + const configs: VideoEncoderConfig[] = [ + { ...base, codec: avcCodec(w, h, fps), avc: { format: 'avc' }, latencyMode: 'realtime' }, + { ...base, codec: vp9Codec(w, h), latencyMode: 'realtime' }, + ]; + let config: VideoEncoderConfig | null = null; + for (const candidate of configs) + if ( + await VideoEncoder.isConfigSupported(candidate).then( + (s) => s.supported === true, + () => false, + ) + ) { + config = candidate; + break; + } + if (!config) throw new Error(`this browser cannot encode ${w}x${h} at ${fps} fps`); + this.codec = config.codec; + await this.openSound(); + const writer = new MatroskaWriter(this.audio ? 2 : 1); + this.writer = writer; + const video = new VideoEncoder({ + output: (chunk, meta) => { + const head = meta?.decoderConfig?.description; + if (head && !this.videoHead) this.videoHead = copyOf(head); + this.colour ??= meta?.decoderConfig?.colorSpace ?? null; + writer.add(1, chunk.timestamp, chunk.type === 'key', chunkBytes(chunk)); + this.collect(); + }, + error: (error) => this.fail(error.message), + }); + video.configure(config); + this.video = video; + } + + /** The master behind the limiter, as Opus; without it the recording goes on silent and says so. */ + private async openSound(): Promise { + const tap = await soundLibrary.outputTap().catch(() => null); + const config: AudioEncoderConfig = { + codec: 'opus', + sampleRate: tap?.rate ?? 48000, + numberOfChannels: 2, + bitrate: OPUS_BITRATE, + }; + const supported = + tap !== null && + typeof AudioEncoder !== 'undefined' && + (await AudioEncoder.isConfigSupported(config).then( + (s) => s.supported === true, + () => false, + )); + if (!tap || !supported) { + tap?.release(); + this.host.log(`Recording "${this.name}" has no sound: this browser cannot record it`); + return; + } + const audio = new AudioEncoder({ + output: (chunk, meta) => { + const head = meta?.decoderConfig?.description; + if (head && !this.audioHead) this.audioHead = copyOf(head); + this.writer?.add(2, chunk.timestamp, true, chunkBytes(chunk)); + this.collect(); + }, + error: (error) => this.fail(error.message), + }); + audio.configure(config); + this.audio = audio; + this.tap = tap; + tap.node.port.onmessage = (event: MessageEvent) => this.heard(event.data); + } + + setPaused(paused: boolean): void { + this.stand(() => (this.paused = paused)); + } + + /** + * A frame that lacks a video is not taken (#417): a composition's clock stands until its video has + * its picture, and the frames meanwhile were empty where the video goes. Like a pause, but the game + * plays on, so what it sounds then is left out too. At most `VIDEO_WAIT_MS` at a stretch, as long + * as the composition waits. + */ + private holdForVideo(): void { + const now = performance.now(); + if (this.source.holds()) this.heldSince ??= now; + else this.heldSince = null; + const held = this.heldSince !== null && now - this.heldSince < VIDEO_WAIT_MS; + this.stand(() => (this.held = held)); + } + + /** The clock stands while paused or held, whichever came first and whichever ends last. */ + private stand(change: () => void): void { + const was = this.standing; + change(); + if (this.standing === was) return; + const now = performance.now(); + if (this.standing) this.clock.pause(now); + else this.clock.resume(now); + } + + private get standing(): boolean { + return this.paused || this.held; + } + + /** The picture is drawn for this frame: the take takes it, if it is due. */ + capture(): void { + if (this.state === 'waiting') { + this.open(); + return; + } + const video = this.video; + if (this.state !== 'recording' || !video) return; + this.source.collect((frame) => this.encode(frame)); + // A frame taken may have reached the most a file holds and ended the recording. + if (!this.taking) return; + this.holdForVideo(); + if (this.standing) return; + const now = performance.now(); + if (!this.clock.started) this.clock.start(now); + const ms = this.clock.at(now); + if (!this.gate.take(ms)) return; + if (this.source.busy || video.encodeQueueSize >= MOST_QUEUED || !this.source.take(ms)) { + this.dropped++; + return; + } + this.lastFrameAt = now; + } + + private get taking(): boolean { + return this.state === 'recording'; + } + + /** A frame from the source, into the video encoder. */ + private encode(frame: VideoFrame): void { + const video = this.video; + if (!video || video.state !== 'configured') { + frame.close(); + return; + } + video.encode(frame, { keyFrame: isKeyFrame(this.frames, this.fps) }); + this.lastMs = frame.timestamp / 1000; + frame.close(); + this.frames++; + if (this.taking && (this.writer?.bytes ?? 0) > RECORDING_MOST_BYTES) + this.finish(`Recording "${this.name}" reached 2 GB, the most the server takes, and was stopped`); + } + + /** A piece of the master from the worklet; an empty one is the last. */ + private heard(sides: Float32Array[]): void { + const audio = this.audio; + const frames = sides[0]?.length ?? 0; + if (frames === 0) { + this.soundEnded?.(); + return; + } + // Before the first frame there is no time to put it at. Pieces keep coming during a pause only + // as long as they were on their way, so they belong before it and are kept. A hold leaves them + // out: the game plays on meanwhile. + if (!audio || !this.tap || this.state !== 'recording' || !this.clock.started || this.held) return; + const rate = this.tap.rate; + this.soundStart ??= Math.max(0, this.clock.at(performance.now()) - (frames / rate) * 1000); + const planar = new Float32Array(frames * 2); + planar.set(sides[0]!, 0); + planar.set(sides[1] ?? sides[0]!, frames); + const data = new AudioData({ + format: 'f32-planar', + sampleRate: rate, + numberOfFrames: frames, + numberOfChannels: 2, + timestamp: Math.round(this.soundStart * 1000 + (this.soundFrames / rate) * 1e6), + data: planar, + }); + audio.encode(data); + data.close(); + this.soundFrames += frames; + } + + /** Ends the recording and saves what it has; `why` goes to the log when it is not the node's stop. */ + finish(why: string): void { + if (why) this.host.log(why); + if (this.state === 'waiting') { + this.state = 'done'; + this.close(); + this.gone(); + this.host.log(`Recording "${this.name}" ended before its first frame, so nothing was saved`); + return; + } + if (this.state === 'starting') { + this.stopAsked = true; + return; + } + if (this.state !== 'recording') return; + this.state = 'finishing'; + this.gone(); + void this.save().catch((error: unknown) => + this.fail(error instanceof Error ? error.message : String(error)), + ); + } + + private async save(): Promise { + await Promise.all([this.lastFrames(), this.lastSound()]); + if (this.video?.state === 'configured') await this.video.flush(); + if (this.audio?.state === 'configured') await this.audio.flush(); + this.close(); + const writer = this.writer; + if (!writer || this.frames === 0) { + this.state = 'done'; + this.host.log(`Recording "${this.name}" ended before its first frame, so nothing was saved`); + return; + } + const frame = 1000 / this.fps; + const { head, rest } = writer.finish({ + video: { + codec: this.codec.startsWith('avc1') ? AVC_CODEC : VP9_CODEC, + width: this.size.w, + height: this.size.h, + private: this.videoHead, + fps: this.fps, + colour: colourOf(this.colour), + }, + audio: this.audio + ? { + channels: 2, + private: this.audioHead, + preSkip: preSkipOf(this.audioHead ?? new Uint8Array()) ?? 0, + } + : null, + durationMs: this.lastMs + frame, + app: 'game-maker', + }); + this.collect(); + const blob = new Blob([head, ...this.folded, ...this.loose, ...rest] as BlobPart[], { + type: 'video/x-matroska', + }); + this.folded = []; + this.loose = []; + this.state = 'done'; + this.host.saved({ + name: this.name, + blob, + durationMs: this.lastMs + frame, + width: this.size.w, + height: this.size.h, + }); + } + + /** Takes the frames still on their way to the encoder, a moment at most. */ + private async lastFrames(): Promise { + const until = performance.now() + LAST_FRAMES_MS; + for (;;) { + this.source.collect((frame) => this.encode(frame)); + if (!this.source.pending || performance.now() > until) return; + await new Promise((resolve) => setTimeout(resolve, 4)); + } + } + + /** Asks the worklet for what it still holds and waits for it, a moment at most. */ + private lastSound(): Promise { + const tap = this.tap; + if (!tap) return Promise.resolve(); + return new Promise((resolve) => { + const timer = setTimeout(done, LAST_SOUND_MS); + function done(): void { + clearTimeout(timer); + resolve(); + } + this.soundEnded = done; + tap.node.port.postMessage('stop'); + }).finally(() => { + tap.release(); + this.tap = null; + }); + } + + /** Moves the clusters the writer finished into the file, folding them into Blobs as they pile up. */ + private collect(): void { + for (const part of this.writer?.take() ?? []) { + this.loose.push(part); + this.looseBytes += part.length; + } + if (this.looseBytes < FOLD_BYTES) return; + this.folded.push(new Blob(this.loose as BlobPart[])); + this.loose = []; + this.looseBytes = 0; + } + + private close(): void { + for (const encoder of [this.video, this.audio]) + if (encoder && encoder.state !== 'closed') encoder.close(); + this.source.close(); + } + + private fail(reason: string): void { + if (this.state === 'done' || this.state === 'failed') return; + this.failure = reason; + this.state = 'failed'; + this.tap?.release(); + this.tap = null; + this.close(); + this.gone(); + this.host.log(`Recording "${this.name}" failed: ${reason}`); + } + + info(): TakeInfo { + return { + name: this.name, + state: this.state, + width: this.size.w, + height: this.size.h, + fps: this.fps, + codec: this.codec, + sound: this.audio !== null, + frames: this.frames, + dropped: this.dropped, + soundFrames: this.soundFrames, + durationMs: this.frames > 0 ? Math.round(this.lastMs + 1000 / this.fps) : 0, + lastFrameAt: this.lastFrameAt, + colour: this.colour, + bytes: this.writer?.bytes ?? 0, + failure: this.failure, + }; + } +} + +function chunkBytes(chunk: EncodedVideoChunk | EncodedAudioChunk): Uint8Array { + const data = new Uint8Array(chunk.byteLength); + chunk.copyTo(data); + return data; +} + +function copyOf(source: AllowSharedBufferSource): Uint8Array { + const view = ArrayBuffer.isView(source) + ? new Uint8Array(source.buffer, source.byteOffset, source.byteLength) + : new Uint8Array(source); + return view.slice(); +} diff --git a/tests/core/recording.test.ts b/tests/core/recording.test.ts index 591b70ba..6c1a763b 100644 --- a/tests/core/recording.test.ts +++ b/tests/core/recording.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from 'vitest'; +import { cursorShape } from '../../src/core/recording/cursors'; import { avcCodec, + editorRecordingSize, + elapsedText, + fitPicture, FrameGate, isKeyFrame, pictureTransform, @@ -138,3 +142,36 @@ describe('the recording nodes', () => { expect(ctx.recordings).toEqual(['stop']); }); }); + +describe('recording the editor', () => { + it('records the whole canvas, an odd row or column cut off', () => { + expect(editorRecordingSize({ w: 2560, h: 1369 })).toEqual({ w: 2560, h: 1368 }); + expect(editorRecordingSize({ w: 1281, h: 721 })).toEqual({ w: 1280, h: 720 }); + }); + + it('puts the canvas where it is while its size is the one it started with', () => { + expect(fitPicture({ w: 1281, h: 720 }, { w: 1280, h: 720 })).toEqual({ x: 0, y: 0, w: 1281, h: 720 }); + }); + + it('fits a resized window in whole and centres it', () => { + expect(fitPicture({ w: 1000, h: 720 }, { w: 1280, h: 720 })).toEqual({ x: 140, y: 0, w: 1000, h: 720 }); + expect(fitPicture({ w: 2560, h: 720 }, { w: 1280, h: 720 })).toEqual({ x: 0, y: 180, w: 1280, h: 360 }); + }); + + it('shows the time it runs in minutes and seconds, hours once there are any', () => { + expect(elapsedText(0)).toBe('0:00'); + expect(elapsedText(7999)).toBe('0:07'); + expect(elapsedText(765_000)).toBe('12:45'); + expect(elapsedText(3_723_000)).toBe('1:02:03'); + }); + + it('draws the pointer the canvas shows, the arrow for what it does not know', () => { + expect(cursorShape('')).toBe(cursorShape('default')); + expect(cursorShape('inherit')).toBe(cursorShape('default')); + expect(cursorShape('ew-resize')).toEqual(cursorShape('col-resize')); + expect(cursorShape('ns-resize')!.turn).toBe(90); + expect(cursorShape('url(a.png) 4 4, grab')).toBe(cursorShape('grab')); + expect(cursorShape('grab')).not.toBe(cursorShape('default')); + expect(cursorShape('none')).toBeNull(); + }); +}); diff --git a/tools/e2e/46-topbar.mjs b/tools/e2e/46-topbar.mjs index e16e7d87..136dec0d 100644 --- a/tools/e2e/46-topbar.mjs +++ b/tools/e2e/46-topbar.mjs @@ -61,11 +61,13 @@ export async function topbar(t) { 'Master routing', 'Clear saved data', null, + 'Start recording', + null, 'General settings…', 'Plugins', ]; check( - 'the menu button opens new project, the project picker, the settings, the master routing, clearing saves, the general settings and the plugins', + 'the menu button opens new project, the project picker, the settings, the master routing, clearing saves, recording the editor, the general settings and the plugins', JSON.stringify(rows) === JSON.stringify(expected), JSON.stringify(rows), ); diff --git a/tools/e2e/67-recording.mjs b/tools/e2e/67-recording.mjs index b3f8c29c..642cb5a2 100644 --- a/tools/e2e/67-recording.mjs +++ b/tools/e2e/67-recording.mjs @@ -174,9 +174,162 @@ export async function recording(t) { colors, ); + // 67e. The editor records itself from the main menu: the whole canvas at 60 fps, the mouse pointer + // drawn in where the system draws it over the canvas, and the red dot beside the menu on + // screen only. The frame is read where the pointer rested on the empty middle of the top bar. + const { mouse, cdp, clickToolbar, clickText } = t; + await clickToolbar('menu'); + const idle = await json('__gm.menuRows()'); + await clickText('Start recording'); + await waitUntil(`__gm.recording().editor.takes[0]?.state === 'recording'`, { label: 'the editor records' }); + const redo = await json('__gm.toolbarAt("redo")'); + const playAt = await json('__gm.toolbarAt("play")'); + const rest = { x: Math.round((redo.x + playAt.x) / 2), y: Math.round(playAt.y) }; + await mouse('mouseMoved', rest.x, rest.y); + await sleep(200); + const restFrom = (await json('__gm.recording().editor.takes[0]')).durationMs; + await sleep(1000); + const restTo = (await json('__gm.recording().editor.takes[0]')).durationMs; + const canvas = await json( + "({ w: document.querySelector('canvas').width, h: document.querySelector('canvas').height })", + ); + // Points of the canvas below the top bar, whose notice changes, while the editor rests: the frame + // of that moment has to show the same. Only points amid one colour count: H.264 shares a colour + // between 2×2 pixels, so a hairline over red comes out pink. + const near = []; + for (let gy = 1; gy < 14; gy++) + for (let gx = 1; gx < 24; gx++) { + const x = Math.round((canvas.w * gx) / 24); + const y = 40 + Math.round(((canvas.h - 40) * gy) / 14); + for (let dy = -2; dy <= 2; dy += 2) + for (let dx = -2; dx <= 2; dx += 2) near.push({ x: x + dx, y: y + dy }); + } + const around = await json(`__gm.pixels(${JSON.stringify(near)})`); + const grid = []; + const live = []; + for (let i = 0; i < near.length; i += 9) { + const patch = around.slice(i, i + 9); + if (patch.every((c) => c === patch[0])) { + grid.push(near[i + 4]); + live.push(patch[4]); + } + } + const badge = (await json('__gm.recording().editor')).badge; + const dot = { x: Math.round(badge.x + 7), y: Math.round(badge.y + badge.h / 2) }; + await evaluate('window.__gm.wake()'); + await sleep(100); + const { data: screen } = await cdp.send('Page.captureScreenshot', { format: 'png' }); + const onScreen = JSON.parse( + await evaluate(`new Promise((resolve) => { + const image = new Image(); + image.onload = () => { + const canvas = document.createElement('canvas'); + canvas.width = image.width; + canvas.height = image.height; + const pen = canvas.getContext('2d'); + pen.drawImage(image, 0, 0); + resolve(JSON.stringify([...pen.getImageData(${dot.x}, ${dot.y}, 1, 1).data].slice(0, 3))); + }; + image.src = 'data:image/png;base64,${screen}'; + })`), + ); + await clickToolbar('menu'); + const busy = await json('__gm.menuRows()'); + await clickText('Stop recording'); + await waitUntil(`__gm.recording().editor.takes[0]?.state === 'done'`, { + label: 'the editor take is saved', + }); + const take = (await json('__gm.recording().editor')).takes[0]; + await waitUntil( + `__gm.recording().uploads.some((u) => u.name === ${JSON.stringify(take.name)} && (u.state === 'added' || u.state === 'failed'))`, + { timeout: 60_000, label: 'the editor recording is in the project' }, + ); + const upload = (await recordings()).uploads.find((u) => u.name === take.name); + const made = await json( + `__gm.state.project.assets.filter((a) => a.id === ${JSON.stringify(upload.assetId)}).map((a) => ({ file: a.file, fps: a.fps }))[0] ?? null`, + ); + const file = made?.file ?? ''; + const at = (restFrom + restTo) / 2000; + const frame = JSON.parse( + await evaluate(`new Promise((resolve) => { + const video = document.createElement('video'); + video.muted = true; + video.onerror = () => resolve('null'); + video.src = '/game-maker/data/videos/${file}'; + video.play().then(() => video.requestVideoFrameCallback(() => { + video.pause(); + video.onseeked = () => setTimeout(() => { + const canvas = document.createElement('canvas'); + canvas.width = video.videoWidth; + canvas.height = video.videoHeight; + const pen = canvas.getContext('2d'); + pen.drawImage(video, 0, 0); + const px = (x, y) => [...pen.getImageData(x, y, 1, 1).data].slice(0, 3); + resolve(JSON.stringify({ + w: canvas.width, + h: canvas.height, + inside: px(${rest.x + 2}, ${rest.y + 8}), + beside: px(${rest.x - 4}, ${rest.y + 8}), + dot: px(${dot.x}, ${dot.y}), + grid: ${JSON.stringify(grid)}.map((p) => px(p.x, p.y)), + })); + }, 150); + video.currentTime = ${at}; + }), () => resolve('null')); + })`), + ); + const hex = (rgb) => rgb.map((v) => v.toString(16).padStart(2, '0')).join(''); + const off = frame + ? frame.grid.filter((rgb, i) => { + const want = live[i] + .slice(1) + .match(/../g) + .map((v) => parseInt(v, 16)); + return rgb.some((v, c) => Math.abs(v - want[c]) > 24); + }).length + : -1; + check( + 'Start recording in the main menu records the whole editor at 60 fps with the pointer drawn in, the red dot on screen only, and Stop recording saves it', + idle.includes('Start recording') && + busy.includes('Stop recording') && + take.width === canvas.w - (canvas.w % 2) && + take.height === canvas.h - (canvas.h % 2) && + take.fps === 60 && + made?.fps === 60 && + upload.state === 'added' && + frame !== null && + frame.w === take.width && + Math.max(...frame.inside) < 60 && + Math.min(...frame.beside) > 25 && + onScreen[0] > 180 && + onScreen[1] < 120 && + frame.dot[0] < 120 && + grid.length >= 50 && + off <= grid.length / 20, + JSON.stringify({ + idle, + busy, + take, + made, + upload, + canvas, + at, + onScreen, + frame: frame && { + ...frame, + inside: hex(frame.inside), + beside: hex(frame.beside), + dot: hex(frame.dot), + grid: undefined, + }, + flat: grid.length, + off, + }), + ); + await mutate(` const p = __gm.state.project; - const made = ${JSON.stringify(assets.map((a) => a.id))}; + const made = ${JSON.stringify([...assets.map((a) => a.id), upload.assetId])}; p.assets = p.assets.filter((a) => a.id !== '${CONTROL}' && !made.includes(a.id)); const scene = __gm.state.activeScene; scene.objects = scene.objects.filter((o) => !o.id.startsWith('rc67-'));`);