diff --git a/README.md b/README.md index 4f4c3a64..9e8818d0 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ npm run build # writes dist/ and regenerates index.html; the folder is wh The browser suite and the three example games need a machine with a GPU: ```bash -node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2080 checks (GM_STEPS=20 for one step) +node tools/seed-e2e.mjs && node tools/e2e.mjs # the editor, 2083 checks (GM_STEPS=20 for one step) # every tool signs in first – see the Accounts page node tools/seed-jumprun.mjs && node tools/smoke-jumprun.mjs node tools/seed-tetris.mjs && node tools/smoke-tetris.mjs diff --git a/src/core/graph/nodes/action/recordingNodes.ts b/src/core/graph/nodes/action/recordingNodes.ts index 7f996233..b27e6247 100644 --- a/src/core/graph/nodes/action/recordingNodes.ts +++ b/src/core/graph/nodes/action/recordingNodes.ts @@ -25,8 +25,12 @@ export const StartRecording: NodeDef = { 'after the moment it starts. "resolution" Game view records at the size the picture has on ' + 'screen when it starts, 720p, 1080p, 1440p and 4K at that height with the width of the camera, ' + 'both at most 3840 × 2160: a wider picture is scaled down to fit. "sound" off records the ' + - 'picture alone. The recording goes on through Load scene and Restart game, holds while the ' + - 'editor pauses the run, and ends with Stop recording or with the run. In the editor it is ' + + 'picture alone. "real time" off moves the game on by one frame of the recording at a time, ' + + 'however long the computer takes for it, so not a frame is missed even at 4K and 60 fps: the ' + + 'game view runs slower meanwhile, and the sound is made apart, in step with the picture. Such a ' + + 'recording holds at most ten minutes. The recording goes on through Load scene and Restart ' + + 'game, holds while the editor pauses the run, and ends with Stop recording or with the run. ' + + 'In the editor it is ' + 'always uploaded: "convert" has the server make an MP4 of it, "import" makes that a video asset ' + 'of the project once it is ready; without them the video picker has it. A released game ' + 'downloads it as an MKV file and takes neither. A second start while one records is ignored and ' + @@ -44,6 +48,7 @@ export const StartRecording: NodeDef = { checkPort('sound', 'sound', 'Sound', true), checkPort('convert', 'convert', 'Convert', true), checkPort('adopt', 'import', 'Import after conversion', true), + checkPort('realTime', 'real time', 'Real time', true), ], outputs: [exec()], params: [], @@ -55,6 +60,7 @@ export const StartRecording: NodeDef = { sound: flag('sound'), convert: flag('convert'), adopt: flag('adopt'), + realTime: flag('realTime'), }), ); return { next: 'exec' }; diff --git a/src/core/recording/recording.ts b/src/core/recording/recording.ts index 6130e4d4..5b7dd027 100644 --- a/src/core/recording/recording.ts +++ b/src/core/recording/recording.ts @@ -17,6 +17,11 @@ export interface RecordingSettings { convert: boolean; /** Whether the converted video becomes an asset of the project; only with `convert`. */ adopt: boolean; + /** + * Off, the run moves one frame of the recording at a time however long each takes (#423), so + * none is missed: see `SteadyPace`. + */ + realTime: boolean; } /** What becomes of a recording on the server, as Start recording asked; a released game downloads it instead. */ @@ -61,7 +66,7 @@ export function recordingSettingsOf( name: string, resolution: string, fps: string | number, - flags: Partial> = {}, + flags: Partial> = {}, ): RecordingSettings { const known = RECORDING_RESOLUTIONS.find((r) => r.value === resolution); return { @@ -71,6 +76,7 @@ export function recordingSettingsOf( sound: flags.sound ?? true, convert: flags.convert ?? true, adopt: flags.adopt ?? true, + realTime: flags.realTime ?? true, }; } @@ -337,6 +343,68 @@ export class FramePace { } } +/** + * The most a recording not in real time holds (#423): its sound is made in an offline context, and + * Chrome takes all of that context's memory once it renders, 230 MB for ten minutes of stereo. + */ +export const STEADY_MOST_SECONDS = 600; + +/** + * The pace of a recording not in real time (#423): the run moves on by one frame of the recording, + * `1 / fps`, however long the computer takes for it. The frame a step draws is owed until it is + * taken; the next step waits for that, for the sound up to it to be made, and for whatever else the + * host says it is not `ready` for. A frame's time is its number, so the video has every one. + */ +export class SteadyPace { + private taken = 0; + private owed = false; + /** Seconds of sound made so far, from the first frame. */ + private made = 0; + + constructor(readonly fps: number) {} + + /** The step the run takes now, in seconds: one frame of the recording, or 0 to hold. */ + step(ready: boolean): number { + if (this.owed || !ready || this.made + 1e-9 < this.taken / this.fps) return 0; + this.owed = true; + return 1 / this.fps; + } + + /** Whether the frame of the last step still waits to be taken. */ + get owes(): boolean { + return this.owed; + } + + /** The time of the frame owed, or of the next one, in ms. */ + get ms(): number { + return (this.taken * 1000) / this.fps; + } + + /** How far the sound is to be made once the frame owed is drawn: up to the frame after it. */ + get soundTo(): number { + return (this.taken + 1) / this.fps; + } + + took(): void { + if (!this.owed) return; + this.owed = false; + this.taken++; + } + + soundMade(seconds: number): void { + this.made = Math.max(this.made, seconds); + } + + get frames(): number { + return this.taken; + } + + /** Seconds of video taken so far. */ + get seconds(): number { + return this.taken / this.fps; + } +} + /** * Where the view's local CSS pixels land when `area` of it (a centre and a size) fills a frame of * `size` pixels: the recording draws into a texture of that size, whatever the window's. diff --git a/src/editor/Editor.ts b/src/editor/Editor.ts index 412788a2..9449e3f3 100644 --- a/src/editor/Editor.ts +++ b/src/editor/Editor.ts @@ -1408,8 +1408,10 @@ export class Editor implements PlayHost { // The game's own pause holds the clock as well, so particles and shaders freeze with the world, // and so does a run that waits for its videos at the start (#404). // A single step out of the pause moves by one fixed step, however long the frame took. + // A recording not in real time (#423) moves it by one frame of the recording, or holds it. const stepping = this.play.beginFrame(); - const frame = stepping ? FIXED_STEP : dt; + const recorded = stepping ? null : this.play.recordingStep(); + const frame = stepping ? FIXED_STEP : (recorded ?? dt); const held = this.state.paused && !stepping; const step = gameClock.advance(frame, held || this.play.isGamePaused || this.play.isWaiting); this.play.tick(step, timeStep(frame, gameClock.scale, held)); diff --git a/src/editor/PlaySession.ts b/src/editor/PlaySession.ts index 3f184d2c..17bfcbce 100644 --- a/src/editor/PlaySession.ts +++ b/src/editor/PlaySession.ts @@ -5,6 +5,7 @@ */ import { emptySharedState, type SharedState, worldPositionOf } from '../core/graph'; import { namedValue } from '../core/graph/values'; +import { AudioSwitch } from '../runtime/audioSwitch'; import { GameRun, type RunServices } from '../runtime/GameRun'; import { AnimationSystem } from '../runtime/systems/AnimationSystem'; import { type BehaviorSystem, behaviorError, type Trace } from '../runtime/systems/BehaviorSystem'; @@ -82,6 +83,13 @@ export class PlaySession { private videoWait: number | null = null; /** Records the game view for `Start recording` (#405); one for the editor, so it outlives a restart. */ private readonly recorder: GameRecorder; + /** The sound of the run, diverted while a recording not in real time goes (#423). */ + private readonly audio = new AudioSwitch(soundLibrary); + /** What a recording not in real time asked the run to move by this frame, see `recordingStep`. */ + private recordStep: number | null = null; + /** When the notice last told how far such a recording has got, by `performance.now()`, and whether it did. */ + private toldPace = 0; + private toldSteady = false; /** `step` asked for a frame; `stepping` is set for the frame that takes it. */ private stepAsked = false; private stepping = false; @@ -99,6 +107,7 @@ export class PlaySession { consoleLine(line, level); }, saved: (recording, options) => host.recordingSaved(recording, options), + audio: this.audio, }); // An object picked during a run is looked at now: the hierarchy has to list it to show it picked. state.on('selection', () => { @@ -132,7 +141,17 @@ export class PlaySession { * recording sets up its encoders. */ get isWaiting(): boolean { - return this.videoWait !== null || this.recorder.isStarting; + return this.videoWait !== null || this.recorder.isStarting || this.recordStep === 0; + } + + /** + * Called before the clock moves: the step a recording not in real time (#423) moves the run by + * this frame, 0 while it holds the run; null when the run keeps the frame's own time. + */ + recordingStep(): number | null { + this.recordStep = + this.world && !this.state.paused && this.videoWait === null ? this.recorder.step() : null; + return this.recordStep; } /** Pause or resume; before Play it switches whether Play starts paused. */ @@ -221,7 +240,7 @@ export class PlaySession { gameClock.reset(); const runtime = this.state.enterPlay(); const services: RunServices = { - audio: soundLibrary, + audio: this.audio, physics, clock: gameClock, saves: saveArea(runtime.id, 'play'), @@ -287,8 +306,24 @@ export class PlaySession { return false; } + /** While a recording not in real time goes (#423), the notice tells how far it has got, twice a second. */ + private tellPace(): void { + const now = performance.now(); + const steady = this.recordStep !== null; + if (steady === this.toldSteady && (!steady || now - this.toldPace < 500)) return; + this.toldSteady = steady; + this.toldPace = now; + this.state.setNotice(this.playNotice(), 'quiet'); + } + private playNotice(): string { if (this.videoWait !== null) return 'Loading the videos… · Esc stops'; + const steady = this.recorder.steadyPace(); + if (steady && !this.state.paused) + return ( + `Recording "${steady.name}" not in real time: ${steady.seconds.toFixed(1)} s of video, ` + + `${steady.slower.toFixed(1)}x real time · Esc stops` + ); if (this.state.paused) { const why = this.breakAt === null ? 'Paused' : `Break in ${this.breakAt}`; return `${why} · Ctrl+Enter resumes · Ctrl+Shift+Enter steps · Esc stops`; @@ -309,7 +344,8 @@ export class PlaySession { gameClock.reset(); // And its sounds end with it, looping music included – nothing may outlive the world. soundLibrary.setSuspended(false); - soundLibrary.stopAll(); + this.audio.stop(''); + this.recordStep = null; videoLibrary.endRun(); this.videoWait = null; this.stepAsked = false; @@ -335,7 +371,13 @@ export class PlaySession { const run = this.run; if (!run) return; const world = run.world; - if ((this.state.paused && !this.stepping) || this.waitsForVideos() || this.recorder.isStarting) { + this.tellPace(); + if ( + (this.state.paused && !this.stepping) || + this.waitsForVideos() || + this.recorder.isStarting || + this.recordStep === 0 + ) { // Keys and clicks that happen while paused are dropped instead of piling up – otherwise // resuming would deliver every one of them in a single frame. this.input.endFrame(); diff --git a/src/player/Player.ts b/src/player/Player.ts index 5ba659f8..bc3c5e10 100644 --- a/src/player/Player.ts +++ b/src/player/Player.ts @@ -2,6 +2,7 @@ import type { SaveArea } from '../core/graph/savedValues'; import { type Project, startScene } from '../core/model/Project'; import { VIDEO_WAIT_MS } from '../core/model/video'; import { gameClock, timeStep } from '../core/time'; +import { AudioSwitch } from '../runtime/audioSwitch'; import { GameRun } from '../runtime/GameRun'; import { loadPhysics, readyPhysics } from '../runtime/physics/engine'; import { behaviorError } from '../runtime/systems/BehaviorSystem'; @@ -25,6 +26,8 @@ export class Player { private videoWait: number | null = null; /** `Start recording` (#405): a released game has no project to add to, so the file is downloaded. */ private readonly recorder: GameRecorder; + /** The sound of a run, diverted while a recording not in real time goes (#423). */ + private readonly audio = new AudioSwitch(soundLibrary); constructor( private readonly project: Project, @@ -39,6 +42,7 @@ export class Player { consoleLine(line, level); }, saved: (recording) => downloadBlob(recording.blob, `${recording.name}.mkv`), + audio: this.audio, }); } @@ -68,7 +72,7 @@ export class Player { if (this.run || !scene || !physics) return; gameClock.reset(); this.run = GameRun.start(this.project, scene.id, this.input, { - audio: soundLibrary, + audio: this.audio, physics, clock: gameClock, saves: this.saves, @@ -91,7 +95,7 @@ export class Player { this.run = null; this.videoWait = null; this.input.detach(); - soundLibrary.stopAll(); + this.audio.stop(''); videoLibrary.endRun(); gameClock.reset(); const error = behaviorError(run.world); @@ -106,15 +110,20 @@ export class Player { const run = this.run; if (!run) return; // A recording that sets up its encoders holds the run too: its frames meanwhile would be lost. - if (this.waitsForVideos() || this.recorder.isStarting) { + // One not in real time (#423) moves it by one of its frames, or holds it until it has taken one. + const waits = this.waitsForVideos() || this.recorder.isStarting; + const recorded = waits ? null : this.recorder.step(); + if (waits || recorded === 0) { this.input.endFrame(); run.world.dropPointerEdges(); this.screen.redraw(); + if (recorded === 0) this.recorder.capture(); requestRender(); return; } - const step = gameClock.advance(dt, run.world.paused); - const request = run.tick(step, timeStep(dt, gameClock.scale, false)); + const frame = recorded ?? dt; + const step = gameClock.advance(frame, run.world.paused); + const request = run.tick(step, timeStep(frame, gameClock.scale, false)); if (request === 'stop') this.stop(); else if (request === 'restart') { this.stop(true); diff --git a/src/runtime/audioSwitch.ts b/src/runtime/audioSwitch.ts new file mode 100644 index 00000000..0ec8db81 --- /dev/null +++ b/src/runtime/audioSwitch.ts @@ -0,0 +1,140 @@ +import type { EaseShare } from '../core/model/animation'; +import type { MixBus } from '../core/music/mixer'; +import type { SequencePlay } from '../core/music/playbacks'; +import type { FxParams, FxPlan, FxVoice } from '../core/soundfx/nodes'; +import type { AudioSink, ClipTiming, NoteTiming } from './systems/Audio'; + +/** The first number a sink diverted to gives out: whatever it numbers is told apart from the live one's. */ +export const DIVERTED_IDS = 1_000_000_000; + +/** + * The sound of a run (#423): what it plays goes to the live sink, until a recording not in real time + * diverts it into one of its own, made apart from the speakers, and back once it ends. A voice is + * reached by its number in the sink that gave it out, so one heard live before goes on and stops as + * before. A stop by name, by owner or of everything reaches both; so do the levels of the mix, which + * the sink diverted to is given as they stand when it takes over. + */ +export class AudioSwitch implements AudioSink { + private aside: AudioSink | null = null; + private mix: { buses: readonly MixBus[]; videos?: ReadonlyMap } | null = null; + private readonly volumes = new Map(); + + constructor(private readonly live: AudioSink) {} + + get diverted(): boolean { + return this.aside !== null; + } + + /** From now on what the run plays goes to `sink`, which numbers from `DIVERTED_IDS`. */ + divert(sink: AudioSink): void { + this.aside = sink; + if (this.mix) sink.setMix(this.mix.buses, this.mix.videos); + for (const [bus, volume] of this.volumes) sink.setMixVolume(bus, volume); + } + + /** Back to the live sink; what the diverted one plays is its own business. */ + restore(): void { + this.aside = null; + } + + private get now(): AudioSink { + return this.aside ?? this.live; + } + + private get both(): AudioSink[] { + return this.aside ? [this.live, this.aside] : [this.live]; + } + + /** The sink that gave out `id`; null for one of a diverted sink gone since. */ + private of(id: number): AudioSink | null { + return id >= DIVERTED_IDS ? this.aside : this.live; + } + + play(file: string, volume: number, loop: boolean, owner: string, fx?: FxVoice, bus?: string): number { + return this.now.play(file, volume, loop, owner, fx, bus); + } + + playNote( + name: string, + layers: readonly FxPlan[], + params: FxParams, + volume: number, + owner: string, + duration: number, + timing?: NoteTiming, + ): number { + return this.now.playNote(name, layers, params, volume, owner, duration, timing); + } + + playClip(file: string, clip: ClipTiming, fx?: FxVoice): number { + return this.now.playClip(file, clip, fx); + } + + playSequence(name: string, play: SequencePlay, owner: string): number { + return this.now.playSequence(name, play, owner); + } + + stopSequence(name: string, letRing?: boolean, fade?: number): void { + for (const sink of this.both) sink.stopSequence(name, letRing, fade); + } + + stopPlayback(id: number, fade: number): void { + this.of(id)?.stopPlayback(id, fade); + } + + release(voice: number): void { + this.of(voice)?.release(voice); + } + + tune(voice: number, params: FxParams, time?: number, ease?: EaseShare): void { + this.of(voice)?.tune(voice, params, time, ease); + } + + /** The live sink knows every file, streamed ones too. */ + length(file: string): number { + return this.live.length(file); + } + + tail(voice: number): number { + return this.of(voice)?.tail(voice) ?? 0; + } + + stop(file: string, letRing?: boolean): void { + for (const sink of this.both) sink.stop(file, letRing); + } + + stopVoice(voice: number, letRing?: boolean): void { + this.of(voice)?.stopVoice(voice, letRing); + } + + stopOwner(owner: string): void { + for (const sink of this.both) sink.stopOwner(owner); + } + + setMix(buses: readonly MixBus[], videos?: ReadonlyMap): void { + this.mix = { buses, ...(videos && { videos }) }; + for (const sink of this.both) sink.setMix(buses, videos); + } + + setMixVolume(bus: string, volume: number, time?: number, ease?: EaseShare): void { + this.volumes.set(bus, volume); + for (const sink of this.both) sink.setMixVolume(bus, volume, time, ease); + } + + mixVolume(bus: string): number { + return this.now.mixVolume(bus); + } + + resetMix(): void { + this.volumes.clear(); + for (const sink of this.both) sink.resetMix(); + } + + tuneMix(bus: string, params: FxParams, time?: number, ease?: EaseShare): void { + for (const sink of this.both) sink.tuneMix(bus, params, time, ease); + } + + clock(): number { + return this.now.clock?.() ?? 0; + } +} diff --git a/src/viewport/GameRecorder.ts b/src/viewport/GameRecorder.ts index b7ef4ba2..573f03d5 100644 --- a/src/viewport/GameRecorder.ts +++ b/src/viewport/GameRecorder.ts @@ -7,9 +7,12 @@ import { recordingSize, type RecordingUploadOptions, } from '../core/recording/recording'; +import type { AudioSwitch } from '../runtime/audioSwitch'; import { ui } from '../ui'; import { PictureReader } from './PictureReader'; import { type FrameSource, type MadeRecording, Take, type TakeHost, type TakeInfo } from './RecordingTake'; +import { type HeardVideo, SteadySound } from './SteadySound'; +import { videoFileUrl, videoLibrary } from './VideoLibrary'; import type { PictureArea } from './WorldViewport'; /** The view a recording takes its frames from (`WorldViewport`). */ @@ -18,6 +21,10 @@ export interface RecordingView { drawPicture(transform: Matrix, target: RenderTexture): void; /** Whether its last redraw left out a video sprite, the video still without its picture. */ missesVideo(): boolean; + /** Whether every driven video it shows has the picture asked of it, not one it is leaving (#423). */ + videosSettled(): boolean; + /** The driven videos it lets be heard now, as a recording not in real time hears them (#423). */ + heardVideos(): HeardVideo[]; } /** Where the recorder looks, and what it tells; `saved` gets what the node asked to become of it. */ @@ -25,6 +32,8 @@ export interface RecordingHost { view(): RecordingView | null; log(line: string, level?: 'info' | 'warn'): void; saved(recording: MadeRecording, options: RecordingUploadOptions): void; + /** The sound of the run, which a recording not in real time diverts while it lasts (#423). */ + audio?: AudioSwitch; } /** @@ -67,6 +76,9 @@ export class GameRecorder implements RecordingSink { log: (line, level) => this.host.log(line, level), saved: (recording) => this.host.saved(recording, upload), }; + const steady = !settings.realTime; + const audio = steady && settings.sound ? (this.host.audio ?? null) : null; + const sound = audio ? new SteadySound(videoFileUrl) : null; const take = new Take( name, settings.fps, @@ -74,15 +86,39 @@ export class GameRecorder implements RecordingSink { host, () => { if (this.take === take) this.take = null; + if (!steady) return; + videoLibrary.setStepped(false); + audio?.restore(); }, - { sound: settings.sound }, + { sound: settings.sound, realTime: settings.realTime, steadySound: sound }, ); this.take = take; this.takes.push(take); if (this.paused) take.setPaused(true); + if (steady) videoLibrary.setStepped(true); + if (audio && sound) audio.divert(sound.library); return ''; } + /** + * The step the run takes this frame while a recording not in real time goes (#423): one frame of + * it, or 0 to hold; null while the run keeps its own time. + */ + step(): number | null { + return this.take?.step() ?? null; + } + + /** + * How far a recording not in real time has got (#423): seconds of video, and how many times + * longer it took; null while none records. + */ + steadyPace(): { name: string; seconds: number; slower: number } | null { + const info = this.take?.info(); + if (!info || info.realTime || info.state !== 'recording') return null; + const seconds = info.durationMs / 1000; + return { name: info.name, seconds, slower: seconds > 0 ? info.tookMs / info.durationMs : 0 }; + } + stop(): string { if (!this.take) return 'Stop recording: nothing is recording'; this.take.finish(''); @@ -155,8 +191,14 @@ class ViewSource implements FrameSource { return this.size; } - holds(): boolean { - return this.host.view()?.missesVideo() ?? false; + holds(exact: boolean): boolean { + const view = this.host.view(); + if (!view) return false; + return view.missesVideo() || (exact && !view.videosSettled()); + } + + heard(): HeardVideo[] { + return this.host.view()?.heardVideos() ?? []; } get busy(): boolean { diff --git a/src/viewport/RecordingTake.ts b/src/viewport/RecordingTake.ts index df3cc95b..5490d0a9 100644 --- a/src/viewport/RecordingTake.ts +++ b/src/viewport/RecordingTake.ts @@ -15,10 +15,13 @@ import { isKeyFrame, RECORDING_MOST_BYTES, RecordingClock, + STEADY_MOST_SECONDS, + SteadyPace, videoBitrate, vp9Codec, } from '../core/recording/recording'; import { soundLibrary } from './SoundLibrary'; +import type { HeardVideo, SteadySound } from './SteadySound'; const OPUS_BITRATE = 160_000; /** Frames the video encoder may have waiting before the next is dropped instead of piling up. */ @@ -69,8 +72,13 @@ export interface FrameSource { * no picture to take the size from. Throws when it cannot take any. */ open(): { w: number; h: number } | null; - /** Whether the picture now lacks something still loading (#417): the frame is not taken. */ - holds(): boolean; + /** + * Whether the picture now lacks something still loading (#417): the frame is not taken. `exact` + * for a recording not in real time (#423), which waits for every video to show the picture asked. + */ + holds(exact: boolean): boolean; + /** The videos heard in the picture now, for a recording that makes its sound apart (#423). */ + heard?(): HeardVideo[]; /** Whether a frame taken now would have nowhere to go. */ readonly busy: boolean; /** Frames taken that `collect` has not handed over yet. */ @@ -102,6 +110,10 @@ export interface TakeInfo { dropped: number; /** Frames no drawn frame came for, see `FrameGate.skipped`. */ skipped: number; + /** False for one that moves the run on frame by frame (#423). */ + realTime: boolean; + /** Milliseconds from its first frame to its last by `performance.now()`. */ + tookMs: number; soundFrames: number; /** From the first frame to the end of the last, in ms. */ durationMs: number; @@ -144,9 +156,20 @@ export class Take { /** Since when the source holds, see `holdForVideo`; null while it does not. */ private heldSince: number | null = null; private lastFrameAt: number | null = null; + private firstFrameAt: number | null = null; private failure: string | null = null; /** When the encoders began to be set up, by `performance.now()`. */ private startingAt = 0; + /** The pace of a recording not in real time (#423); null for one in real time. */ + private readonly steady: SteadyPace | null; + /** Where its sound is made then; null without sound. */ + private readonly steadySound: SteadySound | null; + /** The frame drawn last whose sound is still to be asked for: its time and up to when. */ + private soundDue: { at: number; to: number } | null = null; + /** Since when the frame owed waits, by `performance.now()`. */ + private owedSince = 0; + /** The samples of the file once it ends, so the sound is as long as the picture; null before. */ + private soundLimit: number | null = null; constructor( readonly name: string, @@ -154,11 +177,16 @@ export class Take { private readonly source: FrameSource, private readonly host: TakeHost, private readonly gone: () => void, - /** `sound: false` records the picture alone. */ - private readonly options: { sound?: boolean } = {}, + /** + * `sound: false` records the picture alone. `realTime: false` moves the run on frame by frame + * (#423), its sound made by `steadySound`. + */ + private readonly options: { sound?: boolean; realTime?: boolean; steadySound?: SteadySound | null } = {}, ) { this.gate = new FrameGate(fps); this.pace = new FramePace(fps); + this.steady = options.realTime === false ? new SteadyPace(fps) : null; + this.steadySound = this.steady ? (options.steadySound ?? null) : null; } /** @@ -184,8 +212,9 @@ export class Take { const { w, h } = this.size; const codec = this.codec.startsWith('avc1') ? 'H.264' : 'VP9'; const sound = this.audio ? 'with sound' : 'without sound'; + const pace = this.steady ? ', not in real time' : ''; this.host.log( - `Recording "${this.name}" started: ${w}x${h} at ${this.fps} fps, ${codec}, ${sound}`, + `Recording "${this.name}" started: ${w}x${h} at ${this.fps} fps, ${codec}, ${sound}${pace}`, 'info', ); if (this.stopAsked) this.finish(''); @@ -236,7 +265,8 @@ export class Take { /** The master behind the limiter, as Opus; without it the recording goes on silent and says so. */ private async openSound(): Promise { - const tap = await soundLibrary.outputTap().catch(() => null); + const asked = this.steady ? this.steadySound?.tap() : soundLibrary.outputTap(); + const tap = asked ? await asked.catch(() => null) : null; const config: AudioEncoderConfig = { codec: 'opus', sampleRate: tap?.rate ?? 48000, @@ -295,7 +325,7 @@ export class Take { */ private holdForVideo(): void { const now = performance.now(); - if (this.source.holds()) this.heldSince ??= now; + if (this.source.holds(false)) this.heldSince ??= now; else this.heldSince = null; const held = this.heldSince !== null && now - this.heldSince < VIDEO_WAIT_MS; this.stand(() => (this.held = held)); @@ -326,6 +356,10 @@ export class Take { this.source.collect((frame) => this.encode(frame)); // A frame taken may have reached the most a file holds and ended the recording. if (!this.taking) return; + if (this.steady) { + this.captureSteady(this.steady); + return; + } this.holdForVideo(); if (this.standing) return; const now = performance.now(); @@ -333,7 +367,10 @@ export class Take { const ms = this.clock.at(now); if (!this.gate.take(ms)) return; if (this.source.busy || video.encodeQueueSize >= MOST_QUEUED || !this.source.take(ms)) this.dropped++; - else this.lastFrameAt = now; + else { + this.lastFrameAt = now; + this.firstFrameAt ??= now; + } const got = this.pace.behind(ms, this.missed); if (got !== null) this.host.log( @@ -341,6 +378,62 @@ export class Take { ); } + /** + * For a recording not in real time (#423), the step the run takes now: one frame of it once the + * last is taken, its sound made and nothing else holds; else 0. Null while the run keeps its own + * time: for one in real time, and before the first frame. + */ + step(): number | null { + const pace = this.steady; + if (!pace || this.state !== 'recording') return null; + if (this.paused) return 0; + // A frame whose video has landed since the last tick is taken now, not a tick later. + if (pace.owes || this.soundDue) this.captureSteady(pace); + if (!this.taking) return null; + pace.soundMade(this.steadySound ? this.steadySound.made : Infinity); + const ready = + this.soundDue === null && + !this.source.busy && + (this.video?.encodeQueueSize ?? 0) < MOST_QUEUED && + (this.steadySound?.settled ?? true); + const step = pace.step(ready); + if (step > 0) { + this.soundDue = { at: pace.ms / 1000, to: pace.soundTo }; + this.owedSince = performance.now(); + } + return step; + } + + /** + * The run has drawn the frame its last step owes: its sound is made up to the next one once + * nothing waits for its bytes, and the frame is taken once every video shows the picture asked of + * it, `VIDEO_WAIT_MS` at most. Its time is its number. + */ + private captureSteady(pace: SteadyPace): void { + const sound = this.steadySound; + const due = this.soundDue; + if (due && sound) { + sound.hearVideos(this.source.heard?.() ?? [], due.at); + if (sound.settled) { + sound.renderTo(due.to); + this.soundDue = null; + } + } else this.soundDue = null; + if (!pace.owes) return; + const now = performance.now(); + if ((this.source.busy || this.source.holds(true)) && now - this.owedSince < VIDEO_WAIT_MS) return; + if (this.source.busy || !this.source.take(pace.ms)) this.dropped++; + else { + this.lastFrameAt = now; + this.firstFrameAt ??= now; + } + pace.took(); + if (pace.seconds >= STEADY_MOST_SECONDS) + this.finish( + `Recording "${this.name}" reached ${STEADY_MOST_SECONDS / 60} minutes, the most one not in real time holds, and was stopped`, + ); + } + /** Frames the recording's time has room for that are not in it. */ private get missed(): number { return this.gate.skipped + this.dropped; @@ -388,26 +481,36 @@ export class Take { this.soundEnded?.(); return; } - // Before the first frame there is no time to put it at. Pieces keep coming during a pause only - // as long as they were on their way, so they belong before it and are kept. A hold leaves them - // out: the game plays on meanwhile. - if (!audio || !this.tap || this.state !== 'recording' || !this.clock.started || this.held) return; + if (!audio || !this.tap) return; + let kept = frames; + if (this.steady) { + // Made apart (#423), it starts with the first frame and goes on to the end of the last. + if (this.state !== 'recording' && this.state !== 'finishing') return; + this.soundStart = 0; + if (this.soundLimit !== null) kept = Math.min(frames, this.soundLimit - this.soundFrames); + if (kept <= 0) return; + } else { + // Before the first frame there is no time to put it at. Pieces keep coming during a pause only + // as long as they were on their way, so they belong before it and are kept. A hold leaves them + // out: the game plays on meanwhile. + if (this.state !== 'recording' || !this.clock.started || this.held) return; + this.soundStart ??= Math.max(0, this.clock.at(performance.now()) - (frames / this.tap.rate) * 1000); + } const rate = this.tap.rate; - this.soundStart ??= Math.max(0, this.clock.at(performance.now()) - (frames / rate) * 1000); - const planar = new Float32Array(frames * 2); - planar.set(sides[0]!, 0); - planar.set(sides[1] ?? sides[0]!, frames); + const planar = new Float32Array(kept * 2); + planar.set(sides[0]!.subarray(0, kept), 0); + planar.set((sides[1] ?? sides[0]!).subarray(0, kept), kept); const data = new AudioData({ format: 'f32-planar', sampleRate: rate, - numberOfFrames: frames, + numberOfFrames: kept, numberOfChannels: 2, timestamp: Math.round(this.soundStart * 1000 + (this.soundFrames / rate) * 1e6), data: planar, }); audio.encode(data); data.close(); - this.soundFrames += frames; + this.soundFrames += kept; } /** Ends the recording and saves what it has; `why` goes to the log when it is not the node's stop. */ @@ -481,6 +584,23 @@ export class Take { }); // After the upload's notice, or the editor's would cover this one at once. this.tellMissed(); + this.tellPace(); + } + + /** How much slower than real time a recording not in real time was made (#423). */ + private tellPace(): void { + const pace = this.steady; + if (!pace || pace.seconds <= 0) return; + const took = this.tookMs / 1000; + this.host.log( + `Recording "${this.name}" took ${took.toFixed(1)} s for ${pace.seconds.toFixed(1)} s of video, ` + + `${(took / pace.seconds).toFixed(1)}x real time, and has every frame`, + 'info', + ); + } + + private get tookMs(): number { + return this.firstFrameAt !== null && this.lastFrameAt !== null ? this.lastFrameAt - this.firstFrameAt : 0; } /** @@ -507,10 +627,22 @@ export class Take { } } - /** Asks the worklet for what it still holds and waits for it, a moment at most. */ - private lastSound(): Promise { + /** + * Asks the worklet for what it still holds and waits for it, a moment at most. Sound made apart + * (#423) is made to the end of the last frame first and cut there. + */ + private async lastSound(): Promise { const tap = this.tap; - if (!tap) return Promise.resolve(); + if (!tap) return; + const pace = this.steady; + const sound = this.steadySound; + if (pace && sound) { + this.soundLimit = Math.round(pace.seconds * tap.rate); + sound.renderTo(pace.seconds); + const until = performance.now() + LAST_SOUND_MS; + while (sound.made < pace.seconds && performance.now() < until) + await new Promise((resolve) => setTimeout(resolve, 4)); + } return new Promise((resolve) => { const timer = setTimeout(done, LAST_SOUND_MS); function done(): void { @@ -541,6 +673,7 @@ export class Take { for (const encoder of [this.video, this.audio]) if (encoder && encoder.state !== 'closed') encoder.close(); this.source.close(); + this.steadySound?.close(); } private fail(reason: string): void { @@ -566,6 +699,8 @@ export class Take { frames: this.frames, dropped: this.dropped, skipped: this.gate.skipped, + realTime: this.steady === null, + tookMs: Math.round(this.tookMs), soundFrames: this.soundFrames, durationMs: this.frames > 0 ? Math.round(this.lastMs + 1000 / this.fps) : 0, lastFrameAt: this.lastFrameAt, diff --git a/src/viewport/SoundLibrary.ts b/src/viewport/SoundLibrary.ts index 281b46c3..9e0c2482 100644 --- a/src/viewport/SoundLibrary.ts +++ b/src/viewport/SoundLibrary.ts @@ -310,6 +310,8 @@ export class SoundLibrary implements AudioSink { if (render) { this.context = render.context; this.held = true; + this.nextId = render.firstId ?? 1; + if (guarded) this.guardOut(render.context); } } private decodeContext: OfflineAudioContext | null = null; @@ -379,7 +381,7 @@ export class SoundLibrary implements AudioSink { for (const id of voices) this.stopVoice(id, letRing); }, wake: (pump) => (this.render?.wake ?? wake)(pump), - prepare: (needs, id) => this.ready(needs, id), + prepare: (needs, id) => this.settle(this.ready(needs, id)), ahead: (id) => this.groupAhead(id), level: (id, gain, at, seconds) => this.level(id, gain, at, seconds), after: (at, then) => void this.after(at, then), @@ -390,6 +392,8 @@ export class SoundLibrary implements AudioSink { private readonly groups = new Set(); /** Held by the pause of the toolbar: nothing that reads the clock may wake it then. */ private held = false; + /** Plays and mixes still waiting on what they need, see `settled`. */ + private settling = 0; /** Counts the times buses were let go, so a later use of one can keep it. */ private leavings = 0; /** @@ -658,7 +662,7 @@ export class SoundLibrary implements AudioSink { */ async outputTap(): Promise<{ node: AudioWorkletNode; rate: number; release(): void } | null> { const context = this.audio(); - if (!(context instanceof AudioContext) || !(await this.fxWorklet())) return null; + if ((!this.render && !(context instanceof AudioContext)) || !(await this.fxWorklet())) return null; const guard = this.guard; if (!guard) return null; const node = new AudioWorkletNode(context, 'record-tap', { @@ -876,7 +880,7 @@ export class SoundLibrary implements AudioSink { } this.mix = buses; this.rerouteMedia(); - if (this.context) await this.applyMix(); + if (this.context) await this.settle(this.applyMix()); } /** Every video heard into the bus it goes to now: along the routing of its file, or its pick. */ @@ -1677,6 +1681,46 @@ export class SoundLibrary implements AudioSink { this.stop(''); } + /** + * Whether nothing it was asked to play waits for its bytes, the worklet, or the chains of its + * buses: an offline context that renders now leaves nothing out (#423). + */ + settled(): boolean { + return this.settling === 0 && this.pending.length === 0 && this.loading.size === 0; + } + + private async settle(work: Promise): Promise { + this.settling++; + try { + return await work; + } finally { + this.settling--; + } + } + + /** + * The sound of a video from its decoded file (#423), as `mediaInput` hears its element: into the + * bus its sprite `picked` or else the one the routing has for `file`, from `at` on the audio clock, + * `offset` seconds into it, at `level`. Stopping the source ends it. + */ + mediaBuffer( + buffer: AudioBuffer, + file: string, + picked: string | undefined, + at: number, + offset: number, + level: number, + ): { source: AudioBufferSourceNode; gain: GainNode } { + const context = this.audio(); + const gain = context.createGain(); + gain.gain.value = level; + const source = context.createBufferSource(); + source.buffer = buffer; + source.connect(gain).connect(this.busInput(this.mediaBus(file, true, picked))); + source.start(at, Math.max(0, offset)); + return { source, gain }; + } + /** * A pause holds the sounds where they are; resuming carries them on mid-note. Until then nothing * wakes the clock – a pattern or song reading it every few milliseconds would undo the pause. @@ -2719,7 +2763,7 @@ export class SoundLibrary implements AudioSink { return context; } - private guardOut(context: AudioContext): void { + private guardOut(context: BaseAudioContext): void { const guard: NonNullable = { input: context.createGain(), limiter: null, @@ -2854,6 +2898,8 @@ export interface OfflineRender { context: OfflineAudioContext; files: SoundLibrary; wake(pump: () => void): () => void; + /** The number its first play is given, so its voices are told apart from the live ones' (#423). */ + firstId?: number; } /** Seconds a bake fades out over where its notes would have rung on (#340). */ diff --git a/src/viewport/SteadySound.ts b/src/viewport/SteadySound.ts new file mode 100644 index 00000000..6a0f0366 --- /dev/null +++ b/src/viewport/SteadySound.ts @@ -0,0 +1,185 @@ +import { STEADY_MOST_SECONDS } from '../core/recording/recording'; +import { DIVERTED_IDS } from '../runtime/audioSwitch'; +import { SoundLibrary, soundLibrary } from './SoundLibrary'; + +/** The rate the sound of a recording not in real time is made at: Opus's own. */ +const RATE = 48000; +/** Room behind the most a recording holds, for the last piece the tap still holds back. */ +const ROOM_SECONDS = 1; +/** How far a video's sound may lie off its picture before it is started again where that is. */ +const VIDEO_SLIP = 0.02; +/** How far from 1x a video may run and still be heard, as in a game (`followVideo`). */ +const HEARD_SPEED = 0.1; +/** Seconds a video's sound takes to glide to a new level, and to fall silent. */ +const LEVEL_GLIDE = 0.015; +const VIDEO_FADE = 0.01; + +/** The sound of a video the game view shows this frame (#423). */ +export interface HeardVideo { + /** Its player, one sound each. */ + key: string; + /** The video file, where its sound is decoded from. */ + file: string; + /** Seconds into the file its picture is at. */ + time: number; + /** How loud, 0–1: the video's volume and its sprite's Video volume. */ + level: number; + /** The bus its sprite picked, where it picked one. */ + bus?: string; +} + +/** A video's sound playing: from `at` on the offline clock, `offset` seconds into its file. */ +interface VideoVoice { + source: AudioBufferSourceNode; + gain: GainNode; + at: number; + offset: number; + level: number; +} + +/** + * The sound of a recording not in real time (#423): a library of its own on an offline context, + * the same code a game is heard through, which the run's sound is diverted to (`AudioSwitch`). The + * recording asks it to make the sound up to each next frame (`renderTo`) and the run waits for + * that, so the sound is in step with the picture however slowly the frames come. Videos are heard + * from their decoded files, since their elements play in real time. + */ +export class SteadySound { + readonly library: SoundLibrary; + private readonly context: OfflineAudioContext; + private readonly pumps = new Set<() => void>(); + private started = false; + private rendering = false; + private closed = false; + /** Seconds asked for so far, and those made: a suspend lands on the next block of 128. */ + private asked = 0; + private madeTo = 0; + /** The sound of each video file, null for one without; missing while it is decoded. */ + private readonly files = new Map(); + private readonly decoding = new Set(); + private readonly voices = new Map(); + /** Where each video was asked to be last, to tell a video at 1x from one slowed or jumping. */ + private readonly lastAsked = new Map(); + + constructor(private readonly url: (file: string) => string) { + this.context = new OfflineAudioContext(2, (STEADY_MOST_SECONDS + ROOM_SECONDS) * RATE, RATE); + this.library = new SoundLibrary(true, { + context: this.context, + files: soundLibrary, + wake: (pump) => { + this.pumps.add(pump); + return () => this.pumps.delete(pump); + }, + firstId: DIVERTED_IDS, + }); + } + + /** What leaves its limiter, in pieces of both sides, as `SoundLibrary.outputTap` hands out. */ + tap(): ReturnType { + return this.library.outputTap(); + } + + /** Seconds of sound made so far, from the first frame. */ + get made(): number { + return this.madeTo; + } + + /** Whether nothing it was asked to play still waits for its bytes. */ + get settled(): boolean { + return this.decoding.size === 0 && this.library.settled(); + } + + /** Makes the sound up to `seconds`, a little further where a block of 128 ends after it. */ + renderTo(seconds: number): void { + if (this.closed || seconds <= Math.max(this.asked, this.madeTo)) return; + this.asked = Math.min(seconds, STEADY_MOST_SECONDS + ROOM_SECONDS / 2); + if (!this.rendering) this.render(); + } + + private render(): void { + const target = this.asked; + this.rendering = true; + void this.context.suspend(target).then(() => { + this.rendering = false; + this.madeTo = Math.max(this.madeTo, this.context.currentTime); + if (this.closed) return; + for (const pump of this.pumps) pump(); + if (this.asked > this.madeTo) this.render(); + }); + if (this.started) void this.context.resume(); + else { + this.started = true; + void this.context.startRendering().catch(() => {}); + } + } + + /** + * The sound of the videos shown in the frame of `at` seconds, heard from their files: one that + * plays on at 1x goes on, one that jumped starts again where its picture is, one slowed, sped up + * or gone falls silent. A file not decoded yet is decoded, and `settled` waits for it. + */ + hearVideos(videos: readonly HeardVideo[], at: number): void { + const now = this.context.currentTime; + const seen = new Set(); + for (const video of videos) { + seen.add(video.key); + const last = this.lastAsked.get(video.key); + this.lastAsked.set(video.key, { time: video.time, at }); + const speed = last && at > last.at ? (video.time - last.time) / (at - last.at) : 1; + const buffer = this.files.get(video.file); + if (buffer === undefined) this.decode(video.file); + const voice = this.voices.get(video.key); + if (!buffer || Math.abs(speed - 1) > HEARD_SPEED) { + if (voice) this.end(video.key, voice, now); + continue; + } + const offset = video.time + (now - at); + if (voice && Math.abs(voice.offset + (now - voice.at) - offset) <= VIDEO_SLIP) { + if (voice.level !== video.level) voice.gain.gain.setTargetAtTime(video.level, now, LEVEL_GLIDE); + voice.level = video.level; + continue; + } + if (voice) this.end(video.key, voice, now); + const { source, gain } = this.library.mediaBuffer( + buffer, + video.file, + video.bus, + now, + offset, + video.level, + ); + this.voices.set(video.key, { source, gain, at: now, offset, level: video.level }); + } + for (const [key, voice] of this.voices) if (!seen.has(key)) this.end(key, voice, now); + for (const key of this.lastAsked.keys()) if (!seen.has(key)) this.lastAsked.delete(key); + } + + private end(key: string, voice: VideoVoice, now: number): void { + voice.gain.gain.setTargetAtTime(0, now, VIDEO_FADE / 3); + voice.source.stop(now + VIDEO_FADE); + this.voices.delete(key); + } + + private decode(file: string): void { + if (this.decoding.has(file)) return; + this.decoding.add(file); + void fetch(this.url(file)) + .then((response) => + response.ok ? response.arrayBuffer() : Promise.reject(new Error(`HTTP ${response.status}`)), + ) + .then((bytes) => this.context.decodeAudioData(bytes)) + .then( + (buffer) => this.files.set(file, buffer), + // A video without sound has nothing to decode. + () => this.files.set(file, null), + ) + .finally(() => this.decoding.delete(file)); + } + + /** Nothing more is made; the context stays suspended where it is until it is let go. */ + close(): void { + this.closed = true; + this.pumps.clear(); + this.library.stopAll(); + } +} diff --git a/src/viewport/VideoLibrary.ts b/src/viewport/VideoLibrary.ts index b739f68f..68cca560 100644 --- a/src/viewport/VideoLibrary.ts +++ b/src/viewport/VideoLibrary.ts @@ -25,7 +25,16 @@ import { import { videoDrivenOf, videoLevelOf, videoTimeOf } from '../core/properties/videos'; import { FileLibrary } from './FileLibrary'; import { soundLibrary } from './SoundLibrary'; -import { followVideo, readSpeed, type SpeedReading } from './videoFollow'; +import type { HeardVideo } from './SteadySound'; +import { + followVideo, + PICTURE_NUDGE, + pictureShown, + readSpeed, + type ShownPicture, + showsPictureOf, + type SpeedReading, +} from './videoFollow'; import { chaseSeeks, seekVideo, type SeekChase } from './videoSeek'; const VIDEO_BASE = dataPath('videos'); @@ -74,6 +83,8 @@ interface Drive extends SeekChase { /** How loud the wearer hears it, 0–1, on top of the video's own volume: its Video volume. */ level: number; reading: SpeedReading | null; + /** The picture it was held at last, as Chrome handed it over; null while it plays. */ + shown: ShownPicture | null; } interface Player { @@ -99,6 +110,8 @@ export class VideoLibrary { private readonly listeners = new Set<() => void>(); private heard = false; private paused = false; + /** Driven players hold at every picture asked rather than play: a recording takes each (#423). */ + private stepped = false; private sweeper: ReturnType | null = null; private husher: ReturnType | null = null; /** When the page was last shown, by `performance.now()`; see `sweep`. */ @@ -207,21 +220,54 @@ export class VideoLibrary { return [...this.players.values()].filter((player) => !player.drive); } + /** + * While a recording not in real time goes (#423), every driven player holds at the picture its + * wearer asks for and seeks to the next: the frames come slower than it would play, and each has + * to show its own picture. Their sound is the recording's business then. + */ + setStepped(stepped: boolean): void { + this.stepped = stepped; + } + + /** What a driven player would let be heard in a game now (#423); null for one that is not. */ + heardOf(key: string): HeardVideo | null { + const player = this.players.get(key); + const drive = player?.drive; + if (!player || !drive || player.gone || drive.heard !== 'game') return null; + const { play, element } = player; + return { + key, + file: play.file, + time: drivenTimeOf(play, drive.time, element.duration), + level: play.volume * drive.level, + ...(play.bus !== undefined && { bus: play.bus }), + }; + } + private follow(player: Player, drive: Drive): void { const { element, play } = player; const step = followVideo({ target: drivenTimeOf(play, drive.time, element.duration), - speed: drive.reading?.speed ?? 0, + speed: this.stepped ? 0 : (drive.reading?.speed ?? 0), current: element.currentTime, // A video that ran to the end of its file starts over from its first picture on play(). end: element.ended ? element.currentTime : Math.min(play.end, element.duration), }); - if (step.seek !== null) seekVideo(element, drive, step.seek); + // Held frame by frame for a recording (#423), a seek within the picture shown brings no other. + const shown = + this.stepped && + step.seek !== null && + element.paused && + !drive.landing && + showsPictureOf(drive.shown, step.seek); + if (step.seek !== null && !shown) + seekVideo(element, drive, this.stepped ? step.seek + PICTURE_NUDGE : step.seek); if (step.kind === 'hold') { element.muted = true; if (!element.paused) element.pause(); return; } + drive.shown = null; if (Math.abs(element.playbackRate - step.rate) > 0.001) element.playbackRate = step.rate; if (step.heard && drive.heard !== 'none') this.hear(player, drive.heard === 'game'); else element.muted = true; @@ -315,7 +361,7 @@ export class VideoLibrary { const play = parseVideoKey(key); if (!play) return null; const drive: Drive | null = isDrivenVideoKey(key) - ? { time, heard: 'none', level: 1, reading: null, wanted: null, landing: false } + ? { time, heard: 'none', level: 1, reading: null, wanted: null, landing: false, shown: null } : null; const at = drive ? drivenTimeOf(play, time, NaN) : play.start; const player: Player = { @@ -344,7 +390,10 @@ export class VideoLibrary { player.element, drive, () => player.gone, - () => player.texture?.source.update(), + (shown) => { + player.texture?.source.update(); + if (shown !== null && player.element.paused) drive.shown = pictureShown(drive.shown, shown); + }, ); else { this.watch(player); diff --git a/src/viewport/WorldViewport.ts b/src/viewport/WorldViewport.ts index 4abc4905..f759ef68 100644 --- a/src/viewport/WorldViewport.ts +++ b/src/viewport/WorldViewport.ts @@ -36,6 +36,7 @@ import { snapPlacement } from './pixelGrid'; import { cameraSees } from '../core/model/renderLayers'; import { drivenVideoKey, isVideoKey } from '../core/model/video'; import { videoDrivenOf } from '../core/properties/videos'; +import type { HeardVideo } from './SteadySound'; import { type VideoHearing, videoLibrary } from './VideoLibrary'; /** How strongly the cut-off part of an object shows while the scene view opens its clip area (#260). */ @@ -965,6 +966,21 @@ export abstract class WorldViewport extends W return this.videosLeftOut.has(objectId); } + /** + * Whether every driven video (#404) the last redraw showed has the picture asked of it now, not + * still the one it seeks away from: a recording not in real time waits for that (#423). + */ + videosSettled(): boolean { + for (const key of this.videosShown.values()) if (!videoLibrary.showsAsked(key)) return false; + return true; + } + + /** The driven videos the last redraw showed and lets be heard, as a game hears them (#423). */ + heardVideos(): HeardVideo[] { + if (this.hearsVideo() !== 'game') return []; + return [...this.videosShown.values()].flatMap((key) => videoLibrary.heardOf(key) ?? []); + } + /** Whether a driven video (#404) drawn here may be heard: in a run, and in the animation preview. */ protected hearsVideo(): VideoHearing { return 'none'; diff --git a/src/viewport/videoFollow.ts b/src/viewport/videoFollow.ts index 902a6f26..949cc302 100644 --- a/src/viewport/videoFollow.ts +++ b/src/viewport/videoFollow.ts @@ -97,6 +97,43 @@ export function followVideo({ target, speed, current, end }: FollowState): Follo return { kind: 'play', rate: speed * (1 + nudge), seek: null, heard: heardAt(speed) }; } +/** + * The picture a held video shows (#423): where it begins in the file, and how long a picture of + * the video lasts, as learnt from the pictures shown so far; null while not known. + */ +export interface ShownPicture { + at: number; + span: number | null; +} + +/** + * The picture shown once Chrome hands over the one at `at`: the shortest step between two + * pictures seen is how long one lasts. A seek that steps one picture on teaches it at once. + */ +export function pictureShown(before: ShownPicture | null, at: number): ShownPicture { + const step = before ? Math.abs(at - before.at) : 0; + const span = before?.span ?? null; + return { at, span: step > 1e-6 && (span === null || step < span) ? step : span }; +} + +/** + * Whether a held video shows the picture of `target` already, so a seek there would bring no + * other: Chrome hands over no frame for such a seek, and the frame of a recording waited for one. + * Where the next picture begins belongs to that one, see `PICTURE_NUDGE`. + */ +export function showsPictureOf(shown: ShownPicture | null, target: number): boolean { + if (!shown || shown.span === null) return false; + return target >= shown.at - 1e-6 && target < shown.at + shown.span - 1e-6; +} + +/** + * How far behind the time asked a video held frame by frame is sought (#423). Right onto where a + * picture begins Chrome showed it or the one before, as its time came out in microseconds: in a + * video of 30 fps the one at 3/30 s, but not the one at 2/30 s. Half a millisecond on it is always + * that picture, and still far from the next. + */ +export const PICTURE_NUDGE = 0.0005; + function heardAt(speed: number): boolean { return Math.abs(speed - 1) <= HEARD_SPEED; } diff --git a/src/viewport/videoSeek.ts b/src/viewport/videoSeek.ts index c92572b8..ec9d4e66 100644 --- a/src/viewport/videoSeek.ts +++ b/src/viewport/videoSeek.ts @@ -36,16 +36,29 @@ export function seekVideo(element: HTMLVideoElement, chase: SeekChase, time: num * Once a seek is done, the newest one asked for meanwhile follows. When none does, the seek lands * with the next frame Chrome hands over: at `seeked` the picture was not there yet, and Pixi, which * loads a paused video's picture only then, kept the one before it for a frame. `landed` is where - * the caller loads it again. Chrome hands over no frame for some seeks, though, and a player waiting - * for one stayed empty until the next jump; after `LAND_MS` it lands without. + * the caller loads it again, told the time of that picture in the file. Chrome hands over no frame + * for some seeks, though – one within the picture shown, for one – and a player waiting for one + * stayed empty until the next jump; after `LAND_MS` it lands without. */ export function chaseSeeks( element: HTMLVideoElement, chase: SeekChase, gone: () => boolean, - landed: () => void, + landed: (shown: number | null) => void, ): void { let round = 0; + // A held video may hand over the picture of a seek before `seeked`: a callback asked for only + // then waited for a picture that had come already. One asked for when the seek begins catches it. + let early: number | null = null; + let seeks = 0; + element.addEventListener('seeking', () => { + early = null; + if (!element.paused) return; + const seek = ++seeks; + element.requestVideoFrameCallback((_now, frame) => { + if (seek === seeks && element.paused) early = frame.mediaTime; + }); + }); element.addEventListener('seeked', () => { if (gone()) return; const wanted = chase.wanted; @@ -56,12 +69,13 @@ export function chaseSeeks( } if (!chase.landing) return; const now = ++round; - const land = () => { + const land = (shown: number | null) => { if (gone() || now !== round || element.seeking || !chase.landing) return; - landed(); + landed(shown); chase.landing = false; }; - element.requestVideoFrameCallback(land); - setTimeout(land, LAND_MS); + if (early !== null) return land(early); + element.requestVideoFrameCallback((_now, frame) => land(frame.mediaTime)); + setTimeout(() => land(null), LAND_MS); }); } diff --git a/tests/core/audio-switch.test.ts b/tests/core/audio-switch.test.ts new file mode 100644 index 00000000..72daacab --- /dev/null +++ b/tests/core/audio-switch.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from 'vitest'; +import { AudioSwitch, DIVERTED_IDS } from '../../src/runtime/audioSwitch'; +import { type AudioSink, SILENT_AUDIO } from '../../src/runtime/systems/Audio'; + +/** A sink that numbers its plays from `first` and writes down every call, with its clock at `clock`. */ +function sinkFrom(first: number, clock = 0) { + const calls: string[] = []; + let next = first; + const sink: AudioSink & { calls: string[]; clockAt: number } = { + ...SILENT_AUDIO, + calls, + clockAt: clock, + play: (file) => (calls.push(`play ${file}`), next++), + playSequence: (name) => (calls.push(`sequence ${name}`), next++), + stopVoice: (voice) => void calls.push(`stopVoice ${voice}`), + stopPlayback: (id) => void calls.push(`stopPlayback ${id}`), + tune: (voice) => void calls.push(`tune ${voice}`), + stop: (file) => void calls.push(`stop ${file}`), + stopOwner: (owner) => void calls.push(`stopOwner ${owner}`), + setMix: (buses) => void calls.push(`setMix ${buses.map((bus) => bus.name).join(',')}`), + setMixVolume: (bus, volume) => void calls.push(`setMixVolume ${bus} ${volume}`), + resetMix: () => void calls.push('resetMix'), + length: () => 2, + clock: () => sink.clockAt, + }; + return sink; +} + +describe('AudioSwitch (#423)', () => { + it('plays live until it is diverted, and reaches each voice in the sink that numbered it', () => { + const live = sinkFrom(1); + const aside = sinkFrom(DIVERTED_IDS); + const audio = new AudioSwitch(live); + const before = audio.play('music', 1, true, ''); + audio.divert(aside); + const after = audio.play('hit', 1, false, ''); + expect(before).toBe(1); + expect(after).toBe(DIVERTED_IDS); + audio.tune(before, {}); + audio.stopVoice(after); + expect(live.calls).toEqual(['play music', 'tune 1']); + expect(aside.calls).toEqual(['play hit', `stopVoice ${DIVERTED_IDS}`]); + audio.restore(); + expect(audio.play('again', 1, false, '')).toBe(2); + // One of the diverted sink, gone since, reaches nobody. + audio.stopVoice(after); + expect(aside.calls).toHaveLength(2); + }); + + it('stops by name, owner and everything in both, and hands the mix as it stands to the one it diverts to', () => { + const live = sinkFrom(1); + const aside = sinkFrom(DIVERTED_IDS); + const audio = new AudioSwitch(live); + audio.setMix([{ name: 'mix', volume: 1, into: null, fx: null }]); + audio.setMixVolume('mix:music', 0.5); + audio.divert(aside); + expect(aside.calls).toEqual(['setMix mix', 'setMixVolume mix:music 0.5']); + audio.stop(''); + audio.stopOwner('o1'); + expect(live.calls.slice(-2)).toEqual(['stop ', 'stopOwner o1']); + expect(aside.calls.slice(-2)).toEqual(['stop ', 'stopOwner o1']); + // The live sink knows every file, a streamed one too. + expect(audio.length('long')).toBe(2); + }); + + it('keeps time by the sink it plays to', () => { + const live = sinkFrom(1, 5); + const aside = sinkFrom(DIVERTED_IDS, 0.25); + const audio = new AudioSwitch(live); + const music = audio.playSequence('composition:c', {} as never, ''); + expect(audio.clock()).toBe(5); + audio.divert(aside); + expect(audio.clock()).toBe(0.25); + // A composition that meets its picture again stops its live playing and starts on the other. + audio.stopPlayback(music, 0.02); + audio.playSequence('composition:c', {} as never, ''); + expect(live.calls).toEqual(['sequence composition:c', 'stopPlayback 1']); + expect(aside.calls).toEqual(['sequence composition:c']); + }); +}); diff --git a/tests/core/node-catalog.test.ts b/tests/core/node-catalog.test.ts index 894048c9..6d67c00d 100644 --- a/tests/core/node-catalog.test.ts +++ b/tests/core/node-catalog.test.ts @@ -2431,7 +2431,15 @@ const CASES: Record = { next: 'exec', after: (ctx) => expect(ctx.recordings).toEqual([ - { name: 'Take', resolution: '1080p', fps: 60, sound: true, convert: true, adopt: true }, + { + name: 'Take', + resolution: '1080p', + fps: 60, + sound: true, + convert: true, + adopt: true, + realTime: true, + }, ]), }, ], diff --git a/tests/core/recording.test.ts b/tests/core/recording.test.ts index 6a637459..6989fc5b 100644 --- a/tests/core/recording.test.ts +++ b/tests/core/recording.test.ts @@ -24,6 +24,7 @@ import { recordingName, recordingSettingsOf, recordingSize, + SteadyPace, } from '../../src/core/recording/recording'; import { requireNodeDef } from '../../src/core/commands/require'; import { runDef, fakeContext } from '../helpers'; @@ -44,6 +45,7 @@ describe('recording settings (#405)', () => { sound: false, convert: true, adopt: false, + realTime: true, }); // The flags left out are on, as in "Start recording…". expect(recordingSettingsOf('', 'huge', 25)).toEqual({ @@ -53,7 +55,9 @@ describe('recording settings (#405)', () => { sound: true, convert: true, adopt: true, + realTime: true, }); + expect(recordingSettingsOf('', '4k', '60', { realTime: false }).realTime).toBe(false); }); it('records the game view at its size on screen, even on both sides', () => { @@ -246,6 +250,52 @@ describe('FramePace', () => { }); }); +describe('SteadyPace (#423)', () => { + it('moves the run one frame at a time, each frame timed by its number', () => { + const pace = new SteadyPace(60); + const times: number[] = []; + for (let frame = 0; frame < 120; frame++) { + expect(pace.step(true)).toBeCloseTo(1 / 60, 12); + times.push(pace.ms); + pace.soundMade(pace.soundTo); + pace.took(); + } + expect(pace.frames).toBe(120); + expect(pace.seconds).toBe(2); + times.forEach((ms, i) => expect(ms).toBeCloseTo((i * 1000) / 60, 9)); + }); + + it('holds the run while a frame is owed, its sound not made, or the host is not ready', () => { + const pace = new SteadyPace(30); + expect(pace.step(false)).toBe(0); + expect(pace.step(true)).toBeCloseTo(1 / 30, 12); + expect(pace.owes).toBe(true); + // The frame drawn is not taken yet. + expect(pace.step(true)).toBe(0); + expect(pace.soundTo).toBeCloseTo(1 / 30, 12); + pace.took(); + expect(pace.owes).toBe(false); + // Its sound is not made up to the next frame yet. + expect(pace.step(true)).toBe(0); + pace.soundMade(0.02); + expect(pace.step(true)).toBe(0); + // A suspend lands a little after the frame, on the next block of 128. + pace.soundMade(1664 / 48000); + expect(pace.step(true)).toBeCloseTo(1 / 30, 12); + expect(pace.ms).toBeCloseTo(1000 / 30, 9); + }); + + it('takes a frame owed only once', () => { + const pace = new SteadyPace(60); + pace.took(); + expect(pace.frames).toBe(0); + pace.step(true); + pace.took(); + pace.took(); + expect(pace.frames).toBe(1); + }); +}); + describe('pictureTransform', () => { it('maps the picture onto the whole frame', () => { // A picture of 640x360 CSS pixels centred at (400, 300), recorded at 1080p. @@ -264,7 +314,7 @@ describe('the recording nodes', () => { const result = runDef('StartRecording', ctx, { name: 'Intro', resolution: '720p', fps: '30' }, {}); expect(result.next).toBe('exec'); expect(ctx.recordings).toEqual([ - { name: 'Intro', resolution: '720p', fps: 30, sound: true, convert: true, adopt: true }, + { name: 'Intro', resolution: '720p', fps: 30, sound: true, convert: true, adopt: true, realTime: true }, ]); }); @@ -272,15 +322,15 @@ describe('the recording nodes', () => { const ctx = fakeContext(); runDef('StartRecording', ctx, {}, {}); expect(ctx.recordings).toEqual([ - { name: '', resolution: 'view', fps: 60, sound: true, convert: true, adopt: true }, + { name: '', resolution: 'view', fps: 60, sound: true, convert: true, adopt: true, realTime: true }, ]); }); it('Start recording hands its flags over', () => { const ctx = fakeContext(); - runDef('StartRecording', ctx, { sound: false, convert: true, adopt: false }, {}); + runDef('StartRecording', ctx, { sound: false, convert: true, adopt: false, realTime: false }, {}); expect(ctx.recordings).toEqual([ - { name: '', resolution: 'view', fps: 60, sound: false, convert: true, adopt: false }, + { name: '', resolution: 'view', fps: 60, sound: false, convert: true, adopt: false, realTime: false }, ]); }); @@ -292,6 +342,7 @@ describe('the recording nodes', () => { ['Sound', true], ['Convert', true], ['Import after conversion', true], + ['Real time', true], ]); }); diff --git a/tests/viewport/video-follow.test.ts b/tests/viewport/video-follow.test.ts index 0db73327..67b9ad32 100644 --- a/tests/viewport/video-follow.test.ts +++ b/tests/viewport/video-follow.test.ts @@ -1,8 +1,11 @@ import { describe, expect, it } from 'vitest'; import { followVideo, + PICTURE_NUDGE, + pictureShown, readSpeed, SEEK_DRIFT, + showsPictureOf, SPEED_SPAN_MS, SPEED_WINDOW_MS, } from '../../src/viewport/videoFollow'; @@ -127,3 +130,35 @@ describe('following the asked time', () => { }); }); }); + +describe('the picture a held video shows (#423)', () => { + it('learns how long a picture lasts from the pictures Chrome hands over', () => { + let shown = pictureShown(null, 0); + expect(shown).toEqual({ at: 0, span: null }); + shown = pictureShown(shown, 0); + expect(shown.span).toBeNull(); + shown = pictureShown(shown, 2 / 30); + expect(shown.span).toBeCloseTo(2 / 30, 12); + shown = pictureShown(shown, 3 / 30); + expect(shown.span).toBeCloseTo(1 / 30, 12); + // A jump back or far on teaches nothing shorter. + shown = pictureShown(shown, 1); + expect(shown).toEqual({ at: 1, span: shown.span }); + expect(shown.span).toBeCloseTo(1 / 30, 12); + }); + + it('knows a seek within the picture shown brings no other; where the next begins is the next', () => { + // As measured: frames of a video at 30 fps sought at 60. + const shown = { at: 1 / 30, span: 1 / 30 }; + expect(showsPictureOf(shown, 1 / 30)).toBe(true); + expect(showsPictureOf(shown, 1.5 / 30)).toBe(true); + expect(showsPictureOf(shown, 2 / 30 - 0.0005)).toBe(true); + expect(showsPictureOf(shown, 4 / 60)).toBe(false); + expect(showsPictureOf(shown, 5 / 60)).toBe(false); + expect(showsPictureOf(shown, 0.02)).toBe(false); + expect(showsPictureOf({ at: 1 / 30, span: null }, 1.5 / 30)).toBe(false); + expect(showsPictureOf(null, 0)).toBe(false); + // A seek half a millisecond on stays within its picture, the shortest a video has (60 fps). + expect(showsPictureOf({ at: 0.5, span: 1 / 60 }, 0.5 + PICTURE_NUDGE)).toBe(true); + }); +}); diff --git a/tests/viewport/video-seek.test.ts b/tests/viewport/video-seek.test.ts index e2f10798..d5895002 100644 --- a/tests/viewport/video-seek.test.ts +++ b/tests/viewport/video-seek.test.ts @@ -23,7 +23,8 @@ function fakeVideo() { }, addEventListener: (type: string, listener: () => void) => events.set(type, [...(events.get(type) ?? []), listener]), - requestVideoFrameCallback: (callback: () => void) => frames.push(callback), + requestVideoFrameCallback: (callback: (now: number, frame: { mediaTime: number }) => void) => + frames.push(() => callback(0, { mediaTime: element.currentTime })), }; const seekDone = () => { element.seeking = false; @@ -59,11 +60,15 @@ describe('seeking while scrubbing', () => { const chase = seekChase(); const video = element as unknown as HTMLVideoElement; let landed = 0; + const shown: (number | null)[] = []; chaseSeeks( video, chase, () => false, - () => (landed += 1), + (at) => { + landed += 1; + shown.push(at); + }, ); seekVideo(video, chase, 0.4); seekDone(); @@ -75,5 +80,7 @@ describe('seeking while scrubbing', () => { seekDone(); vi.advanceTimersByTime(200); expect([chase.landing, landed]).toEqual([false, 2]); + // Told the time of the picture Chrome handed over, none where it handed over none. + expect(shown).toEqual([0.4, null]); }); }); diff --git a/tools/e2e/60-release.mjs b/tools/e2e/60-release.mjs index 6154f26a..efd29f63 100644 --- a/tools/e2e/60-release.mjs +++ b/tools/e2e/60-release.mjs @@ -254,7 +254,10 @@ export async function release(t) { screen.frame.h === CAMERA.height * scale && pixels.centre === RED && pixels.beside === BACKGROUND && - JSON.stringify(await player('log()')) === JSON.stringify(['released game started']), + // The recording started in On start says so, once its encoders are set up (77365f1). + JSON.stringify( + (await player('log()')).filter((line) => !line.startsWith('Recording "Released 60" started')), + ) === JSON.stringify(['released game started']), JSON.stringify({ size, auto, screen, pixels, log: await player('log()') }), ); await shot('60-release-playing'); diff --git a/tools/e2e/68-composition.mjs b/tools/e2e/68-composition.mjs index acfb45c2..1e7308f7 100644 --- a/tools/e2e/68-composition.mjs +++ b/tools/e2e/68-composition.mjs @@ -1,4 +1,7 @@ -import { readFileSync } from 'node:fs'; +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { uploadVideo } from '../lib/projects.mjs'; import { MAKE_OBJECT } from './lib/scaffold.mjs'; @@ -11,7 +14,8 @@ import { MAKE_OBJECT } from './lib/scaffold.mjs'; * stands until the Game tab has its first picture, for a clip that starts later the picture there, * and then the video plays at once. A recording that runs meanwhile takes no frame. When the item * ends, its video stops and goes quiet with it. A sound item is planned once the clock runs, timed - * to the cut. + * to the cut. Recorded not in real time from a slowed editor, the file still has every frame, the + * video's picture at each, and its sound and the sound item's in step. * * Everything it adds carries a `cp68` id and goes again at the end. */ @@ -441,6 +445,191 @@ export async function composition(t) { JSON.stringify(routes), ); + // 68j (#423). Recorded not in real time while the editor draws a frame every 45 ms, about 20 a + // second: the file still has every frame, 1/60 s apart. The video item covers the picture, + // its colours turn every half second of its file and so every 30 frames of the recording, and + // its sound is heard from the start. The blip item starts as green does. + await mutate(` + const p = __gm.state.project; + p.assets.find((a) => a.id === 'cp68-screen').props = { width: 4000, height: 3000, color: 0xffffff, texture: 'cp68-video' }; + const c = p.assets.find((a) => a.id === 'cp68-cut'); + c.items = [ + { id: 'cp68-v', track: 'cp68-front', start: 0, length: 120, offset: 0 }, + { id: 'cp68-tone', track: 'cp68-back', start: 30, length: 30, offset: 0, + audio: { kind: 'sound', asset: 'cp68-blip', volume: 0, bus: '#composition' } }, + ]; + p.assets.find((a) => a.id === 'cp68-director').graph = ${JSON.stringify({ + nodes: [ + node('start', 'OnStart', 0, 0), + node('owner', 'Owner', 0, 150), + node('record', 'StartRecording', 300, 0, { + name: 'cp68 steady', + resolution: '720p', + fps: '60', + convert: false, + adopt: false, + realTime: false, + }), + node('play', 'PlayComposition', 600, 0), + node('finished', 'OnCompositionFinished', 0, 300, { composition: 'cp68-cut' }), + node('stop', 'StopRecording', 300, 300), + ], + edges: [ + link('e1', 'start', 'exec', 'record', 'exec'), + link('e2', 'record', 'exec', 'play', 'exec'), + link('e3', 'owner', 'object', 'play', 'object'), + link('e4', 'finished', 'exec', 'stop', 'exec'), + ], + })};`); + const before = await json('__gm.recording()'); + const steadyTake = `__gm.recording().takes[${before.takes.length}]`; + await evaluate(`(() => { + window.__cp68slow = true; + const slow = () => { + if (!window.__cp68slow) return; + const until = performance.now() + 45; + while (performance.now() < until); + requestAnimationFrame(slow); + }; + requestAnimationFrame(slow); + })()`); + await play(); + await waitUntil(`${steadyTake}?.state === 'done' || ${steadyTake}?.state === 'failed'`, { + timeout: 60_000, + label: 'the recording not in real time is saved', + }); + await evaluate('window.__cp68slow = false; 0'); + await stop(); + await waitUntil( + `__gm.recording().uploads.length > ${before.uploads.length} && ['kept', 'failed'].includes(__gm.recording().uploads.at(-1).state)`, + { timeout: 30_000, label: 'the recording is uploaded' }, + ); + const steady = await json(steadyTake); + const steadyUpload = (await json('__gm.recording()')).uploads.at(-1); + const steadyLines = (await json('__gm.console.lines()')) + .map((line) => line.text) + .filter((text) => text.includes('"cp68 steady"')); + const original = await fetch( + `${t.baseUrl}data/videos/${steadyUpload.file.replace(/\.mp4$/, '.source.mkv')}`, + ); + const file = { frames: 0, gaps: [], turns: [], toneBefore: null, toneDuring: null, blipAt: null }; + if (original.ok) { + const dir = mkdtempSync(join(tmpdir(), 'cp68-')); + const mkv = join(dir, 'steady.mkv'); + writeFileSync(mkv, new Uint8Array(await original.arrayBuffer())); + const pts = execFileSync('ffprobe', [ + '-v', + 'error', + '-select_streams', + 'v:0', + '-show_entries', + 'packet=pts_time', + '-of', + 'csv=p=0', + mkv, + ]) + .toString() + .trim() + .split('\n') + .map(Number) + .sort((a, b) => a - b); + file.frames = pts.length; + file.gaps = pts.slice(1).flatMap((at, i) => (Math.abs(at - pts[i] - 1 / 60) > 0.0015 ? [i + 1] : [])); + // Each frame as 8x4 pixels: the colour of the video item covering it, or of what lies behind. + const pixels = execFileSync( + 'ffmpeg', + [ + '-v', + 'error', + '-i', + mkv, + '-map', + '0:v', + '-vf', + 'scale=8:4', + '-f', + 'rawvideo', + '-pix_fmt', + 'rgb24', + '-', + ], + { maxBuffer: 64 * 1024 * 1024 }, + ); + const colours = []; + for (let at = 0; at + 96 <= pixels.length; at += 96) { + const [r, g, b] = [0, 1, 2].map((c) => pixels[at + 13 * 3 + c]); + colours.push(r > 180 && g > 180 && b > 180 ? 'W' : r > 150 ? 'R' : g > 150 ? 'G' : b > 150 ? 'B' : '-'); + } + file.turns = colours.flatMap((colour, i) => + i === 0 || colour !== colours[i - 1] ? [`${i}${colour}`] : [], + ); + const sound = execFileSync( + 'ffmpeg', + ['-v', 'error', '-i', mkv, '-map', '0:a', '-ac', '1', '-ar', '48000', '-f', 'f32le', '-'], + { maxBuffer: 64 * 1024 * 1024 }, + ); + const samples = new Float32Array(sound.buffer, sound.byteOffset, sound.length / 4); + const rms = (from, to) => { + let sum = 0; + const a = Math.max(0, Math.round(from * 48000)); + const b = Math.min(samples.length, Math.round(to * 48000)); + for (let i = a; i < b; i++) sum += samples[i] * samples[i]; + return b > a ? Math.sqrt(sum / (b - a)) : 0; + }; + const red = Number.parseInt(file.turns.find((turn) => turn.endsWith('R')) ?? '-1', 10); + const green = Number.parseInt(file.turns.find((turn) => turn.endsWith('G')) ?? '-1', 10); + if (red >= 0 && green > red) { + file.toneBefore = red > 6 ? rms(0, (red - 3) / 60) : 0; + file.toneDuring = rms((red + 3) / 60, (green - 3) / 60); + // The blip is loud over the tone of the video: the first 2 ms that are. + for (let at = red / 60; at < (green + 30) / 60; at += 0.002) + if (rms(at, at + 0.002) > 3 * file.toneDuring + 0.05) { + file.blipAt = Math.round((at - green / 60) * 1000); + break; + } + } + rmSync(dir, { recursive: true, force: true }); + } + // The video's four colours one after the other, whatever lay behind it before and after. + const colourOrder = file.turns.map((turn) => turn.at(-1)).join(''); + const turned = file.turns + .slice(colourOrder.indexOf('RGBW'), colourOrder.indexOf('RGBW') + 4) + .map((turn) => Number.parseInt(turn, 10)); + check( + 'a recording not in real time has every frame, 1/60 s apart, though the editor drew about 20 a second', + original.ok && + steady.realTime === false && + steady.failure === null && + steady.skipped === 0 && + steady.dropped === 0 && + file.frames === steady.frames && + file.frames >= 120 && + file.gaps.length === 0 && + steady.tookMs > steady.durationMs * 1.5, + JSON.stringify({ status: original.status, steady, frames: file.frames, gaps: file.gaps.slice(0, 5) }), + ); + check( + 'its picture is the video at every frame: the colours turn exactly every 30 frames', + colourOrder.includes('RGBW') && turned.slice(1).every((at, i) => at - turned[i] === 30), + JSON.stringify(file.turns), + ); + check( + 'and its sound is in step: the video heard from its first picture, the blip as green begins', + file.toneBefore !== null && + file.toneBefore < 0.005 && + file.toneDuring > 0.02 && + file.blipAt !== null && + Math.abs(file.blipAt) <= 20 && + steadyLines.some((text) => text.endsWith('with sound, not in real time')) && + steadyLines.some((text) => text.includes('x real time, and has every frame')), + JSON.stringify({ + toneBefore: file.toneBefore, + toneDuring: file.toneDuring, + blipAtMs: file.blipAt, + steadyLines, + }), + ); + await mutate(` const p = __gm.state.project; p.assets = p.assets.filter((a) => !a.id.startsWith('cp68-') && a.id !== ${JSON.stringify(madeId)});