import { DRY_VOICE, fxChainOf, type FxChain, type FxParams, type FxStage, type FxVoice, } from '../core/soundfx/nodes'; import type { EaseShare } from '../core/model/animation'; import { pitchShift } from '../core/soundfx/pitchShift'; import type { AudioSink } from '../runtime/systems/Audio'; const DEFAULT_BASE = '/game-maker/data/sounds'; /** How many pitched copies are kept at once. */ const SHIFTED_KEPT = 32; /** Seconds a changed parameter takes to get most of the way there: quick, but no click (#315). */ const GLIDE = 0.03; /** The rate a sound loaded ahead of its first play is decoded at; a source resamples it on the way out. */ const DECODE_RATE = 48000; /** * One number of a voice that can still change while it plays (#315): a level, a share of a Mix, the * gain of a band – or the pitch of a source, which glides by speed from the pitch its copy was made at. * `even` says what a timed change runs evenly in: a share or a band in what it holds, a gain stage in * decibels (its node holds the factor), a pitch in semitones (its node holds the speed). */ type Knob = | { kind: 'level'; param: AudioParam; target: number; even: 'value' | 'ratio' } | { kind: 'pitch'; source: AudioBufferSourceNode; base: number; target: number }; /** * One sound in flight: its sources, its own gain, the file it came from and who is making it. A * voice has a source per way through its sound FX (#316), started together and as long as each * other. Its knobs are in the order `levelsOf` lists the numbers of its chain. */ interface Voice { /** What `play` returned for it, the handle a `tune` reaches it by. */ id: number; file: string; /** The behavior this voice belongs to; `''` for one nothing owns (see `AudioSink.play`). */ owner: string; sources: AudioBufferSourceNode[]; gain: GainNode; fx: FxVoice; knobs: Knob[]; } /** What one voice is wired from: the file, and what has been made for it so far. */ interface Wiring { context: AudioContext; file: string; buffer: AudioBuffer; loop: boolean; sources: { source: AudioBufferSourceNode; pitch: number }[]; knobs: Knob[]; stages: number; } /** * A play still waiting on its bytes: what a quick `tune` in the meantime writes into, and the timed * ones, which have to start with the voice rather than run out before it is heard. */ interface Pending { id: number; fx: FxVoice; timed: { params: FxParams; time: number; ease: EaseShare }[]; } /** * Plays uploaded sounds, keyed by file name – the texture library's twin on the audio side. Files * are content addressed, so a decoded buffer can never go stale and nothing is evicted. * * Everything here is asynchronous on the way in and synchronous on the way out: `play` returns at * once and starts the voice when the bytes have been fetched and decoded. A sound fired in the * first frame of a run is therefore heard a moment later rather than not at all, and a graph never * waits on the network. * * The context is created on the first play, not at start-up: a browser refuses one before the page * has been interacted with, and an editor that was only ever looked at should not be holding an * audio device open. What `preload` fetches before that is decoded by an offline context, which * holds no device. */ export class SoundLibrary implements AudioSink { private base = DEFAULT_BASE; private context: AudioContext | null = null; private offline: OfflineAudioContext | null = null; private readonly buffers = new Map(); /** Pitched copies of decoded sounds, oldest use first (see `pitched`). */ private readonly shifted = new Map(); private readonly loading = new Map>(); /** Never retried, so one broken reference cannot spam the network. */ private readonly failed = new Set(); private voices: Voice[] = []; private pending: Pending[] = []; /** The number the next play is given; 0 stays free for "nothing plays". */ private nextId = 1; private started = 0; private stopped = 0; private tuned = 0; /** The last voice actually started, with the values it was started at – what a check reads back. */ private last: LastVoice | null = null; /** * How often a stop has reached each file, and how often one has silenced everything. Counters * rather than a "cancelled" flag: what a play has to know is whether a stop happened *after it * was asked for*, and a flag cannot tell that from one that happened long before – which would * make the first play of a sound disappear whenever anything had ever stopped it. */ private readonly stops = new Map(); private allStops = 0; /** The same counter per owner, for the same reason – a behavior destroyed while bytes were in * flight must not be given its voice afterwards. */ private readonly ownerStops = new Map(); /** Where the files are served from. Set once at start-up, before anything plays. */ configure(base: string): void { this.base = base.replace(/\/+$/, ''); } /** * The number is handed out at once, before the bytes are there: a graph can tune a voice in the * same frame it asked for it, and the voice then starts with those values. */ play(file: string, volume: number, loop: boolean, owner = '', fx: FxVoice = DRY_VOICE): number { if (file === '' || this.failed.has(file)) return 0; const id = this.nextId++; const buffer = this.buffers.get(file); if (buffer) { this.start(id, file, buffer, volume, loop, owner, fx); return id; } const askedAt = this.cancelsFor(file, owner); const waiting: Pending = { id, fx: { plan: fx.plan, params: { ...fx.params } }, timed: [] }; this.pending.push(waiting); void this.load(file).then((loaded) => { this.pending = this.pending.filter((other) => other !== waiting); // A `Stop sound` – or the destruction of the behavior that asked – that happened while the // bytes were in flight has to win, or a sound the run already ended would start afterwards. if (!loaded || this.cancelsFor(file, owner) !== askedAt) return; this.start(id, file, loaded, volume, loop, owner, waiting.fx); for (const { params, time, ease } of waiting.timed) this.tune(id, params, time, ease); }); return id; } /** Says whether the voice was still there to be reached – playing, or waiting on its bytes. */ tune(id: number, params: FxParams, time = 0, ease: EaseShare = 'linear'): boolean { const waiting = this.pending.find((other) => other.id === id); if (waiting) { // A quick change is where the voice starts; a timed one – and anything after it, so the order // holds – runs from the moment it is heard. if (time > 0 || waiting.timed.length > 0) waiting.timed.push({ params, time, ease }); else waiting.fx = { plan: waiting.fx.plan, params: { ...waiting.fx.params, ...params } }; return true; } const voice = this.voices.find((other) => other.id === id); if (!voice) return false; voice.fx = { plan: voice.fx.plan, params: { ...voice.fx.params, ...params } }; this.glide(voice, time, ease); this.tuned += 1; return true; } stop(file: string): void { this.silence(file === '' ? this.voices : this.voices.filter((voice) => voice.file === file)); if (file === '') this.allStops += 1; else this.stops.set(file, (this.stops.get(file) ?? 0) + 1); } /** * Every voice of one behavior. An empty owner is deliberately no one: those are the voices that * were started to outlive their behavior, so "stop the owner" must not reach them. */ stopOwner(owner: string): void { if (owner === '') return; this.silence(this.voices.filter((voice) => voice.owner === owner)); this.ownerStops.set(owner, (this.ownerStops.get(owner) ?? 0) + 1); } /** Every voice at once: what ending a run does, so nothing outlives the world that started it. */ stopAll(): void { this.stop(''); } /** A pause holds the sounds where they are; resuming carries them on mid-note. */ setSuspended(suspended: boolean): void { const context = this.context; if (!context) return; if (suspended) void context.suspend(); else void context.resume(); } /** What has been played and what is playing – the only witness a muted check script has. */ stats(): { playing: number; started: number; stopped: number; tuned: number; decoded: number; failed: number; last: LastVoice | null; } { return { playing: this.voices.length, started: this.started, stopped: this.stopped, tuned: this.tuned, decoded: this.buffers.size, failed: this.failed.size, last: this.last, }; } /** * Where the knobs of each voice of that file are right now, oldest first, a pitch as the speed it * plays at – what a check reads to see a change arrive at one voice and not another (test hook). */ levels(file: string): number[][] { return this.voices .filter((voice) => voice.file === file) .map((voice) => voice.knobs.map((knob) => (knob.kind === 'pitch' ? knob.source.playbackRate : knob.param).value), ); } /** How long a decoded sound runs, in seconds; null while it has not been loaded (test hook). */ duration(file: string): number | null { return this.buffers.get(file)?.duration ?? null; } length(file: string): number { return this.duration(file) ?? 0; } /** * Starts loading every file not held yet. Waiting for the first play left that play late: its * start was not heard, and a graph tuning it in the meantime had moved on by the time it started. */ preload(files: Iterable): void { for (const file of files) if (file) void this.load(file); } /** Fetches and decodes once per file; a second play while it is in flight waits on the first. */ async load(file: string): Promise { const held = this.buffers.get(file); if (held) return held; if (this.failed.has(file)) return null; const running = this.loading.get(file); if (running) return running; const work = (async () => { try { const response = await fetch(`${this.base}/${file}`); if (!response.ok) throw new Error(`HTTP ${response.status}`); const bytes = await response.arrayBuffer(); const buffer = await this.decoder().decodeAudioData(bytes); this.buffers.set(file, buffer); return buffer; } catch (error: unknown) { this.failed.add(file); console.error(`[sounds] cannot load ${file}`, error); return null; } finally { this.loading.delete(file); } })(); this.loading.set(file, work); return work; } /** Ends these voices now: `onended` only fires next frame and the counts have to be right here. */ private silence(going: Voice[]): void { for (const voice of going) for (const source of voice.sources) source.stop(); this.voices = this.voices.filter((voice) => !going.includes(voice)); this.stopped += going.length; } private start( id: number, file: string, buffer: AudioBuffer, volume: number, loop: boolean, owner: string, fx: FxVoice, ): void { const context = this.audio(); // A context starts suspended where the page has not been interacted with; without this the // voice would run silently and end as if it had played. if (context.state === 'suspended') void context.resume(); const gain = context.createGain(); gain.gain.value = Math.max(0, Math.min(1, volume)); const wiring: Wiring = { context, file, buffer, loop, sources: [], knobs: [], stages: 0 }; this.wire(wiring, fxChainOf(fx.plan, fx.params)).connect(gain).connect(context.destination); const sources = wiring.sources.map((made) => made.source); this.last = { file, volume: gain.gain.value, loop, pitches: wiring.sources.map((made) => made.pitch), stages: wiring.stages, duration: sources[0]!.buffer!.duration, }; const voice: Voice = { id, file, owner, sources, gain, fx, knobs: wiring.knobs }; let playing = sources.length; for (const source of sources) source.onended = () => { if (--playing === 0) this.voices = this.voices.filter((other) => other !== voice); }; this.voices.push(voice); this.started += 1; for (const source of sources) source.start(); } /** * The nodes a chain becomes, up to the one that carries its sound on. Every way has a source of * its own, so each can glide to its own pitch; a Mix weighs each of its sounds by its share before * they are summed. */ private wire(wiring: Wiring, chain: FxChain): AudioNode { const { context, knobs } = wiring; let tail: AudioNode; if ('mix' in chain) { tail = context.createGain(); const wayA = this.wire(wiring, chain.a); const shareA = gainNode(context, 1 - chain.mix); wayA.connect(shareA).connect(tail); knobs.push({ kind: 'level', param: shareA.gain, target: 1 - chain.mix, even: 'value' }); const wayB = this.wire(wiring, chain.b); const shareB = gainNode(context, chain.mix); wayB.connect(shareB).connect(tail); knobs.push({ kind: 'level', param: shareB.gain, target: chain.mix, even: 'value' }); } else tail = this.source(wiring, chain.pitch); for (const stage of chain.stages) { const [node, param] = fxNode(context, stage); tail = tail.connect(node); knobs.push({ kind: 'level', param, target: stage.gain, even: stage.kind === 'gain' ? 'ratio' : 'value', }); } wiring.stages += chain.stages.length; return tail; } private source(wiring: Wiring, pitch: number): AudioBufferSourceNode { const { context, file, buffer, loop } = wiring; const source = context.createBufferSource(); source.buffer = pitch === 0 ? buffer : this.pitched(context, file, buffer, pitch, loop); source.loop = loop; wiring.sources.push({ source, pitch }); wiring.knobs.push({ kind: 'pitch', source, base: pitch, target: pitch }); return source; } /** * Sends every knob of a voice towards what its values say now, in `time` seconds of the audio * clock along `ease`, or else in a quick glide. A pitch cannot get a new copy in the middle of a sound, so it * glides by speed: the voice gets that much shorter or longer. */ private glide(voice: Voice, time: number, ease: EaseShare): void { const levels = levelsOf(fxChainOf(voice.fx.plan, voice.fx.params)); const now = this.audio().currentTime; voice.knobs.forEach((knob, i) => { const level = levels[i]; if (level === undefined || level === knob.target) return; knob.target = level; if (knob.kind === 'pitch') move(knob.source.playbackRate, 2 ** ((level - knob.base) / 12), now, time, true, ease); else move(knob.param, level, now, time, knob.even === 'ratio', ease); }); } /** * The sound at another pitch and its own length (#312), worked out once per file, pitch and loop * and kept for the next voice. The oldest goes first, so a pitch drawn at random every play cannot * fill the memory. */ private pitched( context: AudioContext, file: string, buffer: AudioBuffer, pitch: number, loop: boolean, ): AudioBuffer { const key = `${file}\u0000${pitch}\u0000${loop}`; const held = this.shifted.get(key); if (held) { this.shifted.delete(key); this.shifted.set(key, held); return held; } const shifted = context.createBuffer(buffer.numberOfChannels, buffer.length, buffer.sampleRate); for (let channel = 0; channel < buffer.numberOfChannels; channel++) shifted .getChannelData(channel) .set(pitchShift(buffer.getChannelData(channel), buffer.sampleRate, pitch, loop)); this.shifted.set(key, shifted); if (this.shifted.size > SHIFTED_KEPT) this.shifted.delete(this.shifted.keys().next().value!); return shifted; } private audio(): AudioContext { return (this.context ??= new AudioContext()); } /** The context that is already playing, or else one that decodes without opening a device. */ private decoder(): BaseAudioContext { if (this.context || typeof OfflineAudioContext === 'undefined') return this.audio(); return (this.offline ??= new OfflineAudioContext(1, 1, DECODE_RATE)); } /** Everything that would have silenced this file for this owner, counted rather than flagged. */ private cancelsFor(file: string, owner: string): number { return (this.stops.get(file) ?? 0) + this.allStops + (this.ownerStops.get(owner) ?? 0); } } /** What `stats` says about the last voice: enough for a check to tell which effects it went through. */ interface LastVoice { file: string; volume: number; loop: boolean; /** Semitones of each source, one per way (#316), and how many gain and filter nodes it passed. */ pitches: number[]; stages: number; /** Seconds of what was played: a pitched voice is as long as the sound itself (#312). */ duration: number; } function gainNode(context: AudioContext, level: number): GainNode { const gain = context.createGain(); gain.gain.value = level; return gain; } /** The node a stage becomes, with the number of it a parameter can move. */ function fxNode(context: AudioContext, stage: FxStage): [AudioNode, AudioParam] { if (stage.kind === 'gain') { const gain = gainNode(context, stage.gain); return [gain, gain.gain]; } const filter = context.createBiquadFilter(); filter.type = stage.filter; filter.frequency.value = stage.frequency; filter.gain.value = stage.gain; filter.Q.value = stage.q; return [filter, filter.gain]; } /** * Sends one knob from where it is right now to `to`. Whatever was still planned for it is dropped * first, so a change never queues behind an older one. A timed change runs along `ease` – a `ratio` * one in its logarithm, which is what decibels and semitones are. Linear and step are the clock's * own ramps, every other easing is handed over as a curve of samples. */ function move( param: AudioParam, to: number, now: number, time: number, ratio: boolean, ease: EaseShare = 'linear', ): void { const from = param.value; // Firefox has no cancelAndHoldAtTime: there the value it reads out now is held by hand. const held = param as Omit & Partial>; if (held.cancelAndHoldAtTime) held.cancelAndHoldAtTime(now); else { param.cancelScheduledValues(now); param.setValueAtTime(from, now); } const logarithmic = ratio && from > 0 && to > 0; if (time <= 0) param.setTargetAtTime(to, now, GLIDE); else if (ease === 'step') param.setValueAtTime(to, now + time); else if (ease === 'linear' && logarithmic) param.exponentialRampToValueAtTime(to, now + time); else if (ease === 'linear') param.linearRampToValueAtTime(to, now + time); else param.setValueCurveAtTime(easedCurve(from, to, time, logarithmic, ease), now, time); } /** Samples per second of an eased curve: a bounce keeps its corners, a long sweep stays small. */ const CURVE_RATE = 200; function easedCurve( from: number, to: number, time: number, logarithmic: boolean, share: (t: number) => number, ): Float32Array { const count = Math.min(Math.max(Math.ceil(time * CURVE_RATE) + 1, 16), 4096); const curve = new Float32Array(count); for (let i = 0; i < count; i++) { const s = share(i / (count - 1)); curve[i] = logarithmic ? from * (to / from) ** s : from + (to - from) * s; } return curve; } /** The numbers of a chain in the order `wire` makes their knobs: a stage's `gain` is what its node holds. */ function levelsOf(chain: FxChain, into: number[] = []): number[] { if ('mix' in chain) { levelsOf(chain.a, into); into.push(1 - chain.mix); levelsOf(chain.b, into); into.push(chain.mix); } else into.push(chain.pitch); for (const stage of chain.stages) into.push(stage.gain); return into; } export const soundLibrary = new SoundLibrary();